Merge pull request #1784 from anyproto/release-7

Release 7 merge

core/block/source/marshall_change_test.go

@@ -45,6 +45,18 @@ func TestMarshallChange(t *testing.T) {
 	})
 }
 
+func TestNewUnmarshalTreeChange(t *testing.T) {
+	ch1, _, _ := MarshalChange(changeWithBigSnapshot())
+	ch2, _, _ := MarshalChange(changeWithBigSnapshot())
+	unmarshalF := NewUnmarshalTreeChange()
+	res1, err := unmarshalF(&objecttree.Change{DataType: dataTypeSnappy}, ch1)
+	require.NoError(t, err)
+	assert.NotNil(t, res1.(*pb.Change).Snapshot)
+	res2, err := unmarshalF(&objecttree.Change{DataType: dataTypeSnappy}, ch2)
+	require.NoError(t, err)
+	assert.Nil(t, res2.(*pb.Change).Snapshot)
+}
+
 func TestUnmarshallChange(t *testing.T) {
 	invalidDataType := "invalid"
 

core/block/source/service.go

@@ -113,9 +113,7 @@ func (b *BuildOptions) BuildTreeOpts() objecttreebuilder.BuildTreeOpts {
 			}
 			return ot, nil
 		},
-		TreeValidator: func(payload treestorage.TreeStorageCreatePayload, buildFunc objecttree.BuildObjectTreeFunc, aclList list.AclList) (retPayload treestorage.TreeStorageCreatePayload, err error) {
-			return objecttree.ValidateFilterRawTree(payload, aclList)
-		},
+		TreeValidator: objecttree.ValidateFilterRawTree,
 	}
 }
 

core/block/source/source.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"math/rand"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/anyproto/any-sync/accountservice"
@@ -75,7 +76,16 @@ func MarshalChange(change *pb.Change) (result []byte, dataType string, err error
 }
 
 func UnmarshalChange(treeChange *objecttree.Change, data []byte) (result any, err error) {
-	change := &pb.Change{}
+	return unmarshalChange(treeChange, data, true)
+}
+
+func unmarshalChange(treeChange *objecttree.Change, data []byte, needSnapshot bool) (result any, err error) {
+	var change proto.Message
+	if needSnapshot {
+		change = &pb.Change{}
+	} else {
+		change = &pb.ChangeNoSnapshot{}
+	}
 	if treeChange.DataType == dataTypeSnappy {
 		buf := bytesPool.Get().([]byte)[:0]
 		defer func() {
@@ -92,10 +102,28 @@ func UnmarshalChange(treeChange *objecttree.Change, data []byte) (result any, er
 			}
 		}
 	}
-	if err = proto.Unmarshal(data, change); err == nil {
-		result = change
+	if err = proto.Unmarshal(data, change); err != nil {
+		return
+	}
+	if needSnapshot {
+		return change, nil
+	} else {
+		noSnapshotChange := change.(*pb.ChangeNoSnapshot)
+		return &pb.Change{
+			Content:   noSnapshotChange.Content,
+			FileKeys:  noSnapshotChange.FileKeys,
+			Timestamp: noSnapshotChange.Timestamp,
+			Version:   noSnapshotChange.Version,
+		}, nil
+	}
+}
+
+// NewUnmarshalTreeChange creates UnmarshalChange func that unmarshalls snapshot only for the first change and ignores it for following. It saves some memory
+func NewUnmarshalTreeChange() objecttree.ChangeConvertFunc {
+	var changeCount atomic.Int32
+	return func(treeChange *objecttree.Change, data []byte) (result any, err error) {
+		return unmarshalChange(treeChange, data, changeCount.CompareAndSwap(0, 1))
 	}
-	return
 }
 
 func UnmarshalChangeWithDataType(dataType string, decrypted []byte) (res any, err error) {
@@ -552,7 +580,7 @@ func BuildState(spaceId string, initState *state.State, ot objecttree.ReadableOb
 		return
 	}
 	var lastMigrationVersion uint32
-	err = ot.IterateFrom(startId, UnmarshalChange,
+	err = ot.IterateFrom(startId, NewUnmarshalTreeChange(),
 		func(change *objecttree.Change) bool {
 			count++
 			lastChange = change

docs/proto.md

@@ -36,6 +36,7 @@
     - [Change.StoreSliceUpdate.Add](#anytype-Change-StoreSliceUpdate-Add)
     - [Change.StoreSliceUpdate.Move](#anytype-Change-StoreSliceUpdate-Move)
     - [Change.StoreSliceUpdate.Remove](#anytype-Change-StoreSliceUpdate-Remove)
+    - [ChangeNoSnapshot](#anytype-ChangeNoSnapshot)
     - [DocumentCreate](#anytype-DocumentCreate)
     - [DocumentDelete](#anytype-DocumentDelete)
     - [DocumentModify](#anytype-DocumentModify)
@@ -2216,9 +2217,6 @@ the element of change tree used to store and internal apply smartBlock history
 
 | Field | Type | Label | Description |
 | ----- | ---- | ----- | ----------- |
-| previous_ids | [string](#string) | repeated | ids of previous changes |
-| last_snapshot_id | [string](#string) |  | id of the last snapshot |
-| previous_meta_ids | [string](#string) | repeated | ids of the last changes with details/relations content |
 | content | [Change.Content](#anytype-Change-Content) | repeated | set of actions to apply |
 | snapshot | [Change.Snapshot](#anytype-Change-Snapshot) |  | snapshot - when not null, the Content will be ignored |
 | fileKeys | [Change.FileKeys](#anytype-Change-FileKeys) | repeated | file keys related to changes content |
@@ -2691,6 +2689,24 @@ the element of change tree used to store and internal apply smartBlock history
 
 
 
+<a name="anytype-ChangeNoSnapshot"></a>
+
+### ChangeNoSnapshot
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| content | [Change.Content](#anytype-Change-Content) | repeated | set of actions to apply |
+| fileKeys | [Change.FileKeys](#anytype-Change-FileKeys) | repeated | file keys related to changes content |
+| timestamp | [int64](#int64) |  | creation timestamp |
+| version | [uint32](#uint32) |  | version of business logic |
+
+
+
+
+
+
 <a name="anytype-DocumentCreate"></a>
 
 ### DocumentCreate

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0
 	github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786
 	github.com/anyproto/any-store v0.1.1
-	github.com/anyproto/any-sync v0.5.11
+	github.com/anyproto/any-sync v0.5.13
 	github.com/anyproto/go-chash v0.1.0
 	github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438
 	github.com/anyproto/lexid v0.0.2
@@ -62,7 +62,7 @@ require (
 	github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/klauspost/compress v1.17.11
-	github.com/libp2p/go-libp2p v0.36.5
+	github.com/libp2p/go-libp2p v0.37.0
 	github.com/libp2p/zeroconf/v2 v2.2.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/magiconair/properties v1.8.7
@@ -183,7 +183,7 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
+	github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
@@ -213,7 +213,7 @@ require (
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/miekg/dns v1.1.61 // indirect
+	github.com/miekg/dns v1.1.62 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -230,17 +230,17 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-proto-validators v0.3.2 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/onsi/ginkgo/v2 v2.19.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.20.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/common v0.60.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/pseudomuto/protokit v0.2.1 // indirect
-	github.com/quic-go/quic-go v0.47.0 // indirect
+	github.com/quic-go/quic-go v0.48.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/rs/zerolog v1.29.0 // indirect
@@ -278,7 +278,7 @@ require (
 	golang.org/x/tools v0.26.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect