GO-4239 Merge branch 'main' into go-4239-writing-tools

cmd/archiveunpacker/main.go

@@ -121,6 +121,18 @@ func handleZip(input, output string) {
 	defer r.Close()
 
 	for _, f := range r.File {
+		dir := filepath.Dir(f.Name)
+		if dir != "." {
+			// nolint: gosec
+			outputDir := filepath.Join(output, dir)
+			if _, err := os.Stat(outputDir); os.IsNotExist(err) {
+				if err := os.MkdirAll(outputDir, 0755); err != nil {
+					log.Printf("Failed to create output subdirectory: %v\n", err)
+					return
+				}
+			}
+		}
+
 		// assuming we are only working with files, not directories
 		if f.FileInfo().IsDir() {
 			continue

cmd/usecasevalidator/main.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/gogo/protobuf/jsonpb"
@@ -207,7 +208,7 @@ func collectUseCaseInfo(files []*zip.File, fileName string) (info *useCaseInfo,
 			continue
 		}
 
-		if strings.HasPrefix(f.Name, "files") {
+		if strings.HasPrefix(f.Name, "files") || f.FileInfo().IsDir() {
 			continue
 		}
 
@@ -296,14 +297,21 @@ func processFiles(files []*zip.File, zw *zip.Writer, info *useCaseInfo, flags *c
 		if err != nil {
 			return err
 		}
-		newData, err := processRawData(data, f.Name, info, flags)
-		if err != nil {
-			if !(flags.exclude && errors.Is(err, errValidationFailed)) {
-				// just do not include object that failed validation
-				incorrectFileFound = true
+
+		var newData []byte
+		if f.FileInfo().IsDir() {
+			newData = data
+		} else {
+			newData, err = processRawData(data, f.Name, info, flags)
+			if err != nil {
+				if !(flags.exclude && errors.Is(err, errValidationFailed)) {
+					// just do not include object that failed validation
+					incorrectFileFound = true
+				}
+				continue
 			}
-			continue
 		}
+
 		if newData == nil || !writeNewFile {
 			continue
 		}
@@ -464,7 +472,11 @@ func removeAccountRelatedDetails(s *pb.ChangeSnapshot) {
 			bundle.RelationKeyLinks.String(),
 			bundle.RelationKeyBacklinks.String(),
 			bundle.RelationKeyWorkspaceId.String(),
-			bundle.RelationKeyIdentityProfileLink.String():
+			bundle.RelationKeyIdentityProfileLink.String(),
+			bundle.RelationKeyAddedDate.String(),
+			bundle.RelationKeySyncDate.String(),
+			bundle.RelationKeySyncError.String(),
+			bundle.RelationKeySyncStatus.String():
 
 			delete(s.Data.Details.Fields, key)
 		}
@@ -492,7 +504,7 @@ func processProfile(data []byte, info *useCaseInfo, spaceDashboardId string) ([]
 	}
 
 	fmt.Println("spaceDashboardId = " + profile.SpaceDashboardId)
-	if _, found := info.objects[profile.SpaceDashboardId]; !found {
+	if _, found := info.objects[profile.SpaceDashboardId]; !found && !slices.Contains([]string{"lastOpened"}, profile.SpaceDashboardId) {
 		err := fmt.Errorf("failed to find Space Dashboard object '%s' among provided", profile.SpaceDashboardId)
 		fmt.Println(err)
 		return nil, err

core/block/bookmark/bookmarkimporter/bookmark_importer_decorator.go

@@ -19,7 +19,7 @@ const CName = "bookmark-importer"
 var log = logging.Logger("bookmark-importer")
 
 type Importer interface {
-	ImportWeb(ctx context.Context, req *pb.RpcObjectImportRequest) (string, *types.Struct, error)
+	ImportWeb(ctx context.Context, req *importer.ImportRequest) (string, *types.Struct, error)
 }
 
 type BookmarkImporterDecorator struct {
@@ -40,9 +40,11 @@ func (bd *BookmarkImporterDecorator) Init(a *app.App) (err error) {
 
 func (bd *BookmarkImporterDecorator) CreateBookmarkObject(ctx context.Context, spaceID string, details *types.Struct, getContent bookmarksvc.ContentFuture) (objectId string, newDetails *types.Struct, err error) {
 	url := pbtypes.GetString(details, bundle.RelationKeySource.String())
-	if objectId, newDetails, err = bd.Importer.ImportWeb(nil, &pb.RpcObjectImportRequest{
-		Params:                &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: url}},
-		UpdateExistingObjects: true,
+	if objectId, newDetails, err = bd.Importer.ImportWeb(nil, &importer.ImportRequest{
+		RpcObjectImportRequest: &pb.RpcObjectImportRequest{
+			Params:                &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: url}},
+			UpdateExistingObjects: true,
+		},
 	}); err != nil {
 		log.With(
 			"function", "BookmarkFetch",

core/block/detailservice/relations.go

@@ -14,11 +14,12 @@ import (
 	"github.com/anyproto/anytype-heart/core/block/cache"
 	"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
 	"github.com/anyproto/anytype-heart/core/domain"
+	"github.com/anyproto/anytype-heart/pb"
 	"github.com/anyproto/anytype-heart/pkg/lib/bundle"
 	coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
 	"github.com/anyproto/anytype-heart/pkg/lib/database"
-	"github.com/anyproto/anytype-heart/pkg/lib/localstore/addr"
 	"github.com/anyproto/anytype-heart/space/spacecore/typeprovider"
+	"github.com/anyproto/anytype-heart/util/dateutil"
 	"github.com/anyproto/anytype-heart/util/pbtypes"
 	"github.com/anyproto/anytype-heart/util/slice"
 )
@@ -65,11 +66,11 @@ func (s *service) ObjectTypeRemoveRelations(ctx context.Context, objectTypeId st
 	})
 }
 
-func (s *service) ListRelationsWithValue(spaceId string, value *types.Value) (keys []string, counters []int64, err error) {
+func (s *service) ListRelationsWithValue(spaceId string, value *types.Value) ([]*pb.RpcRelationListWithValueResponseResponseItem, error) {
 	countersByKeys := make(map[string]int64)
 	detailHandlesValue := generateFilter(value)
 
-	err = s.store.SpaceIndex(spaceId).QueryIterate(database.Query{Filters: nil}, func(details *types.Struct) {
+	err := s.store.SpaceIndex(spaceId).QueryIterate(database.Query{Filters: nil}, func(details *types.Struct) {
 		for key, valueToCheck := range details.Fields {
 			if detailHandlesValue(valueToCheck) {
 				if counter, ok := countersByKeys[key]; ok {
@@ -82,17 +83,21 @@ func (s *service) ListRelationsWithValue(spaceId string, value *types.Value) (ke
 	})
 
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to query objects: %w", err)
+		return nil, fmt.Errorf("failed to query objects: %w", err)
 	}
 
-	keys = maps.Keys(countersByKeys)
+	keys := maps.Keys(countersByKeys)
 	slices.Sort(keys)
+	list := make([]*pb.RpcRelationListWithValueResponseResponseItem, len(keys))
 
-	for _, key := range keys {
-		counters = append(counters, countersByKeys[key])
+	for i, key := range keys {
+		list[i] = &pb.RpcRelationListWithValueResponseResponseItem{
+			RelationKey: key,
+			Counter:     countersByKeys[key],
+		}
 	}
 
-	return keys, counters, nil
+	return list, nil
 }
 
 func generateFilter(value *types.Value) func(v *types.Value) bool {
@@ -121,28 +126,39 @@ func generateFilter(value *types.Value) func(v *types.Value) bool {
 		return equalOrHasFilter
 	}
 
-	start, err := dateIDToDayStart(stringValue)
+	ts, err := dateutil.ParseDateId(stringValue)
 	if err != nil {
-		log.Error("failed to convert date id to day start", zap.Error(err))
+		log.Error("failed to parse Date object id", zap.Error(err))
 		return equalOrHasFilter
 	}
 
+	shortId := dateutil.TimeToShortDateId(ts)
+
+	start := ts.Truncate(24 * time.Hour)
 	end := start.Add(24 * time.Hour)
 	startTimestamp := start.Unix()
 	endTimestamp := end.Unix()
 
+	// filter for date objects is able to find relations with values between the borders of queried day
+	// - for relations with number format it checks timestamp value is between timestamps of this day midnights
+	// - for relations carrying string list it checks if some of the strings has day prefix, e.g.
+	// if _date_2023-12-12-08-30-50 is queried, then all relations with prefix _date_2023-12-12 will be returned
 	return func(v *types.Value) bool {
 		numberValue := int64(v.GetNumberValue())
 		if numberValue >= startTimestamp && numberValue < endTimestamp {
 			return true
 		}
-		return equalOrHasFilter(v)
-	}
-}
 
-func dateIDToDayStart(id string) (time.Time, error) {
-	if !strings.HasPrefix(id, addr.DatePrefix) {
-		return time.Time{}, fmt.Errorf("invalid id: date prefix not found")
+		if list := v.GetListValue(); list != nil {
+			for _, element := range list.Values {
+				if element.Equal(value) {
+					return true
+				}
+				if strings.HasPrefix(element.GetStringValue(), shortId) {
+					return true
+				}
+			}
+		}
+		return v.Equal(value)
 	}
-	return time.Parse("2006-01-02", strings.TrimPrefix(id, addr.DatePrefix))
 }

core/block/detailservice/relations_test.go

@@ -13,10 +13,11 @@ import (
 	"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
 	"github.com/anyproto/anytype-heart/core/block/editor/state"
 	"github.com/anyproto/anytype-heart/core/domain"
+	"github.com/anyproto/anytype-heart/pb"
 	"github.com/anyproto/anytype-heart/pkg/lib/bundle"
-	"github.com/anyproto/anytype-heart/pkg/lib/localstore/addr"
 	"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
 	"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
+	"github.com/anyproto/anytype-heart/util/dateutil"
 	"github.com/anyproto/anytype-heart/util/pbtypes"
 )
 
@@ -53,12 +54,12 @@ func TestService_ListRelationsWithValue(t *testing.T) {
 			bundle.RelationKeyLastModifiedDate: pbtypes.Int64(now.Add(-1 * time.Minute).Unix()),
 			bundle.RelationKeyIsFavorite:       pbtypes.Bool(true),
 			"daysTillSummer":                   pbtypes.Int64(300),
-			bundle.RelationKeyLinks:            pbtypes.StringList([]string{"obj2", "obj3"}),
+			bundle.RelationKeyLinks:            pbtypes.StringList([]string{"obj2", "obj3", dateutil.TimeToDateId(now.Add(-30 * time.Minute))}),
 		},
 		{
 			bundle.RelationKeyId:               pbtypes.String("obj2"),
 			bundle.RelationKeySpaceId:          pbtypes.String(spaceId),
-			bundle.RelationKeyName:             pbtypes.String(addr.TimeToID(now)),
+			bundle.RelationKeyName:             pbtypes.String(dateutil.TimeToDateId(now)),
 			bundle.RelationKeyCreatedDate:      pbtypes.Int64(now.Add(-24*time.Hour - 5*time.Minute).Unix()),
 			bundle.RelationKeyAddedDate:        pbtypes.Int64(now.Add(-24*time.Hour - 3*time.Minute).Unix()),
 			bundle.RelationKeyLastModifiedDate: pbtypes.Int64(now.Add(-1 * time.Minute).Unix()),
@@ -72,54 +73,66 @@ func TestService_ListRelationsWithValue(t *testing.T) {
 			bundle.RelationKeyLastModifiedDate: pbtypes.Int64(now.Unix()),
 			bundle.RelationKeyIsFavorite:       pbtypes.Bool(true),
 			bundle.RelationKeyCoverX:           pbtypes.Int64(300),
-			bundle.RelationKeyMentions:         pbtypes.StringList([]string{addr.TimeToID(now), addr.TimeToID(now.Add(-24 * time.Hour))}),
+			bundle.RelationKeyMentions:         pbtypes.StringList([]string{dateutil.TimeToDateId(now), dateutil.TimeToDateId(now.Add(-24 * time.Hour))}),
 		},
 	})
 
 	bs := service{store: store}
 
 	for _, tc := range []struct {
-		name             string
-		value            *types.Value
-		expectedKeys     []string
-		expectedCounters []int64
+		name         string
+		value        *types.Value
+		expectedList []*pb.RpcRelationListWithValueResponseResponseItem
 	}{
 		{
 			"date object - today",
-			pbtypes.String(addr.TimeToID(now)),
-			[]string{bundle.RelationKeyAddedDate.String(), bundle.RelationKeyCreatedDate.String(), bundle.RelationKeyLastModifiedDate.String(), bundle.RelationKeyMentions.String(), bundle.RelationKeyName.String()},
-			[]int64{1, 2, 3, 1, 1},
+			pbtypes.String(dateutil.TimeToDateId(now)),
+			[]*pb.RpcRelationListWithValueResponseResponseItem{
+				{bundle.RelationKeyAddedDate.String(), 1},
+				{bundle.RelationKeyCreatedDate.String(), 2},
+				{bundle.RelationKeyLastModifiedDate.String(), 3},
+				{bundle.RelationKeyLinks.String(), 1},
+				{bundle.RelationKeyMentions.String(), 1},
+				{bundle.RelationKeyName.String(), 1},
+			},
 		},
 		{
 			"date object - yesterday",
-			pbtypes.String(addr.TimeToID(now.Add(-24 * time.Hour))),
-			[]string{bundle.RelationKeyAddedDate.String(), bundle.RelationKeyCreatedDate.String(), bundle.RelationKeyMentions.String()},
-			[]int64{1, 1, 1},
+			pbtypes.String(dateutil.TimeToDateId(now.Add(-24 * time.Hour))),
+			[]*pb.RpcRelationListWithValueResponseResponseItem{
+				{bundle.RelationKeyAddedDate.String(), 1},
+				{bundle.RelationKeyCreatedDate.String(), 1},
+				{bundle.RelationKeyMentions.String(), 1},
+			},
 		},
 		{
 			"number",
 			pbtypes.Int64(300),
-			[]string{bundle.RelationKeyCoverX.String(), "daysTillSummer"},
-			[]int64{2, 1},
+			[]*pb.RpcRelationListWithValueResponseResponseItem{
+				{bundle.RelationKeyCoverX.String(), 2},
+				{"daysTillSummer", 1},
+			},
 		},
 		{
 			"bool",
 			pbtypes.Bool(true),
-			[]string{bundle.RelationKeyIsFavorite.String(), bundle.RelationKeyIsHidden.String()},
-			[]int64{2, 1},
+			[]*pb.RpcRelationListWithValueResponseResponseItem{
+				{bundle.RelationKeyIsFavorite.String(), 2},
+				{bundle.RelationKeyIsHidden.String(), 1},
+			},
 		},
 		{
 			"string list",
-			pbtypes.StringList([]string{"obj2", "obj3"}),
-			[]string{bundle.RelationKeyLinks.String()},
-			[]int64{1},
+			pbtypes.StringList([]string{"obj2", "obj3", dateutil.TimeToDateId(now.Add(-30 * time.Minute))}),
+			[]*pb.RpcRelationListWithValueResponseResponseItem{
+				{bundle.RelationKeyLinks.String(), 1},
+			},
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			keys, counters, err := bs.ListRelationsWithValue(spaceId, tc.value)
+			list, err := bs.ListRelationsWithValue(spaceId, tc.value)
 			assert.NoError(t, err)
-			assert.Equal(t, tc.expectedKeys, keys)
-			assert.Equal(t, tc.expectedCounters, counters)
+			assert.Equal(t, tc.expectedList, list)
 		})
 	}
 }