GO-4486 research migrating filestore

core/block/editor/files.go

@@ -11,11 +11,14 @@
 	"github.com/anyproto/anytype-heart/core/block/migration"
 	"github.com/anyproto/anytype-heart/core/block/source"
 	"github.com/anyproto/anytype-heart/core/domain"
+	"github.com/anyproto/anytype-heart/core/files"
 	"github.com/anyproto/anytype-heart/core/files/fileobject"
+	"github.com/anyproto/anytype-heart/core/files/fileobject/filemodels"
 	"github.com/anyproto/anytype-heart/core/files/reconciler"
 	"github.com/anyproto/anytype-heart/core/filestorage"
 	"github.com/anyproto/anytype-heart/pkg/lib/bundle"
 	coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
+	"github.com/anyproto/anytype-heart/util/pbtypes"
 )
 
 // required relations for files beside the bundle.RequiredInternalRelations
@@ -34,6 +37,7 @@
 		Text:              stext.NewText(sb, store, f.eventSender),
 		fileObjectService: f.fileObjectService,
 		reconciler:        f.fileReconciler,
+		fileService:       f.fileService,
 	}
 }
 
@@ -44,6 +48,7 @@
 	stext.Text
 	fileObjectService fileobject.Service
 	reconciler        reconciler.Reconciler
+	fileService       files.Service
 }
 
 func (f *File) CreationStateMigration(ctx *smartblock.InitContext) migration.Migration {
@@ -84,10 +89,11 @@
 		return err
 	}
 
-	f.SmartBlock.AddHook(f.reconciler.FileObjectHook(domain.FullID{SpaceID: f.SpaceID(), ObjectID: f.Id()}), smartblock.HookBeforeApply)
+	fullId := domain.FullID{SpaceID: f.SpaceID(), ObjectID: f.Id()}
+
+	f.SmartBlock.AddHook(f.reconciler.FileObjectHook(fullId), smartblock.HookBeforeApply)
 
 	if !ctx.IsNewObject {
-		fullId := domain.FullID{ObjectID: f.Id(), SpaceID: f.SpaceID()}
 		err = f.fileObjectService.EnsureFileAddedToSyncQueue(fullId, ctx.State.Details())
 		if err != nil {
 			log.Errorf("failed to ensure file added to sync queue: %v", err)
@@ -96,5 +102,17 @@
 			return f.fileObjectService.EnsureFileAddedToSyncQueue(fullId, applyInfo.State.Details())
 		}, smartblock.HookOnStateRebuild)
 	}
+
+	infos, err := f.fileService.IndexFile(ctx.Ctx, domain.FullFileId{
+		FileId:  domain.FileId(pbtypes.GetString(ctx.State.Details(), bundle.RelationKeyFileId.String())),
+		SpaceId: f.SpaceID(),
+	}, ctx.State.Details(), ctx.State.GetFileInfo().EncryptionKeys)
+	if err != nil {
+		return fmt.Errorf("get infos for indexing: %w", err)
+	}
+	if len(infos) > 0 {
+		filemodels.FileInfosToDetails(infos, ctx.State)
+	}
+
 	return nil
 }

core/block/editor/smartblock/smartblock.go

@@ -1272,7 +1272,7 @@ func (sb *smartBlock) storeFileKeys(doc state.Doc) {
 			EncryptionKeys: k.Keys,
 		}
 	}
-	if err := sb.fileStore.AddFileKeys(fileKeys...); err != nil {
+	if err := sb.objectStore.AddFileKeys(fileKeys...); err != nil {
 		log.Warnf("can't store file keys: %v", err)
 	}
 }
@@ -1468,7 +1468,7 @@ func (sb *smartBlock) injectDerivedDetails(s *state.State, spaceID string, sbt s
 	}
 
 	if info := s.GetFileInfo(); info.FileId != "" {
-		err := sb.fileStore.AddFileKeys(domain.FileEncryptionKeys{
+		err := sb.objectStore.AddFileKeys(domain.FileEncryptionKeys{
 			FileId:         info.FileId,
 			EncryptionKeys: info.EncryptionKeys,
 		})

core/block/export/export.go

@@ -960,7 +960,7 @@ func (e *exportContext) saveFile(ctx context.Context, wr writer, fileObject sb.S
 	if err != nil {
 		return "", err
 	}
-	if strings.HasPrefix(file.Info().Media, "image") {
+	if strings.HasPrefix(file.Media(), "image") {
 		image, err := e.fileService.ImageByHash(context.TODO(), fullId)
 		if err != nil {
 			return "", err
@@ -980,7 +980,7 @@ func (e *exportContext) saveFile(ctx context.Context, wr writer, fileObject sb.S
 	if err != nil {
 		return "", err
 	}
-	return fileName, wr.WriteFile(fileName, rd, file.Info().LastModifiedDate)
+	return fileName, wr.WriteFile(fileName, rd, file.LastModifiedDate())
 }
 
 func (e *exportContext) createProfileFile(spaceID string, wr writer) error {

core/block/files.go

@@ -67,7 +67,7 @@ func (s *Service) DownloadFile(ctx context.Context, req *pb.RpcFileDownloadReque
 	countReader = datacounter.NewReaderCounter(r)
 	fileName := f.Meta().Name
 	if fileName == "" {
-		fileName = f.Info().Name
+		fileName = f.Name()
 	}
 
 	path, err := files.WriteReaderIntoFileReuseSameExistingFile(req.Path+string(os.PathSeparator)+fileName, countReader)

core/block/import/common/objectid/oldfile.go

@@ -17,15 +17,15 @@ import (
 	"github.com/anyproto/anytype-heart/core/files/fileobject"
 	"github.com/anyproto/anytype-heart/pb"
 	"github.com/anyproto/anytype-heart/pkg/lib/bundle"
-	"github.com/anyproto/anytype-heart/pkg/lib/localstore/filestore"
+	"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
 	"github.com/anyproto/anytype-heart/util/pbtypes"
 )
 
 // oldFile represents file in pre Files-as-Objects format
 type oldFile struct {
 	blockService      *block.Service
-	fileStore         filestore.FileStore
 	fileObjectService fileobject.Service
+	objectStore       objectstore.ObjectStore
 }
 
 func (f *oldFile) GetIDAndPayload(ctx context.Context, spaceId string, sn *common.Snapshot, _ time.Time, _ bool, origin objectorigin.ObjectOrigin) (string, treestorage.TreeStorageCreatePayload, error) {
@@ -50,7 +50,7 @@ func (f *oldFile) GetIDAndPayload(ctx context.Context, spaceId string, sn *commo
 		}
 	}
 
-	err := f.fileStore.AddFileKeys(domain.FileEncryptionKeys{
+	err := f.objectStore.AddFileKeys(domain.FileEncryptionKeys{
 		FileId:         domain.FileId(fileId),
 		EncryptionKeys: filesKeys,
 	})

core/block/import/common/objectid/provider.go

@@ -56,7 +56,7 @@ func NewIDProvider(
 	}
 	oldFile := &oldFile{
 		blockService:      blockService,
-		fileStore:         fileStore,
+		objectStore:       objectStore,
 		fileObjectService: fileObjectService,
 	}
 	p.idProviderBySmartBlockType[sb.SmartBlockTypeWorkspace] = newWorkspace(spaceService)

core/block/import/importer.go

@@ -69,6 +69,7 @@ type Import struct {
 	fileSync            filesync.FileSync
 	notificationService notifications.Notifications
 	eventSender         event.Sender
+	objectStore         objectstore.ObjectStore
 
 	importCtx       context.Context
 	importCtxCancel context.CancelFunc
@@ -98,15 +99,15 @@ func (i *Import) Init(a *app.App) (err error) {
 	for _, c := range converters {
 		i.converters[c.Name()] = c
 	}
-	store := app.MustComponent[objectstore.ObjectStore](a)
+	i.objectStore = app.MustComponent[objectstore.ObjectStore](a)
 	i.fileStore = app.MustComponent[filestore.FileStore](a)
 	fileObjectService := app.MustComponent[fileobject.Service](a)
-	i.idProvider = objectid.NewIDProvider(store, spaceService, i.s, i.fileStore, fileObjectService)
+	i.idProvider = objectid.NewIDProvider(i.objectStore, spaceService, i.s, i.fileStore, fileObjectService)
 	factory := syncer.New(syncer.NewFileSyncer(i.s, fileObjectService), syncer.NewBookmarkSyncer(i.s), syncer.NewIconSyncer(i.s, fileObjectService))
 	relationSyncer := syncer.NewFileRelationSyncer(i.s, fileObjectService)
 	objectCreator := app.MustComponent[objectcreator.Service](a)
 	detailsService := app.MustComponent[detailservice.Service](a)
-	i.oc = creator.New(detailsService, factory, store, relationSyncer, spaceService, objectCreator, i.s)
+	i.oc = creator.New(detailsService, factory, i.objectStore, relationSyncer, spaceService, objectCreator, i.s)
 	i.fileSync = app.MustComponent[filesync.FileSync](a)
 	i.notificationService = app.MustComponent[notifications.Notifications](a)
 	i.eventSender = app.MustComponent[event.Sender](a)
@@ -437,7 +438,7 @@ func (i *Import) getObjectID(
 
 	// Preload file keys
 	for _, fileKeys := range snapshot.Snapshot.GetFileKeys() {
-		err := i.fileStore.AddFileKeys(domain.FileEncryptionKeys{
+		err := i.objectStore.AddFileKeys(domain.FileEncryptionKeys{
 			FileId:         domain.FileId(fileKeys.Hash),
 			EncryptionKeys: fileKeys.Keys,
 		})
@@ -450,7 +451,7 @@ func (i *Import) getObjectID(
 		for _, key := range fileInfo.EncryptionKeys {
 			keys[key.Path] = key.Key
 		}
-		err := i.fileStore.AddFileKeys(domain.FileEncryptionKeys{
+		err := i.objectStore.AddFileKeys(domain.FileEncryptionKeys{
 			FileId:         domain.FileId(fileInfo.FileId),
 			EncryptionKeys: keys,
 		})

core/files/file.go

@@ -19,11 +19,14 @@ import (
 )
 
 type File interface {
-	Meta() *FileMeta
+	Meta() *FileMeta // could be taken from Details
 	FileId() domain.FileId
-	Reader(ctx context.Context) (io.ReadSeeker, error)
+	Reader(ctx context.Context) (io.ReadSeeker, error) // getNode(details.FileVariants[idx])
 	Details(ctx context.Context) (*types.Struct, domain.TypeKey, error)
-	Info() *storage.FileInfo
+	Name() string
+	Media() string
+	LastModifiedDate() int64
+	Mill() string
 }
 
 var _ File = (*file)(nil)
@@ -119,8 +122,20 @@ func (f *file) Details(ctx context.Context) (*types.Struct, domain.TypeKey, erro
 	return t, typeKey, nil
 }
 
-func (f *file) Info() *storage.FileInfo {
-	return f.info
+func (f *file) Name() string {
+	return f.info.Name
+}
+
+func (f *file) Media() string {
+	return f.info.Media
+}
+
+func (f *file) LastModifiedDate() int64 {
+	return f.info.LastModifiedDate
+}
+
+func (f *file) Mill() string {
+	return f.info.Mill
 }
 
 func (f *file) Meta() *FileMeta {
@@ -138,7 +153,7 @@ func (f *file) FileId() domain.FileId {
 }
 
 func (f *file) Reader(ctx context.Context) (io.ReadSeeker, error) {
-	return f.node.getContentReader(ctx, f.spaceID, f.info)
+	return f.node.getContentReader(ctx, f.spaceID, f.info.Hash, f.info.Key)
 }
 
 func calculateCommonDetails(

core/files/fileacl/service.go

@@ -9,7 +9,7 @@ import (
 	"github.com/anyproto/anytype-heart/core/domain"
 	"github.com/anyproto/anytype-heart/core/files"
 	"github.com/anyproto/anytype-heart/core/files/fileobject"
-	"github.com/anyproto/anytype-heart/pkg/lib/localstore/filestore"
+	"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
 	"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
 )
 
@@ -25,7 +25,7 @@ type Service interface {
 type service struct {
 	fileService       files.Service
 	fileObjectService fileobject.Service
-	fileStore         filestore.FileStore
+	objectStore       objectstore.ObjectStore
 }
 
 func New() Service {
@@ -35,7 +35,7 @@ func New() Service {
 func (s *service) Init(a *app.App) error {
 	s.fileService = app.MustComponent[files.Service](a)
 	s.fileObjectService = app.MustComponent[fileobject.Service](a)
-	s.fileStore = app.MustComponent[filestore.FileStore](a)
+	s.objectStore = app.MustComponent[objectstore.ObjectStore](a)
 	return nil
 }
 
@@ -76,7 +76,7 @@ func (s *service) StoreFileKeys(fileId domain.FileId, fileKeys []*model.FileEncr
 	for _, key := range fileKeys {
 		keys.EncryptionKeys[key.Path] = key.Key
 	}
-	err := s.fileStore.AddFileKeys(keys)
+	err := s.objectStore.AddFileKeys(keys)
 	if err != nil {
 		return fmt.Errorf("store file encryption keys: %w", err)
 	}

core/files/fileobject/fileindex.go

@@ -20,12 +20,14 @@
 	fileblock "github.com/anyproto/anytype-heart/core/block/simple/file"
 	"github.com/anyproto/anytype-heart/core/domain"
 	"github.com/anyproto/anytype-heart/core/files"
+	"github.com/anyproto/anytype-heart/core/files/fileobject/filemodels"
 	"github.com/anyproto/anytype-heart/core/filestorage/rpcstore"
 	"github.com/anyproto/anytype-heart/pkg/lib/bundle"
 	"github.com/anyproto/anytype-heart/pkg/lib/database"
 	"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
 	"github.com/anyproto/anytype-heart/pkg/lib/mill"
 	"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
+	"github.com/anyproto/anytype-heart/pkg/lib/pb/storage"
 	"github.com/anyproto/anytype-heart/space"
 	"github.com/anyproto/anytype-heart/util/pbtypes"
 )
@@ -235,13 +237,21 @@
 }
 
 func (ind *indexer) injectMetadataToState(ctx context.Context, st *state.State, fileId domain.FullFileId, id domain.FullID) error {
-	details, typeKey, err := ind.buildDetails(ctx, fileId)
+	infos, err := ind.fileService.IndexFile(ctx, fileId, st.Details(), st.GetFileInfo().EncryptionKeys)
+	if err != nil {
+		return fmt.Errorf("get infos for indexing: %w", err)
+	}
+	if len(infos) > 0 {
+		filemodels.FileInfosToDetails(infos, st)
+	}
+
+	prevDetails := st.CombinedDetails()
+	details, typeKey, err := ind.buildDetails(ctx, fileId, infos)
 	if err != nil {
 		return fmt.Errorf("build details: %w", err)
 	}
 
 	st.SetObjectTypeKey(typeKey)
-	prevDetails := st.CombinedDetails()
 
 	keys := make([]domain.RelationKey, 0, len(details.Fields))
 	for k := range details.Fields {
@@ -259,22 +269,16 @@
 	return nil
 }
 
-func (ind *indexer) buildDetails(ctx context.Context, id domain.FullFileId) (details *types.Struct, typeKey domain.TypeKey, err error) {
-	file, err := ind.fileService.FileByHash(ctx, id)
-	if err != nil {
-		return nil, "", err
-	}
+func (ind *indexer) buildDetails(ctx context.Context, id domain.FullFileId, infos []*storage.FileInfo) (details *types.Struct, typeKey domain.TypeKey, err error) {
+	file := ind.fileService.FileFromInfos(id, infos)
 
-	if file.Info().Mill == mill.BlobId {
+	if file.Mill() == mill.BlobId {
 		details, typeKey, err = file.Details(ctx)
 		if err != nil {
 			return nil, "", err
 		}
 	} else {
-		image, err := ind.fileService.ImageByHash(ctx, id)
-		if err != nil {
-			return nil, "", err
-		}
+		image := ind.fileService.ImageFromInfos(id, infos)
 		details, err = image.Details(ctx)
 		if err != nil {
 			return nil, "", err
@@ -284,7 +288,7 @@
 	// Overwrite typeKey for images in case that image is uploaded as file.
 	// That can be possible because some images can't be handled properly and wee fall back to
 	// handling them as files
-	if mill.IsImage(file.Info().Media) {
+	if mill.IsImage(file.Media()) {
 		typeKey = bundle.TypeKeyImage
 	}