Skip to content

Commit

Permalink
GO-4442 Merge remote-tracking branch 'refs/remotes/origin/main' into g…
Browse files Browse the repository at this point in the history 
…o-4442-add-weights
  • Loading branch information
@fat-fellow
fat-fellow committed Nov 7, 2024
2 parents f2b90eb + 2756fb2 commit 81fd9fa
Show file tree
Hide file tree
Showing 44 changed files with 2,014 additions and 1,049 deletions.
11 changes: 2 additions & 9 deletions core/block/detailservice/relations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
"github.com/anyproto/anytype-heart/pkg/lib/database"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/addr"
"github.com/anyproto/anytype-heart/space/spacecore/typeprovider"
"github.com/anyproto/anytype-heart/util/dateutil"
"github.com/anyproto/anytype-heart/util/pbtypes"
"github.com/anyproto/anytype-heart/util/slice"
)
Expand Down Expand Up @@ -121,7 +121,7 @@ func generateFilter(value *types.Value) func(v *types.Value) bool {
return equalOrHasFilter
}

start, err := dateIDToDayStart(stringValue)
start, err := dateutil.ParseDateId(stringValue)
if err != nil {
log.Error("failed to convert date id to day start", zap.Error(err))
return equalOrHasFilter
Expand All @@ -139,10 +139,3 @@ func generateFilter(value *types.Value) func(v *types.Value) bool {
return equalOrHasFilter(v)
}
}

func dateIDToDayStart(id string) (time.Time, error) {
if !strings.HasPrefix(id, addr.DatePrefix) {
return time.Time{}, fmt.Errorf("invalid id: date prefix not found")
}
return time.Parse("2006-01-02", strings.TrimPrefix(id, addr.DatePrefix))
}
10 changes: 5 additions & 5 deletions core/block/detailservice/relations_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ import (
"github.com/anyproto/anytype-heart/core/block/editor/state"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/addr"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/dateutil"
"github.com/anyproto/anytype-heart/util/pbtypes"
)

Expand Down Expand Up @@ -58,7 +58,7 @@ func TestService_ListRelationsWithValue(t *testing.T) {
{
bundle.RelationKeyId: pbtypes.String("obj2"),
bundle.RelationKeySpaceId: pbtypes.String(spaceId),
bundle.RelationKeyName: pbtypes.String(addr.TimeToID(now)),
bundle.RelationKeyName: pbtypes.String(dateutil.TimeToDateId(now)),
bundle.RelationKeyCreatedDate: pbtypes.Int64(now.Add(-24*time.Hour - 5*time.Minute).Unix()),
bundle.RelationKeyAddedDate: pbtypes.Int64(now.Add(-24*time.Hour - 3*time.Minute).Unix()),
bundle.RelationKeyLastModifiedDate: pbtypes.Int64(now.Add(-1 * time.Minute).Unix()),
Expand All @@ -72,7 +72,7 @@ func TestService_ListRelationsWithValue(t *testing.T) {
bundle.RelationKeyLastModifiedDate: pbtypes.Int64(now.Unix()),
bundle.RelationKeyIsFavorite: pbtypes.Bool(true),
bundle.RelationKeyCoverX: pbtypes.Int64(300),
bundle.RelationKeyMentions: pbtypes.StringList([]string{addr.TimeToID(now), addr.TimeToID(now.Add(-24 * time.Hour))}),
bundle.RelationKeyMentions: pbtypes.StringList([]string{dateutil.TimeToDateId(now), dateutil.TimeToDateId(now.Add(-24 * time.Hour))}),
},
})

Expand All @@ -86,13 +86,13 @@ func TestService_ListRelationsWithValue(t *testing.T) {
}{
{
"date object - today",
pbtypes.String(addr.TimeToID(now)),
pbtypes.String(dateutil.TimeToDateId(now)),
[]string{bundle.RelationKeyAddedDate.String(), bundle.RelationKeyCreatedDate.String(), bundle.RelationKeyLastModifiedDate.String(), bundle.RelationKeyMentions.String(), bundle.RelationKeyName.String()},
[]int64{1, 2, 3, 1, 1},
},
{
"date object - yesterday",
pbtypes.String(addr.TimeToID(now.Add(-24 * time.Hour))),
pbtypes.String(dateutil.TimeToDateId(now.Add(-24 * time.Hour))),
[]string{bundle.RelationKeyAddedDate.String(), bundle.RelationKeyCreatedDate.String(), bundle.RelationKeyMentions.String()},
[]int64{1, 1, 1},
},
Expand Down
20 changes: 10 additions & 10 deletions core/block/editor/smartblock/smartblock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/threads"
"github.com/anyproto/anytype-heart/space/spacecore/storage/sqlitestorage"
"github.com/anyproto/anytype-heart/util/anonymize"
"github.com/anyproto/anytype-heart/util/dateutil"
"github.com/anyproto/anytype-heart/util/internalflag"
"github.com/anyproto/anytype-heart/util/pbtypes"
"github.com/anyproto/anytype-heart/util/slice"
Expand Down Expand Up @@ -461,10 +462,7 @@ func (sb *smartBlock) fetchMeta() (details []*model.ObjectViewDetailsSet, err er
depIds := sb.dependentSmartIds(sb.includeRelationObjectsAsDependents, true, true)
sb.setDependentIDs(depIds)

perSpace, err := sb.partitionIdsBySpace(sb.depIds)
if err != nil {
return nil, fmt.Errorf("partiton by space: %w", err)
}
perSpace := sb.partitionIdsBySpace(sb.depIds)

recordsCh := make(chan *types.Struct, 10)
sb.recordsSub = database.NewSubscription(nil, recordsCh)
Expand Down Expand Up @@ -513,9 +511,14 @@ func (sb *smartBlock) fetchMeta() (details []*model.ObjectViewDetailsSet, err er
return
}

func (sb *smartBlock) partitionIdsBySpace(ids []string) (map[string][]string, error) {
func (sb *smartBlock) partitionIdsBySpace(ids []string) map[string][]string {
perSpace := map[string][]string{}
for _, id := range ids {
if _, parseErr := dateutil.ParseDateId(id); parseErr == nil {
perSpace[sb.space.Id()] = append(perSpace[sb.space.Id()], id)
continue
}

spaceId, err := sb.spaceIdResolver.ResolveSpaceID(id)
if errors.Is(err, sqlitestorage.ErrObjectNotFound) || errors.Is(err, badger.ErrKeyNotFound) {
perSpace[sb.space.Id()] = append(perSpace[sb.space.Id()], id)
Expand All @@ -529,7 +532,7 @@ func (sb *smartBlock) partitionIdsBySpace(ids []string) (map[string][]string, er
}
perSpace[spaceId] = append(perSpace[spaceId], id)
}
return perSpace, nil
return perSpace
}

func (sb *smartBlock) Lock() {
Expand Down Expand Up @@ -861,10 +864,7 @@ func (sb *smartBlock) CheckSubscriptions() (changed bool) {
}
newIDs := sb.recordsSub.Subscribe(sb.depIds)

perSpace, err := sb.partitionIdsBySpace(newIDs)
if err != nil {
log.Errorf("partiton by space error: %v", err)
}
perSpace := sb.partitionIdsBySpace(newIDs)

for spaceId, ids := range perSpace {
spaceIndex := sb.objectStore.SpaceIndex(spaceId)
Expand Down
88 changes: 65 additions & 23 deletions core/block/import/common/error.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,29 @@ import (

"github.com/anyproto/any-sync/commonspace/object/acl/list"

"github.com/anyproto/anytype-heart/core/block/import/common/source"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
)

var ErrCancel = fmt.Errorf("import is canceled")
var ErrFailedToReceiveListOfObjects = fmt.Errorf("failed to receive the list of objects")
var ErrNoObjectsToImport = fmt.Errorf("source path doesn't contain objects to import")
var ErrLimitExceeded = fmt.Errorf("Limit of relations or objects are exceeded ")
var ErrFileLoad = fmt.Errorf("file was not synced")
var (
ErrCancel = errors.New("import is canceled")
ErrCsvLimitExceeded = errors.New("Limit of relations or objects are exceeded ")
ErrFileLoad = errors.New("file was not synced")

ErrNoObjectInIntegration = errors.New("no objects added to Notion integration")
ErrNotionServerIsUnavailable = errors.New("notion server is unavailable")
ErrNotionServerExceedRateLimit = errors.New("rate limit exceeded")

ErrFileImportNoObjectsInZipArchive = errors.New("no objects in zip archive")
ErrFileImportNoObjectsInDirectory = errors.New("no objects in directory")

ErrPbNotAnyBlockFormat = errors.New("file doesn't match Anyblock format ")

ErrWrongHTMLFormat = errors.New("html file has wrong structure")

ErrNoSnapshotToImport = errors.New("no snapshot to import") // for external import
)

type ConvertError struct {
errors []error
Expand Down Expand Up @@ -72,23 +86,15 @@ func (ce *ConvertError) GetResultError(importType model.ImportType) error {
var countNoObjectsToImport int
for _, e := range ce.errors {
switch {
case errors.Is(e, ErrCancel):
return fmt.Errorf("import type: %s: %w", importType.String(), ErrCancel)
case errors.Is(e, ErrLimitExceeded):
return fmt.Errorf("import type: %s: %w", importType.String(), ErrLimitExceeded)
case errors.Is(e, ErrFailedToReceiveListOfObjects):
return ErrFailedToReceiveListOfObjects
case errors.Is(e, ErrFileLoad):
case isDefinedError(e):
return fmt.Errorf("import type: %s: %w", importType.String(), e)
case errors.Is(e, list.ErrInsufficientPermissions):
return e
case errors.Is(e, ErrNoObjectsToImport):
case IsNoObjectError(e):
countNoObjectsToImport++
}
}
// we return ErrNoObjectsToImport only if all paths has such error, otherwise we assume that import finished with internal code error
if countNoObjectsToImport == len(ce.errors) {
return fmt.Errorf("import type: %s: %w", importType.String(), ErrNoObjectsToImport)
return fmt.Errorf("import type: %s: %w", importType.String(), ce.errors[0])
}
return fmt.Errorf("import type: %s: %w", importType.String(), ce.Error())
}
Expand All @@ -99,7 +105,7 @@ func (ce *ConvertError) IsNoObjectToImportError(importPathsCount int) bool {
}
var countNoObjectsToImport int
for _, err := range ce.errors {
if errors.Is(err, ErrNoObjectsToImport) {
if IsNoObjectError(err) {
countNoObjectsToImport++
}
}
Expand All @@ -108,30 +114,66 @@ func (ce *ConvertError) IsNoObjectToImportError(importPathsCount int) bool {
func (ce *ConvertError) ShouldAbortImport(pathsCount int, importType model.ImportType) bool {
return !ce.IsEmpty() && ce.mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING ||
ce.IsNoObjectToImportError(pathsCount) ||
errors.Is(ce.GetResultError(importType), ErrLimitExceeded) ||
errors.Is(ce.GetResultError(importType), ErrCsvLimitExceeded) ||
errors.Is(ce.GetResultError(importType), ErrCancel)
}

func GetImportErrorCode(err error) model.ImportErrorCode {
func GetImportNotificationErrorCode(err error) model.ImportErrorCode {
if err == nil {
return model.Import_NULL
}
switch {
case errors.Is(err, ErrNoObjectsToImport):
return model.Import_NO_OBJECTS_TO_IMPORT
case errors.Is(err, ErrNoObjectInIntegration) ||
errors.Is(err, ErrFileImportNoObjectsInDirectory) ||
errors.Is(err, ErrFileImportNoObjectsInZipArchive): // support existing protocol
return model.Import_NOTION_NO_OBJECTS_IN_INTEGRATION
case errors.Is(err, ErrNotionServerIsUnavailable):
return model.Import_NOTION_SERVER_IS_UNAVAILABLE
case errors.Is(err, ErrNotionServerExceedRateLimit):
return model.Import_NOTION_RATE_LIMIT_EXCEEDED
case errors.Is(err, ErrFileImportNoObjectsInDirectory):
return model.Import_FILE_IMPORT_NO_OBJECTS_IN_DIRECTORY
case errors.Is(err, ErrFileImportNoObjectsInZipArchive):
return model.Import_FILE_IMPORT_NO_OBJECTS_IN_ZIP_ARCHIVE
case errors.Is(err, ErrPbNotAnyBlockFormat):
return model.Import_PB_NOT_ANYBLOCK_FORMAT
case errors.Is(err, ErrCancel):
return model.Import_IMPORT_IS_CANCELED
case errors.Is(err, ErrLimitExceeded):
return model.Import_LIMIT_OF_ROWS_OR_RELATIONS_EXCEEDED
case errors.Is(err, ErrCsvLimitExceeded):
return model.Import_CSV_LIMIT_OF_ROWS_OR_RELATIONS_EXCEEDED
case errors.Is(err, ErrFileLoad):
return model.Import_FILE_LOAD_ERROR
case errors.Is(err, ErrWrongHTMLFormat):
return model.Import_HTML_WRONG_HTML_STRUCTURE
case errors.Is(err, list.ErrInsufficientPermissions):
return model.Import_INSUFFICIENT_PERMISSIONS
default:
return model.Import_INTERNAL_ERROR
}
}

func ErrorBySourceType(s source.Source) error {
if _, ok := s.(*source.Directory); ok {
return ErrFileImportNoObjectsInDirectory
}
if _, ok := s.(*source.Zip); ok {
return ErrFileImportNoObjectsInZipArchive
}
return nil
}

func IsNoObjectError(err error) bool {
return errors.Is(err, ErrNoObjectInIntegration) ||
errors.Is(err, ErrFileImportNoObjectsInDirectory) ||
errors.Is(err, ErrFileImportNoObjectsInZipArchive)
}

func isDefinedError(err error) bool {
return errors.Is(err, ErrCancel) || errors.Is(err, ErrCsvLimitExceeded) || errors.Is(err, ErrNotionServerExceedRateLimit) ||
errors.Is(err, ErrNotionServerIsUnavailable) || errors.Is(err, ErrFileLoad) || errors.Is(err, ErrPbNotAnyBlockFormat) ||
errors.Is(err, ErrWrongHTMLFormat)
}

func GetGalleryResponseCode(err error) pb.RpcObjectImportExperienceResponseErrorCode {
if err == nil {
return pb.RpcObjectImportExperienceResponseError_NULL
Expand Down
Loading

0 comments on commit 81fd9fa

Please sign in to comment.