Implement missing Group APIs (#1415)

* Implement missing Group APIs

This PR implement missing group APIs:

- ListDocumentGroups: to search groups a document is published to.
- ListAccountGroups: to search groups an account is member of.

The change is massive, because it also includes a breaking change to make document IDs unforgeable,
so this doesn't include a migration, but instead marks a new beginning in the list of migrations.
I.e. your current data directory needs to be removed.

It also bumps the network protocol version, to prevent being able to talk to older peers, just to avoid any weird issues.

Frontend Protobuf codegen package changed its name, so all the generated proto files for JS are regenerated.

* Fix the build

* frontend: implement new groups APIs

* stupid package dep

* hd -> hm. better url matching

* enable tests in all packages

* HD -> HM in files and variables

* hd:// => hm://

---------

Co-authored-by: Horacio Herrera <[email protected]>

.github/workflows/validate-site.yml

@@ -6,21 +6,21 @@ on:
       - master
       - electron-ci
     paths:
-      - '.github/workflows/validate-site.yml'
-      - 'frontend/apps/site/**'
-      - 'frotend/packages/shared/**'
-      - 'frotend/packages/ui/**'
-      - 'package.json'
-      - 'frontend/apps/site/package.json'
+      - ".github/workflows/validate-site.yml"
+      - "frontend/apps/site/**"
+      - "frotend/packages/shared/**"
+      - "frotend/packages/ui/**"
+      - "package.json"
+      - "frontend/apps/site/package.json"
   pull_request:
     paths:
-      - '.github/workflows/validate-site.yml'
-      - 'frontend/apps/site/**'
-      - 'frotend/packages/**'
-      - 'package.json'
-      - 'frontend/apps/site/package.json'
+      - ".github/workflows/validate-site.yml"
+      - "frontend/apps/site/**"
+      - "frotend/packages/**"
+      - "package.json"
+      - "frontend/apps/site/package.json"
     branches-ignore:
-      - 'renovate/**'
+      - "renovate/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -36,7 +36,7 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: 'yarn'
+          cache: "yarn"
 
       - name: Install Frontend Dependencies
         env:
@@ -49,4 +49,4 @@ jobs:
           yarn validate
 
       - name: Run Site tests
-        run: yarn site:test
+        run: yarn site:test

backend/daemon/api/accounts/v1alpha/accounts.go

@@ -97,7 +97,7 @@ func (srv *Server) GetAccount(ctx context.Context, in *accounts.GetAccountReques
 		return nil, status.Errorf(codes.NotFound, "account %s not found", aids)
 	}
 
-	entity, err := srv.blobs.LoadEntity(ctx, hyper.EntityID("hd://a/"+aids))
+	entity, err := srv.blobs.LoadEntity(ctx, hyper.EntityID("hm://a/"+aids))
 	if err != nil {
 		return nil, err
 	}
@@ -180,7 +180,7 @@ func (srv *Server) UpdateProfile(ctx context.Context, in *accounts.Profile) (*ac
 
 // UpdateProfile is public so it can be called from sites.
 func UpdateProfile(ctx context.Context, me core.Identity, blobs *hyper.Storage, in *accounts.Profile) error {
-	eid := hyper.EntityID("hd://a/" + me.Account().Principal().String())
+	eid := hyper.EntityID("hm://a/" + me.Account().Principal().String())
 
 	e, err := blobs.LoadEntity(ctx, eid)
 	if err != nil {
@@ -279,7 +279,7 @@ func (srv *Server) ListAccounts(ctx context.Context, in *accounts.ListAccountsRe
 		return nil, err
 	}
 
-	entities, err := srv.blobs.ListEntities(ctx, "hd://a/")
+	entities, err := srv.blobs.ListEntities(ctx, "hm://a/")
 	if err != nil {
 		return nil, err
 	}
@@ -291,7 +291,7 @@ func (srv *Server) ListAccounts(ctx context.Context, in *accounts.ListAccountsRe
 	}
 
 	for _, e := range entities {
-		aid := e.TrimPrefix("hd://a/")
+		aid := e.TrimPrefix("hm://a/")
 		if aid == mine {
 			continue
 		}

backend/daemon/api/documents/v1alpha/changes.go

@@ -39,7 +39,7 @@ func (api *Server) ListChanges(ctx context.Context, in *documents.ListChangesReq
 		return nil, status.Errorf(codes.InvalidArgument, "must provide document id")
 	}
 
-	eid := hyper.EntityID("hd://d/" + in.DocumentId)
+	eid := hyper.EntityID("hm://d/" + in.DocumentId)
 
 	out := &documents.ListChangesResponse{}
 

backend/daemon/api/documents/v1alpha/content_graph.go

@@ -20,11 +20,16 @@ func (srv *Server) ListCitations(ctx context.Context, in *documents.ListCitation
 		return nil, status.Error(codes.InvalidArgument, "must specify document ID")
 	}
 
-	eid := hyper.EntityID("hd://d/" + in.DocumentId)
+	targetEntity := "hm://d/" + in.DocumentId
 
 	var backlinks []hypersql.BacklinksForEntityResult
 	if err := srv.blobs.Query(ctx, func(conn *sqlite.Conn) error {
-		list, err := hypersql.BacklinksForEntity(conn, string(eid))
+		edb, err := hypersql.EntitiesLookupID(conn, targetEntity)
+		if err != nil {
+			return err
+		}
+
+		list, err := hypersql.BacklinksForEntity(conn, edb.EntitiesID)
 		backlinks = list
 		return err
 	}); err != nil {
@@ -37,20 +42,20 @@ func (srv *Server) ListCitations(ctx context.Context, in *documents.ListCitation
 
 	for i, link := range backlinks {
 		var ld hyper.LinkData
-		if err := json.Unmarshal(link.ContentLinksViewData, &ld); err != nil {
+		if err := json.Unmarshal(link.BlobAttrsExtra, &ld); err != nil {
 			return nil, fmt.Errorf("failed to decode link data: %w", err)
 		}
 
-		src := cid.NewCidV1(uint64(link.ContentLinksViewSourceBlobCodec), link.ContentLinksViewSourceBlobMultihash)
+		src := cid.NewCidV1(uint64(link.BlobsCodec), link.BlobsMultihash)
 
 		resp.Links[i] = &documents.Link{
 			Source: &documents.LinkNode{
-				DocumentId: hyper.EntityID(link.ContentLinksViewSourceEID).TrimPrefix("hd://d/"),
-				BlockId:    ld.SourceBlock,
+				DocumentId: hyper.EntityID(link.EntitiesEID).TrimPrefix("hm://d/"),
+				BlockId:    link.BlobAttrsAnchor,
 				Version:    src.String(),
 			},
 			Target: &documents.LinkNode{
-				DocumentId: hyper.EntityID(link.ContentLinksViewTargetEID).TrimPrefix("hd://d/"),
+				DocumentId: in.DocumentId,
 				BlockId:    ld.TargetFragment,
 				Version:    ld.TargetVersion,
 			},

backend/daemon/api/documents/v1alpha/content_graph_test.go

@@ -48,7 +48,7 @@ func TestBacklinks(t *testing.T) {
 					Starts: []int32{0},
 					Ends:   []int32{5},
 					Attributes: map[string]string{
-						"url": "hd://d/" + pub.Document.Id + "?v=" + pub.Version + "#b1",
+						"url": "hm://d/" + pub.Document.Id + "?v=" + pub.Version + "#b1",
 					},
 				},
 			},
@@ -64,7 +64,7 @@ func TestBacklinks(t *testing.T) {
 					Starts: []int32{0},
 					Ends:   []int32{5},
 					Attributes: map[string]string{
-						"url": "hd://d/" + pub.Document.Id + "?v=" + pub.Version,
+						"url": "hm://d/" + pub.Document.Id + "?v=" + pub.Version,
 					},
 				},
 			},

backend/daemon/api/documents/v1alpha/document_model.go

@@ -28,6 +28,7 @@ type docModel struct {
 	tree       *Tree
 	patch      map[string]any
 	oldDraft   cid.Cid
+	oldChange  hyper.Change
 	done       bool
 	nextHLC    hlc.Time
 	origins    map[string]cid.Cid // map of abbreviated origin hashes to actual cids; workaround, should not be necessary.
@@ -64,6 +65,7 @@ func (dm *docModel) restoreDraft(c cid.Cid, ch hyper.Change) (err error) {
 		panic("BUG: restoring draft when patch is not empty")
 	}
 	dm.oldDraft = c
+	dm.oldChange = ch
 
 	if len(dm.e.Heads()) != len(ch.Deps) {
 		return fmt.Errorf("failed to restore draft: state has %d heads while draft change has %d deps", len(dm.e.Heads()), len(ch.Deps))
@@ -81,6 +83,7 @@ func (dm *docModel) restoreDraft(c cid.Cid, ch hyper.Change) (err error) {
 	}
 
 	dm.nextHLC = dm.e.NextTimestamp()
+
 	moves := dm.patch["moves"]
 	delete(dm.patch, "moves")
 
@@ -138,18 +141,18 @@ func (dm *docModel) SetCreateTime(ct time.Time) error {
 		return fmt.Errorf("create time is already set")
 	}
 
-	dm.patch["createTime"] = ct.Unix()
+	dm.patch["createTime"] = int(ct.Unix())
 
 	return nil
 }
 
 func (dm *docModel) SetAuthor(author core.Principal) error {
-	_, ok := dm.e.Get("author")
+	_, ok := dm.e.Get("owner")
 	if ok {
 		return fmt.Errorf("author is already set")
 	}
 
-	dm.patch["author"] = []byte(author)
+	dm.patch["owner"] = []byte(author)
 
 	return nil
 }
@@ -212,7 +215,12 @@ func (dm *docModel) Change() (hb hyper.Blob, err error) {
 
 	dm.cleanupPatch()
 
-	return dm.e.CreateChange(dm.nextHLC, dm.signer, dm.delegation, dm.patch)
+	action := dm.oldChange.Action
+	if action == "" {
+		action = "Create"
+	}
+
+	return dm.e.CreateChange(dm.nextHLC, dm.signer, dm.delegation, dm.patch, hyper.WithAction(action))
 }
 
 func (dm *docModel) Commit(ctx context.Context, bs *hyper.Storage) (hb hyper.Blob, err error) {
@@ -305,44 +313,28 @@ func (dm *docModel) cleanupPatch() {
 
 func (dm *docModel) hydrate(ctx context.Context, blobs *hyper.Storage) (*documents.Document, error) {
 	e := dm.e
-	docpb := &documents.Document{
-		Id:  e.ID().TrimPrefix("hd://d/"),
-		Eid: string(e.ID()),
-	}
 
-	{
-		v, ok := e.Get("createTime")
-		if !ok {
-			return nil, fmt.Errorf("all documents must have create time")
-		}
-		switch vv := v.(type) {
-		case time.Time:
-			docpb.CreateTime = timestamppb.New(vv)
-		case int:
-			docpb.CreateTime = timestamppb.New(time.Unix(int64(vv), 0).UTC())
-		default:
-			return nil, fmt.Errorf("unknown type %T for createTime field", v)
-		}
-	}
+	first := e.AppliedChanges()[0]
 
-	docpb.UpdateTime = timestamppb.New(e.LastChangeTime().Time())
+	createTime, ok := first.Data.Patch["createTime"].(int)
+	if !ok {
+		return nil, fmt.Errorf("document must have createTime field")
+	}
 
-	{
-		v, ok := e.Get("author")
-		if !ok {
-			return nil, fmt.Errorf("all documents must have author")
-		}
+	owner, ok := first.Data.Patch["owner"].([]byte)
+	if !ok {
+		return nil, fmt.Errorf("document must have owner field")
+	}
 
-		switch vv := v.(type) {
-		case core.Principal:
-			docpb.Author = vv.String()
-		case []byte:
-			docpb.Author = core.Principal(vv).String()
-		default:
-			return nil, fmt.Errorf("unknown type %T for document author", v)
-		}
+	docpb := &documents.Document{
+		Id:         e.ID().TrimPrefix("hm://d/"),
+		Eid:        string(e.ID()),
+		CreateTime: timestamppb.New(time.Unix(int64(createTime), 0)),
+		Author:     core.Principal(owner).String(),
 	}
 
+	docpb.UpdateTime = timestamppb.New(e.LastChangeTime().Time())
+
 	{
 		v, ok := e.Get("title")
 		if ok {