feat: Doc field encryption (sourcenetwork#2817)

## Relevant issue(s)

Resolves sourcenetwork#2809 sourcenetwork#2812

## Description

Adds field-level encryption which allows separate fields
to be encryption with a dedicated symmetric key.

cli/collection_create.go

@@ -17,25 +17,34 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/datastore"
 	"github.com/sourcenetwork/defradb/internal/db"
+	"github.com/sourcenetwork/defradb/internal/encryption"
 )
 
 func MakeCollectionCreateCommand() *cobra.Command {
 	var file string
-	var shouldEncrypt bool
+	var shouldEncryptDoc bool
+	var encryptedFields []string
 	var cmd = &cobra.Command{
-		Use:   "create [-i --identity] [-e --encrypt] <document>",
+		Use:   "create [-i --identity] [-e --encrypt] [--encrypt-fields] <document>",
 		Short: "Create a new document.",
 		Long: `Create a new document.
 		
 Options:
-    -i, --identity 
-        Marks the document as private and set the identity as the owner. The access to the document
+	-i, --identity 
+		Marks the document as private and set the identity as the owner. The access to the document
 		and permissions are controlled by ACP (Access Control Policy).
 
 	-e, --encrypt
 		Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a
 		symmetric key for encryption using AES-GCM.
+	
+	--encrypt-fields
+		Comma-separated list of fields to encrypt. If set, DefraDB will encrypt only the specified fields
+		and for every field in the list it will generate a symmetric key for encryption using AES-GCM.
+		If combined with '--encrypt' flag, all the fields in the document not listed in '--encrypt-fields' 
+		will be encrypted with the same key.
 
 Example: create from string:
   defradb client collection create --name User '{ "name": "Bob" }'
@@ -81,7 +90,7 @@ Example: create from stdin:
 			}
 
 			txn, _ := db.TryGetContextTxn(cmd.Context())
-			setContextDocEncryption(cmd, shouldEncrypt, txn)
+			setContextDocEncryption(cmd, shouldEncryptDoc, encryptedFields, txn)
 
 			if client.IsJSONArray(docData) {
 				docs, err := client.NewDocsFromJSON(docData, col.Definition())
@@ -98,8 +107,23 @@ Example: create from stdin:
 			return col.Create(cmd.Context(), doc)
 		},
 	}
-	cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false,
+	cmd.PersistentFlags().BoolVarP(&shouldEncryptDoc, "encrypt", "e", false,
 		"Flag to enable encryption of the document")
+	cmd.PersistentFlags().StringSliceVar(&encryptedFields, "encrypt-fields", nil,
+		"Comma-separated list of fields to encrypt")
 	cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)")
 	return cmd
 }
+
+// setContextDocEncryption sets doc encryption for the current command context.
+func setContextDocEncryption(cmd *cobra.Command, shouldEncryptDoc bool, encryptFields []string, txn datastore.Txn) {
+	if !shouldEncryptDoc && len(encryptFields) == 0 {
+		return
+	}
+	ctx := cmd.Context()
+	if txn != nil {
+		ctx = encryption.ContextWithStore(ctx, txn)
+	}
+	ctx = encryption.SetContextConfigFromParams(ctx, shouldEncryptDoc, encryptFields)
+	cmd.SetContext(ctx)
+}

cli/utils.go

@@ -25,10 +25,8 @@ import (
 
 	acpIdentity "github.com/sourcenetwork/defradb/acp/identity"
 	"github.com/sourcenetwork/defradb/client"
-	"github.com/sourcenetwork/defradb/datastore"
 	"github.com/sourcenetwork/defradb/http"
 	"github.com/sourcenetwork/defradb/internal/db"
-	"github.com/sourcenetwork/defradb/internal/encryption"
 	"github.com/sourcenetwork/defradb/keyring"
 )
 
@@ -162,19 +160,6 @@ func setContextIdentity(cmd *cobra.Command, privateKeyHex string) error {
 	return nil
 }
 
-// setContextDocEncryption sets doc encryption for the current command context.
-func setContextDocEncryption(cmd *cobra.Command, shouldEncrypt bool, txn datastore.Txn) {
-	if !shouldEncrypt {
-		return
-	}
-	ctx := cmd.Context()
-	if txn != nil {
-		ctx = encryption.ContextWithStore(ctx, txn)
-	}
-	ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true})
-	cmd.SetContext(ctx)
-}
-
 // setContextRootDir sets the rootdir for the current command context.
 func setContextRootDir(cmd *cobra.Command) error {
 	rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir")

client/request/consts.go

@@ -26,7 +26,8 @@ const (
 	FieldIDName = "fieldId"
 	ShowDeleted = "showDeleted"
 
-	EncryptArgName = "encrypt"
+	EncryptDocArgName    = "encrypt"
+	EncryptFieldsArgName = "encryptFields"
 
 	FilterClause  = "filter"
 	GroupByClause = "groupBy"

client/request/mutation.go

@@ -50,6 +50,9 @@ type ObjectMutation struct {
 
 	// Encrypt is a boolean flag that indicates whether the input data should be encrypted.
 	Encrypt bool
+
+	// EncryptFields is a list of doc fields from input data that should be encrypted.
+	EncryptFields []string
 }
 
 // ToSelect returns a basic Select object, with the same Name, Alias, and Fields as

datastore/prefix_query.go

@@ -13,6 +13,7 @@ package datastore
 import (
 	"context"
 	"encoding/json"
+	"errors"
 
 	ds "github.com/ipfs/go-datastore"
 
@@ -35,15 +36,13 @@ func DeserializePrefix[T any](
 	elements := make([]T, 0)
 	for res := range q.Next() {
 		if res.Error != nil {
-			_ = q.Close()
-			return nil, nil, res.Error
+			return nil, nil, errors.Join(res.Error, q.Close())
 		}
 
 		var element T
 		err = json.Unmarshal(res.Value, &element)
 		if err != nil {
-			_ = q.Close()
-			return nil, nil, NewErrInvalidStoredValue(err)
+			return nil, nil, errors.Join(NewErrInvalidStoredValue(err), q.Close())
 		}
 		keys = append(keys, res.Key)
 		elements = append(elements, element)
@@ -68,8 +67,7 @@ func FetchKeysForPrefix(
 	keys := make([]ds.Key, 0)
 	for res := range q.Next() {
 		if res.Error != nil {
-			_ = q.Close()
-			return nil, res.Error
+			return nil, errors.Join(res.Error, q.Close())
 		}
 		keys = append(keys, ds.NewKey(res.Key))
 	}

docs/data_format_changes/i2817-doc-field-encryption.md

@@ -0,0 +1,3 @@
+# Doc field encryption
+Changed hard-coded constant test encryption key to be dependant on docID and fieldName. 
+This produces different CIDs.

docs/website/references/cli/defradb_client_collection_create.md

@@ -7,13 +7,19 @@ Create a new document.
 Create a new document.
 
 Options:
-    -i, --identity 
-        Marks the document as private and set the identity as the owner. The access to the document
+	-i, --identity 
+		Marks the document as private and set the identity as the owner. The access to the document
 		and permissions are controlled by ACP (Access Control Policy).
 
 	-e, --encrypt
 		Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a
 		symmetric key for encryption using AES-GCM.
+
+	--encrypt-fields
+		Comma-separated list of fields to encrypt. If set, DefraDB will encrypt only the specified fields
+		and for every field in the list it will generate a symmetric key for encryption using AES-GCM.
+		If combined with '--encrypt' flag, all the fields in the document not listed in '--encrypt-fields' 
+		will be encrypted with the same key.
 
 Example: create from string:
   defradb client collection create --name User '{ "name": "Bob" }'
@@ -33,15 +39,16 @@ Example: create from stdin:
 
 
 ```
-defradb client collection create [-i --identity] [-e --encrypt] <document> [flags]
+defradb client collection create [-i --identity] [-e --encrypt] [--encrypt-fields] <document> [flags]
 ```
 
 ### Options
 
 ```
-  -e, --encrypt       Flag to enable encryption of the document
-  -f, --file string   File containing document(s)
-  -h, --help          help for create
+  -e, --encrypt                  Flag to enable encryption of the document
+      --encrypt-fields strings   Comma-separated list of fields to encrypt
+  -f, --file string              File containing document(s)
+  -h, --help                     help for create
 ```
 
 ### Options inherited from parent commands

http/client_collection.go

@@ -132,9 +132,14 @@ func (c *Collection) CreateMany(
 
 func setDocEncryptionFlagIfNeeded(ctx context.Context, req *http.Request) {
 	encConf := encryption.GetContextConfig(ctx)
-	if encConf.HasValue() && encConf.Value().IsEncrypted {
+	if encConf.HasValue() {
 		q := req.URL.Query()
-		q.Set(docEncryptParam, "true")
+		if encConf.Value().IsDocEncrypted {
+			q.Set(docEncryptParam, "true")
+		}
+		if len(encConf.Value().EncryptedFields) > 0 {
+			q.Set(docEncryptFieldsParam, strings.Join(encConf.Value().EncryptedFields, ","))
+		}
 		req.URL.RawQuery = q.Encode()
 	}
 }

http/handler_collection.go

@@ -16,6 +16,7 @@ import (
 	"io"
 	"net/http"
 	"strconv"
+	"strings"
 
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/go-chi/chi/v5"
@@ -25,6 +26,7 @@ import (
 )
 
 const docEncryptParam = "encrypt"
+const docEncryptFieldsParam = "encryptFields"
 
 type collectionHandler struct{}
 
@@ -47,8 +49,16 @@ func (s *collectionHandler) Create(rw http.ResponseWriter, req *http.Request) {
 	}
 
 	ctx := req.Context()
-	if req.URL.Query().Get(docEncryptParam) == "true" {
-		ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true})
+	q := req.URL.Query()
+	encConf := encryption.DocEncConfig{}
+	if q.Get(docEncryptParam) == "true" {
+		encConf.IsDocEncrypted = true
+	}
+	if q.Get(docEncryptFieldsParam) != "" {
+		encConf.EncryptedFields = strings.Split(q.Get(docEncryptFieldsParam), ",")
+	}
+	if encConf.IsDocEncrypted || len(encConf.EncryptedFields) > 0 {
+		ctx = encryption.SetContextConfig(ctx, encConf)
 	}
 
 	switch {

internal/core/key.go

@@ -795,25 +795,25 @@ func bytesPrefixEnd(b []byte) []byte {
 
 // EncStoreDocKey is a key for the encryption store.
 type EncStoreDocKey struct {
-	DocID   string
-	FieldID uint32
+	DocID     string
+	FieldName string
 }
 
 var _ Key = (*EncStoreDocKey)(nil)
 
 // NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID.
-func NewEncStoreDocKey(docID string, fieldID uint32) EncStoreDocKey {
+func NewEncStoreDocKey(docID string, fieldName string) EncStoreDocKey {
 	return EncStoreDocKey{
-		DocID:   docID,
-		FieldID: fieldID,
+		DocID:     docID,
+		FieldName: fieldName,
 	}
 }
 
 func (k EncStoreDocKey) ToString() string {
-	if k.FieldID == 0 {
+	if k.FieldName == "" {
 		return k.DocID
 	}
-	return fmt.Sprintf("%s/%d", k.DocID, k.FieldID)
+	return fmt.Sprintf("%s/%s", k.DocID, k.FieldName)
 }
 
 func (k EncStoreDocKey) Bytes() []byte {

internal/db/collection.go

@@ -33,6 +33,7 @@ import (
 	"github.com/sourcenetwork/defradb/internal/db/base"
 	"github.com/sourcenetwork/defradb/internal/db/description"
 	"github.com/sourcenetwork/defradb/internal/db/fetcher"
+	"github.com/sourcenetwork/defradb/internal/encryption"
 	"github.com/sourcenetwork/defradb/internal/lens"
 	merklecrdt "github.com/sourcenetwork/defradb/internal/merkle/crdt"
 )
@@ -561,6 +562,27 @@ func (c *collection) Save(
 	return txn.Commit(ctx)
 }
 
+func (c *collection) validateEncryptedFields(ctx context.Context) error {
+	encConf := encryption.GetContextConfig(ctx)
+	if !encConf.HasValue() {
+		return nil
+	}
+	fields := encConf.Value().EncryptedFields
+	if len(fields) == 0 {
+		return nil
+	}
+
+	for _, field := range fields {
+		if _, exists := c.Schema().GetFieldByName(field); !exists {
+			return client.NewErrFieldNotExist(field)
+		}
+		if strings.HasPrefix(field, "_") {
+			return NewErrCanNotEncryptBuiltinField(field)
+		}
+	}
+	return nil
+}
+
 // save saves the document state. save MUST not be called outside the `c.create`
 // and `c.update` methods as we wrap the acp logic within those methods. Calling
 // save elsewhere could cause the omission of acp checks.
@@ -569,6 +591,10 @@ func (c *collection) save(
 	doc *client.Document,
 	isCreate bool,
 ) (cid.Cid, error) {
+	if err := c.validateEncryptedFields(ctx); err != nil {
+		return cid.Undef, err
+	}
+
 	if !isCreate {
 		err := c.updateIndexedDoc(ctx, doc)
 		if err != nil {
@@ -657,7 +683,7 @@ func (c *collection) save(
 				return cid.Undef, err
 			}
 
-			link, _, err := merkleCRDT.Save(ctx, &merklecrdt.DocField{DocID: primaryKey.DocID, FieldValue: val})
+			link, _, err := merkleCRDT.Save(ctx, merklecrdt.NewDocField(primaryKey.DocID, k, val))
 			if err != nil {
 				return cid.Undef, err
 			}

internal/db/errors.go

@@ -100,6 +100,7 @@ const (
 	errReplicatorDocID                          string = "failed to get docID for replicator"
 	errReplicatorCollections                    string = "failed to get collections for replicator"
 	errReplicatorNotFound                       string = "replicator not found"
+	errCanNotEncryptBuiltinField                string = "can not encrypt build-in field"
 )
 
 var (
@@ -140,6 +141,7 @@ var (
 	ErrSelfTargetForReplicator                  = errors.New("can't target ourselves as a replicator")
 	ErrReplicatorCollections                    = errors.New(errReplicatorCollections)
 	ErrReplicatorNotFound                       = errors.New(errReplicatorNotFound)
+	ErrCanNotEncryptBuiltinField                = errors.New(errCanNotEncryptBuiltinField)
 )
 
 // NewErrFailedToGetHeads returns a new error indicating that the heads of a document
@@ -330,6 +332,10 @@ func NewErrCannotMoveField(name string, proposedIndex, existingIndex int) error
 	)
 }
 
+func NewErrCanNotEncryptBuiltinField(name string) error {
+	return errors.New(errCanNotEncryptBuiltinField, errors.NewKV("Name", name))
+}
+
 func NewErrCannotDeleteField(name string) error {
 	return errors.New(
 		errCannotDeleteField,

internal/encryption/config.go

@@ -12,5 +12,8 @@ package encryption
 
 // DocEncConfig is the configuration for document encryption.
 type DocEncConfig struct {
-	IsEncrypted bool
+	// IsDocEncrypted is a flag to indicate if the document should be encrypted.
+	IsDocEncrypted bool
+	//  EncryptedFields is a list of fields individual that should be encrypted.
+	EncryptedFields []string
 }