feat: Reverted order for indexed fields (sourcenetwork#2335)

## Relevant issue(s)

Resolves sourcenetwork#2229

## Description

Enable reverted ordering for indexed fields. Which is mostly relevant to
composite indexes at this point.

This PR introduces a whole new package `encoding`. A significant part of
it is taken from CocroachDB which fits well to our needs. Other encoding
approaches (mostly for integers) were also consider: like fixed-length
encoding, avro's zizzag encoding, base128 varints encoding and few
others.

The encoding package can later be used for encoding of other value and 
might potentially replace CBOR.

client/document.go

@@ -172,7 +172,8 @@ func NewDocsFromJSON(obj []byte, sd SchemaDescription) ([]*Document, error) {
 	return docs, nil
 }
 
-func isNillableKind(kind FieldKind) bool {
+// IsNillableKind returns true if the given FieldKind is nillable.
+func IsNillableKind(kind FieldKind) bool {
 	switch kind {
 	case FieldKind_NILLABLE_STRING, FieldKind_NILLABLE_BLOB, FieldKind_NILLABLE_JSON,
 		FieldKind_NILLABLE_BOOL, FieldKind_NILLABLE_FLOAT, FieldKind_NILLABLE_DATETIME,
@@ -188,7 +189,7 @@ func isNillableKind(kind FieldKind) bool {
 // It will do any minor parsing, like dates, and return
 // the typed value again as an interface.
 func validateFieldSchema(val any, field SchemaFieldDescription) (any, error) {
-	if isNillableKind(field.Kind) {
+	if IsNillableKind(field.Kind) {
 		if val == nil {
 			return nil, nil
 		}

client/field.go

@@ -14,23 +14,18 @@ package client
 type Field interface {
 	Name() string
 	Type() CType //TODO Abstract into a Field Type interface
-	SchemaType() string
 }
 
 type simpleField struct {
-	name       string
-	crdtType   CType
-	schemaType string
+	name     string
+	crdtType CType
 }
 
-func (doc *Document) newField(t CType, name string, schemaType ...string) Field {
+func (doc *Document) newField(t CType, name string) Field {
 	f := simpleField{
 		name:     name,
 		crdtType: t,
 	}
-	if len(schemaType) > 0 {
-		f.schemaType = schemaType[0]
-	}
 	return f
 }
 
@@ -43,8 +38,3 @@ func (field simpleField) Name() string {
 func (field simpleField) Type() CType {
 	return field.crdtType
 }
-
-// SchemaType returns the schema type of the field.
-func (field simpleField) SchemaType() string {
-	return field.schemaType
-}

client/index.go

@@ -10,22 +10,12 @@
 
 package client
 
-// IndexDirection is the direction of an index.
-type IndexDirection string
-
-const (
-	// Ascending is the value to use for an ascending fields
-	Ascending IndexDirection = "ASC"
-	// Descending is the value to use for an descending fields
-	Descending IndexDirection = "DESC"
-)
-
 // IndexFieldDescription describes how a field is being indexed.
 type IndexedFieldDescription struct {
 	// Name contains the name of the field.
 	Name string
-	// Direction contains the direction of the index.
-	Direction IndexDirection
+	// Descending indicates whether the field is indexed in descending order.
+	Descending bool
 }
 
 // IndexDescription describes an index.

client/index_test.go

@@ -38,7 +38,7 @@ func TestCollectIndexesOnField(t *testing.T) {
 					{
 						Name: "index1",
 						Fields: []IndexedFieldDescription{
-							{Name: "test", Direction: Ascending},
+							{Name: "test"},
 						},
 					},
 				},
@@ -48,7 +48,7 @@ func TestCollectIndexesOnField(t *testing.T) {
 				{
 					Name: "index1",
 					Fields: []IndexedFieldDescription{
-						{Name: "test", Direction: Ascending},
+						{Name: "test"},
 					},
 				},
 			},
@@ -60,13 +60,13 @@ func TestCollectIndexesOnField(t *testing.T) {
 					{
 						Name: "index1",
 						Fields: []IndexedFieldDescription{
-							{Name: "test", Direction: Ascending},
+							{Name: "test"},
 						},
 					},
 					{
 						Name: "index2",
 						Fields: []IndexedFieldDescription{
-							{Name: "test", Direction: Descending},
+							{Name: "test", Descending: true},
 						},
 					},
 				},
@@ -76,13 +76,13 @@ func TestCollectIndexesOnField(t *testing.T) {
 				{
 					Name: "index1",
 					Fields: []IndexedFieldDescription{
-						{Name: "test", Direction: Ascending},
+						{Name: "test"},
 					},
 				},
 				{
 					Name: "index2",
 					Fields: []IndexedFieldDescription{
-						{Name: "test", Direction: Descending},
+						{Name: "test", Descending: true},
 					},
 				},
 			},
@@ -94,7 +94,7 @@ func TestCollectIndexesOnField(t *testing.T) {
 					{
 						Name: "index1",
 						Fields: []IndexedFieldDescription{
-							{Name: "other", Direction: Ascending},
+							{Name: "other"},
 						},
 					},
 				},
@@ -109,8 +109,8 @@ func TestCollectIndexesOnField(t *testing.T) {
 					{
 						Name: "index1",
 						Fields: []IndexedFieldDescription{
-							{Name: "other", Direction: Ascending},
-							{Name: "test", Direction: Ascending},
+							{Name: "other"},
+							{Name: "test"},
 						},
 					},
 				},

core/encoding.go

@@ -17,11 +17,13 @@ import (
 	"github.com/sourcenetwork/immutable"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/client/request"
+	"github.com/sourcenetwork/defradb/encoding"
 )
 
-// DecodeFieldValue takes a field value and description and converts it to the
+// NormalizeFieldValue takes a field value and description and converts it to the
 // standardized Defra Go type.
-func DecodeFieldValue(fieldDesc client.FieldDefinition, val any) (any, error) {
+func NormalizeFieldValue(fieldDesc client.FieldDefinition, val any) (any, error) {
 	if val == nil {
 		return nil, nil
 	}
@@ -125,6 +127,16 @@ func DecodeFieldValue(fieldDesc client.FieldDefinition, val any) (any, error) {
 			case string:
 				return time.Parse(time.RFC3339, v)
 			}
+		case client.FieldKind_NILLABLE_BOOL:
+			switch v := val.(type) {
+			case int64:
+				return v != 0, nil
+			}
+		case client.FieldKind_NILLABLE_STRING:
+			switch v := val.(type) {
+			case []byte:
+				return string(v), nil
+			}
 		}
 	}
 
@@ -179,3 +191,126 @@ func convertToInt(propertyName string, untypedValue any) (int64, error) {
 		return 0, client.NewErrUnexpectedType[string](propertyName, untypedValue)
 	}
 }
+
+// DecodeIndexDataStoreKey decodes a IndexDataStoreKey from bytes.
+// It expects the input bytes is in the following format:
+//
+// /[CollectionID]/[IndexID]/[FieldValue](/[FieldValue]...)
+//
+// Where [CollectionID] and [IndexID] are integers
+//
+// All values of the fields are converted to standardized Defra Go type
+// according to fields description.
+func DecodeIndexDataStoreKey(
+	data []byte,
+	indexDesc *client.IndexDescription,
+	fields []client.FieldDefinition,
+) (IndexDataStoreKey, error) {
+	if len(data) == 0 {
+		return IndexDataStoreKey{}, ErrEmptyKey
+	}
+
+	if data[0] != '/' {
+		return IndexDataStoreKey{}, ErrInvalidKey
+	}
+	data = data[1:]
+
+	data, colID, err := encoding.DecodeUvarintAscending(data)
+	if err != nil {
+		return IndexDataStoreKey{}, err
+	}
+
+	key := IndexDataStoreKey{CollectionID: uint32(colID)}
+
+	if data[0] != '/' {
+		return IndexDataStoreKey{}, ErrInvalidKey
+	}
+	data = data[1:]
+
+	data, indID, err := encoding.DecodeUvarintAscending(data)
+	if err != nil {
+		return IndexDataStoreKey{}, err
+	}
+	key.IndexID = uint32(indID)
+
+	if len(data) == 0 {
+		return key, nil
+	}
+
+	for len(data) > 0 {
+		if data[0] != '/' {
+			return IndexDataStoreKey{}, ErrInvalidKey
+		}
+		data = data[1:]
+
+		i := len(key.Fields)
+		descending := false
+		// If the key has more values encoded then fields on the index description, the last
+		// value must be the docID and we treat it as a string.
+		if i < len(indexDesc.Fields) {
+			descending = indexDesc.Fields[i].Descending
+		} else if i > len(indexDesc.Fields) {
+			return IndexDataStoreKey{}, ErrInvalidKey
+		}
+
+		var val any
+		data, val, err = encoding.DecodeFieldValue(data, descending)
+		if err != nil {
+			return IndexDataStoreKey{}, err
+		}
+
+		key.Fields = append(key.Fields, IndexedField{Value: val, Descending: descending})
+	}
+
+	err = normalizeIndexDataStoreKeyValues(&key, fields)
+	return key, err
+}
+
+// normalizeIndexDataStoreKeyValues converts all field values  to standardized
+// Defra Go type according to fields description.
+func normalizeIndexDataStoreKeyValues(key *IndexDataStoreKey, fields []client.FieldDefinition) error {
+	for i := range key.Fields {
+		if key.Fields[i].Value == nil {
+			continue
+		}
+		var err error
+		var val any
+		if i == len(key.Fields)-1 && len(key.Fields)-len(fields) == 1 {
+			bytes, ok := key.Fields[i].Value.([]byte)
+			if !ok {
+				return client.NewErrUnexpectedType[[]byte](request.DocIDArgName, key.Fields[i].Value)
+			}
+			val = string(bytes)
+		} else {
+			val, err = NormalizeFieldValue(fields[i], key.Fields[i].Value)
+		}
+		if err != nil {
+			return err
+		}
+		key.Fields[i].Value = val
+	}
+	return nil
+}
+
+// EncodeIndexDataStoreKey encodes a IndexDataStoreKey to bytes to be stored as a key
+// for secondary indexes.
+func EncodeIndexDataStoreKey(key *IndexDataStoreKey) []byte {
+	if key.CollectionID == 0 {
+		return []byte{}
+	}
+
+	b := encoding.EncodeUvarintAscending([]byte{'/'}, uint64(key.CollectionID))
+
+	if key.IndexID == 0 {
+		return b
+	}
+	b = append(b, '/')
+	b = encoding.EncodeUvarintAscending(b, uint64(key.IndexID))
+
+	for _, field := range key.Fields {
+		b = append(b, '/')
+		b = encoding.EncodeFieldValue(b, field.Value, field.Descending)
+	}
+
+	return b
+}

core/errors.go

@@ -16,15 +16,22 @@ import (
 
 const (
 	errFailedToGetFieldIdOfKey string = "failed to get FieldID of Key"
+	errInvalidFieldIndex       string = "invalid field index"
 )
 
 var (
 	ErrFailedToGetFieldIdOfKey = errors.New(errFailedToGetFieldIdOfKey)
 	ErrEmptyKey                = errors.New("received empty key string")
 	ErrInvalidKey              = errors.New("invalid key string")
+	ErrInvalidFieldIndex       = errors.New(errInvalidFieldIndex)
 )
 
 // NewErrFailedToGetFieldIdOfKey returns the error indicating failure to get FieldID of Key.
 func NewErrFailedToGetFieldIdOfKey(inner error) error {
 	return errors.Wrap(errFailedToGetFieldIdOfKey, inner)
 }
+
+// NewErrInvalidFieldIndex returns the error indicating invalid field index.
+func NewErrInvalidFieldIndex(i int) error {
+	return errors.New(errInvalidFieldIndex, errors.NewKV("index", i))
+}