Allow expanded keys in the BPT

.golangci.yml

@@ -51,6 +51,11 @@ issues:
     linters:
     - noprint
 
+  - path: _gen.go$
+    linters:
+    - goheader
+    - gosimple
+
 linters-settings:
   goheader:
     values:

internal/database/bpt_account.go

@@ -47,7 +47,7 @@ func (a *Account) putBpt() error {
 	}
 
 	hash := *(*[32]byte)(hasher.MerkleHash())
-	return a.parent.BPT().Insert(a.key.Hash(), hash)
+	return a.parent.BPT().Insert(a.key, hash)
 }
 
 // BptReceipt builds a BPT receipt for the account.

internal/database/events.go

@@ -167,7 +167,7 @@ func (e *eventsSet[T]) postRestore() error {
 	// Update the BPT
 	for _, v := range v {
 		err = e.bpt.Insert(
-			e.baseKey.AppendKey(e.getKey(v)).Hash(),
+			e.baseKey.AppendKey(e.getKey(v)),
 			*(*[32]byte)(e.getHash(v)))
 		if err != nil {
 			return errors.UnknownError.Wrap(err)
@@ -203,7 +203,7 @@ func (e *eventsSet[T]) Put(v []T) error {
 		k := e.getKey(v)
 		kh := k.Hash()
 		err = e.bpt.Insert(
-			e.baseKey.AppendKey(k).Hash(),
+			e.baseKey.AppendKey(k),
 			*(*[32]byte)(e.getHash(v)))
 		if err != nil {
 			return errors.UnknownError.Wrap(err)
@@ -213,7 +213,7 @@ func (e *eventsSet[T]) Put(v []T) error {
 
 	// Remove any entries that no longer exist
 	for _, k := range m {
-		err = e.bpt.Delete(e.baseKey.AppendKey(k).Hash())
+		err = e.bpt.Delete(e.baseKey.AppendKey(k))
 		if err != nil {
 			return errors.UnknownError.Wrap(err)
 		}
@@ -226,7 +226,7 @@ func (e *eventsSet[T]) Add(v ...T) error {
 	// Update the BPT
 	for _, v := range v {
 		err := e.bpt.Insert(
-			e.baseKey.AppendKey(e.getKey(v)).Hash(),
+			e.baseKey.AppendKey(e.getKey(v)),
 			*(*[32]byte)(e.getHash(v)))
 		if err != nil {
 			return errors.UnknownError.Wrap(err)
@@ -238,7 +238,7 @@ func (e *eventsSet[T]) Add(v ...T) error {
 
 func (e *eventsSet[T]) Remove(v T) error {
 	// Update the BPT
-	err := e.bpt.Delete(e.baseKey.AppendKey(e.getKey(v)).Hash())
+	err := e.bpt.Delete(e.baseKey.AppendKey(e.getKey(v)))
 	if err != nil {
 		return errors.UnknownError.Wrap(err)
 	}

pkg/database/bpt/bench_test.go

@@ -13,6 +13,7 @@ import (
 	"gitlab.com/accumulatenetwork/accumulate/internal/database/smt/common"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/memory"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/types/record"
 )
 
 func BenchmarkInsert(b *testing.B) {
@@ -23,7 +24,7 @@ func BenchmarkInsert(b *testing.B) {
 
 	var rh common.RandHash
 	for i := 0; i < b.N; i++ {
-		err := bpt.Insert(rh.NextA(), rh.NextA())
+		err := bpt.Insert(record.KeyFromHash(rh.NextA()), rh.NextA())
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -40,7 +41,7 @@ func BenchmarkInsertSubbatch(b *testing.B) {
 	var rh common.RandHash
 	for i := 0; i < b.N; i++ {
 		sub := model.Begin()
-		err := sub.BPT().Insert(rh.NextA(), rh.NextA())
+		err := sub.BPT().Insert(record.KeyFromHash(rh.NextA()), rh.NextA())
 		if err != nil {
 			b.Fatal(err)
 		}

pkg/database/bpt/bpt.go

@@ -8,6 +8,7 @@ package bpt
 
 import (
 	"github.com/cometbft/cometbft/libs/log"
+	"gitlab.com/accumulatenetwork/accumulate/internal/database/record"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/database"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/database/values"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/errors"
@@ -140,15 +141,15 @@ func (b *BPT) newRoot() *rootRecord {
 }
 
 // Get retrieves the latest hash associated with the given key.
-func (b *BPT) Get(key [32]byte) ([32]byte, error) {
-	if v, ok := b.pending[key]; ok {
+func (b *BPT) Get(key *record.Key) ([32]byte, error) {
+	if v, ok := b.pending[key.Hash()]; ok {
 		if v.delete {
 			return [32]byte{}, errors.NotFound
 		}
 		return v.value, nil
 	}
 
-	e, err := b.getRoot().getLeaf(key)
+	e, err := b.getRoot().getLeaf(key.Hash())
 	if err != nil {
 		return [32]byte{}, errors.UnknownError.Wrap(err)
 	}
@@ -165,7 +166,7 @@ again:
 
 	switch f := (*f).(type) {
 	case *leaf:
-		if f.Key == key {
+		if f.Key.Hash() == key {
 			return f, nil
 		}
 	case *branch:

pkg/database/bpt/bpt_receipt.go

@@ -36,7 +36,7 @@ func (b *BPT) collectReceipt(BIdx, bit byte, n *branch, key [32]byte, r *merkle.
 
 	value, ok := entry.(*leaf)
 	if ok {
-		if value.Key == key {
+		if value.Key.Hash() == key {
 			r.Start = append(r.Start[:0], value.Hash[:]...)
 			if other != nil { // If other isn't nil, then add it to the node list of the receipt
 				h, _ := other.getHash()

pkg/database/bpt/bpt_savestate.go

@@ -14,6 +14,7 @@ import (
 	"gitlab.com/accumulatenetwork/accumulate/internal/database/smt/storage"
 	ioutil2 "gitlab.com/accumulatenetwork/accumulate/internal/util/io"
 	"gitlab.com/accumulatenetwork/accumulate/pkg/errors"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/types/record"
 )
 
 // SaveSnapshotV1
@@ -128,7 +129,7 @@ func LoadSnapshotV1(b *BPT, file ioutil2.SectionReader, storeState func(key stor
 	}
 
 	return ReadSnapshotV1(file, func(key storage.Key, hash [32]byte, reader ioutil2.SectionReader) error {
-		err := b.Insert(key, hash) // Insert the key/hash into the BPT
+		err := b.Insert(record.KeyFromHash(key), hash) // Insert the key/hash into the BPT
 		if err != nil {
 			return err
 		}

pkg/database/bpt/enums.yml

@@ -10,4 +10,8 @@ nodeType:
     value: 3
   Boundary:
     description: is the boundary between blocks
-    value: 4
+    value: 4
+  LeafWithExpandedKey:
+    description: is a leaf node with an expanded key
+    label: leaf+key
+    value: 5

pkg/database/bpt/enums_gen.go

@@ -26,14 +26,17 @@ const nodeTypeLeaf nodeType = 3
 // nodeTypeBoundary is the boundary between blocks.
 const nodeTypeBoundary nodeType = 4
 
+// nodeTypeLeafWithExpandedKey is a leaf node with an expanded key.
+const nodeTypeLeafWithExpandedKey nodeType = 5
+
 // GetEnumValue returns the value of the node Type
 func (v nodeType) GetEnumValue() uint64 { return uint64(v) }
 
 // SetEnumValue sets the value. SetEnumValue returns false if the value is invalid.
 func (v *nodeType) SetEnumValue(id uint64) bool {
 	u := nodeType(id)
 	switch u {
-	case nodeTypeEmpty, nodeTypeBranch, nodeTypeLeaf, nodeTypeBoundary:
+	case nodeTypeEmpty, nodeTypeBranch, nodeTypeLeaf, nodeTypeBoundary, nodeTypeLeafWithExpandedKey:
 		*v = u
 		return true
 	}
@@ -51,6 +54,8 @@ func (v nodeType) String() string {
 		return "leaf"
 	case nodeTypeBoundary:
 		return "boundary"
+	case nodeTypeLeafWithExpandedKey:
+		return "leaf+key"
 	}
 	return fmt.Sprintf("nodeType:%d", v)
 }
@@ -66,6 +71,10 @@ func nodeTypeByName(name string) (nodeType, bool) {
 		return nodeTypeLeaf, true
 	case "boundary":
 		return nodeTypeBoundary, true
+	case "leafwithexpandedkey":
+		return nodeTypeLeafWithExpandedKey, true
+	case "leaf+key":
+		return nodeTypeLeafWithExpandedKey, true
 	}
 	return 0, false
 }

pkg/database/bpt/iterate.go

@@ -89,12 +89,12 @@ func walkNode(n node, found *bool, key [32]byte, values []KeyValuePair, pos *int
 		return n.walkRange(found, key, values, pos)
 
 	case *leaf: //                                                If not a node, not nil, it is a value.
-		*found = true     //                                      No matter what, start collecting the range
-		if n.Key == key { //                                      But don't collect if equal to the key
+		*found = true            //                                      No matter what, start collecting the range
+		if n.Key.Hash() == key { //                                      But don't collect if equal to the key
 			break //                                              because we use the last key to get the
 		} //                                                      next range
-		values[*pos] = KeyValuePair{Key: n.Key, Value: n.Hash} // Otherwise copy the value out of the BPT
-		*pos++                                                 // and stuff it in the values list
+		values[*pos] = KeyValuePair{Key: n.Key.Hash(), Value: n.Hash} // Otherwise copy the value out of the BPT
+		*pos++                                                        // and stuff it in the values list
 	}
 
 	return nil

pkg/database/bpt/key_test.go

@@ -0,0 +1,130 @@
+// Copyright 2023 The Accumulate Authors
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+package bpt
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/memory"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/types/record"
+)
+
+// TestOldBptStillWorks verifies that the BPT will still work with data recorded
+// using a previous version.
+func TestOldBptStillWorks(t *testing.T) {
+	unhex := func(s string) []byte {
+		b, err := hex.DecodeString(s)
+		require.NoError(t, err)
+		return b
+	}
+
+	// Load the recorded entries into a database
+	store := memory.New(nil)
+	require.NoError(t, store.Import([]memory.Entry{
+		{
+			Key: record.NewKey("BPT", "Root"),
+			Value: unhex(
+				"010008000770e98b7497b5e25de11190dad4af85cd3d8fedd4a93061171e13e42c3855f37c"),
+		},
+		{
+			Key: record.NewKey("BPT", [32]byte{0x80}),
+			Value: unhex("" +
+				"02c000000000000000000000000000000000000000000000000000000000000000017cd1f9868114" +
+				"069f943607a9cd41d1c8b3317f44b2522133ae6f9ee17c28a52b03c0000000000000000000000000" +
+				"00000000000000000000000000000000000000040000000000000000000000000000000000000000" +
+				"00000000000000000000000380000000000000000000000000000000000000000000000000000000" +
+				"00000000020000000000000000000000000000000000000000000000000000000000000002400000" +
+				"0000000000000000000000000000000000000000000000000000000000015b0f47b11478610c5abc" +
+				"bc912b34ac5f7f5740b6d8b27169b1eb2aea9f1c909c034000000000000000000000000000000000" +
+				"00000000000000000000000000000003000000000000000000000000000000000000000000000000" +
+				"00000000000000030000000000000000000000000000000000000000000000000000000000000000" +
+				"0100000000000000000000000000000000000000000000000000000000000000"),
+		},
+	}))
+
+	// Check the root
+	model := new(ChangeSet)
+	model.store = keyvalue.RecordStore{Store: store.Begin(nil, true)}
+	root, err := model.BPT().GetRootHash()
+	require.NoError(t, err)
+	require.Equal(t, testRoot, root)
+
+	// Check the entries
+	for _, entry := range testEntries {
+		value, err := model.BPT().Get(entry.Key)
+		require.NoError(t, err)
+		require.Equal(t, entry.Hash, value)
+	}
+
+	testEntryMap := map[[32]byte][32]byte{}
+	for _, entry := range testEntries {
+		testEntryMap[entry.Key.Hash()] = entry.Hash
+	}
+
+	for it := model.BPT().Iterate(100); it.Next(); {
+		for _, leaf := range it.Value() {
+			require.Equal(t, testEntryMap[leaf.Key], leaf.Value)
+		}
+	}
+}
+
+// TestSwitchKeyType verifies that nothing explodes when BPT entry keys are
+// changed from compressed to expanded.
+func TestSwitchKeyType(t *testing.T) {
+	kvs := memory.New(nil).Begin(nil, true)
+	store := keyvalue.RecordStore{Store: kvs}
+	bpt := newBPT(nil, nil, store, nil, "BPT")
+
+	entries := []*record.Key{
+		record.NewKey(1),
+		record.NewKey(2),
+		record.NewKey(3),
+		record.NewKey(4),
+	}
+
+	// Insert entries using compressed keys
+	for _, e := range entries {
+		require.NoError(t, bpt.Insert(record.KeyFromHash(e.Hash()), e.Hash()))
+	}
+	require.NoError(t, bpt.Commit())
+	before := allBptKeys(bpt)
+
+	// Reload the tree and update using expanded keys
+	bpt = newBPT(nil, nil, store, nil, "BPT")
+	for _, e := range entries[:len(entries)/2] {
+		require.NoError(t, bpt.Insert(e, e.Append("foo").Hash()))
+	}
+	require.NoError(t, bpt.Commit())
+
+	// Verify the leaf now uses an expanded key
+	l, err := bpt.getRoot().getLeaf(entries[0].Hash())
+	require.NoError(t, err)
+	require.True(t, isExpandedKey(l.Key))
+
+	// Reload the tree and verify
+	bpt = newBPT(nil, nil, store, nil, "BPT")
+	l, err = bpt.getRoot().getLeaf(entries[0].Hash())
+	require.NoError(t, err)
+	require.True(t, isExpandedKey(l.Key))
+
+	// Keys did not change
+	after := allBptKeys(bpt)
+	require.Equal(t, before, after)
+}
+
+func allBptKeys(bpt *BPT) [][32]byte {
+	var entries [][32]byte
+	for it := bpt.Iterate(100); it.Next(); {
+		for _, leaf := range it.Value() {
+			entries = append(entries, leaf.Key)
+		}
+	}
+	return entries
+}