diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1de81dc..80c020f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: env: # Even though we can test against multiple versions, this one is considered a target version. - TARGET_GOLANG_VERSION: "1.18" + TARGET_GOLANG_VERSION: "1.19" jobs: tests: @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - go: ["1.18"] + go: ["1.19"] name: Go Tests steps: - uses: actions/checkout@v3 @@ -83,7 +83,7 @@ jobs: fail-fast: false matrix: goarch: ["arm64", "amd64"] - go: ["1.18"] + go: ["1.19"] timeout-minutes: 5 name: Build for ${{ matrix.goarch }} steps: diff --git a/KVStore.md b/KVStore.md new file mode 100644 index 0000000..b8bc1a4 --- /dev/null +++ b/KVStore.md @@ -0,0 +1,82 @@ +# KVStore + +- [Overview](#overview) +- [Implementation](#implementation) + - [In-Memory and Persistent](#in-memory-and-persistent) + - [Store methods](#store-methods) + - [Lifecycle Methods](#lifecycle-methods) + - [Data Methods](#data-methods) + - [Backups](#backups) + - [Restorations](#restorations) + - [Accessor Methods](#accessor-methods) + - [Prefixed and Sorted Get All](#prefixed-and-sorted-get-all) + - [Clear All Key-Value Pairs](#clear-all-key-value-pairs) + - [Len](#len) + +## Overview + +The `KVStore` interface is a key-value store that is used by the `SMT` and `SMST` as its underlying database for its nodes. However, it is an independent key-value store that can be used for any purpose. + +## Implementation + +The `KVStore` is implemented in [`kvstore.go`](./kvstore.go) and is a wrapper around the [BadgerDB](https://github.com/dgraph-io/badger) key-value database. + +The interface defines simple key-value store accessor methods as well as other methods desired from a key-value database in general, this can be found in [`kvstore.go`](./kvstore.go). + +_NOTE: The `KVStore` interface can be implemented by any key-value store that satisfies the interface and used as the underlying database store for the `SM(S)T`_ + +### In-Memory and Persistent + +The `KVStore` implementation can be used as an in-memory or persistent key-value store. The `NewKVStore` function takes a `path` argument that can be used to specify a path to a directory to store the database files. If the `path` is an empty string, the database will be stored in-memory. + +_NOTE: When providing a path for a persistent database, the directory must exist and be writeable by the user running the application._ + +### Store methods + +As a key-value store the `KVStore` interface defines the simple `Get`, `Set` and `Delete` methods to access and modify the underlying database. + +### Lifecycle Methods + +The `Stop` method **must** be called when the `KVStore` is no longer needed. This method closes the underlying database and frees up any resources used by the `KVStore`. + +For persistent databases, the `Stop` method should be called when the application no longer needs to access the database. For in-memory databases, the `Stop` method should be called when the `KVStore` is no longer needed. + +_NOTE: A persistent `KVStore` that is not stopped will stop another `KVStore` from opening the database._ + +### Data Methods + +The `KVStore` interface provides two methods to allow backups and restorations. + +#### Backups + +The `Backup` method takes an `io.Writer` and a `bool` to indicate whether the backup should be incremental or not. The `io.Writer` is then filled with the contents of the database in an opaque format used by the underlying database for this purpose. + +When the `incremental` bool is `false` a full backup will be performed, otherwise an incremental backup will be performed. This is enabled by the `KVStore` keeping the timestamp of its last backup and only backing up data that has been modified since the last backup. + +#### Restorations + +The `Restore` method takes an `io.Reader` and restores the database from this reader. + +The `KVStore` calling the `Restore` method is expected to be initialised and open, otherwise the restore will fail. + +_NOTE: Any data contained in the `KVStore` when calling restore will be overwritten._ + +### Accessor Methods + +The accessor methods enable simpler access to the underlying database for certain tasks that are desirable in a key-value store. + +#### Prefixed and Sorted Get All + +The `GetAll` method supports the retrieval of all keys and values, where the key has a specific prefix. The `descending` bool indicates whether the keys should be returned in descending order or not. + +_NOTE: In order to retrieve all keys and values the empty prefix `[]byte{}` should be used to match all keys_ + +#### Clear All Key-Value Pairs + +The `ClearAll` method removes all key-value pairs from the database. + +_NOTE: The `ClearAll` method is intended to debug purposes and should not be used in production unless necessary_ + +#### Len + +The `Len` method returns the number of keys in the database, similarly to how the `len` function can return the length of a map. diff --git a/MerkleSumTree.md b/MerkleSumTree.md index f20f371..807d429 100644 --- a/MerkleSumTree.md +++ b/MerkleSumTree.md @@ -233,31 +233,37 @@ import ( ) func main() { - // Initialise a new key-value store to store the nodes of the tree - // (Note: the tree only stores hashed values, not raw value data) - nodeStore := smt.NewSimpleMap() - - // Initialise the tree - tree := smt.NewSparseMerkleSumTree(nodeStore, sha256.New()) - - // Update tree with keys, values and their sums - _ = tree.Update([]byte("foo"), []byte("oof"), 10) - _ = tree.Update([]byte("baz"), []byte("zab"), 7) - _ = tree.Update([]byte("bin"), []byte("nib"), 3) - - sum := tree.Sum() - fmt.Println(sum == 20) // true - - // Generate a Merkle proof for "foo" - proof, _ := tree.Prove([]byte("foo")) - root := tree.Root() // We also need the current tree root for the proof - - // Verify the Merkle proof for "foo"="oof" where "foo" has a sum of 10 - if valid := smt.VerifySumProof(proof, root, []byte("foo"), []byte("oof"), 10, tree.Spec()); valid { - fmt.Println("Proof verification succeeded.") - } else { - fmt.Println("Proof verification failed.") - } + // Initialise a new in-memory key-value store to store the nodes of the tree + // (Note: the tree only stores hashed values, not raw value data) + nodeStore := smt.NewKVStore("") + + // Ensure the database connection closes + defer nodeStore.Stop() + + // Initialise the tree + tree := smt.NewSparseMerkleSumTree(nodeStore, sha256.New()) + + // Update tree with keys, values and their sums + _ = tree.Update([]byte("foo"), []byte("oof"), 10) + _ = tree.Update([]byte("baz"), []byte("zab"), 7) + _ = tree.Update([]byte("bin"), []byte("nib"), 3) + + // Commit the changes to the nodeStore + _ = tree.Commit() + + sum := tree.Sum() + fmt.Println(sum == 20) // true + + // Generate a Merkle proof for "foo" + proof, _ := tree.Prove([]byte("foo")) + root := tree.Root() // We also need the current tree root for the proof + + // Verify the Merkle proof for "foo"="oof" where "foo" has a sum of 10 + if valid := smt.VerifySumProof(proof, root, []byte("foo"), []byte("oof"), 10, tree.Spec()); valid { + fmt.Println("Proof verification succeeded.") + } else { + fmt.Println("Proof verification failed.") + } } ``` diff --git a/README.md b/README.md index da17680..99671c4 100644 --- a/README.md +++ b/README.md @@ -6,27 +6,27 @@ [![Tests](https://github.com/pokt-network/smt/actions/workflows/test.yml/badge.svg)](https://github.com/pokt-network/smt/actions/workflows/test.yml) [![codecov](https://codecov.io/gh/pokt-network/smt/branch/main/graph/badge.svg)](https://codecov.io/gh/pokt-network/smt) -Note: **Requires Go 1.18+** +Note: **Requires Go 1.19+** - [Overview](#overview) - [Implementation](#implementation) - - [Inner Nodes](#inner-nodes) - - [Extension Nodes](#extension-nodes) - - [Leaf Nodes](#leaf-nodes) - - [Lazy Nodes](#lazy-nodes) - - [Lazy Loading](#lazy-loading) - - [Visualisations](#visualisations) - - [General Tree Structure](#general-tree-structure) - - [Lazy Nodes](#lazy-nodes-1) + - [Inner Nodes](#inner-nodes) + - [Extension Nodes](#extension-nodes) + - [Leaf Nodes](#leaf-nodes) + - [Lazy Nodes](#lazy-nodes) + - [Lazy Loading](#lazy-loading) + - [Visualisations](#visualisations) + - [General Tree Structure](#general-tree-structure) + - [Lazy Nodes](#lazy-nodes-1) - [Paths](#paths) - - [Visualisation](#visualisation) + - [Visualisation](#visualisation) - [Values](#values) - - [Nil values](#nil-values) + - [Nil values](#nil-values) - [Hashers \& Digests](#hashers--digests) - [Proofs](#proofs) - - [Verification](#verification) + - [Verification](#verification) - [Database](#database) - - [Data Loss](#data-loss) + - [Data Loss](#data-loss) - [Sparse Merkle Sum Tree](#sparse-merkle-sum-tree) - [Example](#example) @@ -295,20 +295,12 @@ The verification step simply uses the proof data to recompute the root hash with ## Database -This library defines the `MapStore` interface, in [mapstore.go](./mapstore.go) - -```go -type MapStore interface { - Get(key []byte) ([]byte, error) - Set(key []byte, value []byte) error - Delete(key []byte) error -} -``` - -This interface abstracts the `SimpleMap` key-value store and can be used by the SMT to store the nodes of the tree. Any key-value store that implements the `MapStore` interface can be used with this library. +This library defines the `KVStore` interface which by default is implemented using [BadgerDB](https://github.com/dgraph-io/badger), however any databse that implements this interface can be used as a drop in replacement. The `KVStore` allows for both in memory and persisted databases to be used to store the nodes for the SMT. When changes are commited to the underlying database using `Commit()` the digests of the leaf nodes are stored at their respective paths. If retrieved manually from the database the returned value will be the digest of the leaf node, **not** the leaf node's value, even when `WithValueHasher(nil)` is used. The node value can be parsed from this value, as the tree `Get` function does by removing the prefix and path bytes from the returned value. +See [KVStore.md](./KVStore.md) for the details of the implementation. + ### Data Loss In the event of a system crash or unexpected failure of the program utilising the SMT, if the `Commit()` function has not been called, any changes to the tree will be lost. This is due to the underlying database not being changed **until** the `Commit()` function is called and changes are persisted. @@ -330,26 +322,32 @@ import ( ) func main() { - // Initialise a new key-value store to store the nodes of the tree - // (Note: the tree only stores hashed values, not raw value data) - nodeStore := smt.NewSimpleMap() + // Initialise a new in-memory key-value store to store the nodes of the tree + // (Note: the tree only stores hashed values, not raw value data) + nodeStore := smt.NewKVStore("") + + // Ensure the database connection closes + defer nodeStore.Stop() + + // Initialise the tree + tree := smt.NewSparseMerkleTree(nodeStore, sha256.New()) - // Initialise the tree - tree := smt.NewSparseMerkleTree(nodeStore, sha256.New()) + // Update the key "foo" with the value "bar" + _ = tree.Update([]byte("foo"), []byte("bar")) - // Update the key "foo" with the value "bar" - _ = tree.Update([]byte("foo"), []byte("bar")) + // Commit the changes to the node store + _ = tree.Commit() // Generate a Merkle proof for "foo" - proof, _ := tree.Prove([]byte("foo")) - root := tree.Root() // We also need the current tree root for the proof - - // Verify the Merkle proof for "foo"="bar" - if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), tree.Spec()) { - fmt.Println("Proof verification succeeded.") - } else { - fmt.Println("Proof verification failed.") - } + proof, _ := tree.Prove([]byte("foo")) + root := tree.Root() // We also need the current tree root for the proof + + // Verify the Merkle proof for "foo"="bar" + if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), tree.Spec()) { + fmt.Println("Proof verification succeeded.") + } else { + fmt.Println("Proof verification failed.") + } } ``` diff --git a/bench_test.go b/bench_test.go index b7897f3..103b07b 100644 --- a/bench_test.go +++ b/bench_test.go @@ -4,10 +4,15 @@ import ( "crypto/sha256" "strconv" "testing" + + "github.com/stretchr/testify/require" ) func BenchmarkSparseMerkleTree_Update(b *testing.B) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(b, err) + smv, err := NewKVStore("") + require.NoError(b, err) smt := NewSMTWithStorage(smn, smv, sha256.New()) b.ResetTimer() @@ -16,10 +21,16 @@ func BenchmarkSparseMerkleTree_Update(b *testing.B) { s := strconv.Itoa(i) _ = smt.Update([]byte(s), []byte(s)) } + + require.NoError(b, smn.Stop()) + require.NoError(b, smv.Stop()) } func BenchmarkSparseMerkleTree_Delete(b *testing.B) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(b, err) + smv, err := NewKVStore("") + require.NoError(b, err) smt := NewSMTWithStorage(smn, smv, sha256.New()) for i := 0; i < 100000; i++ { @@ -33,4 +44,7 @@ func BenchmarkSparseMerkleTree_Delete(b *testing.B) { s := strconv.Itoa(i) _ = smt.Delete([]byte(s)) } + + require.NoError(b, smn.Stop()) + require.NoError(b, smv.Stop()) } diff --git a/bulk_test.go b/bulk_test.go index 33003c0..47ab8ca 100644 --- a/bulk_test.go +++ b/bulk_test.go @@ -32,7 +32,10 @@ func TestBulkOperations(t *testing.T) { // Test all tree operations in bulk, with specified ratio probabilities of insert, update and delete. func bulkOperations(t *testing.T, operations int, insert int, update int, delete int) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) smt := NewSMTWithStorage(smn, smv, sha256.New()) max := insert + update + delete @@ -85,7 +88,10 @@ func bulkOperations(t *testing.T, operations int, insert int, update int, delete kv[ki].val = defaultValue } } + bulkCheckAll(t, smt, kv) + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } func bulkCheckAll(t *testing.T, smt *SMTWithStorage, kv []bulkop) { diff --git a/fuzz_test.go b/fuzz_test.go index a734d56..f3f4054 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -25,7 +25,8 @@ func FuzzSMT_DetectUnexpectedFailures(f *testing.F) { f.Add(s) } f.Fuzz(func(t *testing.T, input []byte) { - smn := NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) tree := NewSparseMerkleTree(smn, sha256.New()) r := bytes.NewReader(input) @@ -51,7 +52,7 @@ func FuzzSMT_DetectUnexpectedFailures(f *testing.F) { return keys[int(b)%len(keys)] } - // `i` is the loop counter but also used as the input value to `Update` operations + // `i` is the loop counter but also used as the input value to `Update` operations for i := 0; r.Len() != 0; i++ { originalRoot := tree.Root() b, err := r.ReadByte() @@ -97,6 +98,8 @@ func FuzzSMT_DetectUnexpectedFailures(f *testing.F) { newRoot := tree.Root() require.Greater(t, len(newRoot), 0, "new root is empty while err is nil") } + + require.NoError(t, smn.Stop()) }) } diff --git a/go.mod b/go.mod index 59f0403..4ffaade 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,31 @@ module github.com/pokt-network/smt -go 1.18 +go 1.19 -require github.com/stretchr/testify v1.7.1 +require ( + github.com/dgraph-io/badger/v4 v4.2.0 + github.com/stretchr/testify v1.7.1 +) require ( - github.com/davecgh/go-spew v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/google/go-cmp v0.5.8 // indirect + github.com/klauspost/compress v1.12.3 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + go.opencensus.io v0.22.5 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.11.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index 2dca7c9..a3b8661 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,121 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/kvstore.go b/kvstore.go new file mode 100644 index 0000000..27bf2e7 --- /dev/null +++ b/kvstore.go @@ -0,0 +1,211 @@ +package smt + +import ( + "errors" + "fmt" + "io" + + badger "github.com/dgraph-io/badger/v4" +) + +// KVStore is an interface that defines a key-value store +// that can be used standalone or as the node store for an SMT. +type KVStore interface { + // Store methods + Get(key []byte) ([]byte, error) + Set(key, value []byte) error + Delete(key []byte) error + + // Lifecycle methods + Stop() error + + // Data methods + Backup(writer io.Writer, incremental bool) error + Restore(io.Reader) error + + // Accessors + GetAll(prefixKey []byte, descending bool) (keys, values [][]byte, err error) + Exists(key []byte) (bool, error) + ClearAll() error + Len() int +} + +const ( + maxPendingWrites = 16 // used in backup restoration +) + +var _ KVStore = &badgerKVStore{} + +var ( + ErrKVStoreExists = errors.New("kvstore already exists") + ErrKVStoreNotExists = errors.New("kvstore does not exist") +) + +type badgerKVStore struct { + db *badger.DB + last_backup uint64 // timestamp of the most recent backup +} + +// NewKVStore creates a new KVStore using badger as the underlying database +// if no path for a peristence database is provided it will create one in-memory +func NewKVStore(path string) (KVStore, error) { + var db *badger.DB + var err error + if path == "" { + db, err = badger.Open(badgerOptions("").WithInMemory(true)) + } else { + db, err = badger.Open(badgerOptions(path)) + } + if err != nil { + return nil, err + } + return &badgerKVStore{db: db}, nil +} + +// Set sets/updates the value for a given key +func (store *badgerKVStore) Set(key, value []byte) error { + return store.db.Update(func(tx *badger.Txn) error { + return tx.Set(key, value) + }) +} + +// Get returns the value for a given key +func (store *badgerKVStore) Get(key []byte) ([]byte, error) { + var val []byte + if err := store.db.View(func(tx *badger.Txn) error { + item, err := tx.Get(key) + if err != nil { + return err + } + val, err = item.ValueCopy(nil) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return val, nil +} + +// Delete removes a key and its value from the store +func (store *badgerKVStore) Delete(key []byte) error { + return store.db.Update(func(tx *badger.Txn) error { + return tx.Delete(key) + }) +} + +// GetAll returns all keys and values with the given prefix in the specified order +// if the prefix []byte{} is given then all key-value pairs are returned +func (store *badgerKVStore) GetAll(prefix []byte, descending bool) (keys, values [][]byte, err error) { + if err := store.db.View(func(tx *badger.Txn) error { + opt := badger.DefaultIteratorOptions + opt.Prefix = prefix + opt.Reverse = descending + if descending { + prefix = prefixEndBytes(prefix) + } + it := tx.NewIterator(opt) + defer it.Close() + keys = make([][]byte, 0) + values = make([][]byte, 0) + for it.Seek(prefix); it.Valid(); it.Next() { + item := it.Item() + err = item.Value(func(v []byte) error { + b := make([]byte, len(v)) + copy(b, v) + keys = append(keys, item.Key()) + values = append(values, b) + return nil + }) + if err != nil { + return err + } + } + return nil + }); err != nil { + return nil, nil, err + } + return keys, values, nil +} + +// Exists checks whether the key exists in the store +func (store *badgerKVStore) Exists(key []byte) (bool, error) { + val, err := store.Get(key) + if err != nil { + return false, err + } + return val != nil, nil +} + +// ClearAll deletes all key-value pairs in the store +func (store *badgerKVStore) ClearAll() error { + return store.db.DropAll() +} + +// Backup creates a full backup of the store written to the provided writer +// if incremental is true then only the changes since the last backup are written +func (store *badgerKVStore) Backup(w io.Writer, incremental bool) error { + version := uint64(0) + if incremental { + version = store.last_backup + } + timestamp, err := store.db.Backup(w, version) + if err != nil { + return err + } + store.last_backup = timestamp + return nil +} + +// Restore loads the store from a backup in the reader provided +// NOTE: Do not call on a database that is running other concurrent transactions +func (store *badgerKVStore) Restore(r io.Reader) error { + return store.db.Load(r, maxPendingWrites) +} + +// Stop closes the database connection, disabling any access to the store +func (store *badgerKVStore) Stop() error { + return store.db.Close() +} + +// Len gives the number of keys in the store +func (store *badgerKVStore) Len() int { + count := 0 + if err := store.db.View(func(tx *badger.Txn) error { + opt := badger.DefaultIteratorOptions + opt.Prefix = []byte{} + opt.Reverse = false + it := tx.NewIterator(opt) + defer it.Close() + for it.Seek(nil); it.Valid(); it.Next() { + count++ + } + return nil + }); err != nil { + panic(fmt.Sprintf("error getting key count: %v", err)) + } + return count +} + +// PrefixEndBytes returns the end byteslice for a noninclusive range +// that would include all byte slices for which the input is the prefix +func prefixEndBytes(prefix []byte) []byte { + if len(prefix) == 0 { + return nil + } + if prefix[len(prefix)-1] == byte(255) { + return prefixEndBytes(prefix[:len(prefix)-1]) + } + end := make([]byte, len(prefix)) + copy(end, prefix) + end[len(end)-1]++ + return end +} + +// badgerOptions returns the badger options for the store being created +func badgerOptions(path string) badger.Options { + opts := badger.DefaultOptions(path) + opts.Logger = nil // disable badger's logger since it's very noisy + return opts +} diff --git a/kvstore_test.go b/kvstore_test.go new file mode 100644 index 0000000..6c37d61 --- /dev/null +++ b/kvstore_test.go @@ -0,0 +1,403 @@ +package smt + +import ( + "bytes" + "encoding/hex" + "fmt" + "strings" + "testing" + + badger "github.com/dgraph-io/badger/v4" + "github.com/stretchr/testify/require" +) + +func TestKVStore_BasicOperations(t *testing.T) { + store, err := NewKVStore("") + require.NoError(t, err) + require.NotNil(t, store) + + invalidKey := [65001]byte{} + testCases := []struct { + name string + op string + key []byte + value []byte + fail bool + expected error + }{ + { + name: "Successfully sets a value in the store", + op: "set", + key: []byte("testKey"), + value: []byte("testValue"), + fail: false, + expected: nil, + }, + { + name: "Successfully updates a value in the store", + op: "set", + key: []byte("foo"), + value: []byte("new value"), + fail: false, + expected: nil, + }, + { + name: "Fails to set value to nil key", + op: "set", + key: nil, + value: []byte("bar"), + fail: true, + expected: badger.ErrEmptyKey, + }, + { + name: "Fails to set a value to a key that is too large", + op: "set", + key: invalidKey[:], + value: []byte("bar"), + fail: true, + expected: fmt.Errorf("Key with size 65001 exceeded 65000 limit. Key:\n%s", hex.Dump(invalidKey[:1<<10])), + }, + { + name: "Successfully retrieve a value from the store", + op: "get", + key: []byte("foo"), + value: []byte("bar"), + fail: false, + expected: nil, + }, + { + name: "Fails to get a value that is not stored", + op: "get", + key: []byte("bar"), + value: nil, + fail: true, + expected: badger.ErrKeyNotFound, + }, + { + name: "Fails when the key is empty", + op: "get", + key: nil, + value: nil, + fail: true, + expected: badger.ErrEmptyKey, + }, + { + name: "Successfully deletes a value in the store", + op: "delete", + key: []byte("foo"), + value: nil, + fail: false, + expected: nil, + }, + { + name: "Fails to delete a value not in the store", + op: "delete", + key: []byte("bar"), + value: nil, + fail: false, + expected: nil, + }, + { + name: "Fails to set value to nil key", + op: "delete", + key: nil, + value: nil, + fail: true, + expected: badger.ErrEmptyKey, + }, + { + name: "Fails to set a value to a key that is too large", + op: "delete", + key: invalidKey[:], + value: nil, + fail: true, + expected: fmt.Errorf("Key with size 65001 exceeded 65000 limit. Key:\n%s", hex.Dump(invalidKey[:1<<10])), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := store.ClearAll() + require.NoError(t, err) + setupStore(t, store) + switch tc.op { + case "set": + err := store.Set(tc.key, tc.value) + if tc.fail { + require.Error(t, err) + require.EqualError(t, tc.expected, err.Error()) + } else { + require.NoError(t, err) + got, err := store.Get(tc.key) + require.NoError(t, err) + require.Equal(t, tc.value, got) + } + case "get": + got, err := store.Get(tc.key) + if tc.fail { + require.Error(t, err) + require.EqualError(t, tc.expected, err.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tc.value, got) + } + case "delete": + err := store.Delete(tc.key) + if tc.fail { + require.Error(t, err) + require.EqualError(t, tc.expected, err.Error()) + } else { + require.NoError(t, err) + _, err := store.Get(tc.key) + require.EqualError(t, err, badger.ErrKeyNotFound.Error()) + } + } + }) + } + + err = store.Stop() + require.NoError(t, err) +} + +func TestKVStore_GetAllBasic(t *testing.T) { + store, err := NewKVStore("") + require.NoError(t, err) + require.NotNil(t, store) + + keys := [][]byte{ + []byte("foo"), + []byte("bar"), + []byte("baz"), + []byte("bin"), + } + values := [][]byte{ + []byte("oof"), + []byte("rab"), + []byte("zab"), + []byte("nib"), + } + + for i := 0; i < len(keys); i++ { + err := store.Set(keys[i], values[i]) + require.NoError(t, err) + } + + allKeys, allValues, err := store.GetAll([]byte{}, false) + require.NoError(t, err) + require.Equal(t, len(keys), len(allKeys)) + require.Equal(t, len(values), len(allValues)) + + for i := 0; i < len(keys); i++ { + require.Contains(t, allKeys, keys[i]) + require.Contains(t, allValues, values[i]) + } + + err = store.Stop() + require.NoError(t, err) +} + +func TestKVStore_GetAllPrefixed(t *testing.T) { + store, err := NewKVStore("") + require.NoError(t, err) + require.NotNil(t, store) + + keys := [][]byte{ + []byte("foo"), + []byte("bar"), + []byte("baz"), + []byte("bin"), + []byte("testKey1"), + []byte("testKey2"), + []byte("testKey3"), + []byte("testKey4"), + } + values := [][]byte{ + []byte("oof"), + []byte("rab"), + []byte("zab"), + []byte("nib"), + []byte("testValue1"), + []byte("testValue2"), + []byte("testValue3"), + []byte("testValue4"), + } + + for i := 0; i < len(keys); i++ { + err := store.Set(keys[i], values[i]) + require.NoError(t, err) + } + + allKeys, allValues, err := store.GetAll([]byte("testKey"), false) + require.NoError(t, err) + require.Equal(t, 4, len(allKeys)) + require.Equal(t, 4, len(allValues)) + + for i := 0; i < len(keys); i++ { + if strings.HasPrefix(string(keys[i]), "testKey") { + require.Contains(t, allKeys, keys[i]) + require.Contains(t, allValues, values[i]) + } else { + require.NotContains(t, allKeys, keys[i]) + require.NotContains(t, allValues, values[i]) + } + } + + err = store.Stop() + require.NoError(t, err) +} + +func TestKVStore_Exists(t *testing.T) { + store, err := NewKVStore("") + require.NoError(t, err) + require.NotNil(t, store) + + keys := [][]byte{ + []byte("foo"), + []byte("bar"), + []byte("baz"), + []byte("bin"), + } + values := [][]byte{ + []byte("oof"), + nil, + []byte("zab"), + []byte("nib"), + } + + for i := 0; i < len(keys); i++ { + err := store.Set(keys[i], values[i]) + require.NoError(t, err) + } + + // Key exists in store with a value + exists, err := store.Exists([]byte("foo")) + require.NoError(t, err) + require.True(t, exists) + + // Key exists but has nil value + exists, err = store.Exists([]byte("bar")) + require.NoError(t, err) + require.False(t, exists) + + // Key does not exist + exists, err = store.Exists([]byte("oof")) + require.EqualError(t, err, badger.ErrKeyNotFound.Error()) + require.False(t, exists) + + err = store.Stop() + require.NoError(t, err) +} + +func TestKVStore_ClearAll(t *testing.T) { + store, err := NewKVStore("") + require.NoError(t, err) + require.NotNil(t, store) + + keys := [][]byte{ + []byte("foo"), + []byte("bar"), + []byte("baz"), + []byte("bin"), + []byte("testKey1"), + []byte("testKey2"), + []byte("testKey3"), + []byte("testKey4"), + } + values := [][]byte{ + []byte("oof"), + []byte("rab"), + []byte("zab"), + []byte("nib"), + []byte("testValue1"), + []byte("testValue2"), + []byte("testValue3"), + []byte("testValue4"), + } + + for i := 0; i < len(keys); i++ { + err := store.Set(keys[i], values[i]) + require.NoError(t, err) + } + + allKeys, allValues, err := store.GetAll([]byte{}, false) + require.NoError(t, err) + require.Equal(t, len(keys), len(allKeys)) + require.Equal(t, len(values), len(allValues)) + + err = store.ClearAll() + require.NoError(t, err) + + allKeys, allValues, err = store.GetAll([]byte{}, false) + require.NoError(t, err) + require.Equal(t, 0, len(allKeys)) + require.Equal(t, 0, len(allValues)) + + err = store.Stop() + require.NoError(t, err) +} + +func TestKVStore_BackupAndRestore(t *testing.T) { + store, err := NewKVStore("") + require.NoError(t, err) + require.NotNil(t, store) + + setupStore(t, store) + + keys, values, err := store.GetAll([]byte{}, false) + require.NoError(t, err) + + buf := bytes.NewBuffer(nil) + err = store.Backup(buf, false) + require.NoError(t, err) + + require.NoError(t, store.ClearAll()) + err = store.Restore(buf) + require.NoError(t, err) + + newKeys, newValues, err := store.GetAll([]byte{}, false) + require.NoError(t, err) + + require.Equal(t, keys, newKeys) + require.Equal(t, values, newValues) +} + +func TestKVStore_Len(t *testing.T) { + store, err := NewKVStore("") + require.NoError(t, err) + require.NotNil(t, store) + + tests := []struct { + key []byte + value []byte + size int + }{ + { + key: []byte("foo"), + value: []byte("bar"), + size: 1, + }, + { + key: []byte("baz"), + value: []byte("bin"), + size: 2, + }, + { + key: []byte("testKey1"), + value: []byte("testValue1"), + size: 3, + }, + } + + for _, tc := range tests { + require.NoError(t, store.Set(tc.key, tc.value)) + require.Equal(t, tc.size, store.Len()) + } +} + +func setupStore(t *testing.T, store KVStore) { + t.Helper() + err := store.Set([]byte("foo"), []byte("bar")) + require.NoError(t, err) + err = store.Set([]byte("baz"), []byte("bin")) + require.NoError(t, err) +} diff --git a/mapstore.go b/mapstore.go deleted file mode 100644 index 0182bca..0000000 --- a/mapstore.go +++ /dev/null @@ -1,57 +0,0 @@ -package smt - -import ( - "fmt" -) - -// MapStore is a key-value store. -type MapStore interface { - Get(key []byte) ([]byte, error) // Get gets the value for a key. - Set(key, value []byte) error // Set updates the value for a key. - Delete(key []byte) error // Delete deletes a key. -} - -// InvalidKeyError is thrown when a key that does not exist is being accessed. -type InvalidKeyError struct { - Key []byte -} - -func (e *InvalidKeyError) Error() string { - return fmt.Sprintf("invalid key: %x", e.Key) -} - -// SimpleMap is a simple in-memory map. -type SimpleMap struct { - m map[string][]byte -} - -// NewSimpleMap creates a new empty SimpleMap. -func NewSimpleMap() *SimpleMap { - return &SimpleMap{ - m: make(map[string][]byte), - } -} - -// Get gets the value for a key. -func (sm *SimpleMap) Get(key []byte) ([]byte, error) { - if value, ok := sm.m[string(key)]; ok { - return value, nil - } - return nil, &InvalidKeyError{Key: key} -} - -// Set updates the value for a key. -func (sm *SimpleMap) Set(key []byte, value []byte) error { - sm.m[string(key)] = value - return nil -} - -// Delete deletes a key. -func (sm *SimpleMap) Delete(key []byte) error { - _, ok := sm.m[string(key)] - if ok { - delete(sm.m, string(key)) - return nil - } - return &InvalidKeyError{Key: key} -} diff --git a/mapstore_test.go b/mapstore_test.go deleted file mode 100644 index ea945cc..0000000 --- a/mapstore_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package smt - -import ( - "bytes" - "crypto/sha256" - "testing" -) - -func TestSimpleMap(t *testing.T) { - sm := NewSimpleMap() - h := sha256.New() - var value []byte - var err error - - h.Write([]byte("test")) - - // Tests for Get. - _, err = sm.Get(h.Sum(nil)) - if err == nil { - t.Error("did not return an error when getting a non-existent key") - } - - // Tests for Put. - err = sm.Set(h.Sum(nil), []byte("hello")) - if err != nil { - t.Error("updating a key returned an error") - } - value, err = sm.Get(h.Sum(nil)) - if err != nil { - t.Error("getting a key returned an error") - } - if !bytes.Equal(value, []byte("hello")) { - t.Error("failed to update key") - } - - // Tests for Del. - err = sm.Delete(h.Sum(nil)) - if err != nil { - t.Error("deleting a key returned an error") - } - _, err = sm.Get(h.Sum(nil)) - if err == nil { - t.Error("failed to delete key") - } - err = sm.Delete([]byte("nonexistent")) - if err == nil { - t.Error("deleting a key did not return an error on a non-existent key") - } -} diff --git a/proofs.go b/proofs.go index 78bbb1f..2874657 100644 --- a/proofs.go +++ b/proofs.go @@ -3,10 +3,16 @@ package smt import ( "bytes" "encoding/binary" + "encoding/gob" "errors" "math" ) +func init() { + gob.Register(SparseMerkleProof{}) + gob.Register(SparseCompactMerkleProof{}) +} + // ErrBadProof is returned when an invalid Merkle proof is supplied. var ErrBadProof = errors.New("bad proof") @@ -25,6 +31,23 @@ type SparseMerkleProof struct { SiblingData []byte } +// Marshal serialises the SparseMerkleProof to bytes +func (proof *SparseMerkleProof) Marshal() ([]byte, error) { + buf := bytes.NewBuffer(nil) + enc := gob.NewEncoder(buf) + if err := enc.Encode(proof); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Unmarshal deserialises the SparseMerkleProof from bytes +func (proof *SparseMerkleProof) Unmarshal(bz []byte) error { + buf := bytes.NewBuffer(bz) + dec := gob.NewDecoder(buf) + return dec.Decode(proof) +} + func (proof *SparseMerkleProof) sanityCheck(spec *TreeSpec) bool { // Do a basic sanity check on the proof, so that a malicious proof cannot // cause the verifier to fatally exit (e.g. due to an index out-of-range @@ -78,6 +101,23 @@ type SparseCompactMerkleProof struct { SiblingData []byte } +// Marshal serialises the SparseCompactMerkleProof to bytes +func (proof *SparseCompactMerkleProof) Marshal() ([]byte, error) { + buf := bytes.NewBuffer(nil) + enc := gob.NewEncoder(buf) + if err := enc.Encode(proof); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Unmarshal deserialises the SparseCompactMerkleProof from bytes +func (proof *SparseCompactMerkleProof) Unmarshal(bz []byte) error { + buf := bytes.NewBuffer(bz) + dec := gob.NewDecoder(buf) + return dec.Decode(proof) +} + func (proof *SparseCompactMerkleProof) sanityCheck(spec *TreeSpec) bool { // Do a basic sanity check on the proof on the fields of the proof specific to // the compact proof only. diff --git a/proofs_test.go b/proofs_test.go index 5451ab3..4ab9fd7 100644 --- a/proofs_test.go +++ b/proofs_test.go @@ -3,9 +3,168 @@ package smt import ( "bytes" "crypto/rand" + "crypto/sha256" "testing" + + "github.com/stretchr/testify/require" ) +func TestSparseMerkleProof_Marshal(t *testing.T) { + tree := setupTree(t) + + proof, err := tree.Prove([]byte("key")) + require.NoError(t, err) + bz, err := proof.Marshal() + require.NoError(t, err) + require.NotNil(t, bz) + require.Greater(t, len(bz), 0) + + proof2, err := tree.Prove([]byte("key2")) + require.NoError(t, err) + bz2, err := proof2.Marshal() + require.NoError(t, err) + require.NotNil(t, bz2) + require.Greater(t, len(bz2), 0) + require.NotEqual(t, bz, bz2) + + proof3 := randomiseProof(proof) + bz3, err := proof3.Marshal() + require.NoError(t, err) + require.NotNil(t, bz3) + require.Greater(t, len(bz3), 0) + require.NotEqual(t, bz, bz3) +} + +func TestSparseMerkleProof_Unmarshal(t *testing.T) { + tree := setupTree(t) + + proof, err := tree.Prove([]byte("key")) + require.NoError(t, err) + bz, err := proof.Marshal() + require.NoError(t, err) + require.NotNil(t, bz) + require.Greater(t, len(bz), 0) + uproof := new(SparseMerkleProof) + require.NoError(t, uproof.Unmarshal(bz)) + require.Equal(t, proof, uproof) + + proof2, err := tree.Prove([]byte("key2")) + require.NoError(t, err) + bz2, err := proof2.Marshal() + require.NoError(t, err) + require.NotNil(t, bz2) + require.Greater(t, len(bz2), 0) + uproof2 := new(SparseMerkleProof) + require.NoError(t, uproof2.Unmarshal(bz2)) + require.Equal(t, proof2, uproof2) + + proof3 := randomiseProof(proof) + bz3, err := proof3.Marshal() + require.NoError(t, err) + require.NotNil(t, bz3) + require.Greater(t, len(bz3), 0) + uproof3 := new(SparseMerkleProof) + require.NoError(t, uproof3.Unmarshal(bz3)) + require.Equal(t, proof3, uproof3) +} + +func TestSparseCompactMerkletProof_Marshal(t *testing.T) { + tree := setupTree(t) + + proof, err := tree.Prove([]byte("key")) + require.NoError(t, err) + compactProof, err := CompactProof(proof, tree.Spec()) + require.NoError(t, err) + bz, err := compactProof.Marshal() + require.NoError(t, err) + require.NotNil(t, bz) + require.Greater(t, len(bz), 0) + + proof2, err := tree.Prove([]byte("key2")) + require.NoError(t, err) + compactProof2, err := CompactProof(proof2, tree.Spec()) + require.NoError(t, err) + bz2, err := compactProof2.Marshal() + require.NoError(t, err) + require.NotNil(t, bz2) + require.Greater(t, len(bz2), 0) + require.NotEqual(t, bz, bz2) + + proof3 := randomiseProof(proof) + compactProof3, err := CompactProof(proof3, tree.Spec()) + require.NoError(t, err) + bz3, err := compactProof3.Marshal() + require.NoError(t, err) + require.NotNil(t, bz3) + require.Greater(t, len(bz3), 0) + require.NotEqual(t, bz, bz3) +} + +func TestSparseCompactMerkleProof_Unmarshal(t *testing.T) { + tree := setupTree(t) + + proof, err := tree.Prove([]byte("key")) + require.NoError(t, err) + compactProof, err := CompactProof(proof, tree.Spec()) + require.NoError(t, err) + bz, err := compactProof.Marshal() + require.NoError(t, err) + require.NotNil(t, bz) + require.Greater(t, len(bz), 0) + uCproof := new(SparseCompactMerkleProof) + require.NoError(t, uCproof.Unmarshal(bz)) + require.Equal(t, compactProof, uCproof) + uproof, err := DecompactProof(uCproof, tree.Spec()) + require.NoError(t, err) + require.Equal(t, proof, uproof) + + proof2, err := tree.Prove([]byte("key2")) + require.NoError(t, err) + compactProof2, err := CompactProof(proof2, tree.Spec()) + require.NoError(t, err) + bz2, err := compactProof2.Marshal() + require.NoError(t, err) + require.NotNil(t, bz2) + require.Greater(t, len(bz2), 0) + uCproof2 := new(SparseCompactMerkleProof) + require.NoError(t, uCproof2.Unmarshal(bz2)) + require.Equal(t, compactProof2, uCproof2) + uproof2, err := DecompactProof(uCproof2, tree.Spec()) + require.NoError(t, err) + require.Equal(t, proof2, uproof2) + + proof3 := randomiseProof(proof) + compactProof3, err := CompactProof(proof3, tree.Spec()) + require.NoError(t, err) + bz3, err := compactProof3.Marshal() + require.NoError(t, err) + require.NotNil(t, bz3) + require.Greater(t, len(bz3), 0) + uCproof3 := new(SparseCompactMerkleProof) + require.NoError(t, uCproof3.Unmarshal(bz3)) + require.Equal(t, compactProof3, uCproof3) + uproof3, err := DecompactProof(uCproof3, tree.Spec()) + require.NoError(t, err) + require.Equal(t, proof3, uproof3) +} + +func setupTree(t *testing.T) *SMT { + t.Helper() + + db, err := NewKVStore("") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, db.Stop()) + }) + + tree := NewSparseMerkleTree(db, sha256.New()) + require.NoError(t, tree.Update([]byte("key"), []byte("value"))) + require.NoError(t, tree.Update([]byte("key2"), []byte("value2"))) + require.NoError(t, tree.Update([]byte("key3"), []byte("value3"))) + + return tree +} + func randomiseProof(proof *SparseMerkleProof) *SparseMerkleProof { sideNodes := make([][]byte, len(proof.SideNodes)) for i := range sideNodes { diff --git a/smst.go b/smst.go index ea3f466..327c6a8 100644 --- a/smst.go +++ b/smst.go @@ -15,7 +15,7 @@ type SMST struct { } // NewSparseMerkleSumTree returns a pointer to an SMST struct -func NewSparseMerkleSumTree(nodes MapStore, hasher hash.Hash, options ...Option) *SMST { +func NewSparseMerkleSumTree(nodes KVStore, hasher hash.Hash, options ...Option) *SMST { smt := &SMT{ TreeSpec: newTreeSpec(hasher, true), nodes: nodes, @@ -36,7 +36,7 @@ func NewSparseMerkleSumTree(nodes MapStore, hasher hash.Hash, options ...Option) } // ImportSparseMerkleSumTree returns a pointer to an SMST struct with the root hash provided -func ImportSparseMerkleSumTree(nodes MapStore, hasher hash.Hash, root []byte, options ...Option) *SMST { +func ImportSparseMerkleSumTree(nodes KVStore, hasher hash.Hash, root []byte, options ...Option) *SMST { smst := NewSparseMerkleSumTree(nodes, hasher, options...) smst.tree = &lazyNode{root} smst.savedRoot = root diff --git a/smst_proofs_test.go b/smst_proofs_test.go index 4f966b5..46dbecc 100644 --- a/smst_proofs_test.go +++ b/smst_proofs_test.go @@ -10,14 +10,17 @@ import ( // Test base case Merkle proof operations. func TestSMST_ProofsBasic(t *testing.T) { - var smn, smv *SimpleMap + var smn, smv KVStore var smst *SMSTWithStorage var proof *SparseMerkleProof var result bool var root []byte var err error - smn, smv = NewSimpleMap(), NewSimpleMap() + smn, err = NewKVStore("") + require.NoError(t, err) + smv, err = NewKVStore("") + require.NoError(t, err) smst = NewSMSTWithStorage(smn, smv, sha256.New()) base := smst.Spec() @@ -103,15 +106,21 @@ func TestSMST_ProofsBasic(t *testing.T) { require.False(t, result) result = VerifySumProof(randomiseSumProof(proof), root, []byte("testKey3"), defaultValue, 0, base) // invalid proof require.False(t, result) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test sanity check cases for non-compact proofs. func TestSMST_ProofsSanityCheck(t *testing.T) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) smst := NewSMSTWithStorage(smn, smv, sha256.New()) base := smst.Spec() - err := smst.Update([]byte("testKey1"), []byte("testValue1"), 1) + err = smst.Update([]byte("testKey1"), []byte("testValue1"), 1) require.NoError(t, err) err = smst.Update([]byte("testKey2"), []byte("testValue2"), 2) require.NoError(t, err) @@ -161,4 +170,7 @@ func TestSMST_ProofsSanityCheck(t *testing.T) { require.False(t, result) _, err = CompactProof(proof, base) require.Error(t, err) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } diff --git a/smst_test.go b/smst_test.go index d6ce78f..9ffa6cf 100644 --- a/smst_test.go +++ b/smst_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" ) -func NewSMSTWithStorage(nodes, preimages MapStore, hasher hash.Hash, options ...Option) *SMSTWithStorage { +func NewSMSTWithStorage(nodes, preimages KVStore, hasher hash.Hash, options ...Option) *SMSTWithStorage { return &SMSTWithStorage{ SMST: NewSparseMerkleSumTree(nodes, hasher, options...), preimages: preimages, @@ -19,13 +19,15 @@ func NewSMSTWithStorage(nodes, preimages MapStore, hasher hash.Hash, options ... } func TestSMST_TreeUpdateBasic(t *testing.T) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) lazy := NewSparseMerkleSumTree(smn, sha256.New()) smst := &SMSTWithStorage{SMST: lazy, preimages: smv} var value []byte var sum uint64 var has bool - var err error // Test getting an empty key. value, sum, err = smst.GetValueSum([]byte("testKey")) @@ -85,7 +87,7 @@ func TestSMST_TreeUpdateBasic(t *testing.T) { require.NoError(t, lazy.Commit()) - // Test that a tree can be imported from a MapStore. + // Test that a tree can be imported from a KVStore. lazy = ImportSparseMerkleSumTree(smn, sha256.New(), smst.Root()) require.NoError(t, err) smst = &SMSTWithStorage{SMST: lazy, preimages: smv} @@ -104,17 +106,23 @@ func TestSMST_TreeUpdateBasic(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("testValue3"), value) require.Equal(t, uint64(5), sum) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test base case tree delete operations with a few keys. func TestSMST_TreeDeleteBasic(t *testing.T) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) lazy := NewSparseMerkleSumTree(smn, sha256.New()) smst := &SMSTWithStorage{SMST: lazy, preimages: smv} rootEmpty := smst.Root() // Testing inserting, deleting a key, and inserting it again. - err := smst.Update([]byte("testKey"), []byte("testValue"), 5) + err = smst.Update([]byte("testKey"), []byte("testValue"), 5) require.NoError(t, err) root1 := smst.Root() @@ -209,16 +217,21 @@ func TestSMST_TreeDeleteBasic(t *testing.T) { require.Equal(t, []byte("testValue"), value) require.Equal(t, uint64(5), sum) require.Equal(t, root1, smst.Root(), "re-inserting key after deletion") + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test tree ops with known paths func TestSMST_TreeKnownPath(t *testing.T) { ph := dummyPathHasher{32} - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) smst := NewSMSTWithStorage(smn, smv, sha256.New(), WithPathHasher(ph)) var value []byte var sum uint64 - var err error baseKey := make([]byte, ph.PathSize()) keys := make([][]byte, 7) @@ -288,16 +301,21 @@ func TestSMST_TreeKnownPath(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("testValue6"), value) require.Equal(t, uint64(6), sum) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test tree operations when two leafs are immediate neighbors. func TestSMST_TreeMaxHeightCase(t *testing.T) { ph := dummyPathHasher{32} - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) smst := NewSMSTWithStorage(smn, smv, sha256.New(), WithPathHasher(ph)) var value []byte var sum uint64 - var err error // Make two neighboring keys. // The dummy hash function will return the preimage itself as the digest. @@ -329,20 +347,26 @@ func TestSMST_TreeMaxHeightCase(t *testing.T) { proof, err := smst.Prove(key1) require.NoError(t, err) require.Equal(t, 256, len(proof.SideNodes), "unexpected proof size") + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } func TestSMST_OrphanRemoval(t *testing.T) { - var smn, smv *SimpleMap + var smn, smv KVStore var impl *SMST var smst *SMSTWithStorage var err error nodeCount := func(t *testing.T) int { require.NoError(t, impl.Commit()) - return len(smn.m) + return smn.Len() } setup := func() { - smn, smv = NewSimpleMap(), NewSimpleMap() + smn, err = NewKVStore("") + require.NoError(t, err) + smv, err = NewKVStore("") + require.NoError(t, err) impl = NewSparseMerkleSumTree(smn, sha256.New()) smst = &SMSTWithStorage{SMST: impl, preimages: smv} @@ -417,12 +441,16 @@ func TestSMST_OrphanRemoval(t *testing.T) { require.Equal(t, 1, nodeCount(t), tci) } }) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } func TestSMST_TotalSum(t *testing.T) { - snm := NewSimpleMap() + snm, err := NewKVStore("") + require.NoError(t, err) smst := NewSparseMerkleSumTree(snm, sha256.New()) - err := smst.Update([]byte("key1"), []byte("value1"), 5) + err = smst.Update([]byte("key1"), []byte("value1"), 5) require.NoError(t, err) err = smst.Update([]byte("key2"), []byte("value2"), 5) require.NoError(t, err) @@ -461,7 +489,8 @@ func TestSMST_TotalSum(t *testing.T) { require.Equal(t, sum, uint64(10)) // Calculate the total sum of a larger tree - snm = NewSimpleMap() + snm, err = NewKVStore("") + require.NoError(t, err) smst = NewSparseMerkleSumTree(snm, sha256.New()) for i := 1; i < 10000; i++ { err := smst.Update([]byte(fmt.Sprintf("testKey%d", i)), []byte(fmt.Sprintf("testValue%d", i)), uint64(i)) @@ -470,13 +499,16 @@ func TestSMST_TotalSum(t *testing.T) { require.NoError(t, smst.Commit()) sum = smst.Sum() require.Equal(t, sum, uint64(49995000)) + + require.NoError(t, snm.Stop()) } func TestSMST_Retrieval(t *testing.T) { - snm := NewSimpleMap() + snm, err := NewKVStore("") + require.NoError(t, err) smst := NewSparseMerkleSumTree(snm, sha256.New(), WithValueHasher(nil)) - err := smst.Update([]byte("key1"), []byte("value1"), 5) + err = smst.Update([]byte("key1"), []byte("value1"), 5) require.NoError(t, err) err = smst.Update([]byte("key2"), []byte("value2"), 5) require.NoError(t, err) @@ -538,4 +570,6 @@ func TestSMST_Retrieval(t *testing.T) { sum = lazy.Sum() require.Equal(t, sum, uint64(15)) + + require.NoError(t, snm.Stop()) } diff --git a/smst_utils_test.go b/smst_utils_test.go index 5d047f4..065b294 100644 --- a/smst_utils_test.go +++ b/smst_utils_test.go @@ -5,16 +5,18 @@ import ( "encoding/binary" "errors" "fmt" + + "github.com/dgraph-io/badger/v4" ) // SMSTWithStorage wraps an SMST with a mapping of value hashes to values with sums (preimages), for use in tests. // Note: this doesn't delete from preimages (inputs to hashing functions), since there could be duplicate stored values. type SMSTWithStorage struct { *SMST - preimages MapStore + preimages KVStore } -// Update updates a key with a new value in the tree and adds the value to the preimages MapStore +// Update updates a key with a new value in the tree and adds the value to the preimages KVStore func (smst *SMSTWithStorage) Update(key, value []byte, sum uint64) error { if err := smst.SMST.Update(key, value, sum); err != nil { return err @@ -35,16 +37,18 @@ func (smst *SMSTWithStorage) Delete(key []byte) error { } // GetValueSum returns the value and sum of the key stored in the tree, by looking up -// the value hash in the preimages MapStore and extracting the sum +// the value hash in the preimages KVStore and extracting the sum func (smst *SMSTWithStorage) GetValueSum(key []byte) ([]byte, uint64, error) { valueHash, sum, err := smst.Get(key) if err != nil { return nil, 0, err } + if valueHash == nil { + return nil, 0, nil + } value, err := smst.preimages.Get(valueHash) if err != nil { - var invalidKeyError *InvalidKeyError - if errors.As(err, &invalidKeyError) { + if errors.Is(err, badger.ErrKeyNotFound) { // If key isn't found, return default value and sum return defaultValue, 0, nil } else { diff --git a/smt.go b/smt.go index 7fd5384..ddcaeec 100644 --- a/smt.go +++ b/smt.go @@ -51,7 +51,7 @@ type lazyNode struct { type SMT struct { TreeSpec - nodes MapStore + nodes KVStore // Last persisted root hash savedRoot []byte // Current state of tree @@ -64,7 +64,7 @@ type SMT struct { type orphanNodes = [][]byte // NewSparseMerkleTree returns a new pointer to an SMT struct, and applys any options provided -func NewSparseMerkleTree(nodes MapStore, hasher hash.Hash, options ...Option) *SMT { +func NewSparseMerkleTree(nodes KVStore, hasher hash.Hash, options ...Option) *SMT { smt := SMT{ TreeSpec: newTreeSpec(hasher, false), nodes: nodes, @@ -76,7 +76,7 @@ func NewSparseMerkleTree(nodes MapStore, hasher hash.Hash, options ...Option) *S } // ImportSparseMerkleTree returns a pointer to an SMT struct with the provided root hash -func ImportSparseMerkleTree(nodes MapStore, hasher hash.Hash, root []byte, options ...Option) *SMT { +func ImportSparseMerkleTree(nodes KVStore, hasher hash.Hash, root []byte, options ...Option) *SMT { smt := NewSparseMerkleTree(nodes, hasher, options...) smt.tree = &lazyNode{root} smt.savedRoot = root diff --git a/smt_proofs_test.go b/smt_proofs_test.go index 3f39798..ee93bfc 100644 --- a/smt_proofs_test.go +++ b/smt_proofs_test.go @@ -9,14 +9,17 @@ import ( // Test base case Merkle proof operations. func TestSMT_ProofsBasic(t *testing.T) { - var smn, smv *SimpleMap + var smn, smv KVStore var smt *SMTWithStorage var proof *SparseMerkleProof var result bool var root []byte var err error - smn, smv = NewSimpleMap(), NewSimpleMap() + smn, err = NewKVStore("") + require.NoError(t, err) + smv, err = NewKVStore("") + require.NoError(t, err) smt = NewSMTWithStorage(smn, smv, sha256.New()) base := smt.Spec() @@ -84,15 +87,21 @@ func TestSMT_ProofsBasic(t *testing.T) { require.False(t, result) result = VerifyProof(randomiseProof(proof), root, []byte("testKey3"), defaultValue, base) require.False(t, result) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test sanity check cases for non-compact proofs. func TestSMT_ProofsSanityCheck(t *testing.T) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) smt := NewSMTWithStorage(smn, smv, sha256.New()) base := smt.Spec() - err := smt.Update([]byte("testKey1"), []byte("testValue1")) + err = smt.Update([]byte("testKey1"), []byte("testValue1")) require.NoError(t, err) err = smt.Update([]byte("testKey2"), []byte("testValue2")) require.NoError(t, err) @@ -142,4 +151,7 @@ func TestSMT_ProofsSanityCheck(t *testing.T) { require.False(t, result) _, err = CompactProof(proof, base) require.Error(t, err) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } diff --git a/smt_test.go b/smt_test.go index 4f2ec08..8a8cd74 100644 --- a/smt_test.go +++ b/smt_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -func NewSMTWithStorage(nodes, preimages MapStore, hasher hash.Hash, options ...Option) *SMTWithStorage { +func NewSMTWithStorage(nodes, preimages KVStore, hasher hash.Hash, options ...Option) *SMTWithStorage { return &SMTWithStorage{ SMT: NewSparseMerkleTree(nodes, hasher, options...), preimages: preimages, @@ -17,12 +17,14 @@ func NewSMTWithStorage(nodes, preimages MapStore, hasher hash.Hash, options ...O } func TestSMT_TreeUpdateBasic(t *testing.T) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) lazy := NewSparseMerkleTree(smn, sha256.New()) smt := &SMTWithStorage{SMT: lazy, preimages: smv} var value []byte var has bool - var err error // Test getting an empty key. value, err = smt.GetValue([]byte("testKey")) @@ -76,7 +78,7 @@ func TestSMT_TreeUpdateBasic(t *testing.T) { require.NoError(t, lazy.Commit()) - // Test that a tree can be imported from a MapStore. + // Test that a tree can be imported from a KVStore lazy = ImportSparseMerkleTree(smn, sha256.New(), smt.Root()) require.NoError(t, err) smt = &SMTWithStorage{SMT: lazy, preimages: smv} @@ -92,17 +94,23 @@ func TestSMT_TreeUpdateBasic(t *testing.T) { value, err = smt.GetValue([]byte("testKey2")) require.NoError(t, err) require.Equal(t, []byte("testValue"), value) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test base case tree delete operations with a few keys. func TestSMT_TreeDeleteBasic(t *testing.T) { - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) lazy := NewSparseMerkleTree(smn, sha256.New()) smt := &SMTWithStorage{SMT: lazy, preimages: smv} rootEmpty := smt.Root() // Testing inserting, deleting a key, and inserting it again. - err := smt.Update([]byte("testKey"), []byte("testValue")) + err = smt.Update([]byte("testKey"), []byte("testValue")) require.NoError(t, err) root1 := smt.Root() @@ -189,15 +197,20 @@ func TestSMT_TreeDeleteBasic(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("testValue"), value) require.Equal(t, root1, smt.Root(), "re-inserting key after deletion") + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test tree ops with known paths func TestSMT_TreeKnownPath(t *testing.T) { ph := dummyPathHasher{32} - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) smt := NewSMTWithStorage(smn, smv, sha256.New(), WithPathHasher(ph)) var value []byte - var err error baseKey := make([]byte, ph.PathSize()) keys := make([][]byte, 7) @@ -260,15 +273,20 @@ func TestSMT_TreeKnownPath(t *testing.T) { value, err = smt.GetValue(keys[5]) require.NoError(t, err) require.Equal(t, []byte("testValue6"), value) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } // Test tree operations when two leafs are immediate neighbors. func TestSMT_TreeMaxHeightCase(t *testing.T) { ph := dummyPathHasher{32} - smn, smv := NewSimpleMap(), NewSimpleMap() + smn, err := NewKVStore("") + require.NoError(t, err) + smv, err := NewKVStore("") + require.NoError(t, err) smt := NewSMTWithStorage(smn, smv, sha256.New(), WithPathHasher(ph)) var value []byte - var err error // Make two neighboring keys. // The dummy hash function will return the preimage itself as the digest. @@ -298,20 +316,26 @@ func TestSMT_TreeMaxHeightCase(t *testing.T) { proof, err := smt.Prove(key1) require.NoError(t, err) require.Equal(t, 256, len(proof.SideNodes), "unexpected proof size") + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } func TestSMT_OrphanRemoval(t *testing.T) { - var smn, smv *SimpleMap + var smn, smv KVStore var impl *SMT var smt *SMTWithStorage var err error nodeCount := func(t *testing.T) int { require.NoError(t, impl.Commit()) - return len(smn.m) + return smn.Len() } setup := func() { - smn, smv = NewSimpleMap(), NewSimpleMap() + smn, err = NewKVStore("") + require.NoError(t, err) + smv, err = NewKVStore("") + require.NoError(t, err) impl = NewSparseMerkleTree(smn, sha256.New()) smt = &SMTWithStorage{SMT: impl, preimages: smv} @@ -386,4 +410,7 @@ func TestSMT_OrphanRemoval(t *testing.T) { require.Equal(t, 1, nodeCount(t), tci) } }) + + require.NoError(t, smn.Stop()) + require.NoError(t, smv.Stop()) } diff --git a/smt_utils_test.go b/smt_utils_test.go index 79120ad..40f8e68 100644 --- a/smt_utils_test.go +++ b/smt_utils_test.go @@ -3,16 +3,18 @@ package smt import ( "bytes" "errors" + + badger "github.com/dgraph-io/badger/v4" ) // SMTWithStorage wraps an SMT with a mapping of value hashes to values (preimages), for use in tests. // Note: this doesn't delete from preimages (inputs to hashing functions), since there could be duplicate stored values. type SMTWithStorage struct { *SMT - preimages MapStore + preimages KVStore } -// Update updates a key with a new value in the tree and adds the value to the preimages MapStore +// Update updates a key with a new value in the tree and adds the value to the preimages KVStore func (smt *SMTWithStorage) Update(key, value []byte) error { if err := smt.SMT.Update(key, value); err != nil { return err @@ -35,10 +37,12 @@ func (smt *SMTWithStorage) GetValue(key []byte) ([]byte, error) { if err != nil { return nil, err } + if valueHash == nil { + return nil, nil + } value, err := smt.preimages.Get(valueHash) if err != nil { - var invalidKeyError *InvalidKeyError - if errors.As(err, &invalidKeyError) { + if errors.Is(err, badger.ErrKeyNotFound) { // If key isn't found, return default value value = defaultValue } else {