feat: Replace MapStore and SimpleMap with KVStore and BadgerDB (#19)

.github/workflows/test.yml

@@ -9,15 +9,15 @@ on:
 
 env:
   # Even though we can test against multiple versions, this one is considered a target version.
-  TARGET_GOLANG_VERSION: "1.18"
+  TARGET_GOLANG_VERSION: "1.19"
 
 jobs:
   tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        go: ["1.18"]
+        go: ["1.19"]
     name: Go Tests
     steps:
       - uses: actions/checkout@v3
@@ -83,7 +83,7 @@ jobs:
       fail-fast: false
       matrix:
         goarch: ["arm64", "amd64"]
-        go: ["1.18"]
+        go: ["1.19"]
     timeout-minutes: 5
     name: Build for ${{ matrix.goarch }}
     steps:

KVStore.md

@@ -0,0 +1,82 @@
+# KVStore <!-- omit in toc -->
+
+- [Overview](#overview)
+- [Implementation](#implementation)
+  - [In-Memory and Persistent](#in-memory-and-persistent)
+  - [Store methods](#store-methods)
+  - [Lifecycle Methods](#lifecycle-methods)
+  - [Data Methods](#data-methods)
+    - [Backups](#backups)
+    - [Restorations](#restorations)
+  - [Accessor Methods](#accessor-methods)
+    - [Prefixed and Sorted Get All](#prefixed-and-sorted-get-all)
+    - [Clear All Key-Value Pairs](#clear-all-key-value-pairs)
+    - [Len](#len)
+
+## Overview
+
+The `KVStore` interface is a key-value store that is used by the `SMT` and `SMST` as its underlying database for its nodes. However, it is an independent key-value store that can be used for any purpose.
+
+## Implementation
+
+The `KVStore` is implemented in [`kvstore.go`](./kvstore.go) and is a wrapper around the [BadgerDB](https://github.com/dgraph-io/badger) key-value database.
+
+The interface defines simple key-value store accessor methods as well as other methods desired from a key-value database in general, this can be found in [`kvstore.go`](./kvstore.go).
+
+_NOTE: The `KVStore` interface can be implemented by any key-value store that satisfies the interface and used as the underlying database store for the `SM(S)T`_
+
+### In-Memory and Persistent
+
+The `KVStore` implementation can be used as an in-memory or persistent key-value store. The `NewKVStore` function takes a `path` argument that can be used to specify a path to a directory to store the database files. If the `path` is an empty string, the database will be stored in-memory.
+
+_NOTE: When providing a path for a persistent database, the directory must exist and be writeable by the user running the application._
+
+### Store methods
+
+As a key-value store the `KVStore` interface defines the simple `Get`, `Set` and `Delete` methods to access and modify the underlying database.
+
+### Lifecycle Methods
+
+The `Stop` method **must** be called when the `KVStore` is no longer needed. This method closes the underlying database and frees up any resources used by the `KVStore`.
+
+For persistent databases, the `Stop` method should be called when the application no longer needs to access the database. For in-memory databases, the `Stop` method should be called when the `KVStore` is no longer needed.
+
+_NOTE: A persistent `KVStore` that is not stopped will stop another `KVStore` from opening the database._
+
+### Data Methods
+
+The `KVStore` interface provides two methods to allow backups and restorations.
+
+#### Backups
+
+The `Backup` method takes an `io.Writer` and a `bool` to indicate whether the backup should be incremental or not. The `io.Writer` is then filled with the contents of the database in an opaque format used by the underlying database for this purpose.
+
+When the `incremental` bool is `false` a full backup will be performed, otherwise an incremental backup will be performed. This is enabled by the `KVStore` keeping the timestamp of its last backup and only backing up data that has been modified since the last backup.
+
+#### Restorations
+
+The `Restore` method takes an `io.Reader` and restores the database from this reader.
+
+The `KVStore` calling the `Restore` method is expected to be initialised and open, otherwise the restore will fail.
+
+_NOTE: Any data contained in the `KVStore` when calling restore will be overwritten._
+
+### Accessor Methods
+
+The accessor methods enable simpler access to the underlying database for certain tasks that are desirable in a key-value store.
+
+#### Prefixed and Sorted Get All
+
+The `GetAll` method supports the retrieval of all keys and values, where the key has a specific prefix. The `descending` bool indicates whether the keys should be returned in descending order or not.
+
+_NOTE: In order to retrieve all keys and values the empty prefix `[]byte{}` should be used to match all keys_
+
+#### Clear All Key-Value Pairs
+
+The `ClearAll` method removes all key-value pairs from the database.
+
+_NOTE: The `ClearAll` method is intended to debug purposes and should not be used in production unless necessary_
+
+#### Len
+
+The `Len` method returns the number of keys in the database, similarly to how the `len` function can return the length of a map.

MerkleSumTree.md

@@ -233,31 +233,37 @@ import (
 )
 
 func main() {
-	// Initialise a new key-value store to store the nodes of the tree
-	// (Note: the tree only stores hashed values, not raw value data)
-	nodeStore := smt.NewSimpleMap()
-
-	// Initialise the tree
-	tree := smt.NewSparseMerkleSumTree(nodeStore, sha256.New())
-
-	// Update tree with keys, values and their sums
-	_ = tree.Update([]byte("foo"), []byte("oof"), 10)
-	_ = tree.Update([]byte("baz"), []byte("zab"), 7)
-	_ = tree.Update([]byte("bin"), []byte("nib"), 3)
-
-	sum := tree.Sum()
-	fmt.Println(sum == 20) // true
-
-	// Generate a Merkle proof for "foo"
-	proof, _ := tree.Prove([]byte("foo"))
-	root := tree.Root() // We also need the current tree root for the proof
-
-	// Verify the Merkle proof for "foo"="oof" where "foo" has a sum of 10
-	if valid := smt.VerifySumProof(proof, root, []byte("foo"), []byte("oof"), 10, tree.Spec()); valid {
-		fmt.Println("Proof verification succeeded.")
-	} else {
-		fmt.Println("Proof verification failed.")
-	}
+    // Initialise a new in-memory key-value store to store the nodes of the tree
+    // (Note: the tree only stores hashed values, not raw value data)
+    nodeStore := smt.NewKVStore("")
+
+    // Ensure the database connection closes
+    defer nodeStore.Stop()
+
+    // Initialise the tree
+    tree := smt.NewSparseMerkleSumTree(nodeStore, sha256.New())
+
+    // Update tree with keys, values and their sums
+    _ = tree.Update([]byte("foo"), []byte("oof"), 10)
+    _ = tree.Update([]byte("baz"), []byte("zab"), 7)
+    _ = tree.Update([]byte("bin"), []byte("nib"), 3)
+
+    // Commit the changes to the nodeStore
+    _ = tree.Commit()
+
+    sum := tree.Sum()
+    fmt.Println(sum == 20) // true
+
+    // Generate a Merkle proof for "foo"
+    proof, _ := tree.Prove([]byte("foo"))
+    root := tree.Root() // We also need the current tree root for the proof
+
+    // Verify the Merkle proof for "foo"="oof" where "foo" has a sum of 10
+    if valid := smt.VerifySumProof(proof, root, []byte("foo"), []byte("oof"), 10, tree.Spec()); valid {
+   	    fmt.Println("Proof verification succeeded.")
+    } else {
+        fmt.Println("Proof verification failed.")
+    }
 }
 ```
 

README.md

@@ -6,27 +6,27 @@
 [![Tests](https://github.com/pokt-network/smt/actions/workflows/test.yml/badge.svg)](https://github.com/pokt-network/smt/actions/workflows/test.yml)
 [![codecov](https://codecov.io/gh/pokt-network/smt/branch/main/graph/badge.svg)](https://codecov.io/gh/pokt-network/smt)
 
-Note: **Requires Go 1.18+**
+Note: **Requires Go 1.19+**
 
 - [Overview](#overview)
 - [Implementation](#implementation)
-	- [Inner Nodes](#inner-nodes)
-	- [Extension Nodes](#extension-nodes)
-	- [Leaf Nodes](#leaf-nodes)
-	- [Lazy Nodes](#lazy-nodes)
-	- [Lazy Loading](#lazy-loading)
-	- [Visualisations](#visualisations)
-		- [General Tree Structure](#general-tree-structure)
-		- [Lazy Nodes](#lazy-nodes-1)
+  - [Inner Nodes](#inner-nodes)
+  - [Extension Nodes](#extension-nodes)
+  - [Leaf Nodes](#leaf-nodes)
+  - [Lazy Nodes](#lazy-nodes)
+  - [Lazy Loading](#lazy-loading)
+  - [Visualisations](#visualisations)
+    - [General Tree Structure](#general-tree-structure)
+    - [Lazy Nodes](#lazy-nodes-1)
 - [Paths](#paths)
-	- [Visualisation](#visualisation)
+  - [Visualisation](#visualisation)
 - [Values](#values)
-	- [Nil values](#nil-values)
+  - [Nil values](#nil-values)
 - [Hashers \& Digests](#hashers--digests)
 - [Proofs](#proofs)
-	- [Verification](#verification)
+  - [Verification](#verification)
 - [Database](#database)
-	- [Data Loss](#data-loss)
+  - [Data Loss](#data-loss)
 - [Sparse Merkle Sum Tree](#sparse-merkle-sum-tree)
 - [Example](#example)
 
@@ -295,20 +295,12 @@ The verification step simply uses the proof data to recompute the root hash with
 
 ## Database
 
-This library defines the `MapStore` interface, in [mapstore.go](./mapstore.go)
-
-```go
-type MapStore interface {
-	Get(key []byte) ([]byte, error)
-	Set(key []byte, value []byte) error
-	Delete(key []byte) error
-}
-```
-
-This interface abstracts the `SimpleMap` key-value store and can be used by the SMT to store the nodes of the tree. Any key-value store that implements the `MapStore` interface can be used with this library.
+This library defines the `KVStore` interface which by default is implemented using [BadgerDB](https://github.com/dgraph-io/badger), however any databse that implements this interface can be used as a drop in replacement. The `KVStore` allows for both in memory and persisted databases to be used to store the nodes for the SMT.
 
 When changes are commited to the underlying database using `Commit()` the digests of the leaf nodes are stored at their respective paths. If retrieved manually from the database the returned value will be the digest of the leaf node, **not** the leaf node's value, even when `WithValueHasher(nil)` is used. The node value can be parsed from this value, as the tree `Get` function does by removing the prefix and path bytes from the returned value.
 
+See [KVStore.md](./KVStore.md) for the details of the implementation.
+
 ### Data Loss
 
 In the event of a system crash or unexpected failure of the program utilising the SMT, if the `Commit()` function has not been called, any changes to the tree will be lost. This is due to the underlying database not being changed **until** the `Commit()` function is called and changes are persisted.
@@ -330,26 +322,32 @@ import (
 )
 
 func main() {
-	// Initialise a new key-value store to store the nodes of the tree
-	// (Note: the tree only stores hashed values, not raw value data)
-	nodeStore := smt.NewSimpleMap()
+    // Initialise a new in-memory key-value store to store the nodes of the tree
+    // (Note: the tree only stores hashed values, not raw value data)
+    nodeStore := smt.NewKVStore("")
+
+    // Ensure the database connection closes
+    defer nodeStore.Stop()
+
+    // Initialise the tree
+    tree := smt.NewSparseMerkleTree(nodeStore, sha256.New())
 
-	// Initialise the tree
-	tree := smt.NewSparseMerkleTree(nodeStore, sha256.New())
+    // Update the key "foo" with the value "bar"
+    _ = tree.Update([]byte("foo"), []byte("bar"))
 
-	// Update the key "foo" with the value "bar"
-	_ = tree.Update([]byte("foo"), []byte("bar"))
+    // Commit the changes to the node store
+    _ = tree.Commit()
 
 	// Generate a Merkle proof for "foo"
-	proof, _ := tree.Prove([]byte("foo"))
-	root := tree.Root() // We also need the current tree root for the proof
-
-	// Verify the Merkle proof for "foo"="bar"
-	if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), tree.Spec()) {
-		fmt.Println("Proof verification succeeded.")
-	} else {
-		fmt.Println("Proof verification failed.")
-	}
+    proof, _ := tree.Prove([]byte("foo"))
+    root := tree.Root() // We also need the current tree root for the proof
+
+    // Verify the Merkle proof for "foo"="bar"
+    if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), tree.Spec()) {
+        fmt.Println("Proof verification succeeded.")
+    } else {
+        fmt.Println("Proof verification failed.")
+    }
 }
 ```
 

bench_test.go

@@ -4,10 +4,15 @@ import (
 	"crypto/sha256"
 	"strconv"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func BenchmarkSparseMerkleTree_Update(b *testing.B) {
-	smn, smv := NewSimpleMap(), NewSimpleMap()
+	smn, err := NewKVStore("")
+	require.NoError(b, err)
+	smv, err := NewKVStore("")
+	require.NoError(b, err)
 	smt := NewSMTWithStorage(smn, smv, sha256.New())
 
 	b.ResetTimer()
@@ -16,10 +21,16 @@ func BenchmarkSparseMerkleTree_Update(b *testing.B) {
 		s := strconv.Itoa(i)
 		_ = smt.Update([]byte(s), []byte(s))
 	}
+
+	require.NoError(b, smn.Stop())
+	require.NoError(b, smv.Stop())
 }
 
 func BenchmarkSparseMerkleTree_Delete(b *testing.B) {
-	smn, smv := NewSimpleMap(), NewSimpleMap()
+	smn, err := NewKVStore("")
+	require.NoError(b, err)
+	smv, err := NewKVStore("")
+	require.NoError(b, err)
 	smt := NewSMTWithStorage(smn, smv, sha256.New())
 
 	for i := 0; i < 100000; i++ {
@@ -33,4 +44,7 @@ func BenchmarkSparseMerkleTree_Delete(b *testing.B) {
 		s := strconv.Itoa(i)
 		_ = smt.Delete([]byte(s))
 	}
+
+	require.NoError(b, smn.Stop())
+	require.NoError(b, smv.Stop())
 }

bulk_test.go

@@ -32,7 +32,10 @@ func TestBulkOperations(t *testing.T) {
 
 // Test all tree operations in bulk, with specified ratio probabilities of insert, update and delete.
 func bulkOperations(t *testing.T, operations int, insert int, update int, delete int) {
-	smn, smv := NewSimpleMap(), NewSimpleMap()
+	smn, err := NewKVStore("")
+	require.NoError(t, err)
+	smv, err := NewKVStore("")
+	require.NoError(t, err)
 	smt := NewSMTWithStorage(smn, smv, sha256.New())
 
 	max := insert + update + delete
@@ -85,7 +88,10 @@ func bulkOperations(t *testing.T, operations int, insert int, update int, delete
 			kv[ki].val = defaultValue
 		}
 	}
+
 	bulkCheckAll(t, smt, kv)
+	require.NoError(t, smn.Stop())
+	require.NoError(t, smv.Stop())
 }
 
 func bulkCheckAll(t *testing.T, smt *SMTWithStorage, kv []bulkop) {

fuzz_test.go

@@ -25,7 +25,8 @@ func FuzzSMT_DetectUnexpectedFailures(f *testing.F) {
 		f.Add(s)
 	}
 	f.Fuzz(func(t *testing.T, input []byte) {
-		smn := NewSimpleMap()
+		smn, err := NewKVStore("")
+		require.NoError(t, err)
 		tree := NewSparseMerkleTree(smn, sha256.New())
 
 		r := bytes.NewReader(input)
@@ -51,7 +52,7 @@ func FuzzSMT_DetectUnexpectedFailures(f *testing.F) {
 			return keys[int(b)%len(keys)]
 		}
 
-                // `i` is the loop counter but also used as the input value to `Update` operations
+		// `i` is the loop counter but also used as the input value to `Update` operations
 		for i := 0; r.Len() != 0; i++ {
 			originalRoot := tree.Root()
 			b, err := r.ReadByte()
@@ -97,6 +98,8 @@ func FuzzSMT_DetectUnexpectedFailures(f *testing.F) {
 			newRoot := tree.Root()
 			require.Greater(t, len(newRoot), 0, "new root is empty while err is nil")
 		}
+
+		require.NoError(t, smn.Stop())
 	})
 }