From da7f632001775c200899aae457b465ae3d7e2f20 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 12:40:20 -0400
Subject: [PATCH 01/42] update

---
 x/merkledb/benchmarks/benchmark.go   | 294 +++++++++++++++++++++++++++
 x/merkledb/benchmarks/directories.go | 119 +++++++++++
 2 files changed, 413 insertions(+)
 create mode 100644 x/merkledb/benchmarks/benchmark.go
 create mode 100644 x/merkledb/benchmarks/directories.go

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
new file mode 100644
index 000000000000..61b6c984ac99
--- /dev/null
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -0,0 +1,294 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/pflag"
+
+	"github.com/ava-labs/avalanchego/database"
+	"github.com/ava-labs/avalanchego/database/leveldb"
+	"github.com/ava-labs/avalanchego/trace"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/ava-labs/avalanchego/utils/units"
+	"github.com/ava-labs/avalanchego/x/merkledb"
+)
+
+const (
+	defaultDatabaseEntries    = 1000000
+	databaseCreationBatchSize = 10000
+	defaultHistoryLength      = 300
+	databaseRunningBatchSize  = 2500
+	databaseRunningUpdateSize = 5000
+)
+
+var databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
+
+func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
+	return merkledb.Config{
+		BranchFactor:                merkledb.BranchFactor16,
+		Hasher:                      merkledb.DefaultHasher,
+		RootGenConcurrency:          0,
+		HistoryLength:               defaultHistoryLength,
+		ValueNodeCacheSize:          units.MiB,
+		IntermediateNodeCacheSize:   units.MiB,
+		IntermediateWriteBufferSize: units.KiB,
+		IntermediateWriteBatchSize:  256 * units.KiB,
+		Reg:                         promRegistry,
+		TraceLevel:                  merkledb.InfoTrace,
+		Tracer:                      trace.Noop,
+	}
+}
+
+func getGoldenDatabaseDirectory() string {
+	wd, _ := os.Getwd()
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", databaseEntries))
+}
+
+func getRunningDatabaseDirectory() string {
+	wd, _ := os.Getwd()
+	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", databaseEntries))
+}
+
+func calculateIndexEncoding(idx uint64) []byte {
+	var entryEncoding [8]byte
+	binary.NativeEndian.PutUint64(entryEncoding[:], idx)
+	entryHash := sha256.Sum256(entryEncoding[:])
+	return entryHash[:]
+}
+
+func createGoldenDatabase() error {
+	promRegistry := prometheus.NewRegistry()
+
+	levelDB, err := leveldb.New(
+		getGoldenDatabaseDirectory(),
+		nil,
+		logging.NoLog{},
+		promRegistry,
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create levelDB database : %v\n", err)
+		return err
+	}
+
+	mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(promRegistry))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create merkleDB database : %v\n", err)
+		return err
+	}
+
+	startInsertTime := time.Now()
+	var currentBatch database.Batch
+	for entryIdx := uint64(0); entryIdx < *databaseEntries; entryIdx++ {
+		if entryIdx%databaseCreationBatchSize == 0 {
+			currentBatch = mdb.NewBatch()
+		}
+
+		entryHash := calculateIndexEncoding(entryIdx)
+		err = currentBatch.Put(entryHash, entryHash)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to put value in merkleDB database : %v\n", err)
+			return err
+		}
+
+		if entryIdx%databaseCreationBatchSize == (databaseCreationBatchSize - 1) {
+			err = currentBatch.Write()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err)
+				return err
+			}
+		}
+	}
+	if (*databaseEntries)%databaseCreationBatchSize != 0 {
+		err = currentBatch.Write()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err)
+			return err
+		}
+	}
+	rootID, err := mdb.GetMerkleRoot(context.Background())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to get root ID : %v\n", err)
+		return err
+	}
+
+	fmt.Printf("Generated and inserted %d batches of size %d in %v\n",
+		(*databaseEntries)/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
+	err = mdb.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to close levelDB database : %v\n", err)
+		return err
+	}
+	err = levelDB.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to close merkleDB database : %v\n", err)
+		return err
+	}
+	fmt.Printf("Completed initialization with hash of %v\n", rootID.Hex())
+	return nil
+}
+
+func resetRunningDatabaseDirectory() error {
+	runningDir := getRunningDatabaseDirectory()
+	if _, err := os.Stat(runningDir); err == nil {
+		err := os.RemoveAll(runningDir)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
+			return err
+		}
+	}
+	err := os.Mkdir(runningDir, 0o777)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create running directory : %v\n", err)
+		return err
+	}
+	err = CopyDirectory(getGoldenDatabaseDirectory(), runningDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to duplicate golden directory : %v\n", err)
+		return err
+	}
+	return nil
+}
+
+func runBenchmark() error {
+	promRegistry := prometheus.NewRegistry()
+	http.Handle("/metrics", promhttp.Handler())
+
+	server := &http.Server{
+		Addr:              ":8080",
+		ReadHeaderTimeout: 3 * time.Second,
+	}
+	go func() {
+		err := server.ListenAndServe()
+		if err != nil && err != http.ErrServerClosed {
+			fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err)
+		}
+	}()
+
+	writesCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "writes",
+		Help:      "Total number of keys written",
+	})
+	batchCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "batch",
+		Help:      "Total number of batches written",
+	})
+	promRegistry.MustRegister(writesCounter)
+	promRegistry.MustRegister(batchCounter)
+
+	levelDB, err := leveldb.New(
+		getRunningDatabaseDirectory(),
+		nil,
+		logging.NoLog{},
+		promRegistry,
+	)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create levelDB database : %v\n", err)
+		return err
+	}
+
+	mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(promRegistry))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create merkleDB database : %v\n", err)
+		return err
+	}
+	defer func() {
+		mdb.Close()
+		levelDB.Close()
+	}()
+
+	low := uint64(0)
+	var deleteDuration, addDuration, updateDuration, batchDuration time.Duration
+	for {
+		startBatchTime := time.Now()
+		batch := mdb.NewBatch()
+
+		// delete first 2.5k keys from the beginning
+		startDeleteTime := time.Now()
+		for keyToDeleteIdx := low; keyToDeleteIdx < low+databaseRunningBatchSize; keyToDeleteIdx++ {
+			entryHash := calculateIndexEncoding(keyToDeleteIdx)
+			err = batch.Delete(entryHash)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to delete merkleDB entry : %v\n", err)
+				return err
+			}
+		}
+		deleteDuration = time.Since(startDeleteTime)
+
+		// add 2.5k past end.
+		startInsertTime := time.Now()
+		for keyToAddIdx := low + (*databaseEntries); keyToAddIdx < low+(*databaseEntries)+databaseRunningBatchSize; keyToAddIdx++ {
+			entryHash := calculateIndexEncoding(keyToAddIdx)
+			err = batch.Put(entryHash, entryHash)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to insert merkleDB entry : %v\n", err)
+				return err
+			}
+		}
+		addDuration = time.Since(startInsertTime)
+
+		// update middle 5k entries
+		startUpdateTime := time.Now()
+		for keyToUpdateIdx := low + ((*databaseEntries) / 2); keyToUpdateIdx < low+((*databaseEntries)/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
+			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
+			updateEntryValue := calculateIndexEncoding(keyToUpdateIdx - ((*databaseEntries) / 2))
+			err = batch.Put(updateEntryKey, updateEntryValue)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to update merkleDB entry : %v\n", err)
+				return err
+			}
+		}
+		updateDuration = time.Since(startUpdateTime)
+
+		err = batch.Write()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to write batch : %v\n", err)
+			return err
+		}
+		batchDuration = time.Since(startBatchTime)
+
+		fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
+			time.Second/deleteDuration,
+			time.Second/updateDuration,
+			time.Second/addDuration,
+			time.Second/batchDuration)
+		writesCounter.Add(databaseRunningBatchSize + databaseRunningBatchSize + databaseRunningUpdateSize)
+		batchCounter.Inc()
+	}
+
+	// return nil
+}
+
+func main() {
+	pflag.Parse()
+
+	goldenDir := getGoldenDatabaseDirectory()
+	if _, err := os.Stat(goldenDir); os.IsNotExist(err) {
+		// create golden image.
+		if createGoldenDatabase() != nil {
+			os.Exit(1)
+			return
+		}
+	}
+	if resetRunningDatabaseDirectory() != nil {
+		os.Exit(1)
+		return
+	}
+	if runBenchmark() != nil {
+		os.Exit(1)
+		return
+	}
+}
diff --git a/x/merkledb/benchmarks/directories.go b/x/merkledb/benchmarks/directories.go
new file mode 100644
index 000000000000..e65899326afa
--- /dev/null
+++ b/x/merkledb/benchmarks/directories.go
@@ -0,0 +1,119 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+func CopyDirectory(scrDir, dest string) error {
+	entries, err := os.ReadDir(scrDir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		sourcePath := filepath.Join(scrDir, entry.Name())
+		destPath := filepath.Join(dest, entry.Name())
+
+		fileInfo, err := os.Stat(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+		if !ok {
+			return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath)
+		}
+
+		switch fileInfo.Mode() & os.ModeType {
+		case os.ModeDir:
+			if err := CreateIfNotExists(destPath, 0o755); err != nil {
+				return err
+			}
+			if err := CopyDirectory(sourcePath, destPath); err != nil {
+				return err
+			}
+		case os.ModeSymlink:
+			if err := CopySymLink(sourcePath, destPath); err != nil {
+				return err
+			}
+		default:
+			if err := Copy(sourcePath, destPath); err != nil {
+				return err
+			}
+		}
+
+		if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil {
+			return err
+		}
+
+		fInfo, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
+		isSymlink := fInfo.Mode()&os.ModeSymlink != 0
+		if !isSymlink {
+			if err := os.Chmod(destPath, fInfo.Mode()); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func Copy(srcFile, dstFile string) error {
+	out, err := os.Create(dstFile)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	in, err := os.Open(srcFile)
+	if err != nil {
+		return err
+	}
+
+	defer in.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func Exists(filePath string) bool {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return false
+	}
+
+	return true
+}
+
+func CreateIfNotExists(dir string, perm os.FileMode) error {
+	if Exists(dir) {
+		return nil
+	}
+
+	if err := os.MkdirAll(dir, perm); err != nil {
+		return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
+	}
+
+	return nil
+}
+
+func CopySymLink(source, dest string) error {
+	link, err := os.Readlink(source)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(link, dest)
+}

From 67b83d91dc843124f30c57710a9ac8813bcc481c Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 13:14:01 -0400
Subject: [PATCH 02/42] update

---
 x/merkledb/benchmarks/benchmark.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 61b6c984ac99..781142b89fec 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -31,9 +31,13 @@ const (
 	defaultHistoryLength      = 300
 	databaseRunningBatchSize  = 2500
 	databaseRunningUpdateSize = 5000
+	defaultMetricsPort        = 3000
 )
 
-var databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
+var (
+	databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
+	httpMetricPort  = pflag.Uint64("p", defaultMetricsPort, "default metrics port")
+)
 
 func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
 	return merkledb.Config{
@@ -166,7 +170,7 @@ func runBenchmark() error {
 	http.Handle("/metrics", promhttp.Handler())
 
 	server := &http.Server{
-		Addr:              ":8080",
+		Addr:              fmt.Sprintf(":%d", *httpMetricPort),
 		ReadHeaderTimeout: 3 * time.Second,
 	}
 	go func() {

From 60a1bd72449eff5b4188e443c64efce80988847b Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 15:13:03 -0400
Subject: [PATCH 03/42] update

---
 x/merkledb/benchmarks/benchmark.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 781142b89fec..6c71d8d625c9 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -57,12 +57,12 @@ func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
 
 func getGoldenDatabaseDirectory() string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", *databaseEntries))
 }
 
 func getRunningDatabaseDirectory() string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", *databaseEntries))
 }
 
 func calculateIndexEncoding(idx uint64) []byte {

From e9f6f205c8fc6d643e8e43945a293a5e84fe9020 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 15:49:32 -0400
Subject: [PATCH 04/42] registration order.

---
 x/merkledb/benchmarks/benchmark.go | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 6c71d8d625c9..d65b2589ca2c 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -167,18 +167,7 @@ func resetRunningDatabaseDirectory() error {
 
 func runBenchmark() error {
 	promRegistry := prometheus.NewRegistry()
-	http.Handle("/metrics", promhttp.Handler())
-
-	server := &http.Server{
-		Addr:              fmt.Sprintf(":%d", *httpMetricPort),
-		ReadHeaderTimeout: 3 * time.Second,
-	}
-	go func() {
-		err := server.ListenAndServe()
-		if err != nil && err != http.ErrServerClosed {
-			fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err)
-		}
-	}()
+	prometheus.Register(promRegistry)
 
 	writesCounter := prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "merkledb_bench",
@@ -193,6 +182,19 @@ func runBenchmark() error {
 	promRegistry.MustRegister(writesCounter)
 	promRegistry.MustRegister(batchCounter)
 
+	http.Handle("/metrics", promhttp.Handler())
+
+	server := &http.Server{
+		Addr:              fmt.Sprintf(":%d", *httpMetricPort),
+		ReadHeaderTimeout: 3 * time.Second,
+	}
+	go func() {
+		err := server.ListenAndServe()
+		if err != nil && err != http.ErrServerClosed {
+			fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err)
+		}
+	}()
+
 	levelDB, err := leveldb.New(
 		getRunningDatabaseDirectory(),
 		nil,

From 2ce08b8ac0687ce818a48397e6a6b16a12f89f5f Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 16:25:45 -0400
Subject: [PATCH 05/42] add few more metrics.

---
 x/merkledb/benchmarks/benchmark.go | 45 ++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 6 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index d65b2589ca2c..05f3d40c6c52 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -37,6 +37,7 @@ const (
 var (
 	databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
 	httpMetricPort  = pflag.Uint64("p", defaultMetricsPort, "default metrics port")
+	verbose         = pflag.Bool("v", false, "verbose")
 )
 
 func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
@@ -179,8 +180,32 @@ func runBenchmark() error {
 		Name:      "batch",
 		Help:      "Total number of batches written",
 	})
+	deleteRate := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_delete_rate",
+		Help:      "The rate at which elements are deleted",
+	})
+	updateRate := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_update_rate",
+		Help:      "The rate at which elements are updated",
+	})
+	insertRate := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_insert_rate",
+		Help:      "The rate at which elements are inserted",
+	})
+	batchWriteRate := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "batch_write_rate",
+		Help:      "The rate at which the batch was written",
+	})
 	promRegistry.MustRegister(writesCounter)
 	promRegistry.MustRegister(batchCounter)
+	promRegistry.MustRegister(deleteRate)
+	promRegistry.MustRegister(updateRate)
+	promRegistry.MustRegister(insertRate)
+	promRegistry.MustRegister(batchWriteRate)
 
 	http.Handle("/metrics", promhttp.Handler())
 
@@ -233,6 +258,7 @@ func runBenchmark() error {
 			}
 		}
 		deleteDuration = time.Since(startDeleteTime)
+		deleteRate.Set(float64(time.Second) / float64(deleteDuration))
 
 		// add 2.5k past end.
 		startInsertTime := time.Now()
@@ -245,6 +271,7 @@ func runBenchmark() error {
 			}
 		}
 		addDuration = time.Since(startInsertTime)
+		insertRate.Set(float64(time.Second) / float64(addDuration))
 
 		// update middle 5k entries
 		startUpdateTime := time.Now()
@@ -258,19 +285,25 @@ func runBenchmark() error {
 			}
 		}
 		updateDuration = time.Since(startUpdateTime)
+		updateRate.Set(float64(time.Second) / float64(updateDuration))
 
+		batchWriteStartTime := time.Now()
 		err = batch.Write()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to write batch : %v\n", err)
 			return err
 		}
 		batchDuration = time.Since(startBatchTime)
-
-		fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
-			time.Second/deleteDuration,
-			time.Second/updateDuration,
-			time.Second/addDuration,
-			time.Second/batchDuration)
+		batchWriteDuration := time.Since(batchWriteStartTime)
+		batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+
+		if *verbose {
+			fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
+				time.Second/deleteDuration,
+				time.Second/updateDuration,
+				time.Second/addDuration,
+				time.Second/batchDuration)
+		}
 		writesCounter.Add(databaseRunningBatchSize + databaseRunningBatchSize + databaseRunningUpdateSize)
 		batchCounter.Inc()
 	}

From a8b673f4f912e2ebf79bd191c125891ce592a676 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 16:58:55 -0400
Subject: [PATCH 06/42] update

---
 x/merkledb/benchmarks/dashboard.grafana | 298 ++++++++++++++++++++++++
 1 file changed, 298 insertions(+)
 create mode 100644 x/merkledb/benchmarks/dashboard.grafana

diff --git a/x/merkledb/benchmarks/dashboard.grafana b/x/merkledb/benchmarks/dashboard.grafana
new file mode 100644
index 000000000000..83b01b5fe91c
--- /dev/null
+++ b/x/merkledb/benchmarks/dashboard.grafana
@@ -0,0 +1,298 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "bdxazslv08jcwd"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 25,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "merkledb_bench_entry_delete_rate",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Deletes",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "merkledb_bench_entry_insert_rate",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Inserts",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_entry_update_rate",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Updates",
+          "useBackend": false
+        }
+      ],
+      "title": "Rows Operations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "bdxazslv08jcwd"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "rate(merkledb_bench_batch[$__rate_interval])",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "Effective Rate",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_batch_write_rate",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Batch Write Operation",
+          "useBackend": false
+        }
+      ],
+      "title": "Batch Rate",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "MerkleDB operations benchmark",
+  "uid": "edwwwnfme46ioa",
+  "version": 4,
+  "weekStart": ""
+}
\ No newline at end of file

From e888f9e4228430ed74ec84d7bee3b10155895dc8 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 17:31:40 -0400
Subject: [PATCH 07/42] fix metric

---
 x/merkledb/benchmarks/benchmark.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 05f3d40c6c52..21b545f7ae33 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -258,7 +258,7 @@ func runBenchmark() error {
 			}
 		}
 		deleteDuration = time.Since(startDeleteTime)
-		deleteRate.Set(float64(time.Second) / float64(deleteDuration))
+		deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration))
 
 		// add 2.5k past end.
 		startInsertTime := time.Now()
@@ -271,7 +271,7 @@ func runBenchmark() error {
 			}
 		}
 		addDuration = time.Since(startInsertTime)
-		insertRate.Set(float64(time.Second) / float64(addDuration))
+		insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration))
 
 		// update middle 5k entries
 		startUpdateTime := time.Now()
@@ -285,7 +285,7 @@ func runBenchmark() error {
 			}
 		}
 		updateDuration = time.Since(startUpdateTime)
-		updateRate.Set(float64(time.Second) / float64(updateDuration))
+		updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
 
 		batchWriteStartTime := time.Now()
 		err = batch.Write()

From a34e94952f899840a2d46ffabbf6f04f0592cfb7 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 8 Sep 2024 17:39:49 -0400
Subject: [PATCH 08/42] update

---
 x/merkledb/benchmarks/benchmark.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 21b545f7ae33..fa0a4b003f28 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -168,7 +168,9 @@ func resetRunningDatabaseDirectory() error {
 
 func runBenchmark() error {
 	promRegistry := prometheus.NewRegistry()
-	prometheus.Register(promRegistry)
+	if err := prometheus.Register(promRegistry); err != nil {
+		return err
+	}
 
 	writesCounter := prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "merkledb_bench",

From 8984569d64353b62d7eca29f28b352d4b7bacee5 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Mon, 9 Sep 2024 11:15:01 -0400
Subject: [PATCH 09/42] few more updates.

---
 x/merkledb/benchmarks/benchmark.go | 52 ++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index fa0a4b003f28..58410097d1a9 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -26,9 +26,8 @@ import (
 )
 
 const (
-	defaultDatabaseEntries    = 1000000
-	databaseCreationBatchSize = 10000
-	defaultHistoryLength      = 300
+	defaultDatabaseEntries    = 2000000
+	databaseCreationBatchSize = 1000000
 	databaseRunningBatchSize  = 2500
 	databaseRunningUpdateSize = 5000
 	defaultMetricsPort        = 3000
@@ -41,6 +40,7 @@ var (
 )
 
 func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
+	const defaultHistoryLength = 300
 	return merkledb.Config{
 		BranchFactor:                merkledb.BranchFactor16,
 		Hasher:                      merkledb.DefaultHasher,
@@ -51,11 +51,16 @@ func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
 		IntermediateWriteBufferSize: units.KiB,
 		IntermediateWriteBatchSize:  256 * units.KiB,
 		Reg:                         promRegistry,
-		TraceLevel:                  merkledb.InfoTrace,
+		TraceLevel:                  merkledb.NoTrace,
 		Tracer:                      trace.Noop,
 	}
 }
 
+func getGoldenStagingDatabaseDirectory() string {
+	wd, _ := os.Getwd()
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", *databaseEntries))
+}
+
 func getGoldenDatabaseDirectory() string {
 	wd, _ := os.Getwd()
 	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", *databaseEntries))
@@ -74,10 +79,21 @@ func calculateIndexEncoding(idx uint64) []byte {
 }
 
 func createGoldenDatabase() error {
+	err := os.RemoveAll(getGoldenStagingDatabaseDirectory())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
+		return err
+	}
+	err = os.Mkdir(getGoldenStagingDatabaseDirectory(), 0o777)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create golden staging directory : %v\n", err)
+		return err
+	}
+
 	promRegistry := prometheus.NewRegistry()
 
 	levelDB, err := leveldb.New(
-		getGoldenDatabaseDirectory(),
+		getGoldenStagingDatabaseDirectory(),
 		nil,
 		logging.NoLog{},
 		promRegistry,
@@ -87,12 +103,27 @@ func createGoldenDatabase() error {
 		return err
 	}
 
-	mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(promRegistry))
+	mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(nil))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create merkleDB database : %v\n", err)
 		return err
 	}
 
+	fmt.Printf("Initializing database.")
+	ticksCh := make(chan interface{})
+	go func() {
+		t := time.NewTicker(time.Second)
+		defer t.Stop()
+		for {
+			select {
+			case <-ticksCh:
+				return
+			case <-t.C:
+				fmt.Printf(".")
+			}
+		}
+	}()
+
 	startInsertTime := time.Now()
 	var currentBatch database.Batch
 	for entryIdx := uint64(0); entryIdx < *databaseEntries; entryIdx++ {
@@ -113,6 +144,7 @@ func createGoldenDatabase() error {
 				fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err)
 				return err
 			}
+			currentBatch = nil
 		}
 	}
 	if (*databaseEntries)%databaseCreationBatchSize != 0 {
@@ -122,6 +154,9 @@ func createGoldenDatabase() error {
 			return err
 		}
 	}
+	close(ticksCh)
+
+	fmt.Printf(" done!\n")
 	rootID, err := mdb.GetMerkleRoot(context.Background())
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to get root ID : %v\n", err)
@@ -140,6 +175,11 @@ func createGoldenDatabase() error {
 		fmt.Fprintf(os.Stderr, "unable to close merkleDB database : %v\n", err)
 		return err
 	}
+	err = os.Rename(getGoldenStagingDatabaseDirectory(), getGoldenDatabaseDirectory())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to rename golden staging directory : %v\n", err)
+		return err
+	}
 	fmt.Printf("Completed initialization with hash of %v\n", rootID.Hex())
 	return nil
 }

From c844c707c64aba5ea038fe9b3dbd25c860d0efe8 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Mon, 9 Sep 2024 12:12:24 -0400
Subject: [PATCH 10/42] reuse batch

---
 x/merkledb/benchmarks/benchmark.go | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 58410097d1a9..95267c45ef3b 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -17,7 +17,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/pflag"
 
-	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/database/leveldb"
 	"github.com/ava-labs/avalanchego/trace"
 	"github.com/ava-labs/avalanchego/utils/logging"
@@ -125,12 +124,8 @@ func createGoldenDatabase() error {
 	}()
 
 	startInsertTime := time.Now()
-	var currentBatch database.Batch
+	currentBatch := mdb.NewBatch()
 	for entryIdx := uint64(0); entryIdx < *databaseEntries; entryIdx++ {
-		if entryIdx%databaseCreationBatchSize == 0 {
-			currentBatch = mdb.NewBatch()
-		}
-
 		entryHash := calculateIndexEncoding(entryIdx)
 		err = currentBatch.Put(entryHash, entryHash)
 		if err != nil {
@@ -144,7 +139,7 @@ func createGoldenDatabase() error {
 				fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err)
 				return err
 			}
-			currentBatch = nil
+			currentBatch.Reset()
 		}
 	}
 	if (*databaseEntries)%databaseCreationBatchSize != 0 {
@@ -285,9 +280,9 @@ func runBenchmark() error {
 
 	low := uint64(0)
 	var deleteDuration, addDuration, updateDuration, batchDuration time.Duration
+	batch := mdb.NewBatch()
 	for {
 		startBatchTime := time.Now()
-		batch := mdb.NewBatch()
 
 		// delete first 2.5k keys from the beginning
 		startDeleteTime := time.Now()
@@ -339,6 +334,8 @@ func runBenchmark() error {
 		batchWriteDuration := time.Since(batchWriteStartTime)
 		batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
 
+		batch.Reset()
+
 		if *verbose {
 			fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
 				time.Second/deleteDuration,

From 30f05e74c41a922d55c10f183987f7c6b9729ce0 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Mon, 9 Sep 2024 16:28:45 -0400
Subject: [PATCH 11/42] update

---
 x/merkledb/benchmarks/benchmark.go | 161 +++++++++++++++++------------
 1 file changed, 94 insertions(+), 67 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 95267c45ef3b..4792c6ee23d4 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -25,10 +25,10 @@ import (
 )
 
 const (
-	defaultDatabaseEntries    = 2000000
-	databaseCreationBatchSize = 1000000
-	databaseRunningBatchSize  = 2500
-	databaseRunningUpdateSize = 5000
+	defaultDatabaseEntries    = 1000000000 // 1B
+	databaseCreationBatchSize = 10000      // 10k
+	databaseRunningBatchSize  = 2500       // 2.5k
+	databaseRunningUpdateSize = 5000       // 5k
 	defaultMetricsPort        = 3000
 )
 
@@ -36,17 +36,59 @@ var (
 	databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
 	httpMetricPort  = pflag.Uint64("p", defaultMetricsPort, "default metrics port")
 	verbose         = pflag.Bool("v", false, "verbose")
+
+	insertsCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "insert_counter",
+		Help:      "Total number of inserts",
+	})
+	deletesCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "deletes_counter",
+		Help:      "Total number of deletes",
+	})
+	updatesCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "updates_counter",
+		Help:      "Total number of updates",
+	})
+	batchCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "batch",
+		Help:      "Total number of batches written",
+	})
+	deleteRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_delete_rate",
+		Help:      "The rate at which elements are deleted",
+	})
+	updateRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_update_rate",
+		Help:      "The rate at which elements are updated",
+	})
+	insertRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_insert_rate",
+		Help:      "The rate at which elements are inserted",
+	})
+	batchWriteRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "batch_write_rate",
+		Help:      "The rate at which the batch was written",
+	})
+	promRegistry = prometheus.NewRegistry()
 )
 
 func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
-	const defaultHistoryLength = 300
+	const defaultHistoryLength = 120
 	return merkledb.Config{
 		BranchFactor:                merkledb.BranchFactor16,
 		Hasher:                      merkledb.DefaultHasher,
 		RootGenConcurrency:          0,
 		HistoryLength:               defaultHistoryLength,
 		ValueNodeCacheSize:          units.MiB,
-		IntermediateNodeCacheSize:   units.MiB,
+		IntermediateNodeCacheSize:   1024 * units.MiB,
 		IntermediateWriteBufferSize: units.KiB,
 		IntermediateWriteBatchSize:  256 * units.KiB,
 		Reg:                         promRegistry,
@@ -89,8 +131,6 @@ func createGoldenDatabase() error {
 		return err
 	}
 
-	promRegistry := prometheus.NewRegistry()
-
 	levelDB, err := leveldb.New(
 		getGoldenStagingDatabaseDirectory(),
 		nil,
@@ -108,7 +148,7 @@ func createGoldenDatabase() error {
 		return err
 	}
 
-	fmt.Printf("Initializing database.")
+	fmt.Print("Initializing database.")
 	ticksCh := make(chan interface{})
 	go func() {
 		t := time.NewTicker(time.Second)
@@ -118,12 +158,13 @@ func createGoldenDatabase() error {
 			case <-ticksCh:
 				return
 			case <-t.C:
-				fmt.Printf(".")
+				fmt.Print(".")
 			}
 		}
 	}()
 
 	startInsertTime := time.Now()
+	startInsertBatchTime := startInsertTime
 	currentBatch := mdb.NewBatch()
 	for entryIdx := uint64(0); entryIdx < *databaseEntries; entryIdx++ {
 		entryHash := calculateIndexEncoding(entryIdx)
@@ -140,6 +181,11 @@ func createGoldenDatabase() error {
 				return err
 			}
 			currentBatch.Reset()
+
+			addDuration := time.Since(startInsertBatchTime)
+			insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration))
+			insertsCounter.Add(databaseCreationBatchSize)
+			startInsertBatchTime = time.Now()
 		}
 	}
 	if (*databaseEntries)%databaseCreationBatchSize != 0 {
@@ -151,7 +197,7 @@ func createGoldenDatabase() error {
 	}
 	close(ticksCh)
 
-	fmt.Printf(" done!\n")
+	fmt.Print(" done!\n")
 	rootID, err := mdb.GetMerkleRoot(context.Background())
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to get root ID : %v\n", err)
@@ -202,61 +248,6 @@ func resetRunningDatabaseDirectory() error {
 }
 
 func runBenchmark() error {
-	promRegistry := prometheus.NewRegistry()
-	if err := prometheus.Register(promRegistry); err != nil {
-		return err
-	}
-
-	writesCounter := prometheus.NewCounter(prometheus.CounterOpts{
-		Namespace: "merkledb_bench",
-		Name:      "writes",
-		Help:      "Total number of keys written",
-	})
-	batchCounter := prometheus.NewCounter(prometheus.CounterOpts{
-		Namespace: "merkledb_bench",
-		Name:      "batch",
-		Help:      "Total number of batches written",
-	})
-	deleteRate := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: "merkledb_bench",
-		Name:      "entry_delete_rate",
-		Help:      "The rate at which elements are deleted",
-	})
-	updateRate := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: "merkledb_bench",
-		Name:      "entry_update_rate",
-		Help:      "The rate at which elements are updated",
-	})
-	insertRate := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: "merkledb_bench",
-		Name:      "entry_insert_rate",
-		Help:      "The rate at which elements are inserted",
-	})
-	batchWriteRate := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: "merkledb_bench",
-		Name:      "batch_write_rate",
-		Help:      "The rate at which the batch was written",
-	})
-	promRegistry.MustRegister(writesCounter)
-	promRegistry.MustRegister(batchCounter)
-	promRegistry.MustRegister(deleteRate)
-	promRegistry.MustRegister(updateRate)
-	promRegistry.MustRegister(insertRate)
-	promRegistry.MustRegister(batchWriteRate)
-
-	http.Handle("/metrics", promhttp.Handler())
-
-	server := &http.Server{
-		Addr:              fmt.Sprintf(":%d", *httpMetricPort),
-		ReadHeaderTimeout: 3 * time.Second,
-	}
-	go func() {
-		err := server.ListenAndServe()
-		if err != nil && err != http.ErrServerClosed {
-			fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err)
-		}
-	}()
-
 	levelDB, err := leveldb.New(
 		getRunningDatabaseDirectory(),
 		nil,
@@ -296,6 +287,7 @@ func runBenchmark() error {
 		}
 		deleteDuration = time.Since(startDeleteTime)
 		deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration))
+		deletesCounter.Add(databaseRunningBatchSize)
 
 		// add 2.5k past end.
 		startInsertTime := time.Now()
@@ -309,6 +301,7 @@ func runBenchmark() error {
 		}
 		addDuration = time.Since(startInsertTime)
 		insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration))
+		insertsCounter.Add(databaseRunningBatchSize)
 
 		// update middle 5k entries
 		startUpdateTime := time.Now()
@@ -323,6 +316,7 @@ func runBenchmark() error {
 		}
 		updateDuration = time.Since(startUpdateTime)
 		updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
+		updatesCounter.Add(databaseRunningUpdateSize)
 
 		batchWriteStartTime := time.Now()
 		err = batch.Write()
@@ -343,16 +337,49 @@ func runBenchmark() error {
 				time.Second/addDuration,
 				time.Second/batchDuration)
 		}
-		writesCounter.Add(databaseRunningBatchSize + databaseRunningBatchSize + databaseRunningUpdateSize)
+
 		batchCounter.Inc()
 	}
 
 	// return nil
 }
 
+func setupMetrics() error {
+	if err := prometheus.Register(promRegistry); err != nil {
+		return err
+	}
+	promRegistry.MustRegister(insertsCounter)
+	promRegistry.MustRegister(deletesCounter)
+	promRegistry.MustRegister(updatesCounter)
+	promRegistry.MustRegister(batchCounter)
+	promRegistry.MustRegister(deleteRate)
+	promRegistry.MustRegister(updateRate)
+	promRegistry.MustRegister(insertRate)
+	promRegistry.MustRegister(batchWriteRate)
+
+	http.Handle("/metrics", promhttp.Handler())
+
+	server := &http.Server{
+		Addr:              fmt.Sprintf(":%d", *httpMetricPort),
+		ReadHeaderTimeout: 3 * time.Second,
+	}
+	go func() {
+		err := server.ListenAndServe()
+		if err != nil && err != http.ErrServerClosed {
+			fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err)
+		}
+	}()
+	return nil
+}
+
 func main() {
 	pflag.Parse()
 
+	if setupMetrics() != nil {
+		os.Exit(1)
+		return
+	}
+
 	goldenDir := getGoldenDatabaseDirectory()
 	if _, err := os.Stat(goldenDir); os.IsNotExist(err) {
 		// create golden image.

From e2ed004ebf948c74cff78772cf9d31048f47b340 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Mon, 9 Sep 2024 16:33:15 -0400
Subject: [PATCH 12/42] bugfix

---
 x/merkledb/benchmarks/benchmark.go | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 4792c6ee23d4..83aa30993868 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -175,16 +175,21 @@ func createGoldenDatabase() error {
 		}
 
 		if entryIdx%databaseCreationBatchSize == (databaseCreationBatchSize - 1) {
+			addDuration := time.Since(startInsertBatchTime)
+			insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration))
+			insertsCounter.Add(databaseCreationBatchSize)
+
+			batchWriteStartTime := time.Now()
 			err = currentBatch.Write()
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err)
 				return err
 			}
 			currentBatch.Reset()
+			batchWriteDuration := time.Since(batchWriteStartTime)
+			batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+			batchCounter.Inc()
 
-			addDuration := time.Since(startInsertBatchTime)
-			insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration))
-			insertsCounter.Add(databaseCreationBatchSize)
 			startInsertBatchTime = time.Now()
 		}
 	}

From 24da31dfd5fefaeae0461a517e4bc7a9643e4df6 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Tue, 10 Sep 2024 11:12:56 -0400
Subject: [PATCH 13/42] avoid double registration

---
 x/merkledb/benchmarks/benchmark.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 83aa30993868..57c9b74b4708 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -135,7 +135,7 @@ func createGoldenDatabase() error {
 		getGoldenStagingDatabaseDirectory(),
 		nil,
 		logging.NoLog{},
-		promRegistry,
+		prometheus.NewRegistry(),
 	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create levelDB database : %v\n", err)

From 171c3b5b0cc5bba09d19ca6851276b09d349286b Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Tue, 10 Sep 2024 13:23:31 -0400
Subject: [PATCH 14/42] add leveldb configuration; optimize writes.

---
 x/merkledb/benchmarks/benchmark.go     |   8 +-
 x/merkledb/benchmarks/levelDBConfig.go | 122 +++++++++++++++++++++++++
 2 files changed, 126 insertions(+), 4 deletions(-)
 create mode 100644 x/merkledb/benchmarks/levelDBConfig.go

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 57c9b74b4708..d38d0f0e911c 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -133,7 +133,7 @@ func createGoldenDatabase() error {
 
 	levelDB, err := leveldb.New(
 		getGoldenStagingDatabaseDirectory(),
-		nil,
+		getLevelDbConfig(),
 		logging.NoLog{},
 		prometheus.NewRegistry(),
 	)
@@ -213,12 +213,12 @@ func createGoldenDatabase() error {
 		(*databaseEntries)/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
 	err = mdb.Close()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to close levelDB database : %v\n", err)
+		fmt.Fprintf(os.Stderr, "unable to close merkleDB database : %v\n", err)
 		return err
 	}
 	err = levelDB.Close()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to close merkleDB database : %v\n", err)
+		fmt.Fprintf(os.Stderr, "unable to close levelDB database : %v\n", err)
 		return err
 	}
 	err = os.Rename(getGoldenStagingDatabaseDirectory(), getGoldenDatabaseDirectory())
@@ -255,7 +255,7 @@ func resetRunningDatabaseDirectory() error {
 func runBenchmark() error {
 	levelDB, err := leveldb.New(
 		getRunningDatabaseDirectory(),
-		nil,
+		getLevelDbConfig(),
 		logging.NoLog{},
 		promRegistry,
 	)
diff --git a/x/merkledb/benchmarks/levelDBConfig.go b/x/merkledb/benchmarks/levelDBConfig.go
new file mode 100644
index 000000000000..6c7f0f296d5c
--- /dev/null
+++ b/x/merkledb/benchmarks/levelDBConfig.go
@@ -0,0 +1,122 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/ava-labs/avalanchego/database/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+type config struct {
+	// BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
+	// Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher.
+	//
+	// The default value is 12MiB.
+	BlockCacheCapacity int `json:"blockCacheCapacity"`
+	// BlockSize is the minimum uncompressed size in bytes of each 'sorted table'
+	// block.
+	//
+	// The default value is 4KiB.
+	BlockSize int `json:"blockSize"`
+	// CompactionExpandLimitFactor limits compaction size after expanded.
+	// This will be multiplied by table size limit at compaction target level.
+	//
+	// The default value is 25.
+	CompactionExpandLimitFactor int `json:"compactionExpandLimitFactor"`
+	// CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2)
+	// that a single 'sorted table' generates.  This will be multiplied by
+	// table size limit at grandparent level.
+	//
+	// The default value is 10.
+	CompactionGPOverlapsFactor int `json:"compactionGPOverlapsFactor"`
+	// CompactionL0Trigger defines number of 'sorted table' at level-0 that will
+	// trigger compaction.
+	//
+	// The default value is 4.
+	CompactionL0Trigger int `json:"compactionL0Trigger"`
+	// CompactionSourceLimitFactor limits compaction source size. This doesn't apply to
+	// level-0.
+	// This will be multiplied by table size limit at compaction target level.
+	//
+	// The default value is 1.
+	CompactionSourceLimitFactor int `json:"compactionSourceLimitFactor"`
+	// CompactionTableSize limits size of 'sorted table' that compaction generates.
+	// The limits for each level will be calculated as:
+	//   CompactionTableSize * (CompactionTableSizeMultiplier ^ Level)
+	// The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel.
+	//
+	// The default value is 2MiB.
+	CompactionTableSize int `json:"compactionTableSize"`
+	// CompactionTableSizeMultiplier defines multiplier for CompactionTableSize.
+	//
+	// The default value is 1.
+	CompactionTableSizeMultiplier float64 `json:"compactionTableSizeMultiplier"`
+	// CompactionTableSizeMultiplierPerLevel defines per-level multiplier for
+	// CompactionTableSize.
+	// Use zero to skip a level.
+	//
+	// The default value is nil.
+	CompactionTableSizeMultiplierPerLevel []float64 `json:"compactionTableSizeMultiplierPerLevel"`
+	// CompactionTotalSize limits total size of 'sorted table' for each level.
+	// The limits for each level will be calculated as:
+	//   CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level)
+	// The multiplier for each level can also fine-tuned using
+	// CompactionTotalSizeMultiplierPerLevel.
+	//
+	// The default value is 10MiB.
+	CompactionTotalSize int `json:"compactionTotalSize"`
+	// CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize.
+	//
+	// The default value is 10.
+	CompactionTotalSizeMultiplier float64 `json:"compactionTotalSizeMultiplier"`
+	// DisableSeeksCompaction allows disabling 'seeks triggered compaction'.
+	// The purpose of 'seeks triggered compaction' is to optimize database so
+	// that 'level seeks' can be minimized, however this might generate many
+	// small compaction which may not preferable.
+	//
+	// The default is true.
+	DisableSeeksCompaction bool `json:"disableSeeksCompaction"`
+	// OpenFilesCacheCapacity defines the capacity of the open files caching.
+	// Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher.
+	//
+	// The default value is 1024.
+	OpenFilesCacheCapacity int `json:"openFilesCacheCapacity"`
+	// WriteBuffer defines maximum size of a 'memdb' before flushed to
+	// 'sorted table'. 'memdb' is an in-memory DB backed by an on-disk
+	// unsorted journal.
+	//
+	// LevelDB may held up to two 'memdb' at the same time.
+	//
+	// The default value is 6MiB.
+	WriteBuffer      int `json:"writeBuffer"`
+	FilterBitsPerKey int `json:"filterBitsPerKey"`
+
+	// MaxManifestFileSize is the maximum size limit of the MANIFEST-****** file.
+	// When the MANIFEST-****** file grows beyond this size, LevelDB will create
+	// a new MANIFEST file.
+	//
+	// The default value is infinity.
+	MaxManifestFileSize int64 `json:"maxManifestFileSize"`
+
+	// MetricUpdateFrequency is the frequency to poll LevelDB metrics.
+	// If <= 0, LevelDB metrics aren't polled.
+	MetricUpdateFrequency time.Duration `json:"metricUpdateFrequency"`
+}
+
+func getLevelDbConfig() []byte {
+	parsedConfig := config{
+		BlockCacheCapacity:     16 * opt.MiB,
+		DisableSeeksCompaction: true,
+		OpenFilesCacheCapacity: 64,
+		WriteBuffer:            256 * opt.MiB,
+		FilterBitsPerKey:       leveldb.DefaultBitsPerKey,
+		MaxManifestFileSize:    leveldb.DefaultMaxManifestFileSize,
+		MetricUpdateFrequency:  leveldb.DefaultMetricUpdateFrequency,
+	}
+	bytes, _ := json.Marshal(parsedConfig)
+	return bytes
+}

From c16e8916197b1d1588bb266adb1bea0a8e77e4d3 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Tue, 10 Sep 2024 14:40:34 -0400
Subject: [PATCH 15/42] update

---
 x/merkledb/benchmarks/benchmark.go            |   4 +-
 ...{directories.go => directories_default.go} |   2 +
 x/merkledb/benchmarks/directories_windows.go  | 107 ++++++++++++++++++
 x/merkledb/benchmarks/levelDBConfig.go        |   5 +-
 4 files changed, 114 insertions(+), 4 deletions(-)
 rename x/merkledb/benchmarks/{directories.go => directories_default.go} (98%)
 create mode 100644 x/merkledb/benchmarks/directories_windows.go

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index d38d0f0e911c..dbe999aed03a 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -133,7 +133,7 @@ func createGoldenDatabase() error {
 
 	levelDB, err := leveldb.New(
 		getGoldenStagingDatabaseDirectory(),
-		getLevelDbConfig(),
+		getLevelDBConfig(),
 		logging.NoLog{},
 		prometheus.NewRegistry(),
 	)
@@ -255,7 +255,7 @@ func resetRunningDatabaseDirectory() error {
 func runBenchmark() error {
 	levelDB, err := leveldb.New(
 		getRunningDatabaseDirectory(),
-		getLevelDbConfig(),
+		getLevelDBConfig(),
 		logging.NoLog{},
 		promRegistry,
 	)
diff --git a/x/merkledb/benchmarks/directories.go b/x/merkledb/benchmarks/directories_default.go
similarity index 98%
rename from x/merkledb/benchmarks/directories.go
rename to x/merkledb/benchmarks/directories_default.go
index e65899326afa..976f02311072 100644
--- a/x/merkledb/benchmarks/directories.go
+++ b/x/merkledb/benchmarks/directories_default.go
@@ -1,6 +1,8 @@
 // Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
 // See the file LICENSE for licensing terms.
 
+//go:build linux || darwin || unix
+
 package main
 
 import (
diff --git a/x/merkledb/benchmarks/directories_windows.go b/x/merkledb/benchmarks/directories_windows.go
new file mode 100644
index 000000000000..d3fc82b445ca
--- /dev/null
+++ b/x/merkledb/benchmarks/directories_windows.go
@@ -0,0 +1,107 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build windows
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func CopyDirectory(scrDir, dest string) error {
+	entries, err := os.ReadDir(scrDir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		sourcePath := filepath.Join(scrDir, entry.Name())
+		destPath := filepath.Join(dest, entry.Name())
+
+		fileInfo, err := os.Stat(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		switch fileInfo.Mode() & os.ModeType {
+		case os.ModeDir:
+			if err := CreateIfNotExists(destPath, 0o755); err != nil {
+				return err
+			}
+			if err := CopyDirectory(sourcePath, destPath); err != nil {
+				return err
+			}
+		default:
+			if err := Copy(sourcePath, destPath); err != nil {
+				return err
+			}
+		}
+
+		fInfo, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
+		isSymlink := fInfo.Mode()&os.ModeSymlink != 0
+		if !isSymlink {
+			if err := os.Chmod(destPath, fInfo.Mode()); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func Copy(srcFile, dstFile string) error {
+	out, err := os.Create(dstFile)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	in, err := os.Open(srcFile)
+	if err != nil {
+		return err
+	}
+
+	defer in.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func Exists(filePath string) bool {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return false
+	}
+
+	return true
+}
+
+func CreateIfNotExists(dir string, perm os.FileMode) error {
+	if Exists(dir) {
+		return nil
+	}
+
+	if err := os.MkdirAll(dir, perm); err != nil {
+		return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
+	}
+
+	return nil
+}
+
+func CopySymLink(source, dest string) error {
+	link, err := os.Readlink(source)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(link, dest)
+}
diff --git a/x/merkledb/benchmarks/levelDBConfig.go b/x/merkledb/benchmarks/levelDBConfig.go
index 6c7f0f296d5c..319b0baf79cc 100644
--- a/x/merkledb/benchmarks/levelDBConfig.go
+++ b/x/merkledb/benchmarks/levelDBConfig.go
@@ -7,8 +7,9 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/ava-labs/avalanchego/database/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
+
+	"github.com/ava-labs/avalanchego/database/leveldb"
 )
 
 type config struct {
@@ -107,7 +108,7 @@ type config struct {
 	MetricUpdateFrequency time.Duration `json:"metricUpdateFrequency"`
 }
 
-func getLevelDbConfig() []byte {
+func getLevelDBConfig() []byte {
 	parsedConfig := config{
 		BlockCacheCapacity:     16 * opt.MiB,
 		DisableSeeksCompaction: true,

From bfe6c7cfd884573cfc106ece9abdfa167b679d54 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Tue, 10 Sep 2024 15:02:11 -0400
Subject: [PATCH 16/42] update

---
 x/merkledb/benchmarks/dashboard.grafana | 256 ++++++++++++++++++++++--
 1 file changed, 238 insertions(+), 18 deletions(-)

diff --git a/x/merkledb/benchmarks/dashboard.grafana b/x/merkledb/benchmarks/dashboard.grafana
index 83b01b5fe91c..d89544d13828 100644
--- a/x/merkledb/benchmarks/dashboard.grafana
+++ b/x/merkledb/benchmarks/dashboard.grafana
@@ -22,11 +22,7 @@
   "links": [],
   "panels": [
     {
-      "datasource": {
-        "default": true,
-        "type": "prometheus",
-        "uid": "bdxazslv08jcwd"
-      },
+      "datasource": {},
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -157,14 +153,14 @@
           "useBackend": false
         }
       ],
-      "title": "Rows Operations",
+      "title": "Rows Operations Rate",
       "type": "timeseries"
     },
     {
       "datasource": {
         "default": true,
         "type": "prometheus",
-        "uid": "bdxazslv08jcwd"
+        "uid": "adxe5ao03b1mob"
       },
       "fieldConfig": {
         "defaults": {
@@ -180,7 +176,7 @@
             "barAlignment": 0,
             "barWidthFactor": 0.6,
             "drawStyle": "line",
-            "fillOpacity": 0,
+            "fillOpacity": 25,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
@@ -198,7 +194,7 @@
             "spanNulls": false,
             "stacking": {
               "group": "A",
-              "mode": "none"
+              "mode": "normal"
             },
             "thresholdsStyle": {
               "mode": "off"
@@ -224,10 +220,10 @@
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 0,
-        "y": 8
+        "x": 12,
+        "y": 0
       },
-      "id": 2,
+      "id": 5,
       "options": {
         "legend": {
           "calcs": [],
@@ -247,16 +243,139 @@
             "uid": "bdxazslv08jcwd"
           },
           "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "rate(merkledb_bench_batch[$__rate_interval])",
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "rate(merkledb_bench_deletes_counter[$__rate_interval])",
+          "format": "time_series",
           "fullMetaSearch": false,
-          "includeNullMetadata": true,
+          "hide": false,
+          "includeNullMetadata": false,
           "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "{{__name__}}",
           "range": true,
-          "refId": "Effective Rate",
+          "refId": "Deletes",
           "useBackend": false
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "rate(merkledb_bench_insert_counter[$__rate_interval])",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Inserts",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "rate(merkledb_bench_updates_counter[$__rate_interval])",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Updates",
+          "useBackend": false
+        }
+      ],
+      "title": "Rows Operations - Counter Interval",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
         {
           "datasource": {
             "type": "prometheus",
@@ -277,6 +396,107 @@
       ],
       "title": "Batch Rate",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "rate(merkledb_bench_batch[$__rate_interval])",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "Effective Rate",
+          "useBackend": false
+        }
+      ],
+      "title": "Batch Operation - Counter Interval",
+      "type": "timeseries"
     }
   ],
   "refresh": "5s",
@@ -293,6 +513,6 @@
   "timezone": "browser",
   "title": "MerkleDB operations benchmark",
   "uid": "edwwwnfme46ioa",
-  "version": 4,
+  "version": 7,
   "weekStart": ""
 }
\ No newline at end of file

From 4a0ae2b02420cbc2da52d1c7e9dea74eacc434c4 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Thu, 12 Sep 2024 09:53:55 -0400
Subject: [PATCH 17/42] update per CR.

---
 x/merkledb/benchmarks/benchmark.go      |   3 +-
 x/merkledb/benchmarks/dashboard.grafana | 234 +++++++++++++++++++++++-
 2 files changed, 229 insertions(+), 8 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index dbe999aed03a..14c3f5b77010 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -310,9 +310,9 @@ func runBenchmark() error {
 
 		// update middle 5k entries
 		startUpdateTime := time.Now()
+		updateEntryValue := calculateIndexEncoding(low)
 		for keyToUpdateIdx := low + ((*databaseEntries) / 2); keyToUpdateIdx < low+((*databaseEntries)/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
 			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
-			updateEntryValue := calculateIndexEncoding(keyToUpdateIdx - ((*databaseEntries) / 2))
 			err = batch.Put(updateEntryKey, updateEntryValue)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "unable to update merkleDB entry : %v\n", err)
@@ -344,6 +344,7 @@ func runBenchmark() error {
 		}
 
 		batchCounter.Inc()
+		low += databaseRunningBatchSize
 	}
 
 	// return nil
diff --git a/x/merkledb/benchmarks/dashboard.grafana b/x/merkledb/benchmarks/dashboard.grafana
index d89544d13828..b34f4698771a 100644
--- a/x/merkledb/benchmarks/dashboard.grafana
+++ b/x/merkledb/benchmarks/dashboard.grafana
@@ -226,7 +226,9 @@
       "id": 5,
       "options": {
         "legend": {
-          "calcs": [],
+          "calcs": [
+            "mean"
+          ],
           "displayMode": "list",
           "placement": "bottom",
           "showLegend": true
@@ -245,7 +247,7 @@
           "disableTextWrap": false,
           "editorMode": "builder",
           "exemplar": false,
-          "expr": "rate(merkledb_bench_deletes_counter[$__rate_interval])",
+          "expr": "rate(merkledb_bench_deletes_counter[10m])",
           "format": "time_series",
           "fullMetaSearch": false,
           "hide": false,
@@ -264,7 +266,7 @@
           "disableTextWrap": false,
           "editorMode": "builder",
           "exemplar": false,
-          "expr": "rate(merkledb_bench_insert_counter[$__rate_interval])",
+          "expr": "rate(merkledb_bench_insert_counter[10m])",
           "fullMetaSearch": false,
           "hide": false,
           "includeNullMetadata": false,
@@ -281,7 +283,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(merkledb_bench_updates_counter[$__rate_interval])",
+          "expr": "rate(merkledb_bench_updates_counter[10m])",
           "fullMetaSearch": false,
           "hide": false,
           "includeNullMetadata": false,
@@ -467,7 +469,9 @@
       "id": 4,
       "options": {
         "legend": {
-          "calcs": [],
+          "calcs": [
+            "mean"
+          ],
           "displayMode": "list",
           "placement": "bottom",
           "showLegend": true
@@ -497,6 +501,222 @@
       ],
       "title": "Batch Operation - Counter Interval",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "adxe5ao03b1mob"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_insert_counter",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Inserted Rows",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 4,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "adxe5ao03b1mob"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_updates_counter",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Updated Rows",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 8,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "adxe5ao03b1mob"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_deletes_counter",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Deleted Rows",
+      "type": "gauge"
     }
   ],
   "refresh": "5s",
@@ -506,13 +726,13 @@
     "list": []
   },
   "time": {
-    "from": "now-1h",
+    "from": "now-24h",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "browser",
   "title": "MerkleDB operations benchmark",
   "uid": "edwwwnfme46ioa",
-  "version": 7,
+  "version": 8,
   "weekStart": ""
 }
\ No newline at end of file

From 2014fdc6e51bea3068f08374f59ceca0b287b4a2 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Mon, 16 Sep 2024 17:19:51 -0400
Subject: [PATCH 18/42] add a benchmark to compare with eth's merkle trie.

---
 x/merkledb/benchmarks_eth/benchmark.go        | 456 +++++++++++
 x/merkledb/benchmarks_eth/dashboard.grafana   | 738 ++++++++++++++++++
 .../benchmarks_eth/directories_default.go     | 121 +++
 .../benchmarks_eth/directories_windows.go     | 107 +++
 x/merkledb/benchmarks_eth/go.mod              |  66 ++
 x/merkledb/benchmarks_eth/go.sum              | 267 +++++++
 6 files changed, 1755 insertions(+)
 create mode 100644 x/merkledb/benchmarks_eth/benchmark.go
 create mode 100644 x/merkledb/benchmarks_eth/dashboard.grafana
 create mode 100644 x/merkledb/benchmarks_eth/directories_default.go
 create mode 100644 x/merkledb/benchmarks_eth/directories_windows.go
 create mode 100644 x/merkledb/benchmarks_eth/go.mod
 create mode 100644 x/merkledb/benchmarks_eth/go.sum

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
new file mode 100644
index 000000000000..a1f4b242a9d9
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -0,0 +1,456 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/pflag"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-ethereum/triedb"
+)
+
+const (
+	defaultDatabaseEntries    = 1000000000 // 1B
+	databaseCreationBatchSize = 10000      // 10k
+	databaseRunningBatchSize  = 2500       // 2.5k
+	databaseRunningUpdateSize = 5000       // 5k
+	defaultMetricsPort        = 3000
+)
+
+var (
+	databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
+	httpMetricPort  = pflag.Uint64("p", defaultMetricsPort, "default metrics port")
+	verbose         = pflag.Bool("v", false, "verbose")
+
+	insertsCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "insert_counter",
+		Help:      "Total number of inserts",
+	})
+	deletesCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "deletes_counter",
+		Help:      "Total number of deletes",
+	})
+	updatesCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "updates_counter",
+		Help:      "Total number of updates",
+	})
+	batchCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "merkledb_bench",
+		Name:      "batch",
+		Help:      "Total number of batches written",
+	})
+	deleteRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_delete_rate",
+		Help:      "The rate at which elements are deleted",
+	})
+	updateRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_update_rate",
+		Help:      "The rate at which elements are updated",
+	})
+	insertRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "entry_insert_rate",
+		Help:      "The rate at which elements are inserted",
+	})
+	batchWriteRate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "merkledb_bench",
+		Name:      "batch_write_rate",
+		Help:      "The rate at which the batch was written",
+	})
+	promRegistry = prometheus.NewRegistry()
+)
+
+/*
+func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
+	const defaultHistoryLength = 120
+	return merkledb.Config{
+		BranchFactor:                merkledb.BranchFactor16,
+		Hasher:                      merkledb.DefaultHasher,
+		RootGenConcurrency:          0,
+		HistoryLength:               defaultHistoryLength,
+		ValueNodeCacheSize:          units.MiB,
+		IntermediateNodeCacheSize:   1024 * units.MiB,
+		IntermediateWriteBufferSize: units.KiB,
+		IntermediateWriteBatchSize:  256 * units.KiB,
+		Reg:                         promRegistry,
+		TraceLevel:                  merkledb.NoTrace,
+		Tracer:                      trace.Noop,
+	}
+}*/
+
+func getGoldenStagingDatabaseDirectory() string {
+	wd, _ := os.Getwd()
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", *databaseEntries))
+}
+
+func getGoldenDatabaseDirectory() string {
+	wd, _ := os.Getwd()
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", *databaseEntries))
+}
+
+func getRunningDatabaseDirectory() string {
+	wd, _ := os.Getwd()
+	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", *databaseEntries))
+}
+
+func calculateIndexEncoding(idx uint64) []byte {
+	var entryEncoding [8]byte
+	binary.NativeEndian.PutUint64(entryEncoding[:], idx)
+	entryHash := sha256.Sum256(entryEncoding[:])
+	return entryHash[:]
+}
+
+func createGoldenDatabase() error {
+	stagingDirectory := getGoldenStagingDatabaseDirectory()
+	err := os.RemoveAll(stagingDirectory)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
+		return err
+	}
+	err = os.Mkdir(stagingDirectory, 0o777)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create golden staging directory : %v\n", err)
+		return err
+	}
+
+	ldb, err := rawdb.NewLevelDBDatabase(stagingDirectory, 500, 200, "metrics_prefix", false)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create level db database : %v\n", err)
+		return err
+	}
+
+	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
+		Preimages: false,
+		IsVerkle:  false,
+		HashDB:    nil,
+		PathDB:    nil,
+	})
+	tdb := trie.NewEmpty(trieDb)
+
+	fmt.Print("Initializing database.")
+	ticksCh := make(chan interface{})
+	go func() {
+		t := time.NewTicker(time.Second)
+		defer t.Stop()
+		for {
+			select {
+			case <-ticksCh:
+				return
+			case <-t.C:
+				fmt.Print(".")
+			}
+		}
+	}()
+
+	var root common.Hash
+	writeBatch := func() error {
+		var nodes *trienode.NodeSet
+		root, nodes = tdb.Commit(false)
+		err = trieDb.Update(root, types.EmptyRootHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
+			return err
+		}
+		err = trieDb.Commit(root, false)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to commit trie : %v\n", err)
+			return err
+		}
+		tdb, err = trie.New(trie.TrieID(root), trieDb)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to create new trie : %v\n", err)
+			return err
+		}
+		return nil
+	}
+
+	startInsertTime := time.Now()
+	startInsertBatchTime := startInsertTime
+	for entryIdx := uint64(0); entryIdx < *databaseEntries; entryIdx++ {
+		entryHash := calculateIndexEncoding(entryIdx)
+		tdb.Update(entryHash, entryHash)
+
+		if entryIdx%databaseCreationBatchSize == (databaseCreationBatchSize - 1) {
+			addDuration := time.Since(startInsertBatchTime)
+			insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration))
+			insertsCounter.Add(databaseCreationBatchSize)
+
+			batchWriteStartTime := time.Now()
+
+			err = writeBatch()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to write value in database : %v\n", err)
+				return err
+			}
+
+			batchWriteDuration := time.Since(batchWriteStartTime)
+			batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+			batchCounter.Inc()
+
+			startInsertBatchTime = time.Now()
+		}
+	}
+	if (*databaseEntries)%databaseCreationBatchSize != 0 {
+		err = writeBatch()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to write value in database : %v\n", err)
+			return err
+		}
+	}
+	close(ticksCh)
+	fmt.Print(" done!\n")
+
+	fmt.Printf("Generated and inserted %d batches of size %d in %v\n",
+		(*databaseEntries)/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
+	err = trieDb.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to close trie database : %v\n", err)
+		return err
+	}
+	err = ldb.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to close levelDB database : %v\n", err)
+		return err
+	}
+
+	err = os.WriteFile(path.Join(stagingDirectory, "root.txt"), root.Bytes(), 0o644)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to save root : %v\n", err)
+		return err
+	}
+
+	err = os.Rename(getGoldenStagingDatabaseDirectory(), getGoldenDatabaseDirectory())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to rename golden staging directory : %v\n", err)
+		return err
+	}
+	fmt.Printf("Completed initialization with hash of %v\n", root.Hex())
+	return nil
+}
+
+func resetRunningDatabaseDirectory() error {
+	runningDir := getRunningDatabaseDirectory()
+	if _, err := os.Stat(runningDir); err == nil {
+		err := os.RemoveAll(runningDir)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
+			return err
+		}
+	}
+	err := os.Mkdir(runningDir, 0o777)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create running directory : %v\n", err)
+		return err
+	}
+	err = CopyDirectory(getGoldenDatabaseDirectory(), runningDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to duplicate golden directory : %v\n", err)
+		return err
+	}
+	return nil
+}
+
+func runBenchmark() error {
+	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(), "root.txt"))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to read root : %v\n", err)
+		return err
+	}
+
+	ldb, err := rawdb.Open(rawdb.OpenOptions{
+		Type:              "leveldb",
+		Directory:         getRunningDatabaseDirectory(),
+		AncientsDirectory: "",
+		Namespace:         "metrics_prefix",
+		Cache:             500,
+		Handles:           200,
+		ReadOnly:          false,
+
+		Ephemeral: false,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create level db database : %v\n", err)
+		return err
+	}
+
+	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
+		Preimages: false,
+		IsVerkle:  false,
+		HashDB:    nil,
+		PathDB:    nil,
+	})
+
+	root := common.BytesToHash(rootBytes)
+	tdb, err := trie.New(trie.TrieID(root), trieDb)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to create trie database : %v\n", err)
+		return err
+	}
+
+	writeBatch := func() error {
+		var nodes *trienode.NodeSet
+		root, nodes = tdb.Commit(false)
+		err = trieDb.Update(root, types.EmptyRootHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
+			return err
+		}
+		err = trieDb.Commit(root, false)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to commit trie : %v\n", err)
+			return err
+		}
+		tdb, err = trie.New(trie.TrieID(root), trieDb)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to create new trie : %v\n", err)
+			return err
+		}
+		return nil
+	}
+
+	low := uint64(0)
+	var deleteDuration, addDuration, updateDuration, batchDuration time.Duration
+	for {
+		startBatchTime := time.Now()
+
+		// delete first 2.5k keys from the beginning
+		startDeleteTime := time.Now()
+		for keyToDeleteIdx := low; keyToDeleteIdx < low+databaseRunningBatchSize; keyToDeleteIdx++ {
+			entryHash := calculateIndexEncoding(keyToDeleteIdx)
+			err = tdb.Delete(entryHash)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to delete merkleDB entry : %v\n", err)
+				return err
+			}
+		}
+		deleteDuration = time.Since(startDeleteTime)
+		deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration))
+		deletesCounter.Add(databaseRunningBatchSize)
+
+		// add 2.5k past end.
+		startInsertTime := time.Now()
+		for keyToAddIdx := low + (*databaseEntries); keyToAddIdx < low+(*databaseEntries)+databaseRunningBatchSize; keyToAddIdx++ {
+			entryHash := calculateIndexEncoding(keyToAddIdx)
+			err = tdb.Update(entryHash, entryHash)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to insert merkleDB entry : %v\n", err)
+				return err
+			}
+		}
+		addDuration = time.Since(startInsertTime)
+		insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration))
+		insertsCounter.Add(databaseRunningBatchSize)
+
+		// update middle 5k entries
+		startUpdateTime := time.Now()
+		updateEntryValue := calculateIndexEncoding(low)
+		for keyToUpdateIdx := low + ((*databaseEntries) / 2); keyToUpdateIdx < low+((*databaseEntries)/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
+			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
+			err = tdb.Update(updateEntryKey, updateEntryValue)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "unable to update merkleDB entry : %v\n", err)
+				return err
+			}
+		}
+		updateDuration = time.Since(startUpdateTime)
+		updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
+		updatesCounter.Add(databaseRunningUpdateSize)
+
+		batchWriteStartTime := time.Now()
+		err = writeBatch()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to write batch : %v\n", err)
+			return err
+		}
+		batchDuration = time.Since(startBatchTime)
+		batchWriteDuration := time.Since(batchWriteStartTime)
+		batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+
+		if *verbose {
+			fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
+				time.Second/deleteDuration,
+				time.Second/updateDuration,
+				time.Second/addDuration,
+				time.Second/batchDuration)
+		}
+
+		batchCounter.Inc()
+		low += databaseRunningBatchSize
+	}
+
+	// return nil
+}
+
+func setupMetrics() error {
+	if err := prometheus.Register(promRegistry); err != nil {
+		return err
+	}
+	promRegistry.MustRegister(insertsCounter)
+	promRegistry.MustRegister(deletesCounter)
+	promRegistry.MustRegister(updatesCounter)
+	promRegistry.MustRegister(batchCounter)
+	promRegistry.MustRegister(deleteRate)
+	promRegistry.MustRegister(updateRate)
+	promRegistry.MustRegister(insertRate)
+	promRegistry.MustRegister(batchWriteRate)
+
+	http.Handle("/metrics", promhttp.Handler())
+
+	server := &http.Server{
+		Addr:              fmt.Sprintf(":%d", *httpMetricPort),
+		ReadHeaderTimeout: 3 * time.Second,
+	}
+	go func() {
+		err := server.ListenAndServe()
+		if err != nil && err != http.ErrServerClosed {
+			fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err)
+		}
+	}()
+	return nil
+}
+
+func main() {
+	pflag.Parse()
+
+	if setupMetrics() != nil {
+		os.Exit(1)
+		return
+	}
+
+	goldenDir := getGoldenDatabaseDirectory()
+	if _, err := os.Stat(goldenDir); os.IsNotExist(err) {
+		// create golden image.
+		if createGoldenDatabase() != nil {
+			os.Exit(1)
+			return
+		}
+	}
+	if resetRunningDatabaseDirectory() != nil {
+		os.Exit(1)
+		return
+	}
+	if runBenchmark() != nil {
+		os.Exit(1)
+		return
+	}
+}
diff --git a/x/merkledb/benchmarks_eth/dashboard.grafana b/x/merkledb/benchmarks_eth/dashboard.grafana
new file mode 100644
index 000000000000..b34f4698771a
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/dashboard.grafana
@@ -0,0 +1,738 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {},
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 25,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "merkledb_bench_entry_delete_rate",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Deletes",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "merkledb_bench_entry_insert_rate",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Inserts",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_entry_update_rate",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Updates",
+          "useBackend": false
+        }
+      ],
+      "title": "Rows Operations Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 25,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "rate(merkledb_bench_deletes_counter[10m])",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Deletes",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "rate(merkledb_bench_insert_counter[10m])",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Inserts",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "rate(merkledb_bench_updates_counter[10m])",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Updates",
+          "useBackend": false
+        }
+      ],
+      "title": "Rows Operations - Counter Interval",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_batch_write_rate",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{__name__}}",
+          "range": true,
+          "refId": "Batch Write Operation",
+          "useBackend": false
+        }
+      ],
+      "title": "Batch Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean"
+          ],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "bdxazslv08jcwd"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "rate(merkledb_bench_batch[$__rate_interval])",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "Effective Rate",
+          "useBackend": false
+        }
+      ],
+      "title": "Batch Operation - Counter Interval",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "adxe5ao03b1mob"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_insert_counter",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Inserted Rows",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 4,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "adxe5ao03b1mob"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_updates_counter",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Updated Rows",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "default": true,
+        "type": "prometheus",
+        "uid": "adxe5ao03b1mob"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 8,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.2.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "adxe5ao03b1mob"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "merkledb_bench_deletes_counter",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Deleted Rows",
+      "type": "gauge"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "MerkleDB operations benchmark",
+  "uid": "edwwwnfme46ioa",
+  "version": 8,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/x/merkledb/benchmarks_eth/directories_default.go b/x/merkledb/benchmarks_eth/directories_default.go
new file mode 100644
index 000000000000..976f02311072
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/directories_default.go
@@ -0,0 +1,121 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build linux || darwin || unix
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+func CopyDirectory(scrDir, dest string) error {
+	entries, err := os.ReadDir(scrDir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		sourcePath := filepath.Join(scrDir, entry.Name())
+		destPath := filepath.Join(dest, entry.Name())
+
+		fileInfo, err := os.Stat(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+		if !ok {
+			return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath)
+		}
+
+		switch fileInfo.Mode() & os.ModeType {
+		case os.ModeDir:
+			if err := CreateIfNotExists(destPath, 0o755); err != nil {
+				return err
+			}
+			if err := CopyDirectory(sourcePath, destPath); err != nil {
+				return err
+			}
+		case os.ModeSymlink:
+			if err := CopySymLink(sourcePath, destPath); err != nil {
+				return err
+			}
+		default:
+			if err := Copy(sourcePath, destPath); err != nil {
+				return err
+			}
+		}
+
+		if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil {
+			return err
+		}
+
+		fInfo, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
+		isSymlink := fInfo.Mode()&os.ModeSymlink != 0
+		if !isSymlink {
+			if err := os.Chmod(destPath, fInfo.Mode()); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func Copy(srcFile, dstFile string) error {
+	out, err := os.Create(dstFile)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	in, err := os.Open(srcFile)
+	if err != nil {
+		return err
+	}
+
+	defer in.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func Exists(filePath string) bool {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return false
+	}
+
+	return true
+}
+
+func CreateIfNotExists(dir string, perm os.FileMode) error {
+	if Exists(dir) {
+		return nil
+	}
+
+	if err := os.MkdirAll(dir, perm); err != nil {
+		return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
+	}
+
+	return nil
+}
+
+func CopySymLink(source, dest string) error {
+	link, err := os.Readlink(source)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(link, dest)
+}
diff --git a/x/merkledb/benchmarks_eth/directories_windows.go b/x/merkledb/benchmarks_eth/directories_windows.go
new file mode 100644
index 000000000000..d3fc82b445ca
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/directories_windows.go
@@ -0,0 +1,107 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build windows
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func CopyDirectory(scrDir, dest string) error {
+	entries, err := os.ReadDir(scrDir)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		sourcePath := filepath.Join(scrDir, entry.Name())
+		destPath := filepath.Join(dest, entry.Name())
+
+		fileInfo, err := os.Stat(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		switch fileInfo.Mode() & os.ModeType {
+		case os.ModeDir:
+			if err := CreateIfNotExists(destPath, 0o755); err != nil {
+				return err
+			}
+			if err := CopyDirectory(sourcePath, destPath); err != nil {
+				return err
+			}
+		default:
+			if err := Copy(sourcePath, destPath); err != nil {
+				return err
+			}
+		}
+
+		fInfo, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
+		isSymlink := fInfo.Mode()&os.ModeSymlink != 0
+		if !isSymlink {
+			if err := os.Chmod(destPath, fInfo.Mode()); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func Copy(srcFile, dstFile string) error {
+	out, err := os.Create(dstFile)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	in, err := os.Open(srcFile)
+	if err != nil {
+		return err
+	}
+
+	defer in.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func Exists(filePath string) bool {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return false
+	}
+
+	return true
+}
+
+func CreateIfNotExists(dir string, perm os.FileMode) error {
+	if Exists(dir) {
+		return nil
+	}
+
+	if err := os.MkdirAll(dir, perm); err != nil {
+		return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
+	}
+
+	return nil
+}
+
+func CopySymLink(source, dest string) error {
+	link, err := os.Readlink(source)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(link, dest)
+}
diff --git a/x/merkledb/benchmarks_eth/go.mod b/x/merkledb/benchmarks_eth/go.mod
new file mode 100644
index 000000000000..4a7e29464c22
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/go.mod
@@ -0,0 +1,66 @@
+module github.com/ava-labs/avalanchego/x/merkledb/benchmarks_eth
+
+go 1.22.3
+
+require (
+	github.com/prometheus/client_golang v1.20.3
+	github.com/spf13/pflag v1.0.5
+)
+
+require (
+	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v1.1.1 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
+	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
+	github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/holiman/uint256 v1.3.1 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/supranational/blst v0.3.11 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/ethereum/go-ethereum v1.14.8
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+)
diff --git a/x/merkledb/benchmarks_eth/go.sum b/x/merkledb/benchmarks_eth/go.sum
new file mode 100644
index 000000000000..18cff77e093f
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/go.sum
@@ -0,0 +1,267 @@
+github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
+github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
+github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
+github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
+github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=
+github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ=
+github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
+github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
+github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
+github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
+github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
+github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
+github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
+github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw=
+github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
+github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
+github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
+github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
+github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ=
+github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
+github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
+github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
+github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I=
+github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
+github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI=
+github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
+github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
+github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA=
+github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
+github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ1VTIig=
+github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs=
+github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4=
+github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
+github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
+github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk=
+github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
+github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs=
+github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
+github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
+github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
+github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
+github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
+github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
+github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
+github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI=
+github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
+github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
+rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=

From af5c4ab61aa83dfad38b46222fce28ce6f06e434 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Mon, 16 Sep 2024 17:33:04 -0400
Subject: [PATCH 19/42] udpate

---
 x/merkledb/benchmarks_eth/benchmark.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index a1f4b242a9d9..82d71364c246 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -285,8 +285,7 @@ func runBenchmark() error {
 		Cache:             500,
 		Handles:           200,
 		ReadOnly:          false,
-
-		Ephemeral: false,
+		Ephemeral:         false,
 	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create level db database : %v\n", err)
@@ -339,7 +338,7 @@ func runBenchmark() error {
 			entryHash := calculateIndexEncoding(keyToDeleteIdx)
 			err = tdb.Delete(entryHash)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to delete merkleDB entry : %v\n", err)
+				fmt.Fprintf(os.Stderr, "unable to delete trie entry : %v\n", err)
 				return err
 			}
 		}
@@ -353,7 +352,7 @@ func runBenchmark() error {
 			entryHash := calculateIndexEncoding(keyToAddIdx)
 			err = tdb.Update(entryHash, entryHash)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to insert merkleDB entry : %v\n", err)
+				fmt.Fprintf(os.Stderr, "unable to insert trie entry : %v\n", err)
 				return err
 			}
 		}
@@ -368,7 +367,7 @@ func runBenchmark() error {
 			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
 			err = tdb.Update(updateEntryKey, updateEntryValue)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to update merkleDB entry : %v\n", err)
+				fmt.Fprintf(os.Stderr, "unable to update trie entry : %v\n", err)
 				return err
 			}
 		}

From 4f807c82d039c67d599816020e70c3b6c3765413 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Wed, 18 Sep 2024 10:01:01 -0400
Subject: [PATCH 20/42] update caches

---
 x/merkledb/benchmarks/benchmark.go     |  4 +--
 x/merkledb/benchmarks/levelDBConfig.go |  6 +++--
 x/merkledb/benchmarks_eth/benchmark.go | 37 +++++++++++---------------
 3 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 14c3f5b77010..75d2aeddc1b2 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -30,6 +30,7 @@ const (
 	databaseRunningBatchSize  = 2500       // 2.5k
 	databaseRunningUpdateSize = 5000       // 5k
 	defaultMetricsPort        = 3000
+	firewoodNumberOfRevisions = 120
 )
 
 var (
@@ -81,12 +82,11 @@ var (
 )
 
 func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
-	const defaultHistoryLength = 120
 	return merkledb.Config{
 		BranchFactor:                merkledb.BranchFactor16,
 		Hasher:                      merkledb.DefaultHasher,
 		RootGenConcurrency:          0,
-		HistoryLength:               defaultHistoryLength,
+		HistoryLength:               firewoodNumberOfRevisions,
 		ValueNodeCacheSize:          units.MiB,
 		IntermediateNodeCacheSize:   1024 * units.MiB,
 		IntermediateWriteBufferSize: units.KiB,
diff --git a/x/merkledb/benchmarks/levelDBConfig.go b/x/merkledb/benchmarks/levelDBConfig.go
index 319b0baf79cc..ff096907a09a 100644
--- a/x/merkledb/benchmarks/levelDBConfig.go
+++ b/x/merkledb/benchmarks/levelDBConfig.go
@@ -109,11 +109,13 @@ type config struct {
 }
 
 func getLevelDBConfig() []byte {
+	// align with geth test and use 6gb as cache.
+	cacheSize := int(6)
 	parsedConfig := config{
-		BlockCacheCapacity:     16 * opt.MiB,
+		BlockCacheCapacity:     (cacheSize / 2) * opt.MiB,
 		DisableSeeksCompaction: true,
 		OpenFilesCacheCapacity: 64,
-		WriteBuffer:            256 * opt.MiB,
+		WriteBuffer:            (cacheSize / 4) * opt.MiB,
 		FilterBitsPerKey:       leveldb.DefaultBitsPerKey,
 		MaxManifestFileSize:    leveldb.DefaultMaxManifestFileSize,
 		MetricUpdateFrequency:  leveldb.DefaultMetricUpdateFrequency,
diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 82d71364c246..abfa42745dde 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -22,6 +22,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/triedb"
+	"github.com/ethereum/go-ethereum/triedb/pathdb"
 )
 
 const (
@@ -30,6 +31,8 @@ const (
 	databaseRunningBatchSize  = 2500       // 2.5k
 	databaseRunningUpdateSize = 5000       // 5k
 	defaultMetricsPort        = 3000
+	levelDBCacheSize          = 6_000
+	firewoodNumberOfRevisions = 120
 )
 
 var (
@@ -80,24 +83,6 @@ var (
 	promRegistry = prometheus.NewRegistry()
 )
 
-/*
-func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
-	const defaultHistoryLength = 120
-	return merkledb.Config{
-		BranchFactor:                merkledb.BranchFactor16,
-		Hasher:                      merkledb.DefaultHasher,
-		RootGenConcurrency:          0,
-		HistoryLength:               defaultHistoryLength,
-		ValueNodeCacheSize:          units.MiB,
-		IntermediateNodeCacheSize:   1024 * units.MiB,
-		IntermediateWriteBufferSize: units.KiB,
-		IntermediateWriteBatchSize:  256 * units.KiB,
-		Reg:                         promRegistry,
-		TraceLevel:                  merkledb.NoTrace,
-		Tracer:                      trace.Noop,
-	}
-}*/
-
 func getGoldenStagingDatabaseDirectory() string {
 	wd, _ := os.Getwd()
 	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", *databaseEntries))
@@ -120,6 +105,14 @@ func calculateIndexEncoding(idx uint64) []byte {
 	return entryHash[:]
 }
 
+func getPathDBConfig() *pathdb.Config {
+	return &pathdb.Config{
+		StateHistory:   firewoodNumberOfRevisions, // keep the same requiremenet across each db
+		CleanCacheSize: 6 * 1024 * 1024 * 1024,    // 4GB - this is a guess, but I assume we're currently underutilizing memory by a lot
+		DirtyCacheSize: 6 * 1024 * 1024 * 1024,    // 4GB - this is a guess, but I assume we're currently underutilizing memory by a lot
+	}
+}
+
 func createGoldenDatabase() error {
 	stagingDirectory := getGoldenStagingDatabaseDirectory()
 	err := os.RemoveAll(stagingDirectory)
@@ -133,7 +126,7 @@ func createGoldenDatabase() error {
 		return err
 	}
 
-	ldb, err := rawdb.NewLevelDBDatabase(stagingDirectory, 500, 200, "metrics_prefix", false)
+	ldb, err := rawdb.NewLevelDBDatabase(stagingDirectory, levelDBCacheSize, 200, "metrics_prefix", false)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create level db database : %v\n", err)
 		return err
@@ -143,7 +136,7 @@ func createGoldenDatabase() error {
 		Preimages: false,
 		IsVerkle:  false,
 		HashDB:    nil,
-		PathDB:    nil,
+		PathDB:    getPathDBConfig(),
 	})
 	tdb := trie.NewEmpty(trieDb)
 
@@ -282,7 +275,7 @@ func runBenchmark() error {
 		Directory:         getRunningDatabaseDirectory(),
 		AncientsDirectory: "",
 		Namespace:         "metrics_prefix",
-		Cache:             500,
+		Cache:             levelDBCacheSize,
 		Handles:           200,
 		ReadOnly:          false,
 		Ephemeral:         false,
@@ -296,7 +289,7 @@ func runBenchmark() error {
 		Preimages: false,
 		IsVerkle:  false,
 		HashDB:    nil,
-		PathDB:    nil,
+		PathDB:    getPathDBConfig(),
 	})
 
 	root := common.BytesToHash(rootBytes)

From 291f2dad3ee032b2f638d48bb450fe681bea731d Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Wed, 18 Sep 2024 10:54:18 -0400
Subject: [PATCH 21/42] fix bench

---
 x/merkledb/benchmarks_eth/benchmark.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index abfa42745dde..220bf774c41d 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -156,10 +156,11 @@ func createGoldenDatabase() error {
 	}()
 
 	var root common.Hash
+	parentHash := types.EmptyRootHash
 	writeBatch := func() error {
 		var nodes *trienode.NodeSet
 		root, nodes = tdb.Commit(false)
-		err = trieDb.Update(root, types.EmptyRootHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		err = trieDb.Update(root, parentHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
 			return err
@@ -174,6 +175,7 @@ func createGoldenDatabase() error {
 			fmt.Fprintf(os.Stderr, "unable to create new trie : %v\n", err)
 			return err
 		}
+		parentHash = root
 		return nil
 	}
 
@@ -292,17 +294,17 @@ func runBenchmark() error {
 		PathDB:    getPathDBConfig(),
 	})
 
-	root := common.BytesToHash(rootBytes)
-	tdb, err := trie.New(trie.TrieID(root), trieDb)
+	parentHash := common.BytesToHash(rootBytes)
+	tdb, err := trie.New(trie.TrieID(parentHash), trieDb)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create trie database : %v\n", err)
 		return err
 	}
-
+	var root common.Hash
 	writeBatch := func() error {
 		var nodes *trienode.NodeSet
 		root, nodes = tdb.Commit(false)
-		err = trieDb.Update(root, types.EmptyRootHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		err = trieDb.Update(root, parentHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
 			return err
@@ -317,6 +319,7 @@ func runBenchmark() error {
 			fmt.Fprintf(os.Stderr, "unable to create new trie : %v\n", err)
 			return err
 		}
+		parentHash = root
 		return nil
 	}
 

From 058ec5591db3af45138d8f74e00406cc542ffa48 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Wed, 18 Sep 2024 11:38:58 -0400
Subject: [PATCH 22/42] fix typo

---
 x/merkledb/benchmarks/levelDBConfig.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x/merkledb/benchmarks/levelDBConfig.go b/x/merkledb/benchmarks/levelDBConfig.go
index ff096907a09a..3a2a2d3115b1 100644
--- a/x/merkledb/benchmarks/levelDBConfig.go
+++ b/x/merkledb/benchmarks/levelDBConfig.go
@@ -110,7 +110,7 @@ type config struct {
 
 func getLevelDBConfig() []byte {
 	// align with geth test and use 6gb as cache.
-	cacheSize := int(6)
+	cacheSize := int(6000)
 	parsedConfig := config{
 		BlockCacheCapacity:     (cacheSize / 2) * opt.MiB,
 		DisableSeeksCompaction: true,

From 0c80953c12fe604b3881250f130f2b34d3369f77 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Thu, 19 Sep 2024 11:18:19 -0400
Subject: [PATCH 23/42] update dashboard.

---
 x/merkledb/benchmarks/dashboard.grafana     |  51 +-
 x/merkledb/benchmarks_eth/dashboard.grafana | 738 --------------------
 2 files changed, 17 insertions(+), 772 deletions(-)
 delete mode 100644 x/merkledb/benchmarks_eth/dashboard.grafana

diff --git a/x/merkledb/benchmarks/dashboard.grafana b/x/merkledb/benchmarks/dashboard.grafana
index b34f4698771a..4f8c52618aa3 100644
--- a/x/merkledb/benchmarks/dashboard.grafana
+++ b/x/merkledb/benchmarks/dashboard.grafana
@@ -100,8 +100,7 @@
       "targets": [
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -119,8 +118,7 @@
         },
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -137,8 +135,7 @@
         },
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -159,8 +156,7 @@
     {
       "datasource": {
         "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
+        "type": "prometheus"
       },
       "fieldConfig": {
         "defaults": {
@@ -241,8 +237,7 @@
       "targets": [
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -260,8 +255,7 @@
         },
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -278,8 +272,7 @@
         },
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -300,8 +293,7 @@
     {
       "datasource": {
         "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
+        "type": "prometheus"
       },
       "fieldConfig": {
         "defaults": {
@@ -380,8 +372,7 @@
       "targets": [
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -402,8 +393,7 @@
     {
       "datasource": {
         "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
+        "type": "prometheus"
       },
       "fieldConfig": {
         "defaults": {
@@ -484,8 +474,7 @@
       "targets": [
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "code",
@@ -505,8 +494,7 @@
     {
       "datasource": {
         "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
+        "type": "prometheus"
       },
       "fieldConfig": {
         "defaults": {
@@ -556,8 +544,7 @@
       "targets": [
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "adxe5ao03b1mob"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -577,8 +564,7 @@
     {
       "datasource": {
         "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
+        "type": "prometheus"
       },
       "fieldConfig": {
         "defaults": {
@@ -628,8 +614,7 @@
       "targets": [
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "adxe5ao03b1mob"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -649,8 +634,7 @@
     {
       "datasource": {
         "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
+        "type": "prometheus"
       },
       "fieldConfig": {
         "defaults": {
@@ -700,8 +684,7 @@
       "targets": [
         {
           "datasource": {
-            "type": "prometheus",
-            "uid": "adxe5ao03b1mob"
+            "type": "prometheus"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
diff --git a/x/merkledb/benchmarks_eth/dashboard.grafana b/x/merkledb/benchmarks_eth/dashboard.grafana
deleted file mode 100644
index b34f4698771a..000000000000
--- a/x/merkledb/benchmarks_eth/dashboard.grafana
+++ /dev/null
@@ -1,738 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": 1,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {},
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 25,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "normal"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "merkledb_bench_entry_delete_rate",
-          "format": "time_series",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "Deletes",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "merkledb_bench_entry_insert_rate",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "Inserts",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "merkledb_bench_entry_update_rate",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "Updates",
-          "useBackend": false
-        }
-      ],
-      "title": "Rows Operations Rate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 25,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "normal"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [
-            "mean"
-          ],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "rate(merkledb_bench_deletes_counter[10m])",
-          "format": "time_series",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": false,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "Deletes",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "rate(merkledb_bench_insert_counter[10m])",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": false,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "Inserts",
-          "useBackend": false
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "rate(merkledb_bench_updates_counter[10m])",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": false,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "Updates",
-          "useBackend": false
-        }
-      ],
-      "title": "Rows Operations - Counter Interval",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 8
-      },
-      "id": 2,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "merkledb_bench_batch_write_rate",
-          "fullMetaSearch": false,
-          "hide": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "{{__name__}}",
-          "range": true,
-          "refId": "Batch Write Operation",
-          "useBackend": false
-        }
-      ],
-      "title": "Batch Rate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 8
-      },
-      "id": 4,
-      "options": {
-        "legend": {
-          "calcs": [
-            "mean"
-          ],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "bdxazslv08jcwd"
-          },
-          "disableTextWrap": false,
-          "editorMode": "code",
-          "expr": "rate(merkledb_bench_batch[$__rate_interval])",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "Effective Rate",
-          "useBackend": false
-        }
-      ],
-      "title": "Batch Operation - Counter Interval",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 4,
-        "x": 0,
-        "y": 16
-      },
-      "id": 6,
-      "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "sizing": "auto"
-      },
-      "pluginVersion": "11.2.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "adxe5ao03b1mob"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "merkledb_bench_insert_counter",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Inserted Rows",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 4,
-        "x": 4,
-        "y": 16
-      },
-      "id": 7,
-      "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "sizing": "auto"
-      },
-      "pluginVersion": "11.2.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "adxe5ao03b1mob"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "merkledb_bench_updates_counter",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Updated Rows",
-      "type": "gauge"
-    },
-    {
-      "datasource": {
-        "default": true,
-        "type": "prometheus",
-        "uid": "adxe5ao03b1mob"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 5,
-        "w": 4,
-        "x": 8,
-        "y": 16
-      },
-      "id": 8,
-      "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "sizing": "auto"
-      },
-      "pluginVersion": "11.2.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "adxe5ao03b1mob"
-          },
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "merkledb_bench_deletes_counter",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Deleted Rows",
-      "type": "gauge"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 39,
-  "tags": [],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-24h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "browser",
-  "title": "MerkleDB operations benchmark",
-  "uid": "edwwwnfme46ioa",
-  "version": 8,
-  "weekStart": ""
-}
\ No newline at end of file

From a6385b0ab722ced019a0363103b2d12bd5692350 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Thu, 19 Sep 2024 22:39:08 -0400
Subject: [PATCH 24/42] update

---
 x/merkledb/benchmarks/benchmark.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 75d2aeddc1b2..e9c551ce944d 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -82,7 +82,7 @@ var (
 )
 
 func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
-	return merkledb.Config{
+	config := merkledb.Config{
 		BranchFactor:                merkledb.BranchFactor16,
 		Hasher:                      merkledb.DefaultHasher,
 		RootGenConcurrency:          0,
@@ -95,6 +95,11 @@ func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
 		TraceLevel:                  merkledb.NoTrace,
 		Tracer:                      trace.Noop,
 	}
+	if promRegistry == nil {
+		// avoid large history during database construction.
+		config.HistoryLength = 4
+	}
+	return config
 }
 
 func getGoldenStagingDatabaseDirectory() string {

From 6a1817e51371f5d0018ad11d36ac710c21bf7988 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Fri, 20 Sep 2024 09:07:08 -0400
Subject: [PATCH 25/42] update

---
 x/merkledb/benchmarks/benchmark.go     | 7 +------
 x/merkledb/benchmarks/levelDBConfig.go | 2 +-
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index e9c551ce944d..75d2aeddc1b2 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -82,7 +82,7 @@ var (
 )
 
 func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
-	config := merkledb.Config{
+	return merkledb.Config{
 		BranchFactor:                merkledb.BranchFactor16,
 		Hasher:                      merkledb.DefaultHasher,
 		RootGenConcurrency:          0,
@@ -95,11 +95,6 @@ func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
 		TraceLevel:                  merkledb.NoTrace,
 		Tracer:                      trace.Noop,
 	}
-	if promRegistry == nil {
-		// avoid large history during database construction.
-		config.HistoryLength = 4
-	}
-	return config
 }
 
 func getGoldenStagingDatabaseDirectory() string {
diff --git a/x/merkledb/benchmarks/levelDBConfig.go b/x/merkledb/benchmarks/levelDBConfig.go
index 3a2a2d3115b1..7a8a76dec202 100644
--- a/x/merkledb/benchmarks/levelDBConfig.go
+++ b/x/merkledb/benchmarks/levelDBConfig.go
@@ -110,7 +110,7 @@ type config struct {
 
 func getLevelDBConfig() []byte {
 	// align with geth test and use 6gb as cache.
-	cacheSize := int(6000)
+	cacheSize := int(2000)
 	parsedConfig := config{
 		BlockCacheCapacity:     (cacheSize / 2) * opt.MiB,
 		DisableSeeksCompaction: true,

From 716fbbdee9a5061bfe14f90a03c8883ad799cc6f Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 22 Sep 2024 17:19:45 -0400
Subject: [PATCH 26/42] refactor and add few unit test for database creation.

---
 x/merkledb/benchmarks/benchmark.go          | 54 +++++++-------
 x/merkledb/benchmarks/benchmark_test.go     | 64 +++++++++++++++++
 x/merkledb/benchmarks_eth/benchmark.go      | 59 ++++++++-------
 x/merkledb/benchmarks_eth/benchmark_test.go | 79 +++++++++++++++++++++
 x/merkledb/benchmarks_eth/go.mod            |  4 ++
 5 files changed, 207 insertions(+), 53 deletions(-)
 create mode 100644 x/merkledb/benchmarks/benchmark_test.go
 create mode 100644 x/merkledb/benchmarks_eth/benchmark_test.go

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 75d2aeddc1b2..00971ebda750 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -97,19 +97,19 @@ func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config {
 	}
 }
 
-func getGoldenStagingDatabaseDirectory() string {
+func getGoldenStagingDatabaseDirectory(databaseEntries uint64) string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", *databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", databaseEntries))
 }
 
-func getGoldenDatabaseDirectory() string {
+func getGoldenDatabaseDirectory(databaseEntries uint64) string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", *databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", databaseEntries))
 }
 
-func getRunningDatabaseDirectory() string {
+func getRunningDatabaseDirectory(databaseEntries uint64) string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", *databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", databaseEntries))
 }
 
 func calculateIndexEncoding(idx uint64) []byte {
@@ -119,20 +119,20 @@ func calculateIndexEncoding(idx uint64) []byte {
 	return entryHash[:]
 }
 
-func createGoldenDatabase() error {
-	err := os.RemoveAll(getGoldenStagingDatabaseDirectory())
+func createGoldenDatabase(databaseEntries uint64) error {
+	err := os.RemoveAll(getGoldenStagingDatabaseDirectory(databaseEntries))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
 		return err
 	}
-	err = os.Mkdir(getGoldenStagingDatabaseDirectory(), 0o777)
+	err = os.Mkdir(getGoldenStagingDatabaseDirectory(databaseEntries), 0o777)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create golden staging directory : %v\n", err)
 		return err
 	}
 
 	levelDB, err := leveldb.New(
-		getGoldenStagingDatabaseDirectory(),
+		getGoldenStagingDatabaseDirectory(databaseEntries),
 		getLevelDBConfig(),
 		logging.NoLog{},
 		prometheus.NewRegistry(),
@@ -166,7 +166,7 @@ func createGoldenDatabase() error {
 	startInsertTime := time.Now()
 	startInsertBatchTime := startInsertTime
 	currentBatch := mdb.NewBatch()
-	for entryIdx := uint64(0); entryIdx < *databaseEntries; entryIdx++ {
+	for entryIdx := uint64(0); entryIdx < databaseEntries; entryIdx++ {
 		entryHash := calculateIndexEncoding(entryIdx)
 		err = currentBatch.Put(entryHash, entryHash)
 		if err != nil {
@@ -193,7 +193,7 @@ func createGoldenDatabase() error {
 			startInsertBatchTime = time.Now()
 		}
 	}
-	if (*databaseEntries)%databaseCreationBatchSize != 0 {
+	if databaseEntries%databaseCreationBatchSize != 0 {
 		err = currentBatch.Write()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err)
@@ -209,8 +209,6 @@ func createGoldenDatabase() error {
 		return err
 	}
 
-	fmt.Printf("Generated and inserted %d batches of size %d in %v\n",
-		(*databaseEntries)/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
 	err = mdb.Close()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to close merkleDB database : %v\n", err)
@@ -221,7 +219,11 @@ func createGoldenDatabase() error {
 		fmt.Fprintf(os.Stderr, "unable to close levelDB database : %v\n", err)
 		return err
 	}
-	err = os.Rename(getGoldenStagingDatabaseDirectory(), getGoldenDatabaseDirectory())
+
+	fmt.Printf("Generated and inserted %d batches of size %d in %v\n",
+		databaseEntries/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
+
+	err = os.Rename(getGoldenStagingDatabaseDirectory(databaseEntries), getGoldenDatabaseDirectory(databaseEntries))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to rename golden staging directory : %v\n", err)
 		return err
@@ -230,8 +232,8 @@ func createGoldenDatabase() error {
 	return nil
 }
 
-func resetRunningDatabaseDirectory() error {
-	runningDir := getRunningDatabaseDirectory()
+func resetRunningDatabaseDirectory(databaseEntries uint64) error {
+	runningDir := getRunningDatabaseDirectory(databaseEntries)
 	if _, err := os.Stat(runningDir); err == nil {
 		err := os.RemoveAll(runningDir)
 		if err != nil {
@@ -244,7 +246,7 @@ func resetRunningDatabaseDirectory() error {
 		fmt.Fprintf(os.Stderr, "unable to create running directory : %v\n", err)
 		return err
 	}
-	err = CopyDirectory(getGoldenDatabaseDirectory(), runningDir)
+	err = CopyDirectory(getGoldenDatabaseDirectory(databaseEntries), runningDir)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to duplicate golden directory : %v\n", err)
 		return err
@@ -252,9 +254,9 @@ func resetRunningDatabaseDirectory() error {
 	return nil
 }
 
-func runBenchmark() error {
+func runBenchmark(databaseEntries uint64) error {
 	levelDB, err := leveldb.New(
-		getRunningDatabaseDirectory(),
+		getRunningDatabaseDirectory(databaseEntries),
 		getLevelDBConfig(),
 		logging.NoLog{},
 		promRegistry,
@@ -296,7 +298,7 @@ func runBenchmark() error {
 
 		// add 2.5k past end.
 		startInsertTime := time.Now()
-		for keyToAddIdx := low + (*databaseEntries); keyToAddIdx < low+(*databaseEntries)+databaseRunningBatchSize; keyToAddIdx++ {
+		for keyToAddIdx := low + databaseEntries; keyToAddIdx < low+databaseEntries+databaseRunningBatchSize; keyToAddIdx++ {
 			entryHash := calculateIndexEncoding(keyToAddIdx)
 			err = batch.Put(entryHash, entryHash)
 			if err != nil {
@@ -311,7 +313,7 @@ func runBenchmark() error {
 		// update middle 5k entries
 		startUpdateTime := time.Now()
 		updateEntryValue := calculateIndexEncoding(low)
-		for keyToUpdateIdx := low + ((*databaseEntries) / 2); keyToUpdateIdx < low+((*databaseEntries)/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
+		for keyToUpdateIdx := low + (databaseEntries / 2); keyToUpdateIdx < low+(databaseEntries/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
 			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
 			err = batch.Put(updateEntryKey, updateEntryValue)
 			if err != nil {
@@ -386,19 +388,19 @@ func main() {
 		return
 	}
 
-	goldenDir := getGoldenDatabaseDirectory()
+	goldenDir := getGoldenDatabaseDirectory(*databaseEntries)
 	if _, err := os.Stat(goldenDir); os.IsNotExist(err) {
 		// create golden image.
-		if createGoldenDatabase() != nil {
+		if createGoldenDatabase(*databaseEntries) != nil {
 			os.Exit(1)
 			return
 		}
 	}
-	if resetRunningDatabaseDirectory() != nil {
+	if resetRunningDatabaseDirectory(*databaseEntries) != nil {
 		os.Exit(1)
 		return
 	}
-	if runBenchmark() != nil {
+	if runBenchmark(*databaseEntries) != nil {
 		os.Exit(1)
 		return
 	}
diff --git a/x/merkledb/benchmarks/benchmark_test.go b/x/merkledb/benchmarks/benchmark_test.go
new file mode 100644
index 000000000000..6da89c00e90f
--- /dev/null
+++ b/x/merkledb/benchmarks/benchmark_test.go
@@ -0,0 +1,64 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/ava-labs/avalanchego/database/leveldb"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/ava-labs/avalanchego/x/merkledb"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGoldenDatabaseCreation(t *testing.T) {
+	for _, size := range []uint64{9500, 20000, 50000, 103011} {
+		testGoldenDatabaseCreation(t, size)
+	}
+}
+
+func testGoldenDatabaseCreation(t *testing.T, size uint64) {
+	path := getGoldenDatabaseDirectory(size)
+	if Exists(getGoldenDatabaseDirectory(size)) {
+		require.NoError(t, os.RemoveAll(path))
+	}
+	require.NoError(t, createGoldenDatabase(size))
+	resetRunningDatabaseDirectory(size)
+	testGoldenDatabaseContent(t, size)
+	require.NoError(t, os.RemoveAll(path))
+	require.NoError(t, os.RemoveAll(getRunningDatabaseDirectory(size)))
+}
+
+func testGoldenDatabaseContent(t *testing.T, size uint64) {
+	levelDB, err := leveldb.New(
+		getRunningDatabaseDirectory(size),
+		getLevelDBConfig(),
+		logging.NoLog{},
+		prometheus.NewRegistry(),
+	)
+	require.NoError(t, err)
+	mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(nil))
+	require.NoError(t, err)
+	defer func() {
+		mdb.Close()
+		levelDB.Close()
+	}()
+	// read values and make sure they are what we expect them to be.
+	for keyIndex := uint64(0); keyIndex < size; keyIndex++ {
+		entryHash := calculateIndexEncoding(keyIndex)
+		entryValue, err := mdb.Get(entryHash)
+		require.NoError(t, err)
+		require.Equal(t, entryHash, entryValue)
+	}
+
+	// try an entry beyond.
+	entryHash := calculateIndexEncoding(size + 1)
+	_, err = mdb.Get(entryHash)
+	require.Error(t, err)
+
+	mdb.Clear()
+}
diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 220bf774c41d..770920a22292 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -83,19 +83,19 @@ var (
 	promRegistry = prometheus.NewRegistry()
 )
 
-func getGoldenStagingDatabaseDirectory() string {
+func getGoldenStagingDatabaseDirectory(databaseEntries uint64) string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", *databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", databaseEntries))
 }
 
-func getGoldenDatabaseDirectory() string {
+func getGoldenDatabaseDirectory(databaseEntries uint64) string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", *databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", databaseEntries))
 }
 
-func getRunningDatabaseDirectory() string {
+func getRunningDatabaseDirectory(databaseEntries uint64) string {
 	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", *databaseEntries))
+	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", databaseEntries))
 }
 
 func calculateIndexEncoding(idx uint64) []byte {
@@ -113,8 +113,8 @@ func getPathDBConfig() *pathdb.Config {
 	}
 }
 
-func createGoldenDatabase() error {
-	stagingDirectory := getGoldenStagingDatabaseDirectory()
+func createGoldenDatabase(databaseEntries uint64) error {
+	stagingDirectory := getGoldenStagingDatabaseDirectory(databaseEntries)
 	err := os.RemoveAll(stagingDirectory)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
@@ -157,10 +157,11 @@ func createGoldenDatabase() error {
 
 	var root common.Hash
 	parentHash := types.EmptyRootHash
+	blockHeight := uint64(0)
 	writeBatch := func() error {
 		var nodes *trienode.NodeSet
 		root, nodes = tdb.Commit(false)
-		err = trieDb.Update(root, parentHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
 			return err
@@ -176,12 +177,13 @@ func createGoldenDatabase() error {
 			return err
 		}
 		parentHash = root
+		blockHeight++
 		return nil
 	}
 
 	startInsertTime := time.Now()
 	startInsertBatchTime := startInsertTime
-	for entryIdx := uint64(0); entryIdx < *databaseEntries; entryIdx++ {
+	for entryIdx := uint64(0); entryIdx < databaseEntries; entryIdx++ {
 		entryHash := calculateIndexEncoding(entryIdx)
 		tdb.Update(entryHash, entryHash)
 
@@ -205,7 +207,7 @@ func createGoldenDatabase() error {
 			startInsertBatchTime = time.Now()
 		}
 	}
-	if (*databaseEntries)%databaseCreationBatchSize != 0 {
+	if databaseEntries%databaseCreationBatchSize != 0 {
 		err = writeBatch()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to write value in database : %v\n", err)
@@ -215,8 +217,6 @@ func createGoldenDatabase() error {
 	close(ticksCh)
 	fmt.Print(" done!\n")
 
-	fmt.Printf("Generated and inserted %d batches of size %d in %v\n",
-		(*databaseEntries)/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
 	err = trieDb.Close()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to close trie database : %v\n", err)
@@ -228,13 +228,16 @@ func createGoldenDatabase() error {
 		return err
 	}
 
+	fmt.Printf("Generated and inserted %d batches of size %d in %v\n",
+		databaseEntries/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
+
 	err = os.WriteFile(path.Join(stagingDirectory, "root.txt"), root.Bytes(), 0o644)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to save root : %v\n", err)
 		return err
 	}
 
-	err = os.Rename(getGoldenStagingDatabaseDirectory(), getGoldenDatabaseDirectory())
+	err = os.Rename(getGoldenStagingDatabaseDirectory(databaseEntries), getGoldenDatabaseDirectory(databaseEntries))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to rename golden staging directory : %v\n", err)
 		return err
@@ -243,8 +246,8 @@ func createGoldenDatabase() error {
 	return nil
 }
 
-func resetRunningDatabaseDirectory() error {
-	runningDir := getRunningDatabaseDirectory()
+func resetRunningDatabaseDirectory(databaseEntries uint64) error {
+	runningDir := getRunningDatabaseDirectory(databaseEntries)
 	if _, err := os.Stat(runningDir); err == nil {
 		err := os.RemoveAll(runningDir)
 		if err != nil {
@@ -257,7 +260,7 @@ func resetRunningDatabaseDirectory() error {
 		fmt.Fprintf(os.Stderr, "unable to create running directory : %v\n", err)
 		return err
 	}
-	err = CopyDirectory(getGoldenDatabaseDirectory(), runningDir)
+	err = CopyDirectory(getGoldenDatabaseDirectory(databaseEntries), runningDir)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to duplicate golden directory : %v\n", err)
 		return err
@@ -265,8 +268,8 @@ func resetRunningDatabaseDirectory() error {
 	return nil
 }
 
-func runBenchmark() error {
-	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(), "root.txt"))
+func runBenchmark(databaseEntries uint64) error {
+	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to read root : %v\n", err)
 		return err
@@ -274,7 +277,7 @@ func runBenchmark() error {
 
 	ldb, err := rawdb.Open(rawdb.OpenOptions{
 		Type:              "leveldb",
-		Directory:         getRunningDatabaseDirectory(),
+		Directory:         getRunningDatabaseDirectory(databaseEntries),
 		AncientsDirectory: "",
 		Namespace:         "metrics_prefix",
 		Cache:             levelDBCacheSize,
@@ -301,10 +304,11 @@ func runBenchmark() error {
 		return err
 	}
 	var root common.Hash
+	blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize
 	writeBatch := func() error {
 		var nodes *trienode.NodeSet
 		root, nodes = tdb.Commit(false)
-		err = trieDb.Update(root, parentHash, 0 /*block*/, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
 			return err
@@ -320,6 +324,7 @@ func runBenchmark() error {
 			return err
 		}
 		parentHash = root
+		blockHeight++
 		return nil
 	}
 
@@ -344,7 +349,7 @@ func runBenchmark() error {
 
 		// add 2.5k past end.
 		startInsertTime := time.Now()
-		for keyToAddIdx := low + (*databaseEntries); keyToAddIdx < low+(*databaseEntries)+databaseRunningBatchSize; keyToAddIdx++ {
+		for keyToAddIdx := low + databaseEntries; keyToAddIdx < low+databaseEntries+databaseRunningBatchSize; keyToAddIdx++ {
 			entryHash := calculateIndexEncoding(keyToAddIdx)
 			err = tdb.Update(entryHash, entryHash)
 			if err != nil {
@@ -359,7 +364,7 @@ func runBenchmark() error {
 		// update middle 5k entries
 		startUpdateTime := time.Now()
 		updateEntryValue := calculateIndexEncoding(low)
-		for keyToUpdateIdx := low + ((*databaseEntries) / 2); keyToUpdateIdx < low+((*databaseEntries)/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
+		for keyToUpdateIdx := low + (databaseEntries / 2); keyToUpdateIdx < low+(databaseEntries/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
 			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
 			err = tdb.Update(updateEntryKey, updateEntryValue)
 			if err != nil {
@@ -432,19 +437,19 @@ func main() {
 		return
 	}
 
-	goldenDir := getGoldenDatabaseDirectory()
+	goldenDir := getGoldenDatabaseDirectory(*databaseEntries)
 	if _, err := os.Stat(goldenDir); os.IsNotExist(err) {
 		// create golden image.
-		if createGoldenDatabase() != nil {
+		if createGoldenDatabase(*databaseEntries) != nil {
 			os.Exit(1)
 			return
 		}
 	}
-	if resetRunningDatabaseDirectory() != nil {
+	if resetRunningDatabaseDirectory(*databaseEntries) != nil {
 		os.Exit(1)
 		return
 	}
-	if runBenchmark() != nil {
+	if runBenchmark(*databaseEntries) != nil {
 		os.Exit(1)
 		return
 	}
diff --git a/x/merkledb/benchmarks_eth/benchmark_test.go b/x/merkledb/benchmarks_eth/benchmark_test.go
new file mode 100644
index 000000000000..3ba7feeb804d
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/benchmark_test.go
@@ -0,0 +1,79 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/triedb"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGoldenDatabaseCreation(t *testing.T) {
+	for _, size := range []uint64{9500, 20000, 50000, 103011} {
+		testGoldenDatabaseCreation(t, size)
+	}
+}
+
+func testGoldenDatabaseCreation(t *testing.T, size uint64) {
+	path := getGoldenDatabaseDirectory(size)
+	if Exists(getGoldenDatabaseDirectory(size)) {
+		require.NoError(t, os.RemoveAll(path))
+	}
+	require.NoError(t, createGoldenDatabase(size))
+	resetRunningDatabaseDirectory(size)
+	testGoldenDatabaseContent(t, size)
+	require.NoError(t, os.RemoveAll(path))
+	require.NoError(t, os.RemoveAll(getRunningDatabaseDirectory(size)))
+}
+
+func testGoldenDatabaseContent(t *testing.T, size uint64) {
+	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(size), "root.txt"))
+	require.NoError(t, err)
+
+	ldb, err := rawdb.Open(rawdb.OpenOptions{
+		Type:              "leveldb",
+		Directory:         getRunningDatabaseDirectory(size),
+		AncientsDirectory: "",
+		Namespace:         "metrics_prefix",
+		Cache:             levelDBCacheSize,
+		Handles:           200,
+		ReadOnly:          false,
+		Ephemeral:         false,
+	})
+	require.NoError(t, err)
+
+	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
+		Preimages: false,
+		IsVerkle:  false,
+		HashDB:    nil,
+		PathDB:    getPathDBConfig(),
+	})
+
+	parentHash := common.BytesToHash(rootBytes)
+	tdb, err := trie.New(trie.TrieID(parentHash), trieDb)
+	require.NoError(t, err)
+
+	// read values and make sure they are what we expect them to be.
+	for keyIndex := uint64(0); keyIndex < size; keyIndex++ {
+		entryHash := calculateIndexEncoding(keyIndex)
+		entryValue, err := tdb.Get(entryHash)
+		require.NoError(t, err)
+		require.Equal(t, entryHash, entryValue)
+	}
+
+	// try an entry beyond.
+	entryHash := calculateIndexEncoding(size + 1)
+	entryValue, err := tdb.Get(entryHash)
+	require.NoError(t, err) // for geth, we have no error in case of a missing key.
+	require.Equal(t, []byte(nil), entryValue)
+
+	require.NoError(t, trieDb.Close())
+	require.NoError(t, ldb.Close())
+}
diff --git a/x/merkledb/benchmarks_eth/go.mod b/x/merkledb/benchmarks_eth/go.mod
index 4a7e29464c22..08b827dca96c 100644
--- a/x/merkledb/benchmarks_eth/go.mod
+++ b/x/merkledb/benchmarks_eth/go.mod
@@ -5,6 +5,7 @@ go 1.22.3
 require (
 	github.com/prometheus/client_golang v1.20.3
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.9.0
 )
 
 require (
@@ -22,6 +23,7 @@ require (
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
 	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
 	github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
 	github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect
@@ -36,6 +38,7 @@ require (
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
@@ -45,6 +48,7 @@ require (
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	golang.org/x/sync v0.7.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 

From dd25de9f4849a9b311ecdcf35b05090c84acba5d Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 22 Sep 2024 18:10:58 -0400
Subject: [PATCH 27/42] add unit test.

---
 x/merkledb/benchmarks_eth/benchmark_test.go | 67 +++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/x/merkledb/benchmarks_eth/benchmark_test.go b/x/merkledb/benchmarks_eth/benchmark_test.go
index 3ba7feeb804d..d42fb3cddc15 100644
--- a/x/merkledb/benchmarks_eth/benchmark_test.go
+++ b/x/merkledb/benchmarks_eth/benchmark_test.go
@@ -10,8 +10,11 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/triedb"
+	"github.com/ethereum/go-ethereum/triedb/pathdb"
 	"github.com/stretchr/testify/require"
 )
 
@@ -77,3 +80,67 @@ func testGoldenDatabaseContent(t *testing.T, size uint64) {
 	require.NoError(t, trieDb.Close())
 	require.NoError(t, ldb.Close())
 }
+
+func TestRevisions(t *testing.T) {
+	revisionCount := uint64(10)
+	wd, _ := os.Getwd()
+	dbPath := path.Join(wd, "db-test-revisions")
+	require.NoError(t, os.RemoveAll(dbPath))
+	err := os.Mkdir(dbPath, 0o777)
+	require.NoError(t, err)
+	ldb, err := rawdb.NewLevelDBDatabase(dbPath, levelDBCacheSize, 200, "metrics_prefix", false)
+	require.NoError(t, err)
+	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
+		Preimages: false,
+		IsVerkle:  false,
+		HashDB:    nil,
+		PathDB: &pathdb.Config{
+			StateHistory:   revisionCount, // keep the same requiremenet across each db
+			CleanCacheSize: 1024 * 1024,
+			DirtyCacheSize: 1024 * 1024,
+		},
+	})
+	tdb := trie.NewEmpty(trieDb)
+	for entryIdx := uint64(0); entryIdx < 1000000; entryIdx++ {
+		entryHash := calculateIndexEncoding(entryIdx)
+		tdb.Update(entryHash, make([]byte, 32))
+	}
+
+	revisions := make([]common.Hash, 0)
+	var nodes *trienode.NodeSet
+	var root common.Hash
+	root, nodes = tdb.Commit(false)
+	err = trieDb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil /*states*/)
+	require.NoError(t, err)
+	err = trieDb.Commit(root, false)
+	require.NoError(t, err)
+	tdb, err = trie.New(trie.TrieID(root), trieDb)
+	require.NoError(t, err)
+	revisions = append(revisions, root)
+
+	for currentRev := uint64(1); currentRev < 1000; currentRev++ {
+		// update the tree based on the current revision.
+		for entryIdx := uint64(currentRev-1) * 1000; entryIdx < uint64(currentRev)*1000; entryIdx++ {
+			entryHash := calculateIndexEncoding(entryIdx)
+			tdb.Update(entryHash, entryHash)
+		}
+		root, nodes = tdb.Commit(false)
+		err = trieDb.Update(root, revisions[len(revisions)-1], currentRev, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		require.NoError(t, err)
+		err = trieDb.Commit(root, false)
+		require.NoError(t, err)
+		tdb, err = trie.New(trie.TrieID(root), trieDb)
+		require.NoError(t, err)
+		revisions = append(revisions, root)
+
+		// check all the historical revision and see which ones are available and which one aren't.
+		for revIdx, revRoot := range revisions {
+			_, err = trie.New(trie.TrieID(revRoot), trieDb)
+			if uint64(revIdx)+revisionCount >= currentRev {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+			}
+		}
+	}
+}

From ae64496dc7fa4c340ae2f5a9732d627fa30b2e07 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 22 Sep 2024 23:12:24 -0400
Subject: [PATCH 28/42] update revisions

---
 x/merkledb/benchmarks/benchmark.go     | 2 +-
 x/merkledb/benchmarks/levelDBConfig.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go
index 00971ebda750..2bf0c08ff507 100644
--- a/x/merkledb/benchmarks/benchmark.go
+++ b/x/merkledb/benchmarks/benchmark.go
@@ -30,7 +30,7 @@ const (
 	databaseRunningBatchSize  = 2500       // 2.5k
 	databaseRunningUpdateSize = 5000       // 5k
 	defaultMetricsPort        = 3000
-	firewoodNumberOfRevisions = 120
+	firewoodNumberOfRevisions = 5
 )
 
 var (
diff --git a/x/merkledb/benchmarks/levelDBConfig.go b/x/merkledb/benchmarks/levelDBConfig.go
index 7a8a76dec202..3a2a2d3115b1 100644
--- a/x/merkledb/benchmarks/levelDBConfig.go
+++ b/x/merkledb/benchmarks/levelDBConfig.go
@@ -110,7 +110,7 @@ type config struct {
 
 func getLevelDBConfig() []byte {
 	// align with geth test and use 6gb as cache.
-	cacheSize := int(2000)
+	cacheSize := int(6000)
 	parsedConfig := config{
 		BlockCacheCapacity:     (cacheSize / 2) * opt.MiB,
 		DisableSeeksCompaction: true,

From f3191bc5f7bd23ee28bfb0d863f88bf4cd8df3c3 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Mon, 23 Sep 2024 13:28:54 -0400
Subject: [PATCH 29/42] adding historical lookup for geth.

---
 x/merkledb/benchmarks_eth/benchmark_test.go | 24 +++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/benchmark_test.go b/x/merkledb/benchmarks_eth/benchmark_test.go
index d42fb3cddc15..e848450f0d3e 100644
--- a/x/merkledb/benchmarks_eth/benchmark_test.go
+++ b/x/merkledb/benchmarks_eth/benchmark_test.go
@@ -107,6 +107,8 @@ func TestRevisions(t *testing.T) {
 	}
 
 	revisions := make([]common.Hash, 0)
+	revisionsTrie := make([]*trie.Trie, 0)
+
 	var nodes *trienode.NodeSet
 	var root common.Hash
 	root, nodes = tdb.Commit(false)
@@ -114,6 +116,9 @@ func TestRevisions(t *testing.T) {
 	require.NoError(t, err)
 	err = trieDb.Commit(root, false)
 	require.NoError(t, err)
+	tdb2, err := trie.New(trie.TrieID(root), trieDb)
+	require.NoError(t, err)
+	revisionsTrie = append(revisionsTrie, tdb2)
 	tdb, err = trie.New(trie.TrieID(root), trieDb)
 	require.NoError(t, err)
 	revisions = append(revisions, root)
@@ -129,18 +134,29 @@ func TestRevisions(t *testing.T) {
 		require.NoError(t, err)
 		err = trieDb.Commit(root, false)
 		require.NoError(t, err)
+		tdb2, err = trie.New(trie.TrieID(root), trieDb)
+		require.NoError(t, err)
+		revisionsTrie = append(revisionsTrie, tdb2)
 		tdb, err = trie.New(trie.TrieID(root), trieDb)
 		require.NoError(t, err)
 		revisions = append(revisions, root)
 
 		// check all the historical revision and see which ones are available and which one aren't.
-		for revIdx, revRoot := range revisions {
-			_, err = trie.New(trie.TrieID(revRoot), trieDb)
-			if uint64(revIdx)+revisionCount >= currentRev {
+		for revIdx, revTrie := range revisionsTrie {
+			entryHash := calculateIndexEncoding(uint64(revIdx) * 1000)
+			entryValue, err := revTrie.Get(entryHash)
+			if revIdx != 0 {
 				require.NoError(t, err)
+				require.Equal(t, make([]byte, 32), entryValue)
 			} else {
-				require.Error(t, err)
+				require.Error(t, err) // should be "missing node error"
+				continue
 			}
+
+			entryHash = calculateIndexEncoding(uint64(revIdx)*1000 - 1)
+			entryValue, err = revTrie.Get(entryHash)
+			require.NoError(t, err)
+			require.Equal(t, entryHash, entryValue)
 		}
 	}
 }

From a0f31a4d9aa3907489e8b3f5dfcd7825dbae6ab9 Mon Sep 17 00:00:00 2001
From: Stephen Buttolph <stephen@avalabs.org>
Date: Wed, 25 Sep 2024 15:07:05 -0400
Subject: [PATCH 30/42] Fix geth merkledb unit test (#3414)

---
 x/merkledb/benchmarks_eth/benchmark_test.go | 157 ++++++++++++--------
 1 file changed, 94 insertions(+), 63 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/benchmark_test.go b/x/merkledb/benchmarks_eth/benchmark_test.go
index e848450f0d3e..50b4958fe48d 100644
--- a/x/merkledb/benchmarks_eth/benchmark_test.go
+++ b/x/merkledb/benchmarks_eth/benchmark_test.go
@@ -82,81 +82,112 @@ func testGoldenDatabaseContent(t *testing.T, size uint64) {
 }
 
 func TestRevisions(t *testing.T) {
-	revisionCount := uint64(10)
+	require := require.New(t)
+
+	// Clean up the database directory
 	wd, _ := os.Getwd()
 	dbPath := path.Join(wd, "db-test-revisions")
-	require.NoError(t, os.RemoveAll(dbPath))
+	require.NoError(os.RemoveAll(dbPath))
 	err := os.Mkdir(dbPath, 0o777)
-	require.NoError(t, err)
+	require.NoError(err)
+
+	// Create a new leveldb database
 	ldb, err := rawdb.NewLevelDBDatabase(dbPath, levelDBCacheSize, 200, "metrics_prefix", false)
-	require.NoError(t, err)
-	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
-		Preimages: false,
-		IsVerkle:  false,
-		HashDB:    nil,
-		PathDB: &pathdb.Config{
-			StateHistory:   revisionCount, // keep the same requiremenet across each db
-			CleanCacheSize: 1024 * 1024,
-			DirtyCacheSize: 1024 * 1024,
+	require.NoError(err)
+
+	// Create a new trie database
+	const maxDiffs = 128 // triedb will store 128 diffs.
+	trieDB := triedb.NewDatabase(
+		ldb,
+		&triedb.Config{
+			Preimages: false,
+			IsVerkle:  false,
+			HashDB:    nil,
+			PathDB: &pathdb.Config{
+				CleanCacheSize: 1024 * 1024,
+				DirtyCacheSize: 0, // Disable dirty cache to force trieDB to write to disk.
+			},
 		},
-	})
-	tdb := trie.NewEmpty(trieDb)
-	for entryIdx := uint64(0); entryIdx < 1000000; entryIdx++ {
-		entryHash := calculateIndexEncoding(entryIdx)
-		tdb.Update(entryHash, make([]byte, 32))
+	)
+
+	// Initially populate the trie
+	const diffSize = 100
+	var (
+		index uint64
+		tdb   = trie.NewEmpty(trieDB)
+	)
+	for i := 0; i < diffSize; i++ {
+		entryHash := calculateIndexEncoding(index)
+		require.NoError(tdb.Update(entryHash, entryHash))
+		index++
 	}
 
-	revisions := make([]common.Hash, 0)
-	revisionsTrie := make([]*trie.Trie, 0)
+	// Commit the initial state
+	root, nodes := tdb.Commit(false)
+	require.NoError(trieDB.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil /*states*/))
 
-	var nodes *trienode.NodeSet
-	var root common.Hash
-	root, nodes = tdb.Commit(false)
-	err = trieDb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil /*states*/)
-	require.NoError(t, err)
-	err = trieDb.Commit(root, false)
-	require.NoError(t, err)
-	tdb2, err := trie.New(trie.TrieID(root), trieDb)
-	require.NoError(t, err)
-	revisionsTrie = append(revisionsTrie, tdb2)
-	tdb, err = trie.New(trie.TrieID(root), trieDb)
-	require.NoError(t, err)
-	revisions = append(revisions, root)
+	const numDiffs = 150
+	rootHashes := []common.Hash{root}
+	for height := uint64(1); height < numDiffs; height++ {
+		prevRoot := root
+		tdb, err = trie.New(trie.TrieID(prevRoot), trieDB)
+		require.NoError(err)
 
-	for currentRev := uint64(1); currentRev < 1000; currentRev++ {
 		// update the tree based on the current revision.
-		for entryIdx := uint64(currentRev-1) * 1000; entryIdx < uint64(currentRev)*1000; entryIdx++ {
-			entryHash := calculateIndexEncoding(entryIdx)
-			tdb.Update(entryHash, entryHash)
+		for i := 0; i < diffSize; i++ {
+			entryHash := calculateIndexEncoding(index)
+			require.NoError(tdb.Update(entryHash, entryHash))
+			index++
 		}
+
+		// Commit the changes
 		root, nodes = tdb.Commit(false)
-		err = trieDb.Update(root, revisions[len(revisions)-1], currentRev, trienode.NewWithNodeSet(nodes), nil /*states*/)
-		require.NoError(t, err)
-		err = trieDb.Commit(root, false)
-		require.NoError(t, err)
-		tdb2, err = trie.New(trie.TrieID(root), trieDb)
-		require.NoError(t, err)
-		revisionsTrie = append(revisionsTrie, tdb2)
-		tdb, err = trie.New(trie.TrieID(root), trieDb)
-		require.NoError(t, err)
-		revisions = append(revisions, root)
-
-		// check all the historical revision and see which ones are available and which one aren't.
-		for revIdx, revTrie := range revisionsTrie {
-			entryHash := calculateIndexEncoding(uint64(revIdx) * 1000)
-			entryValue, err := revTrie.Get(entryHash)
-			if revIdx != 0 {
-				require.NoError(t, err)
-				require.Equal(t, make([]byte, 32), entryValue)
-			} else {
-				require.Error(t, err) // should be "missing node error"
-				continue
-			}
-
-			entryHash = calculateIndexEncoding(uint64(revIdx)*1000 - 1)
-			entryValue, err = revTrie.Get(entryHash)
-			require.NoError(t, err)
-			require.Equal(t, entryHash, entryValue)
+		require.NoError(trieDB.Update(root, prevRoot, height, trienode.NewWithNodeSet(nodes), nil /*states*/))
+
+		rootHashes = append(rootHashes, root)
+	}
+
+	// Verify that old revisions were pruned.
+	numPruned := max(len(rootHashes)-maxDiffs-1, 0)
+	for _, root := range rootHashes[:numPruned] {
+		_, err = trie.New(trie.TrieID(root), trieDB)
+		require.Error(err)
+	}
+
+	// Verify that all 128 revisions are queryable and return the expected data.
+	for height, root := range rootHashes[numPruned:] {
+		height += numPruned
+
+		tdb, err = trie.New(trie.TrieID(root), trieDB)
+		require.NoError(err)
+
+		maxIndex := diffSize * uint64(height+1)
+		for i := uint64(0); i < maxIndex; i++ {
+			entryHash := calculateIndexEncoding(i)
+			entryValue, err := tdb.Get(entryHash)
+			require.NoError(err)
+			require.Equal(entryHash, entryValue)
 		}
+
+		entryHash := calculateIndexEncoding(maxIndex)
+		entryValue, err := tdb.Get(entryHash)
+		require.NoError(err)
+		require.Empty(entryValue)
 	}
+
+	trieDB = triedb.NewDatabase(
+		ldb,
+		&triedb.Config{
+			Preimages: false,
+			IsVerkle:  false,
+			HashDB:    nil,
+			PathDB: &pathdb.Config{
+				CleanCacheSize: 1024 * 1024,
+				DirtyCacheSize: 0, // Disable dirty cache to force trieDB to write to disk.
+			},
+		},
+	)
+
+	_, err = trieDB.Reader(rootHashes[numPruned])
+	require.NoError(err)
 }

From 4602debd06062cf653e3b59119216994c8936149 Mon Sep 17 00:00:00 2001
From: Ron Kuris <ron.kuris@avalabs.org>
Date: Wed, 25 Sep 2024 17:19:01 -1000
Subject: [PATCH 31/42] Code cleanups (#3416)

---
 x/merkledb/benchmarks/benchmark_test.go     |   7 +-
 x/merkledb/benchmarks_eth/benchmark.go      | 286 ++++++++++----------
 x/merkledb/benchmarks_eth/benchmark_test.go |  18 +-
 x/merkledb/benchmarks_eth/go.mod            |   1 +
 x/merkledb/benchmarks_eth/go.sum            |   2 +
 5 files changed, 160 insertions(+), 154 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark_test.go b/x/merkledb/benchmarks/benchmark_test.go
index 6da89c00e90f..5352ae898064 100644
--- a/x/merkledb/benchmarks/benchmark_test.go
+++ b/x/merkledb/benchmarks/benchmark_test.go
@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/ava-labs/avalanchego/database/leveldb"
+	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/ava-labs/avalanchego/x/merkledb"
 	"github.com/prometheus/client_golang/prometheus"
@@ -27,7 +28,7 @@ func testGoldenDatabaseCreation(t *testing.T, size uint64) {
 		require.NoError(t, os.RemoveAll(path))
 	}
 	require.NoError(t, createGoldenDatabase(size))
-	resetRunningDatabaseDirectory(size)
+	require.NoError(t, resetRunningDatabaseDirectory(size))
 	testGoldenDatabaseContent(t, size)
 	require.NoError(t, os.RemoveAll(path))
 	require.NoError(t, os.RemoveAll(getRunningDatabaseDirectory(size)))
@@ -58,7 +59,7 @@ func testGoldenDatabaseContent(t *testing.T, size uint64) {
 	// try an entry beyond.
 	entryHash := calculateIndexEncoding(size + 1)
 	_, err = mdb.Get(entryHash)
-	require.Error(t, err)
+	require.ErrorIs(t, err, database.ErrNotFound)
 
-	mdb.Clear()
+	require.NoError(t, mdb.Clear())
 }
diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 770920a22292..8a4a4584ab64 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -12,6 +12,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/ava-labs/avalanchego/utils/units"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/pflag"
@@ -26,76 +27,110 @@ import (
 )
 
 const (
-	defaultDatabaseEntries    = 1000000000 // 1B
-	databaseCreationBatchSize = 10000      // 10k
-	databaseRunningBatchSize  = 2500       // 2.5k
-	databaseRunningUpdateSize = 5000       // 5k
-	defaultMetricsPort        = 3000
-	levelDBCacheSize          = 6_000
-	firewoodNumberOfRevisions = 120
+	defaultDatabaseEntries       = 1_000_000_000
+	databaseCreationBatchSize    = 10_000
+	databaseRunningBatchSize     = databaseCreationBatchSize / 4
+	databaseRunningUpdateSize    = databaseCreationBatchSize / 2
+	defaultMetricsPort           = 3_000
+	benchmarkRevisionHistorySize = 128
 )
 
-var (
-	databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
-	httpMetricPort  = pflag.Uint64("p", defaultMetricsPort, "default metrics port")
-	verbose         = pflag.Bool("v", false, "verbose")
+// TODO: Adjust these cache sizes for maximum performance
+const (
+	cleanCacheSizeBytes = 4 * units.GiB
+	levelDBCacheSizeMB  = 6 * units.GiB / units.MiB
+
+	// TODO: Why 200? The default is 500
+	// see https://pkg.go.dev/github.com/syndtr/goleveldb@v1.0.0/leveldb/opt#Options
+	openFilesCacheCapacity = 200
+)
 
-	insertsCounter = prometheus.NewCounter(prometheus.CounterOpts{
+var pathDBConfig = pathdb.Config{
+	StateHistory:   benchmarkRevisionHistorySize,
+	CleanCacheSize: cleanCacheSizeBytes,
+	DirtyCacheSize: 0,
+	ReadOnly:       false,
+}
+
+var stats = struct {
+	inserts        prometheus.Counter
+	deletes        prometheus.Counter
+	updates        prometheus.Counter
+	batches        prometheus.Counter
+	deleteRate     prometheus.Gauge
+	updateRate     prometheus.Gauge
+	insertRate     prometheus.Gauge
+	batchWriteRate prometheus.Gauge
+	registry       *prometheus.Registry
+}{
+	inserts: prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "merkledb_bench",
 		Name:      "insert_counter",
 		Help:      "Total number of inserts",
-	})
-	deletesCounter = prometheus.NewCounter(prometheus.CounterOpts{
+	}),
+	deletes: prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "merkledb_bench",
 		Name:      "deletes_counter",
 		Help:      "Total number of deletes",
-	})
-	updatesCounter = prometheus.NewCounter(prometheus.CounterOpts{
+	}),
+	updates: prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "merkledb_bench",
 		Name:      "updates_counter",
 		Help:      "Total number of updates",
-	})
-	batchCounter = prometheus.NewCounter(prometheus.CounterOpts{
+	}),
+	batches: prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "merkledb_bench",
 		Name:      "batch",
 		Help:      "Total number of batches written",
-	})
-	deleteRate = prometheus.NewGauge(prometheus.GaugeOpts{
+	}),
+	deleteRate: prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "merkledb_bench",
 		Name:      "entry_delete_rate",
 		Help:      "The rate at which elements are deleted",
-	})
-	updateRate = prometheus.NewGauge(prometheus.GaugeOpts{
+	}),
+	updateRate: prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "merkledb_bench",
 		Name:      "entry_update_rate",
 		Help:      "The rate at which elements are updated",
-	})
-	insertRate = prometheus.NewGauge(prometheus.GaugeOpts{
+	}),
+	insertRate: prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "merkledb_bench",
 		Name:      "entry_insert_rate",
 		Help:      "The rate at which elements are inserted",
-	})
-	batchWriteRate = prometheus.NewGauge(prometheus.GaugeOpts{
+	}),
+	batchWriteRate: prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: "merkledb_bench",
 		Name:      "batch_write_rate",
 		Help:      "The rate at which the batch was written",
-	})
-	promRegistry = prometheus.NewRegistry()
+	}),
+
+	registry: prometheus.NewRegistry(),
+}
+
+// command line arguments
+var (
+	databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
+	httpMetricPort  = pflag.Uint64("p", defaultMetricsPort, "default metrics port")
+	verbose         = pflag.Bool("v", false, "verbose")
 )
 
+func Must[T any](obj T, err error) T {
+	if err != nil {
+		panic(err)
+	}
+	return obj
+}
+
 func getGoldenStagingDatabaseDirectory(databaseEntries uint64) string {
-	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", databaseEntries))
+	return path.Join(Must(os.Getwd()), fmt.Sprintf("db-bench-test-golden-staging-%d", databaseEntries))
 }
 
 func getGoldenDatabaseDirectory(databaseEntries uint64) string {
-	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", databaseEntries))
+	return path.Join(Must(os.Getwd()), fmt.Sprintf("db-bench-test-golden-%d", databaseEntries))
 }
 
 func getRunningDatabaseDirectory(databaseEntries uint64) string {
-	wd, _ := os.Getwd()
-	return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", databaseEntries))
+	return path.Join(Must(os.Getwd()), fmt.Sprintf("db-bench-test-running-%d", databaseEntries))
 }
 
 func calculateIndexEncoding(idx uint64) []byte {
@@ -105,38 +140,27 @@ func calculateIndexEncoding(idx uint64) []byte {
 	return entryHash[:]
 }
 
-func getPathDBConfig() *pathdb.Config {
-	return &pathdb.Config{
-		StateHistory:   firewoodNumberOfRevisions, // keep the same requiremenet across each db
-		CleanCacheSize: 6 * 1024 * 1024 * 1024,    // 4GB - this is a guess, but I assume we're currently underutilizing memory by a lot
-		DirtyCacheSize: 6 * 1024 * 1024 * 1024,    // 4GB - this is a guess, but I assume we're currently underutilizing memory by a lot
-	}
-}
-
 func createGoldenDatabase(databaseEntries uint64) error {
 	stagingDirectory := getGoldenStagingDatabaseDirectory(databaseEntries)
 	err := os.RemoveAll(stagingDirectory)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
-		return err
+		return fmt.Errorf("unable to remove running directory : %v", err)
 	}
 	err = os.Mkdir(stagingDirectory, 0o777)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to create golden staging directory : %v\n", err)
-		return err
+		return fmt.Errorf("unable to create golden staging directory : %v", err)
 	}
 
-	ldb, err := rawdb.NewLevelDBDatabase(stagingDirectory, levelDBCacheSize, 200, "metrics_prefix", false)
+	ldb, err := rawdb.NewLevelDBDatabase(stagingDirectory, levelDBCacheSizeMB, openFilesCacheCapacity, "metrics_prefix", false)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to create level db database : %v\n", err)
-		return err
+		return fmt.Errorf("unable to create level db database : %v", err)
 	}
 
 	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
 		Preimages: false,
 		IsVerkle:  false,
 		HashDB:    nil,
-		PathDB:    getPathDBConfig(),
+		PathDB:    &pathDBConfig,
 	})
 	tdb := trie.NewEmpty(trieDb)
 
@@ -157,24 +181,18 @@ func createGoldenDatabase(databaseEntries uint64) error {
 
 	var root common.Hash
 	parentHash := types.EmptyRootHash
-	blockHeight := uint64(0)
+	blockHeight := uint64(1)
 	writeBatch := func() error {
 		var nodes *trienode.NodeSet
 		root, nodes = tdb.Commit(false)
 		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
-			return err
-		}
-		err = trieDb.Commit(root, false)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to commit trie : %v\n", err)
-			return err
+			return fmt.Errorf("unable to update trie : %v", err)
 		}
+
 		tdb, err = trie.New(trie.TrieID(root), trieDb)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to create new trie : %v\n", err)
-			return err
+			return fmt.Errorf("unable to create new trie : %v", err)
 		}
 		parentHash = root
 		blockHeight++
@@ -183,64 +201,63 @@ func createGoldenDatabase(databaseEntries uint64) error {
 
 	startInsertTime := time.Now()
 	startInsertBatchTime := startInsertTime
-	for entryIdx := uint64(0); entryIdx < databaseEntries; entryIdx++ {
+	for entryIdx := range databaseEntries {
 		entryHash := calculateIndexEncoding(entryIdx)
 		tdb.Update(entryHash, entryHash)
 
 		if entryIdx%databaseCreationBatchSize == (databaseCreationBatchSize - 1) {
 			addDuration := time.Since(startInsertBatchTime)
-			insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration))
-			insertsCounter.Add(databaseCreationBatchSize)
+			stats.insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration))
+			stats.inserts.Add(databaseCreationBatchSize)
 
 			batchWriteStartTime := time.Now()
 
 			err = writeBatch()
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to write value in database : %v\n", err)
-				return err
+				return fmt.Errorf("unable to write value in database : %v", err)
 			}
 
 			batchWriteDuration := time.Since(batchWriteStartTime)
-			batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
-			batchCounter.Inc()
+			stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+			stats.batches.Inc()
 
 			startInsertBatchTime = time.Now()
 		}
 	}
+	// write the last batch. In our default case, there won't be a last batch
 	if databaseEntries%databaseCreationBatchSize != 0 {
 		err = writeBatch()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to write value in database : %v\n", err)
-			return err
+			return fmt.Errorf("unable to write value in database : %v", err)
 		}
 	}
+	// Note that without this commit, the final hash is not available
+	// TODO: figure out why this is necessary
+	err = trieDb.Commit(root, false)
+	if err != nil {
+		return fmt.Errorf("unable to commit trie : %v", err)
+	}
 	close(ticksCh)
 	fmt.Print(" done!\n")
 
-	err = trieDb.Close()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to close trie database : %v\n", err)
-		return err
+	if err = trieDb.Close(); err != nil {
+		return fmt.Errorf("unable to close trie database : %v", err)
 	}
-	err = ldb.Close()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to close levelDB database : %v\n", err)
-		return err
+
+	if err = ldb.Close(); err != nil {
+		return fmt.Errorf("unable to close levelDB database : %v", err)
 	}
 
 	fmt.Printf("Generated and inserted %d batches of size %d in %v\n",
 		databaseEntries/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime))
 
-	err = os.WriteFile(path.Join(stagingDirectory, "root.txt"), root.Bytes(), 0o644)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to save root : %v\n", err)
-		return err
+	if err = os.WriteFile(path.Join(stagingDirectory, "root.txt"), root.Bytes(), 0o644); err != nil {
+		return fmt.Errorf("unable to save root : %v", err)
 	}
 
 	err = os.Rename(getGoldenStagingDatabaseDirectory(databaseEntries), getGoldenDatabaseDirectory(databaseEntries))
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to rename golden staging directory : %v\n", err)
-		return err
+		return fmt.Errorf("unable to rename golden staging directory : %v", err)
 	}
 	fmt.Printf("Completed initialization with hash of %v\n", root.Hex())
 	return nil
@@ -251,19 +268,16 @@ func resetRunningDatabaseDirectory(databaseEntries uint64) error {
 	if _, err := os.Stat(runningDir); err == nil {
 		err := os.RemoveAll(runningDir)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err)
-			return err
+			return fmt.Errorf("unable to remove running directory : %v", err)
 		}
 	}
 	err := os.Mkdir(runningDir, 0o777)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to create running directory : %v\n", err)
-		return err
+		return fmt.Errorf("unable to create running directory : %v", err)
 	}
 	err = CopyDirectory(getGoldenDatabaseDirectory(databaseEntries), runningDir)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to duplicate golden directory : %v\n", err)
-		return err
+		return fmt.Errorf("unable to duplicate golden directory : %v", err)
 	}
 	return nil
 }
@@ -271,8 +285,7 @@ func resetRunningDatabaseDirectory(databaseEntries uint64) error {
 func runBenchmark(databaseEntries uint64) error {
 	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to read root : %v\n", err)
-		return err
+		return fmt.Errorf("unable to read root : %v", err)
 	}
 
 	ldb, err := rawdb.Open(rawdb.OpenOptions{
@@ -280,28 +293,26 @@ func runBenchmark(databaseEntries uint64) error {
 		Directory:         getRunningDatabaseDirectory(databaseEntries),
 		AncientsDirectory: "",
 		Namespace:         "metrics_prefix",
-		Cache:             levelDBCacheSize,
+		Cache:             levelDBCacheSizeMB,
 		Handles:           200,
 		ReadOnly:          false,
 		Ephemeral:         false,
 	})
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to create level db database : %v\n", err)
-		return err
+		return fmt.Errorf("unable to create level db database : %v", err)
 	}
 
 	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
 		Preimages: false,
 		IsVerkle:  false,
 		HashDB:    nil,
-		PathDB:    getPathDBConfig(),
+		PathDB:    &pathDBConfig,
 	})
 
 	parentHash := common.BytesToHash(rootBytes)
 	tdb, err := trie.New(trie.TrieID(parentHash), trieDb)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to create trie database : %v\n", err)
-		return err
+		return fmt.Errorf("unable to create trie database : %v", err)
 	}
 	var root common.Hash
 	blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize
@@ -310,18 +321,11 @@ func runBenchmark(databaseEntries uint64) error {
 		root, nodes = tdb.Commit(false)
 		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to update trie : %v\n", err)
-			return err
-		}
-		err = trieDb.Commit(root, false)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to commit trie : %v\n", err)
-			return err
+			return fmt.Errorf("unable to update trie : %v", err)
 		}
 		tdb, err = trie.New(trie.TrieID(root), trieDb)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to create new trie : %v\n", err)
-			return err
+			return fmt.Errorf("unable to create new trie : %v", err)
 		}
 		parentHash = root
 		blockHeight++
@@ -339,13 +343,12 @@ func runBenchmark(databaseEntries uint64) error {
 			entryHash := calculateIndexEncoding(keyToDeleteIdx)
 			err = tdb.Delete(entryHash)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to delete trie entry : %v\n", err)
-				return err
+				return fmt.Errorf("unable to delete trie entry : %v", err)
 			}
 		}
 		deleteDuration = time.Since(startDeleteTime)
-		deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration))
-		deletesCounter.Add(databaseRunningBatchSize)
+		stats.deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration))
+		stats.deletes.Add(databaseRunningBatchSize)
 
 		// add 2.5k past end.
 		startInsertTime := time.Now()
@@ -353,13 +356,12 @@ func runBenchmark(databaseEntries uint64) error {
 			entryHash := calculateIndexEncoding(keyToAddIdx)
 			err = tdb.Update(entryHash, entryHash)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to insert trie entry : %v\n", err)
-				return err
+				return fmt.Errorf("unable to insert trie entry : %v", err)
 			}
 		}
 		addDuration = time.Since(startInsertTime)
-		insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration))
-		insertsCounter.Add(databaseRunningBatchSize)
+		stats.insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration))
+		stats.inserts.Add(databaseRunningBatchSize)
 
 		// update middle 5k entries
 		startUpdateTime := time.Now()
@@ -368,23 +370,21 @@ func runBenchmark(databaseEntries uint64) error {
 			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
 			err = tdb.Update(updateEntryKey, updateEntryValue)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to update trie entry : %v\n", err)
-				return err
+				return fmt.Errorf("unable to update trie entry : %v", err)
 			}
 		}
 		updateDuration = time.Since(startUpdateTime)
-		updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
-		updatesCounter.Add(databaseRunningUpdateSize)
+		stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
+		stats.updates.Add(databaseRunningUpdateSize)
 
 		batchWriteStartTime := time.Now()
 		err = writeBatch()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to write batch : %v\n", err)
-			return err
+			return fmt.Errorf("unable to write batch : %v", err)
 		}
 		batchDuration = time.Since(startBatchTime)
 		batchWriteDuration := time.Since(batchWriteStartTime)
-		batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+		stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
 
 		if *verbose {
 			fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
@@ -394,36 +394,34 @@ func runBenchmark(databaseEntries uint64) error {
 				time.Second/batchDuration)
 		}
 
-		batchCounter.Inc()
+		stats.batches.Inc()
 		low += databaseRunningBatchSize
 	}
-
-	// return nil
 }
 
 func setupMetrics() error {
-	if err := prometheus.Register(promRegistry); err != nil {
+	if err := prometheus.Register(stats.registry); err != nil {
 		return err
 	}
-	promRegistry.MustRegister(insertsCounter)
-	promRegistry.MustRegister(deletesCounter)
-	promRegistry.MustRegister(updatesCounter)
-	promRegistry.MustRegister(batchCounter)
-	promRegistry.MustRegister(deleteRate)
-	promRegistry.MustRegister(updateRate)
-	promRegistry.MustRegister(insertRate)
-	promRegistry.MustRegister(batchWriteRate)
+	stats.registry.MustRegister(stats.inserts)
+	stats.registry.MustRegister(stats.deletes)
+	stats.registry.MustRegister(stats.updates)
+	stats.registry.MustRegister(stats.batches)
+	stats.registry.MustRegister(stats.deleteRate)
+	stats.registry.MustRegister(stats.updateRate)
+	stats.registry.MustRegister(stats.insertRate)
+	stats.registry.MustRegister(stats.batchWriteRate)
 
 	http.Handle("/metrics", promhttp.Handler())
 
-	server := &http.Server{
+	prometheusServer := &http.Server{
 		Addr:              fmt.Sprintf(":%d", *httpMetricPort),
 		ReadHeaderTimeout: 3 * time.Second,
 	}
 	go func() {
-		err := server.ListenAndServe()
+		err := prometheusServer.ListenAndServe()
 		if err != nil && err != http.ErrServerClosed {
-			fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err)
+			panic(fmt.Sprintf("unable to listen and serve : %v\n", err))
 		}
 	}()
 	return nil
@@ -432,25 +430,25 @@ func setupMetrics() error {
 func main() {
 	pflag.Parse()
 
-	if setupMetrics() != nil {
+	if err := setupMetrics(); err != nil {
+		fmt.Fprintf(os.Stderr, "unable to setup metrics : %v\n", err)
 		os.Exit(1)
-		return
 	}
 
 	goldenDir := getGoldenDatabaseDirectory(*databaseEntries)
 	if _, err := os.Stat(goldenDir); os.IsNotExist(err) {
 		// create golden image.
-		if createGoldenDatabase(*databaseEntries) != nil {
+		if err := createGoldenDatabase(*databaseEntries); err != nil {
+			fmt.Fprintf(os.Stderr, "unable to create golden database : %v\n", err)
 			os.Exit(1)
-			return
 		}
 	}
-	if resetRunningDatabaseDirectory(*databaseEntries) != nil {
+	if err := resetRunningDatabaseDirectory(*databaseEntries); err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to reset running database directory: %v\n", err)
 		os.Exit(1)
-		return
 	}
-	if runBenchmark(*databaseEntries) != nil {
+	if err := runBenchmark(*databaseEntries); err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err)
 		os.Exit(1)
-		return
 	}
 }
diff --git a/x/merkledb/benchmarks_eth/benchmark_test.go b/x/merkledb/benchmarks_eth/benchmark_test.go
index 50b4958fe48d..4b588b1a03db 100644
--- a/x/merkledb/benchmarks_eth/benchmark_test.go
+++ b/x/merkledb/benchmarks_eth/benchmark_test.go
@@ -8,6 +8,7 @@ import (
 	"path"
 	"testing"
 
+	"github.com/ava-labs/avalanchego/utils/units"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -18,6 +19,12 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var testPathDbConfig = pathdb.Config{
+	CleanCacheSize: units.MiB, // use a small cache for testing
+	DirtyCacheSize: 0,         // force changes to disk
+	ReadOnly:       false,     // allow writes
+}
+
 func TestGoldenDatabaseCreation(t *testing.T) {
 	for _, size := range []uint64{9500, 20000, 50000, 103011} {
 		testGoldenDatabaseCreation(t, size)
@@ -45,7 +52,7 @@ func testGoldenDatabaseContent(t *testing.T, size uint64) {
 		Directory:         getRunningDatabaseDirectory(size),
 		AncientsDirectory: "",
 		Namespace:         "metrics_prefix",
-		Cache:             levelDBCacheSize,
+		Cache:             levelDBCacheSizeMB,
 		Handles:           200,
 		ReadOnly:          false,
 		Ephemeral:         false,
@@ -56,7 +63,7 @@ func testGoldenDatabaseContent(t *testing.T, size uint64) {
 		Preimages: false,
 		IsVerkle:  false,
 		HashDB:    nil,
-		PathDB:    getPathDBConfig(),
+		PathDB:    &testPathDbConfig,
 	})
 
 	parentHash := common.BytesToHash(rootBytes)
@@ -92,7 +99,7 @@ func TestRevisions(t *testing.T) {
 	require.NoError(err)
 
 	// Create a new leveldb database
-	ldb, err := rawdb.NewLevelDBDatabase(dbPath, levelDBCacheSize, 200, "metrics_prefix", false)
+	ldb, err := rawdb.NewLevelDBDatabase(dbPath, levelDBCacheSizeMB, 200, "metrics_prefix", false)
 	require.NoError(err)
 
 	// Create a new trie database
@@ -103,10 +110,7 @@ func TestRevisions(t *testing.T) {
 			Preimages: false,
 			IsVerkle:  false,
 			HashDB:    nil,
-			PathDB: &pathdb.Config{
-				CleanCacheSize: 1024 * 1024,
-				DirtyCacheSize: 0, // Disable dirty cache to force trieDB to write to disk.
-			},
+			PathDB:    &testPathDbConfig,
 		},
 	)
 
diff --git a/x/merkledb/benchmarks_eth/go.mod b/x/merkledb/benchmarks_eth/go.mod
index 08b827dca96c..b0de373fc702 100644
--- a/x/merkledb/benchmarks_eth/go.mod
+++ b/x/merkledb/benchmarks_eth/go.mod
@@ -53,6 +53,7 @@ require (
 )
 
 require (
+	github.com/ava-labs/avalanchego v1.11.11
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/ethereum/go-ethereum v1.14.8
diff --git a/x/merkledb/benchmarks_eth/go.sum b/x/merkledb/benchmarks_eth/go.sum
index 18cff77e093f..22cb9c888244 100644
--- a/x/merkledb/benchmarks_eth/go.sum
+++ b/x/merkledb/benchmarks_eth/go.sum
@@ -4,6 +4,8 @@ github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjC
 github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/ava-labs/avalanchego v1.11.11 h1:MIQq8xRavRj4ZXHA4G+aMiymig7SOScGOG1SApmMvBc=
+github.com/ava-labs/avalanchego v1.11.11/go.mod h1:yFx3V31Jy9NFa8GZlgGnwiVf8KGjeF2+Uc99l9Scd/8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=

From 72b9d13755d701a0c311704de3d308f4bd2078bb Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Wed, 25 Sep 2024 23:52:44 -0400
Subject: [PATCH 32/42] fix lint

---
 x/merkledb/benchmarks/benchmark_test.go     | 7 ++++---
 x/merkledb/benchmarks_eth/benchmark_test.go | 3 +--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/x/merkledb/benchmarks/benchmark_test.go b/x/merkledb/benchmarks/benchmark_test.go
index 5352ae898064..6ebee9dd0693 100644
--- a/x/merkledb/benchmarks/benchmark_test.go
+++ b/x/merkledb/benchmarks/benchmark_test.go
@@ -8,12 +8,13 @@ import (
 	"os"
 	"testing"
 
-	"github.com/ava-labs/avalanchego/database/leveldb"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+
 	"github.com/ava-labs/avalanchego/database"
+	"github.com/ava-labs/avalanchego/database/leveldb"
 	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/ava-labs/avalanchego/x/merkledb"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/stretchr/testify/require"
 )
 
 func TestGoldenDatabaseCreation(t *testing.T) {
diff --git a/x/merkledb/benchmarks_eth/benchmark_test.go b/x/merkledb/benchmarks_eth/benchmark_test.go
index 4b588b1a03db..e8e658a98a05 100644
--- a/x/merkledb/benchmarks_eth/benchmark_test.go
+++ b/x/merkledb/benchmarks_eth/benchmark_test.go
@@ -95,8 +95,7 @@ func TestRevisions(t *testing.T) {
 	wd, _ := os.Getwd()
 	dbPath := path.Join(wd, "db-test-revisions")
 	require.NoError(os.RemoveAll(dbPath))
-	err := os.Mkdir(dbPath, 0o777)
-	require.NoError(err)
+	require.NoError(os.Mkdir(dbPath, 0o777))
 
 	// Create a new leveldb database
 	ldb, err := rawdb.NewLevelDBDatabase(dbPath, levelDBCacheSizeMB, 200, "metrics_prefix", false)

From b94e8f34dc1c7f571965ac5606a49322b89d60a3 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sat, 28 Sep 2024 16:08:20 -0400
Subject: [PATCH 33/42] update

---
 x/merkledb/benchmarks_eth/benchmark.go | 222 +++++++++++--------------
 x/merkledb/benchmarks_eth/classic.go   | 134 +++++++++++++++
 x/merkledb/benchmarks_eth/go.mod       |   2 +
 x/merkledb/benchmarks_eth/go.sum       |   6 +
 x/merkledb/benchmarks_eth/zipf.go      | 110 ++++++++++++
 5 files changed, 349 insertions(+), 125 deletions(-)
 create mode 100644 x/merkledb/benchmarks_eth/classic.go
 create mode 100644 x/merkledb/benchmarks_eth/zipf.go

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 8a4a4584ab64..d78ca1b82692 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -15,7 +15,7 @@ import (
 	"github.com/ava-labs/avalanchego/utils/units"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/spf13/pflag"
+	"github.com/spf13/cobra"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -27,12 +27,14 @@ import (
 )
 
 const (
-	defaultDatabaseEntries       = 1_000_000_000
+	defaultDatabaseEntries       = 0
 	databaseCreationBatchSize    = 10_000
 	databaseRunningBatchSize     = databaseCreationBatchSize / 4
 	databaseRunningUpdateSize    = databaseCreationBatchSize / 2
 	defaultMetricsPort           = 3_000
 	benchmarkRevisionHistorySize = 128
+	defaultZipfS                 = float64(1.01)
+	defaultZipfV                 = float64(2.7)
 )
 
 // TODO: Adjust these cache sizes for maximum performance
@@ -107,11 +109,13 @@ var stats = struct {
 	registry: prometheus.NewRegistry(),
 }
 
-// command line arguments
+// command line arguments parameters
 var (
-	databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries")
-	httpMetricPort  = pflag.Uint64("p", defaultMetricsPort, "default metrics port")
-	verbose         = pflag.Bool("v", false, "verbose")
+	databaseEntries *uint64
+	httpMetricPort  *uint64
+	verbose         *bool
+	sZipf           *float64
+	vZipf           *float64
 )
 
 func Must[T any](obj T, err error) T {
@@ -282,123 +286,6 @@ func resetRunningDatabaseDirectory(databaseEntries uint64) error {
 	return nil
 }
 
-func runBenchmark(databaseEntries uint64) error {
-	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
-	if err != nil {
-		return fmt.Errorf("unable to read root : %v", err)
-	}
-
-	ldb, err := rawdb.Open(rawdb.OpenOptions{
-		Type:              "leveldb",
-		Directory:         getRunningDatabaseDirectory(databaseEntries),
-		AncientsDirectory: "",
-		Namespace:         "metrics_prefix",
-		Cache:             levelDBCacheSizeMB,
-		Handles:           200,
-		ReadOnly:          false,
-		Ephemeral:         false,
-	})
-	if err != nil {
-		return fmt.Errorf("unable to create level db database : %v", err)
-	}
-
-	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
-		Preimages: false,
-		IsVerkle:  false,
-		HashDB:    nil,
-		PathDB:    &pathDBConfig,
-	})
-
-	parentHash := common.BytesToHash(rootBytes)
-	tdb, err := trie.New(trie.TrieID(parentHash), trieDb)
-	if err != nil {
-		return fmt.Errorf("unable to create trie database : %v", err)
-	}
-	var root common.Hash
-	blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize
-	writeBatch := func() error {
-		var nodes *trienode.NodeSet
-		root, nodes = tdb.Commit(false)
-		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
-		if err != nil {
-			return fmt.Errorf("unable to update trie : %v", err)
-		}
-		tdb, err = trie.New(trie.TrieID(root), trieDb)
-		if err != nil {
-			return fmt.Errorf("unable to create new trie : %v", err)
-		}
-		parentHash = root
-		blockHeight++
-		return nil
-	}
-
-	low := uint64(0)
-	var deleteDuration, addDuration, updateDuration, batchDuration time.Duration
-	for {
-		startBatchTime := time.Now()
-
-		// delete first 2.5k keys from the beginning
-		startDeleteTime := time.Now()
-		for keyToDeleteIdx := low; keyToDeleteIdx < low+databaseRunningBatchSize; keyToDeleteIdx++ {
-			entryHash := calculateIndexEncoding(keyToDeleteIdx)
-			err = tdb.Delete(entryHash)
-			if err != nil {
-				return fmt.Errorf("unable to delete trie entry : %v", err)
-			}
-		}
-		deleteDuration = time.Since(startDeleteTime)
-		stats.deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration))
-		stats.deletes.Add(databaseRunningBatchSize)
-
-		// add 2.5k past end.
-		startInsertTime := time.Now()
-		for keyToAddIdx := low + databaseEntries; keyToAddIdx < low+databaseEntries+databaseRunningBatchSize; keyToAddIdx++ {
-			entryHash := calculateIndexEncoding(keyToAddIdx)
-			err = tdb.Update(entryHash, entryHash)
-			if err != nil {
-				return fmt.Errorf("unable to insert trie entry : %v", err)
-			}
-		}
-		addDuration = time.Since(startInsertTime)
-		stats.insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration))
-		stats.inserts.Add(databaseRunningBatchSize)
-
-		// update middle 5k entries
-		startUpdateTime := time.Now()
-		updateEntryValue := calculateIndexEncoding(low)
-		for keyToUpdateIdx := low + (databaseEntries / 2); keyToUpdateIdx < low+(databaseEntries/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
-			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
-			err = tdb.Update(updateEntryKey, updateEntryValue)
-			if err != nil {
-				return fmt.Errorf("unable to update trie entry : %v", err)
-			}
-		}
-		updateDuration = time.Since(startUpdateTime)
-		stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
-		stats.updates.Add(databaseRunningUpdateSize)
-
-		batchWriteStartTime := time.Now()
-		err = writeBatch()
-		if err != nil {
-			return fmt.Errorf("unable to write batch : %v", err)
-		}
-		batchDuration = time.Since(startBatchTime)
-		batchWriteDuration := time.Since(batchWriteStartTime)
-		stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
-
-		if *verbose {
-			fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
-				time.Second/deleteDuration,
-				time.Second/updateDuration,
-				time.Second/addDuration,
-				time.Second/batchDuration)
-		}
-
-		stats.batches.Inc()
-		low += databaseRunningBatchSize
-	}
-}
-
 func setupMetrics() error {
 	if err := prometheus.Register(stats.registry); err != nil {
 		return err
@@ -428,13 +315,75 @@ func setupMetrics() error {
 }
 
 func main() {
-	pflag.Parse()
+	rootCmd := &cobra.Command{
+		Use:   "benchmarks_eth",
+		Short: "benchmarks_eth",
+		RunE: func(*cobra.Command, []string) error {
+			fmt.Printf("Please specify which type of benchmark you'd like to run : [classic, zipf]\n")
+			return nil
+		},
+	}
+	databaseEntries = rootCmd.PersistentFlags().Uint64("n", defaultDatabaseEntries, "Number of database entries in golden database")
+	httpMetricPort = rootCmd.PersistentFlags().Uint64("p", defaultMetricsPort, "HTTP metrics port")
+	verbose = rootCmd.PersistentFlags().Bool("verbose", false, "Verbose mode")
+	rootCmd.MarkPersistentFlagRequired("n")
+
+	classicCmd := &cobra.Command{
+		Use:   "classic",
+		Short: "Run the classic benchmark",
+		RunE: func(*cobra.Command, []string) error {
+			classicBenchmark()
+			return nil
+		},
+	}
+	zipfCmd := &cobra.Command{
+		Use:   "zipf",
+		Short: "Run the zipf benchmark",
+		RunE: func(*cobra.Command, []string) error {
+			zipfBenchmark()
+			return nil
+		},
+	}
+	sZipf = zipfCmd.PersistentFlags().Float64("s", defaultZipfS, "s (Zipf distribution = [(v+k)^(-s)], Default = 1.00)")
+	vZipf = zipfCmd.PersistentFlags().Float64("v", defaultZipfV, "v (Zipf distribution = [(v+k)^(-s)], Default = 2.7)")
+	miniCmd := &cobra.Command{
+		Use:   "mini",
+		Short: "Run the mini benchmark",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if *sZipf <= 1.0 {
+				fmt.Fprintf(os.Stderr, "The s value for Zipf distribution must be greater than 1")
+				os.Exit(1)
+			}
+			if *vZipf < 1.0 {
+				fmt.Fprintf(os.Stderr, "The v value for Zipf distribution must be greater or equal to 1")
+				os.Exit(1)
+			}
+		},
+		RunE: func(*cobra.Command, []string) error {
+			miniBenchmark()
+			return nil
+		},
+	}
+
+	rootCmd.AddCommand(classicCmd, zipfCmd, miniCmd)
 
 	if err := setupMetrics(); err != nil {
 		fmt.Fprintf(os.Stderr, "unable to setup metrics : %v\n", err)
 		os.Exit(1)
 	}
 
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "tmpnetctl failed: %v\n", err)
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func commonBenchmarkBootstrap() {
+	if *databaseEntries == 0 {
+		fmt.Fprintf(os.Stderr, "Please specify number of database entries by using the --n flag\n")
+		os.Exit(1)
+	}
 	goldenDir := getGoldenDatabaseDirectory(*databaseEntries)
 	if _, err := os.Stat(goldenDir); os.IsNotExist(err) {
 		// create golden image.
@@ -447,8 +396,31 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Unable to reset running database directory: %v\n", err)
 		os.Exit(1)
 	}
-	if err := runBenchmark(*databaseEntries); err != nil {
+}
+
+func classicBenchmark() {
+	commonBenchmarkBootstrap()
+
+	if err := runClassicBenchmark(*databaseEntries); err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func zipfBenchmark() {
+	commonBenchmarkBootstrap()
+
+	if err := runZipfBenchmark(*databaseEntries, *sZipf, *vZipf); err != nil {
 		fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err)
 		os.Exit(1)
 	}
 }
+
+func miniBenchmark() {
+	commonBenchmarkBootstrap()
+
+	/*if err := runMiniBenchmark(*databaseEntries); err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err)
+		os.Exit(1)
+	}*/
+}
diff --git a/x/merkledb/benchmarks_eth/classic.go b/x/merkledb/benchmarks_eth/classic.go
new file mode 100644
index 000000000000..2cfd3264414d
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/classic.go
@@ -0,0 +1,134 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-ethereum/triedb"
+)
+
+func runClassicBenchmark(databaseEntries uint64) error {
+	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
+	if err != nil {
+		return fmt.Errorf("unable to read root : %v", err)
+	}
+
+	ldb, err := rawdb.Open(rawdb.OpenOptions{
+		Type:              "leveldb",
+		Directory:         getRunningDatabaseDirectory(databaseEntries),
+		AncientsDirectory: "",
+		Namespace:         "metrics_prefix",
+		Cache:             levelDBCacheSizeMB,
+		Handles:           200,
+		ReadOnly:          false,
+		Ephemeral:         false,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to create level db database : %v", err)
+	}
+
+	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
+		Preimages: false,
+		IsVerkle:  false,
+		HashDB:    nil,
+		PathDB:    &pathDBConfig,
+	})
+
+	parentHash := common.BytesToHash(rootBytes)
+	tdb, err := trie.New(trie.TrieID(parentHash), trieDb)
+	if err != nil {
+		return fmt.Errorf("unable to create trie database : %v", err)
+	}
+	var root common.Hash
+	blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize
+	writeBatch := func() error {
+		var nodes *trienode.NodeSet
+		root, nodes = tdb.Commit(false)
+		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		if err != nil {
+			return fmt.Errorf("unable to update trie : %v", err)
+		}
+		tdb, err = trie.New(trie.TrieID(root), trieDb)
+		if err != nil {
+			return fmt.Errorf("unable to create new trie : %v", err)
+		}
+		parentHash = root
+		blockHeight++
+		return nil
+	}
+
+	low := uint64(0)
+	var deleteDuration, addDuration, updateDuration, batchDuration time.Duration
+	for {
+		startBatchTime := time.Now()
+
+		// delete first 2.5k keys from the beginning
+		startDeleteTime := time.Now()
+		for keyToDeleteIdx := low; keyToDeleteIdx < low+databaseRunningBatchSize; keyToDeleteIdx++ {
+			entryHash := calculateIndexEncoding(keyToDeleteIdx)
+			err = tdb.Delete(entryHash)
+			if err != nil {
+				return fmt.Errorf("unable to delete trie entry : %v", err)
+			}
+		}
+		deleteDuration = time.Since(startDeleteTime)
+		stats.deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration))
+		stats.deletes.Add(databaseRunningBatchSize)
+
+		// add 2.5k past end.
+		startInsertTime := time.Now()
+		for keyToAddIdx := low + databaseEntries; keyToAddIdx < low+databaseEntries+databaseRunningBatchSize; keyToAddIdx++ {
+			entryHash := calculateIndexEncoding(keyToAddIdx)
+			err = tdb.Update(entryHash, entryHash)
+			if err != nil {
+				return fmt.Errorf("unable to insert trie entry : %v", err)
+			}
+		}
+		addDuration = time.Since(startInsertTime)
+		stats.insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration))
+		stats.inserts.Add(databaseRunningBatchSize)
+
+		// update middle 5k entries
+		startUpdateTime := time.Now()
+		updateEntryValue := calculateIndexEncoding(low)
+		for keyToUpdateIdx := low + (databaseEntries / 2); keyToUpdateIdx < low+(databaseEntries/2)+databaseRunningUpdateSize; keyToUpdateIdx++ {
+			updateEntryKey := calculateIndexEncoding(keyToUpdateIdx)
+			err = tdb.Update(updateEntryKey, updateEntryValue)
+			if err != nil {
+				return fmt.Errorf("unable to update trie entry : %v", err)
+			}
+		}
+		updateDuration = time.Since(startUpdateTime)
+		stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
+		stats.updates.Add(databaseRunningUpdateSize)
+
+		batchWriteStartTime := time.Now()
+		err = writeBatch()
+		if err != nil {
+			return fmt.Errorf("unable to write batch : %v", err)
+		}
+		batchDuration = time.Since(startBatchTime)
+		batchWriteDuration := time.Since(batchWriteStartTime)
+		stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+
+		if *verbose {
+			fmt.Printf("delete rate [%d]	update rate [%d]	insert rate [%d]	batch rate [%d]\n",
+				time.Second/deleteDuration,
+				time.Second/updateDuration,
+				time.Second/addDuration,
+				time.Second/batchDuration)
+		}
+
+		stats.batches.Inc()
+		low += databaseRunningBatchSize
+	}
+}
diff --git a/x/merkledb/benchmarks_eth/go.mod b/x/merkledb/benchmarks_eth/go.mod
index b0de373fc702..391cd7aa789c 100644
--- a/x/merkledb/benchmarks_eth/go.mod
+++ b/x/merkledb/benchmarks_eth/go.mod
@@ -4,6 +4,7 @@ go 1.22.3
 
 require (
 	github.com/prometheus/client_golang v1.20.3
+	github.com/spf13/cobra v1.5.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 )
@@ -32,6 +33,7 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/holiman/uint256 v1.3.1 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
diff --git a/x/merkledb/benchmarks_eth/go.sum b/x/merkledb/benchmarks_eth/go.sum
index 22cb9c888244..0f99057278f4 100644
--- a/x/merkledb/benchmarks_eth/go.sum
+++ b/x/merkledb/benchmarks_eth/go.sum
@@ -38,6 +38,7 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj
 github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I=
 github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI=
@@ -97,6 +98,8 @@ github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs
 github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
@@ -153,8 +156,11 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
 github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
diff --git a/x/merkledb/benchmarks_eth/zipf.go b/x/merkledb/benchmarks_eth/zipf.go
new file mode 100644
index 000000000000..465087b80551
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/zipf.go
@@ -0,0 +1,110 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-ethereum/triedb"
+)
+
+func runZipfBenchmark(databaseEntries uint64, sZipf float64, vZipf float64) error {
+	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
+	if err != nil {
+		return fmt.Errorf("unable to read root : %v", err)
+	}
+
+	ldb, err := rawdb.Open(rawdb.OpenOptions{
+		Type:              "leveldb",
+		Directory:         getRunningDatabaseDirectory(databaseEntries),
+		AncientsDirectory: "",
+		Namespace:         "metrics_prefix",
+		Cache:             levelDBCacheSizeMB,
+		Handles:           200,
+		ReadOnly:          false,
+		Ephemeral:         false,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to create level db database : %v", err)
+	}
+
+	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
+		Preimages: false,
+		IsVerkle:  false,
+		HashDB:    nil,
+		PathDB:    &pathDBConfig,
+	})
+
+	parentHash := common.BytesToHash(rootBytes)
+	tdb, err := trie.New(trie.TrieID(parentHash), trieDb)
+	if err != nil {
+		return fmt.Errorf("unable to create trie database : %v", err)
+	}
+	var root common.Hash
+	blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize
+	writeBatch := func() error {
+		var nodes *trienode.NodeSet
+		root, nodes = tdb.Commit(false)
+		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		if err != nil {
+			return fmt.Errorf("unable to update trie : %v", err)
+		}
+		tdb, err = trie.New(trie.TrieID(root), trieDb)
+		if err != nil {
+			return fmt.Errorf("unable to create new trie : %v", err)
+		}
+		parentHash = root
+		blockHeight++
+		return nil
+	}
+
+	batchIndex := uint64(0)
+	var updateDuration, batchDuration time.Duration
+
+	zipf := rand.NewZipf(rand.New(rand.NewSource(0)), sZipf, vZipf, databaseEntries)
+	for {
+		startBatchTime := time.Now()
+
+		// update 10k entries
+		startUpdateTime := time.Now()
+		updateEntryValue := calculateIndexEncoding(batchIndex)
+
+		for keyToUpdateIdx := 0; keyToUpdateIdx < databaseCreationBatchSize; keyToUpdateIdx++ {
+			updateEntryKey := calculateIndexEncoding(zipf.Uint64())
+			err = tdb.Update(updateEntryKey, updateEntryValue)
+			if err != nil {
+				return fmt.Errorf("unable to update trie entry : %v", err)
+			}
+		}
+		updateDuration = time.Since(startUpdateTime)
+		stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
+		stats.updates.Add(databaseRunningUpdateSize)
+
+		batchWriteStartTime := time.Now()
+		err = writeBatch()
+		if err != nil {
+			return fmt.Errorf("unable to write batch : %v", err)
+		}
+		batchDuration = time.Since(startBatchTime)
+		batchWriteDuration := time.Since(batchWriteStartTime)
+		stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+
+		if *verbose {
+			fmt.Printf("update rate [%d] batch rate [%d]\n",
+				time.Second/updateDuration,
+				time.Second/batchDuration)
+		}
+
+		stats.batches.Inc()
+		batchIndex++
+	}
+}

From 2485f98b05dcae45a447f2921ffe3b9c7b6cdac2 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sat, 28 Sep 2024 16:25:25 -0400
Subject: [PATCH 34/42] update

---
 x/merkledb/benchmarks_eth/benchmark.go |  26 +++---
 x/merkledb/benchmarks_eth/mini.go      | 107 +++++++++++++++++++++++++
 2 files changed, 120 insertions(+), 13 deletions(-)
 create mode 100644 x/merkledb/benchmarks_eth/mini.go

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index d78ca1b82692..24957177edfa 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -33,7 +33,7 @@ const (
 	databaseRunningUpdateSize    = databaseCreationBatchSize / 2
 	defaultMetricsPort           = 3_000
 	benchmarkRevisionHistorySize = 128
-	defaultZipfS                 = float64(1.01)
+	defaultZipfS                 = float64(1.2)
 	defaultZipfV                 = float64(2.7)
 )
 
@@ -339,6 +339,16 @@ func main() {
 	zipfCmd := &cobra.Command{
 		Use:   "zipf",
 		Short: "Run the zipf benchmark",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if *sZipf <= 1.0 {
+				fmt.Fprintf(os.Stderr, "The s value for Zipf distribution must be greater than 1\n")
+				os.Exit(1)
+			}
+			if *vZipf < 1.0 {
+				fmt.Fprintf(os.Stderr, "The v value for Zipf distribution must be greater or equal to 1\n")
+				os.Exit(1)
+			}
+		},
 		RunE: func(*cobra.Command, []string) error {
 			zipfBenchmark()
 			return nil
@@ -349,16 +359,6 @@ func main() {
 	miniCmd := &cobra.Command{
 		Use:   "mini",
 		Short: "Run the mini benchmark",
-		PreRun: func(cmd *cobra.Command, args []string) {
-			if *sZipf <= 1.0 {
-				fmt.Fprintf(os.Stderr, "The s value for Zipf distribution must be greater than 1")
-				os.Exit(1)
-			}
-			if *vZipf < 1.0 {
-				fmt.Fprintf(os.Stderr, "The v value for Zipf distribution must be greater or equal to 1")
-				os.Exit(1)
-			}
-		},
 		RunE: func(*cobra.Command, []string) error {
 			miniBenchmark()
 			return nil
@@ -419,8 +419,8 @@ func zipfBenchmark() {
 func miniBenchmark() {
 	commonBenchmarkBootstrap()
 
-	/*if err := runMiniBenchmark(*databaseEntries); err != nil {
+	if err := runMiniBenchmark(*databaseEntries); err != nil {
 		fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err)
 		os.Exit(1)
-	}*/
+	}
 }
diff --git a/x/merkledb/benchmarks_eth/mini.go b/x/merkledb/benchmarks_eth/mini.go
new file mode 100644
index 000000000000..f6df71022d01
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/mini.go
@@ -0,0 +1,107 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-ethereum/triedb"
+)
+
+func runMiniBenchmark(databaseEntries uint64) error {
+	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
+	if err != nil {
+		return fmt.Errorf("unable to read root : %v", err)
+	}
+
+	ldb, err := rawdb.Open(rawdb.OpenOptions{
+		Type:              "leveldb",
+		Directory:         getRunningDatabaseDirectory(databaseEntries),
+		AncientsDirectory: "",
+		Namespace:         "metrics_prefix",
+		Cache:             levelDBCacheSizeMB,
+		Handles:           200,
+		ReadOnly:          false,
+		Ephemeral:         false,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to create level db database : %v", err)
+	}
+
+	trieDb := triedb.NewDatabase(ldb, &triedb.Config{
+		Preimages: false,
+		IsVerkle:  false,
+		HashDB:    nil,
+		PathDB:    &pathDBConfig,
+	})
+
+	parentHash := common.BytesToHash(rootBytes)
+	tdb, err := trie.New(trie.TrieID(parentHash), trieDb)
+	if err != nil {
+		return fmt.Errorf("unable to create trie database : %v", err)
+	}
+	var root common.Hash
+	blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize
+	writeBatch := func() error {
+		var nodes *trienode.NodeSet
+		root, nodes = tdb.Commit(false)
+		err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/)
+		if err != nil {
+			return fmt.Errorf("unable to update trie : %v", err)
+		}
+		tdb, err = trie.New(trie.TrieID(root), trieDb)
+		if err != nil {
+			return fmt.Errorf("unable to create new trie : %v", err)
+		}
+		parentHash = root
+		blockHeight++
+		return nil
+	}
+
+	rand := rand.New(rand.NewSource(0))
+	batchIdx := uint64(0)
+	var updateDuration, batchDuration time.Duration
+	for {
+		startBatchTime := time.Now()
+
+		// update a single entry, at random.
+		startUpdateTime := time.Now()
+		keyToUpdateIdx := rand.Uint64() % databaseEntries
+		updateEntryValue := calculateIndexEncoding(batchIdx)
+		err = tdb.Update(calculateIndexEncoding(keyToUpdateIdx), updateEntryValue)
+		if err != nil {
+			return fmt.Errorf("unable to update trie entry : %v", err)
+		}
+
+		updateDuration = time.Since(startUpdateTime)
+		stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration))
+		stats.updates.Add(databaseRunningUpdateSize)
+
+		batchWriteStartTime := time.Now()
+		err = writeBatch()
+		if err != nil {
+			return fmt.Errorf("unable to write batch : %v", err)
+		}
+		batchDuration = time.Since(startBatchTime)
+		batchWriteDuration := time.Since(batchWriteStartTime)
+		stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
+
+		if *verbose {
+			fmt.Printf("update rate [%d]	batch rate [%d]\n",
+				time.Second/updateDuration,
+				time.Second/batchDuration)
+		}
+
+		stats.batches.Inc()
+		batchIdx++
+	}
+}

From b2aa0090958f48141b0069a07eecc8a34b15b42d Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sat, 28 Sep 2024 17:06:54 -0400
Subject: [PATCH 35/42] update

---
 x/merkledb/benchmarks_eth/benchmark.go        | 28 +++++++++----------
 .../benchmarks_eth/{mini.go => single.go}     | 10 +++----
 .../{classic.go => tenkrandom.go}             |  2 +-
 x/merkledb/benchmarks_eth/zipf_test.go        | 24 ++++++++++++++++
 4 files changed, 43 insertions(+), 21 deletions(-)
 rename x/merkledb/benchmarks_eth/{mini.go => single.go} (90%)
 rename x/merkledb/benchmarks_eth/{classic.go => tenkrandom.go} (98%)
 create mode 100644 x/merkledb/benchmarks_eth/zipf_test.go

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 24957177edfa..600f23a8600c 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -319,7 +319,7 @@ func main() {
 		Use:   "benchmarks_eth",
 		Short: "benchmarks_eth",
 		RunE: func(*cobra.Command, []string) error {
-			fmt.Printf("Please specify which type of benchmark you'd like to run : [classic, zipf]\n")
+			fmt.Printf("Please specify which type of benchmark you'd like to run : [tenkrandom, zipf, single]\n")
 			return nil
 		},
 	}
@@ -328,11 +328,11 @@ func main() {
 	verbose = rootCmd.PersistentFlags().Bool("verbose", false, "Verbose mode")
 	rootCmd.MarkPersistentFlagRequired("n")
 
-	classicCmd := &cobra.Command{
-		Use:   "classic",
-		Short: "Run the classic benchmark",
+	tenkrandomCmd := &cobra.Command{
+		Use:   "tenkrandom",
+		Short: "Run the tenkrandom benchmark",
 		RunE: func(*cobra.Command, []string) error {
-			classicBenchmark()
+			tenkrandomBenchmark()
 			return nil
 		},
 	}
@@ -356,16 +356,16 @@ func main() {
 	}
 	sZipf = zipfCmd.PersistentFlags().Float64("s", defaultZipfS, "s (Zipf distribution = [(v+k)^(-s)], Default = 1.00)")
 	vZipf = zipfCmd.PersistentFlags().Float64("v", defaultZipfV, "v (Zipf distribution = [(v+k)^(-s)], Default = 2.7)")
-	miniCmd := &cobra.Command{
-		Use:   "mini",
-		Short: "Run the mini benchmark",
+	singleCmd := &cobra.Command{
+		Use:   "single",
+		Short: "Run the single benchmark",
 		RunE: func(*cobra.Command, []string) error {
-			miniBenchmark()
+			singleBenchmark()
 			return nil
 		},
 	}
 
-	rootCmd.AddCommand(classicCmd, zipfCmd, miniCmd)
+	rootCmd.AddCommand(tenkrandomCmd, zipfCmd, singleCmd)
 
 	if err := setupMetrics(); err != nil {
 		fmt.Fprintf(os.Stderr, "unable to setup metrics : %v\n", err)
@@ -398,10 +398,10 @@ func commonBenchmarkBootstrap() {
 	}
 }
 
-func classicBenchmark() {
+func tenkrandomBenchmark() {
 	commonBenchmarkBootstrap()
 
-	if err := runClassicBenchmark(*databaseEntries); err != nil {
+	if err := runTenkrandomBenchmark(*databaseEntries); err != nil {
 		fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err)
 		os.Exit(1)
 	}
@@ -416,10 +416,10 @@ func zipfBenchmark() {
 	}
 }
 
-func miniBenchmark() {
+func singleBenchmark() {
 	commonBenchmarkBootstrap()
 
-	if err := runMiniBenchmark(*databaseEntries); err != nil {
+	if err := runSingleBenchmark(*databaseEntries); err != nil {
 		fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err)
 		os.Exit(1)
 	}
diff --git a/x/merkledb/benchmarks_eth/mini.go b/x/merkledb/benchmarks_eth/single.go
similarity index 90%
rename from x/merkledb/benchmarks_eth/mini.go
rename to x/merkledb/benchmarks_eth/single.go
index f6df71022d01..58c2460f8881 100644
--- a/x/merkledb/benchmarks_eth/mini.go
+++ b/x/merkledb/benchmarks_eth/single.go
@@ -4,8 +4,8 @@
 package main
 
 import (
+	"encoding/binary"
 	"fmt"
-	"math/rand"
 	"os"
 	"path"
 	"time"
@@ -17,7 +17,7 @@ import (
 	"github.com/ethereum/go-ethereum/triedb"
 )
 
-func runMiniBenchmark(databaseEntries uint64) error {
+func runSingleBenchmark(databaseEntries uint64) error {
 	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
 	if err != nil {
 		return fmt.Errorf("unable to read root : %v", err)
@@ -67,17 +67,15 @@ func runMiniBenchmark(databaseEntries uint64) error {
 		return nil
 	}
 
-	rand := rand.New(rand.NewSource(0))
 	batchIdx := uint64(0)
+	updateEntryKey := calculateIndexEncoding(0)
 	var updateDuration, batchDuration time.Duration
 	for {
 		startBatchTime := time.Now()
 
 		// update a single entry, at random.
 		startUpdateTime := time.Now()
-		keyToUpdateIdx := rand.Uint64() % databaseEntries
-		updateEntryValue := calculateIndexEncoding(batchIdx)
-		err = tdb.Update(calculateIndexEncoding(keyToUpdateIdx), updateEntryValue)
+		err = tdb.Update(updateEntryKey, binary.BigEndian.AppendUint64(nil, batchIdx))
 		if err != nil {
 			return fmt.Errorf("unable to update trie entry : %v", err)
 		}
diff --git a/x/merkledb/benchmarks_eth/classic.go b/x/merkledb/benchmarks_eth/tenkrandom.go
similarity index 98%
rename from x/merkledb/benchmarks_eth/classic.go
rename to x/merkledb/benchmarks_eth/tenkrandom.go
index 2cfd3264414d..3e679898d961 100644
--- a/x/merkledb/benchmarks_eth/classic.go
+++ b/x/merkledb/benchmarks_eth/tenkrandom.go
@@ -16,7 +16,7 @@ import (
 	"github.com/ethereum/go-ethereum/triedb"
 )
 
-func runClassicBenchmark(databaseEntries uint64) error {
+func runTenkrandomBenchmark(databaseEntries uint64) error {
 	rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt"))
 	if err != nil {
 		return fmt.Errorf("unable to read root : %v", err)
diff --git a/x/merkledb/benchmarks_eth/zipf_test.go b/x/merkledb/benchmarks_eth/zipf_test.go
new file mode 100644
index 000000000000..e18ec167c5a8
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/zipf_test.go
@@ -0,0 +1,24 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package main
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func TestZipfDist(t *testing.T) {
+	dbSize := uint64(1000000)
+	chunkSize := 10000
+	for _, sVal := range []float64{1.01, 1.1, 1.2, 1.3, 1.4} {
+		for _, vVal := range []float64{1.0} {
+			zipf := rand.NewZipf(rand.New(rand.NewSource(0)), sVal, vVal, dbSize)
+			samples := make(map[uint64]uint64)
+			for i := 0; i < chunkSize; i++ {
+				samples[zipf.Uint64()]++
+			}
+			t.Logf("number of unique keys for sVal = %v and vVal = %v is %d when sampling %d out of %d\n", sVal, vVal, len(samples), chunkSize, dbSize)
+		}
+	}
+}

From ceada1661b484875ee90a9bf2b3a5b67c533f50d Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sat, 28 Sep 2024 17:18:00 -0400
Subject: [PATCH 36/42] update

---
 x/merkledb/benchmarks_eth/setup.sh | 81 ++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 x/merkledb/benchmarks_eth/setup.sh

diff --git a/x/merkledb/benchmarks_eth/setup.sh b/x/merkledb/benchmarks_eth/setup.sh
new file mode 100644
index 000000000000..32bad58a9532
--- /dev/null
+++ b/x/merkledb/benchmarks_eth/setup.sh
@@ -0,0 +1,81 @@
+#### run these commands from the root user ####
+
+mkdir -p /etc/apt/keyrings/
+wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null
+echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
+apt-get update
+apt-get upgrade -y
+  
+mkdir -p /etc/systemd/system/grafana-server.service.d
+cat > /etc/systemd/system/grafana-server.service.d/override.conf <<!
+[Service]
+# Give the CAP_NET_BIND_SERVICE capability
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+  
+# A private user cannot have process capabilities on the host's user
+# namespace and thus CAP_NET_BIND_SERVICE has no effect.
+PrivateUsers=false
+!
+  
+apt install -y git protobuf-compiler build-essential apt-transport-https grafana prometheus net-tools
+  
+perl -pi -e 's/^;?http_port = .*/http_port = 80/' /etc/grafana/grafana.ini
+cat >> /etc/prometheus/prometheus.yml <<!
+  - job_name: firewood
+    static_configs:
+      - targets: ['localhost:3000']
+!
+cat >> /etc/default/prometheus-node-exporter <<!
+ARGS="--collector.filesystem.mount-points-exclude=\"^/(dev|proc|run|sys|media|var/lib/docker/.+)($|/)\""
+!
+
+systemctl daemon-reload
+killall grafana-server
+systemctl start grafana-server
+systemctl enable grafana-server.service
+systemctl restart prometheus
+  
+NVME_DEV="$(realpath /dev/disk/by-id/nvme-Amazon_EC2_NVMe_Instance_Storage_* | uniq)"
+if [ -n "$NVME_DEV" ]; then
+  mkfs.ext4 -E nodiscard -i 6291456 "$NVME_DEV"
+  NVME_MOUNT=/mnt/nvme
+  mkdir -p "$NVME_MOUNT"
+  mount -o noatime "$NVME_DEV" "$NVME_MOUNT"
+  echo "$NVME_DEV $NVME_MOUNT ext4 noatime 0 0" >> /etc/fstab
+  mkdir -p "$NVME_MOUNT/ubuntu/firewood"
+  chown ubuntu:ubuntu "$NVME_MOUNT/ubuntu" "$NVME_MOUNT/ubuntu/firewood"
+  ln -s "$NVME_MOUNT/ubuntu/firewood" /home/ubuntu/firewood
+fi
+
+## install go
+export GO_INSTALLATION_FILE=go1.23.1.linux-amd64.tar.gz
+curl -OL https://go.dev/dl/${GO_INSTALLATION_FILE}
+rm -rf /usr/local/go && tar -C /usr/local -xzf ${GO_INSTALLATION_FILE}
+rm ${GO_INSTALLATION_FILE}
+
+#### you can switch to the ubuntu user here ####
+
+
+git clone https://github.com/ava-labs/avalanchego.git
+cd avalanchego
+git checkout tsachi/bench_merkledb
+export PATH=$PATH:/usr/local/go/bin
+go build ./x/merkledb/benchmarks_eth
+
+#### stop here, these commands are run by hand ####
+
+# 10M rows:
+nohup benchmarks_eth tenkrandom --n 10000000 > output.txt & 
+nohup benchmarks_eth zipf --n 10000000 > output.txt &
+nohup benchmarks_eth single --n 10000000 > output.txt &
+
+# 50M rows:
+nohup benchmarks_eth tenkrandom --n 50000000 > output.txt & 
+nohup benchmarks_eth zipf --n 50000000 > output.txt &
+nohup benchmarks_eth single --n 50000000 > output.txt &
+
+# 100M rows:
+nohup benchmarks_eth tenkrandom --n 100000000 > output.txt & 
+nohup benchmarks_eth zipf --n 100000000 > output.txt &
+nohup benchmarks_eth single --n 100000000 > output.txt &
\ No newline at end of file

From 9ff9dbe0d8d00101ef1a548f89f6a7a715078f11 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sat, 28 Sep 2024 18:13:50 -0400
Subject: [PATCH 37/42] update setup

---
 x/merkledb/benchmarks_eth/setup.sh | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/setup.sh b/x/merkledb/benchmarks_eth/setup.sh
index 32bad58a9532..521a3ec1ef2f 100644
--- a/x/merkledb/benchmarks_eth/setup.sh
+++ b/x/merkledb/benchmarks_eth/setup.sh
@@ -45,7 +45,7 @@ if [ -n "$NVME_DEV" ]; then
   echo "$NVME_DEV $NVME_MOUNT ext4 noatime 0 0" >> /etc/fstab
   mkdir -p "$NVME_MOUNT/ubuntu/firewood"
   chown ubuntu:ubuntu "$NVME_MOUNT/ubuntu" "$NVME_MOUNT/ubuntu/firewood"
-  ln -s "$NVME_MOUNT/ubuntu/firewood" /home/ubuntu/firewood
+  ln -s "$NVME_MOUNT/ubuntu/firewood" /home/ubuntu/avalanchego
 fi
 
 ## install go
@@ -55,27 +55,26 @@ rm -rf /usr/local/go && tar -C /usr/local -xzf ${GO_INSTALLATION_FILE}
 rm ${GO_INSTALLATION_FILE}
 
 #### you can switch to the ubuntu user here ####
+## ----------------------------------------- ###
 
-
-git clone https://github.com/ava-labs/avalanchego.git
+git clone https://github.com/ava-labs/avalanchego.git 
 cd avalanchego
 git checkout tsachi/bench_merkledb
-export PATH=$PATH:/usr/local/go/bin
-go build ./x/merkledb/benchmarks_eth
+cd ./x/merkledb/benchmarks_eth && PATH=$PATH:/usr/local/go/bin go build .
 
 #### stop here, these commands are run by hand ####
 
 # 10M rows:
-nohup benchmarks_eth tenkrandom --n 10000000 > output.txt & 
-nohup benchmarks_eth zipf --n 10000000 > output.txt &
-nohup benchmarks_eth single --n 10000000 > output.txt &
+nohup ./benchmarks_eth tenkrandom --n 10000000 > output.txt & 
+nohup ./benchmarks_eth zipf --n 10000000 > output.txt &
+nohup ./benchmarks_eth single --n 10000000 > output.txt &
 
 # 50M rows:
-nohup benchmarks_eth tenkrandom --n 50000000 > output.txt & 
-nohup benchmarks_eth zipf --n 50000000 > output.txt &
-nohup benchmarks_eth single --n 50000000 > output.txt &
+nohup ./benchmarks_eth tenkrandom --n 50000000 > output.txt & 
+nohup ./benchmarks_eth zipf --n 50000000 > output.txt &
+nohup ./benchmarks_eth single --n 50000000 > output.txt &
 
 # 100M rows:
-nohup benchmarks_eth tenkrandom --n 100000000 > output.txt & 
-nohup benchmarks_eth zipf --n 100000000 > output.txt &
-nohup benchmarks_eth single --n 100000000 > output.txt &
\ No newline at end of file
+nohup ./benchmarks_eth tenkrandom --n 100000000 > output.txt & 
+nohup ./benchmarks_eth zipf --n 100000000 > output.txt &
+nohup ./benchmarks_eth single --n 100000000 > output.txt &
\ No newline at end of file

From ddd0c886d5e711b692ca244e88d05cad7ebbb1d6 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sat, 28 Sep 2024 18:15:34 -0400
Subject: [PATCH 38/42] updatr

---
 x/merkledb/benchmarks_eth/go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x/merkledb/benchmarks_eth/go.mod b/x/merkledb/benchmarks_eth/go.mod
index 391cd7aa789c..ee72b7fcf35d 100644
--- a/x/merkledb/benchmarks_eth/go.mod
+++ b/x/merkledb/benchmarks_eth/go.mod
@@ -5,7 +5,6 @@ go 1.22.3
 require (
 	github.com/prometheus/client_golang v1.20.3
 	github.com/spf13/cobra v1.5.0
-	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 )
 
@@ -44,6 +43,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/supranational/blst v0.3.11 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect

From 1b87ea8d2850c227bf709d6e801baa7f24080461 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 29 Sep 2024 14:22:35 -0400
Subject: [PATCH 39/42] use less memory for cache so that it would fit for 16gb
 machines.

---
 x/merkledb/benchmarks_eth/benchmark.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 600f23a8600c..4d7c524beb18 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -34,13 +34,13 @@ const (
 	defaultMetricsPort           = 3_000
 	benchmarkRevisionHistorySize = 128
 	defaultZipfS                 = float64(1.2)
-	defaultZipfV                 = float64(2.7)
+	defaultZipfV                 = float64(1.0)
 )
 
 // TODO: Adjust these cache sizes for maximum performance
 const (
-	cleanCacheSizeBytes = 4 * units.GiB
-	levelDBCacheSizeMB  = 6 * units.GiB / units.MiB
+	cleanCacheSizeBytes = 2 * units.GiB
+	levelDBCacheSizeMB  = 3 * units.GiB / units.MiB
 
 	// TODO: Why 200? The default is 500
 	// see https://pkg.go.dev/github.com/syndtr/goleveldb@v1.0.0/leveldb/opt#Options

From 78d164b85313c50378a0e4557d72539adb48446b Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 29 Sep 2024 15:04:33 -0400
Subject: [PATCH 40/42] fix linter issues

---
 x/merkledb/benchmarks_eth/setup.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/setup.sh b/x/merkledb/benchmarks_eth/setup.sh
index 521a3ec1ef2f..e2f04cb87adf 100644
--- a/x/merkledb/benchmarks_eth/setup.sh
+++ b/x/merkledb/benchmarks_eth/setup.sh
@@ -1,5 +1,6 @@
-#### run these commands from the root user ####
+#!/usr/bin/env bash
 
+#### run these commands from the root user ####
 mkdir -p /etc/apt/keyrings/
 wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null
 echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
@@ -58,7 +59,7 @@ rm ${GO_INSTALLATION_FILE}
 ## ----------------------------------------- ###
 
 git clone https://github.com/ava-labs/avalanchego.git 
-cd avalanchego
+cd avalanchego || exit
 git checkout tsachi/bench_merkledb
 cd ./x/merkledb/benchmarks_eth && PATH=$PATH:/usr/local/go/bin go build .
 
@@ -77,4 +78,6 @@ nohup ./benchmarks_eth single --n 50000000 > output.txt &
 # 100M rows:
 nohup ./benchmarks_eth tenkrandom --n 100000000 > output.txt & 
 nohup ./benchmarks_eth zipf --n 100000000 > output.txt &
-nohup ./benchmarks_eth single --n 100000000 > output.txt &
\ No newline at end of file
+nohup ./benchmarks_eth single --n 100000000 > output.txt &
+
+cd ~/avalanchego/x/merkledb/benchmarks_eth && git pull && PATH=$PATH:/usr/local/go/bin go build .
\ No newline at end of file

From e3a0002037c685bc667fca308d94fc18b81836eb Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 29 Sep 2024 18:44:54 -0400
Subject: [PATCH 41/42] further reduce cache size

---
 x/merkledb/benchmarks_eth/benchmark.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 4d7c524beb18..3473a0b076c9 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -39,8 +39,8 @@ const (
 
 // TODO: Adjust these cache sizes for maximum performance
 const (
-	cleanCacheSizeBytes = 2 * units.GiB
-	levelDBCacheSizeMB  = 3 * units.GiB / units.MiB
+	cleanCacheSizeBytes = 1 * units.GiB
+	levelDBCacheSizeMB  = 3 * units.GiB / (2 * units.MiB)
 
 	// TODO: Why 200? The default is 500
 	// see https://pkg.go.dev/github.com/syndtr/goleveldb@v1.0.0/leveldb/opt#Options

From d9327aec2f0d678cfa815d535eb1c99cb9128f33 Mon Sep 17 00:00:00 2001
From: tsachiherman <24438559+tsachiherman@users.noreply.github.com>
Date: Sun, 20 Oct 2024 13:27:55 -0400
Subject: [PATCH 42/42] update

---
 x/merkledb/benchmarks_eth/benchmark.go  | 48 ++++++++++++++++++++++++-
 x/merkledb/benchmarks_eth/single.go     | 30 ++++++++++++----
 x/merkledb/benchmarks_eth/tenkrandom.go |  2 +-
 x/merkledb/benchmarks_eth/zipf.go       |  2 +-
 4 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go
index 3473a0b076c9..52d71de923cb 100644
--- a/x/merkledb/benchmarks_eth/benchmark.go
+++ b/x/merkledb/benchmarks_eth/benchmark.go
@@ -16,10 +16,13 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/triedb"
@@ -155,7 +158,7 @@ func createGoldenDatabase(databaseEntries uint64) error {
 		return fmt.Errorf("unable to create golden staging directory : %v", err)
 	}
 
-	ldb, err := rawdb.NewLevelDBDatabase(stagingDirectory, levelDBCacheSizeMB, openFilesCacheCapacity, "metrics_prefix", false)
+	ldb, err := openRawDatabaseNoCompression(stagingDirectory, levelDBCacheSizeMB, openFilesCacheCapacity)
 	if err != nil {
 		return fmt.Errorf("unable to create level db database : %v", err)
 	}
@@ -424,3 +427,46 @@ func singleBenchmark() {
 		os.Exit(1)
 	}
 }
+
+func openRawDatabaseNoCompression(stagingDirectory string, cache int, handles int) (ethdb.Database, error) {
+	udb, err := leveldb.NewCustom(stagingDirectory, "metrics_prefix", func(options *opt.Options) {
+		// cache := levelDBCacheSizeMB
+		// handles := openFilesCacheCapacity
+		const minCache int = 16
+		const minHandles int = 16
+		// Ensure we have some minimal caching and file guarantees
+		if cache < minCache {
+			cache = minCache
+		}
+		if handles < minHandles {
+			handles = minHandles
+		}
+		// Set default options
+		options.OpenFilesCacheCapacity = handles
+		options.BlockCacheCapacity = cache / 2 * opt.MiB
+		options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally
+		// disable compression
+		options.Compression = opt.NoCompression
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to create level db database : %v", err)
+	}
+
+	return rawdb.NewDatabase(udb), nil
+}
+
+func openLevelDBDatabaseNoCompression(opt rawdb.OpenOptions) (ethdb.Database, error) {
+	ldb, err := openRawDatabaseNoCompression(opt.Directory, opt.Cache, opt.Handles)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open level db database : %v", err)
+	}
+	if len(opt.AncientsDirectory) == 0 {
+		return ldb, nil
+	}
+	frdb, err := rawdb.NewDatabaseWithFreezer(ldb, opt.AncientsDirectory, opt.Namespace, opt.ReadOnly)
+	if err != nil {
+		ldb.Close()
+		return nil, err
+	}
+	return frdb, nil
+}
diff --git a/x/merkledb/benchmarks_eth/single.go b/x/merkledb/benchmarks_eth/single.go
index 58c2460f8881..99f5b5ae853b 100644
--- a/x/merkledb/benchmarks_eth/single.go
+++ b/x/merkledb/benchmarks_eth/single.go
@@ -23,7 +23,7 @@ func runSingleBenchmark(databaseEntries uint64) error {
 		return fmt.Errorf("unable to read root : %v", err)
 	}
 
-	ldb, err := rawdb.Open(rawdb.OpenOptions{
+	ldb, err := openLevelDBDatabaseNoCompression(rawdb.OpenOptions{
 		Type:              "leveldb",
 		Directory:         getRunningDatabaseDirectory(databaseEntries),
 		AncientsDirectory: "",
@@ -68,16 +68,23 @@ func runSingleBenchmark(databaseEntries uint64) error {
 	}
 
 	batchIdx := uint64(0)
-	updateEntryKey := calculateIndexEncoding(0)
+	entriesKeys := make([][]byte, 10000)
+	for i := range entriesKeys {
+		entriesKeys[i] = calculateIndexEncoding(uint64(i))
+	}
 	var updateDuration, batchDuration time.Duration
+	entriesToUpdate := uint64(1)
+	lastChangeEntriesCount := time.Now()
 	for {
 		startBatchTime := time.Now()
 
 		// update a single entry, at random.
 		startUpdateTime := time.Now()
-		err = tdb.Update(updateEntryKey, binary.BigEndian.AppendUint64(nil, batchIdx))
-		if err != nil {
-			return fmt.Errorf("unable to update trie entry : %v", err)
+		for i := uint64(0); i < entriesToUpdate; i++ {
+			err = tdb.Update(entriesKeys[i], binary.BigEndian.AppendUint64(nil, batchIdx*10000+i))
+			if err != nil {
+				return fmt.Errorf("unable to update trie entry : %v", err)
+			}
 		}
 
 		updateDuration = time.Since(startUpdateTime)
@@ -94,12 +101,21 @@ func runSingleBenchmark(databaseEntries uint64) error {
 		stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration))
 
 		if *verbose {
-			fmt.Printf("update rate [%d]	batch rate [%d]\n",
+			fmt.Printf("update rate [%d]\tbatch rate [%d]\tbatch size [%d]\n",
 				time.Second/updateDuration,
-				time.Second/batchDuration)
+				time.Second/batchDuration,
+				entriesToUpdate)
 		}
 
 		stats.batches.Inc()
 		batchIdx++
+
+		if time.Since(lastChangeEntriesCount) > 5*time.Minute {
+			lastChangeEntriesCount = time.Now()
+			entriesToUpdate *= 10
+			if entriesToUpdate > 10000 {
+				entriesToUpdate = 1
+			}
+		}
 	}
 }
diff --git a/x/merkledb/benchmarks_eth/tenkrandom.go b/x/merkledb/benchmarks_eth/tenkrandom.go
index 3e679898d961..bf1f07ba0d1c 100644
--- a/x/merkledb/benchmarks_eth/tenkrandom.go
+++ b/x/merkledb/benchmarks_eth/tenkrandom.go
@@ -22,7 +22,7 @@ func runTenkrandomBenchmark(databaseEntries uint64) error {
 		return fmt.Errorf("unable to read root : %v", err)
 	}
 
-	ldb, err := rawdb.Open(rawdb.OpenOptions{
+	ldb, err := openLevelDBDatabaseNoCompression(rawdb.OpenOptions{
 		Type:              "leveldb",
 		Directory:         getRunningDatabaseDirectory(databaseEntries),
 		AncientsDirectory: "",
diff --git a/x/merkledb/benchmarks_eth/zipf.go b/x/merkledb/benchmarks_eth/zipf.go
index 465087b80551..4f86f28ebef7 100644
--- a/x/merkledb/benchmarks_eth/zipf.go
+++ b/x/merkledb/benchmarks_eth/zipf.go
@@ -23,7 +23,7 @@ func runZipfBenchmark(databaseEntries uint64, sZipf float64, vZipf float64) erro
 		return fmt.Errorf("unable to read root : %v", err)
 	}
 
-	ldb, err := rawdb.Open(rawdb.OpenOptions{
+	ldb, err := openLevelDBDatabaseNoCompression(rawdb.OpenOptions{
 		Type:              "leveldb",
 		Directory:         getRunningDatabaseDirectory(databaseEntries),
 		AncientsDirectory: "",