diff --git a/x/merkledb/benchmarks/benchmark.go b/x/merkledb/benchmarks/benchmark.go new file mode 100644 index 000000000000..2bf0c08ff507 --- /dev/null +++ b/x/merkledb/benchmarks/benchmark.go @@ -0,0 +1,407 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "net/http" + "os" + "path" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/spf13/pflag" + + "github.com/ava-labs/avalanchego/database/leveldb" + "github.com/ava-labs/avalanchego/trace" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/units" + "github.com/ava-labs/avalanchego/x/merkledb" +) + +const ( + defaultDatabaseEntries = 1000000000 // 1B + databaseCreationBatchSize = 10000 // 10k + databaseRunningBatchSize = 2500 // 2.5k + databaseRunningUpdateSize = 5000 // 5k + defaultMetricsPort = 3000 + firewoodNumberOfRevisions = 5 +) + +var ( + databaseEntries = pflag.Uint64("n", defaultDatabaseEntries, "number of database entries") + httpMetricPort = pflag.Uint64("p", defaultMetricsPort, "default metrics port") + verbose = pflag.Bool("v", false, "verbose") + + insertsCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "insert_counter", + Help: "Total number of inserts", + }) + deletesCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "deletes_counter", + Help: "Total number of deletes", + }) + updatesCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "updates_counter", + Help: "Total number of updates", + }) + batchCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "batch", + Help: "Total number of batches written", + }) + deleteRate = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "entry_delete_rate", + Help: "The rate at which elements are deleted", + }) + updateRate = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "entry_update_rate", + Help: "The rate at which elements are updated", + }) + insertRate = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "entry_insert_rate", + Help: "The rate at which elements are inserted", + }) + batchWriteRate = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "batch_write_rate", + Help: "The rate at which the batch was written", + }) + promRegistry = prometheus.NewRegistry() +) + +func getMerkleDBConfig(promRegistry prometheus.Registerer) merkledb.Config { + return merkledb.Config{ + BranchFactor: merkledb.BranchFactor16, + Hasher: merkledb.DefaultHasher, + RootGenConcurrency: 0, + HistoryLength: firewoodNumberOfRevisions, + ValueNodeCacheSize: units.MiB, + IntermediateNodeCacheSize: 1024 * units.MiB, + IntermediateWriteBufferSize: units.KiB, + IntermediateWriteBatchSize: 256 * units.KiB, + Reg: promRegistry, + TraceLevel: merkledb.NoTrace, + Tracer: trace.Noop, + } +} + +func getGoldenStagingDatabaseDirectory(databaseEntries uint64) string { + wd, _ := os.Getwd() + return path.Join(wd, fmt.Sprintf("db-bench-test-golden-staging-%d", databaseEntries)) +} + +func getGoldenDatabaseDirectory(databaseEntries uint64) string { + wd, _ := os.Getwd() + return path.Join(wd, fmt.Sprintf("db-bench-test-golden-%d", databaseEntries)) +} + +func getRunningDatabaseDirectory(databaseEntries uint64) string { + wd, _ := os.Getwd() + return path.Join(wd, fmt.Sprintf("db-bench-test-running-%d", databaseEntries)) +} + +func calculateIndexEncoding(idx uint64) []byte { + var entryEncoding [8]byte + binary.NativeEndian.PutUint64(entryEncoding[:], idx) + entryHash := sha256.Sum256(entryEncoding[:]) + return entryHash[:] +} + +func createGoldenDatabase(databaseEntries uint64) error { + err := os.RemoveAll(getGoldenStagingDatabaseDirectory(databaseEntries)) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err) + return err + } + err = os.Mkdir(getGoldenStagingDatabaseDirectory(databaseEntries), 0o777) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to create golden staging directory : %v\n", err) + return err + } + + levelDB, err := leveldb.New( + getGoldenStagingDatabaseDirectory(databaseEntries), + getLevelDBConfig(), + logging.NoLog{}, + prometheus.NewRegistry(), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to create levelDB database : %v\n", err) + return err + } + + mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(nil)) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to create merkleDB database : %v\n", err) + return err + } + + fmt.Print("Initializing database.") + ticksCh := make(chan interface{}) + go func() { + t := time.NewTicker(time.Second) + defer t.Stop() + for { + select { + case <-ticksCh: + return + case <-t.C: + fmt.Print(".") + } + } + }() + + startInsertTime := time.Now() + startInsertBatchTime := startInsertTime + currentBatch := mdb.NewBatch() + for entryIdx := uint64(0); entryIdx < databaseEntries; entryIdx++ { + entryHash := calculateIndexEncoding(entryIdx) + err = currentBatch.Put(entryHash, entryHash) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to put value in merkleDB database : %v\n", err) + return err + } + + if entryIdx%databaseCreationBatchSize == (databaseCreationBatchSize - 1) { + addDuration := time.Since(startInsertBatchTime) + insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration)) + insertsCounter.Add(databaseCreationBatchSize) + + batchWriteStartTime := time.Now() + err = currentBatch.Write() + if err != nil { + fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err) + return err + } + currentBatch.Reset() + batchWriteDuration := time.Since(batchWriteStartTime) + batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration)) + batchCounter.Inc() + + startInsertBatchTime = time.Now() + } + } + if databaseEntries%databaseCreationBatchSize != 0 { + err = currentBatch.Write() + if err != nil { + fmt.Fprintf(os.Stderr, "unable to write value in merkleDB database : %v\n", err) + return err + } + } + close(ticksCh) + + fmt.Print(" done!\n") + rootID, err := mdb.GetMerkleRoot(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to get root ID : %v\n", err) + return err + } + + err = mdb.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "unable to close merkleDB database : %v\n", err) + return err + } + err = levelDB.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "unable to close levelDB database : %v\n", err) + return err + } + + fmt.Printf("Generated and inserted %d batches of size %d in %v\n", + databaseEntries/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime)) + + err = os.Rename(getGoldenStagingDatabaseDirectory(databaseEntries), getGoldenDatabaseDirectory(databaseEntries)) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to rename golden staging directory : %v\n", err) + return err + } + fmt.Printf("Completed initialization with hash of %v\n", rootID.Hex()) + return nil +} + +func resetRunningDatabaseDirectory(databaseEntries uint64) error { + runningDir := getRunningDatabaseDirectory(databaseEntries) + if _, err := os.Stat(runningDir); err == nil { + err := os.RemoveAll(runningDir) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to remove running directory : %v\n", err) + return err + } + } + err := os.Mkdir(runningDir, 0o777) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to create running directory : %v\n", err) + return err + } + err = CopyDirectory(getGoldenDatabaseDirectory(databaseEntries), runningDir) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to duplicate golden directory : %v\n", err) + return err + } + return nil +} + +func runBenchmark(databaseEntries uint64) error { + levelDB, err := leveldb.New( + getRunningDatabaseDirectory(databaseEntries), + getLevelDBConfig(), + logging.NoLog{}, + promRegistry, + ) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to create levelDB database : %v\n", err) + return err + } + + mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(promRegistry)) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to create merkleDB database : %v\n", err) + return err + } + defer func() { + mdb.Close() + levelDB.Close() + }() + + low := uint64(0) + var deleteDuration, addDuration, updateDuration, batchDuration time.Duration + batch := mdb.NewBatch() + for { + startBatchTime := time.Now() + + // delete first 2.5k keys from the beginning + startDeleteTime := time.Now() + for keyToDeleteIdx := low; keyToDeleteIdx < low+databaseRunningBatchSize; keyToDeleteIdx++ { + entryHash := calculateIndexEncoding(keyToDeleteIdx) + err = batch.Delete(entryHash) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to delete merkleDB entry : %v\n", err) + return err + } + } + deleteDuration = time.Since(startDeleteTime) + deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration)) + deletesCounter.Add(databaseRunningBatchSize) + + // add 2.5k past end. + startInsertTime := time.Now() + for keyToAddIdx := low + databaseEntries; keyToAddIdx < low+databaseEntries+databaseRunningBatchSize; keyToAddIdx++ { + entryHash := calculateIndexEncoding(keyToAddIdx) + err = batch.Put(entryHash, entryHash) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to insert merkleDB entry : %v\n", err) + return err + } + } + addDuration = time.Since(startInsertTime) + insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration)) + insertsCounter.Add(databaseRunningBatchSize) + + // update middle 5k entries + startUpdateTime := time.Now() + updateEntryValue := calculateIndexEncoding(low) + for keyToUpdateIdx := low + (databaseEntries / 2); keyToUpdateIdx < low+(databaseEntries/2)+databaseRunningUpdateSize; keyToUpdateIdx++ { + updateEntryKey := calculateIndexEncoding(keyToUpdateIdx) + err = batch.Put(updateEntryKey, updateEntryValue) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to update merkleDB entry : %v\n", err) + return err + } + } + updateDuration = time.Since(startUpdateTime) + updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration)) + updatesCounter.Add(databaseRunningUpdateSize) + + batchWriteStartTime := time.Now() + err = batch.Write() + if err != nil { + fmt.Fprintf(os.Stderr, "unable to write batch : %v\n", err) + return err + } + batchDuration = time.Since(startBatchTime) + batchWriteDuration := time.Since(batchWriteStartTime) + batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration)) + + batch.Reset() + + if *verbose { + fmt.Printf("delete rate [%d] update rate [%d] insert rate [%d] batch rate [%d]\n", + time.Second/deleteDuration, + time.Second/updateDuration, + time.Second/addDuration, + time.Second/batchDuration) + } + + batchCounter.Inc() + low += databaseRunningBatchSize + } + + // return nil +} + +func setupMetrics() error { + if err := prometheus.Register(promRegistry); err != nil { + return err + } + promRegistry.MustRegister(insertsCounter) + promRegistry.MustRegister(deletesCounter) + promRegistry.MustRegister(updatesCounter) + promRegistry.MustRegister(batchCounter) + promRegistry.MustRegister(deleteRate) + promRegistry.MustRegister(updateRate) + promRegistry.MustRegister(insertRate) + promRegistry.MustRegister(batchWriteRate) + + http.Handle("/metrics", promhttp.Handler()) + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", *httpMetricPort), + ReadHeaderTimeout: 3 * time.Second, + } + go func() { + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + fmt.Fprintf(os.Stderr, "unable to listen and serve : %v\n", err) + } + }() + return nil +} + +func main() { + pflag.Parse() + + if setupMetrics() != nil { + os.Exit(1) + return + } + + goldenDir := getGoldenDatabaseDirectory(*databaseEntries) + if _, err := os.Stat(goldenDir); os.IsNotExist(err) { + // create golden image. + if createGoldenDatabase(*databaseEntries) != nil { + os.Exit(1) + return + } + } + if resetRunningDatabaseDirectory(*databaseEntries) != nil { + os.Exit(1) + return + } + if runBenchmark(*databaseEntries) != nil { + os.Exit(1) + return + } +} diff --git a/x/merkledb/benchmarks/benchmark_test.go b/x/merkledb/benchmarks/benchmark_test.go new file mode 100644 index 000000000000..6ebee9dd0693 --- /dev/null +++ b/x/merkledb/benchmarks/benchmark_test.go @@ -0,0 +1,66 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "context" + "os" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/leveldb" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/x/merkledb" +) + +func TestGoldenDatabaseCreation(t *testing.T) { + for _, size := range []uint64{9500, 20000, 50000, 103011} { + testGoldenDatabaseCreation(t, size) + } +} + +func testGoldenDatabaseCreation(t *testing.T, size uint64) { + path := getGoldenDatabaseDirectory(size) + if Exists(getGoldenDatabaseDirectory(size)) { + require.NoError(t, os.RemoveAll(path)) + } + require.NoError(t, createGoldenDatabase(size)) + require.NoError(t, resetRunningDatabaseDirectory(size)) + testGoldenDatabaseContent(t, size) + require.NoError(t, os.RemoveAll(path)) + require.NoError(t, os.RemoveAll(getRunningDatabaseDirectory(size))) +} + +func testGoldenDatabaseContent(t *testing.T, size uint64) { + levelDB, err := leveldb.New( + getRunningDatabaseDirectory(size), + getLevelDBConfig(), + logging.NoLog{}, + prometheus.NewRegistry(), + ) + require.NoError(t, err) + mdb, err := merkledb.New(context.Background(), levelDB, getMerkleDBConfig(nil)) + require.NoError(t, err) + defer func() { + mdb.Close() + levelDB.Close() + }() + // read values and make sure they are what we expect them to be. + for keyIndex := uint64(0); keyIndex < size; keyIndex++ { + entryHash := calculateIndexEncoding(keyIndex) + entryValue, err := mdb.Get(entryHash) + require.NoError(t, err) + require.Equal(t, entryHash, entryValue) + } + + // try an entry beyond. + entryHash := calculateIndexEncoding(size + 1) + _, err = mdb.Get(entryHash) + require.ErrorIs(t, err, database.ErrNotFound) + + require.NoError(t, mdb.Clear()) +} diff --git a/x/merkledb/benchmarks/dashboard.grafana b/x/merkledb/benchmarks/dashboard.grafana new file mode 100644 index 000000000000..4f8c52618aa3 --- /dev/null +++ b/x/merkledb/benchmarks/dashboard.grafana @@ -0,0 +1,721 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "datasource": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "merkledb_bench_entry_delete_rate", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "Deletes", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "merkledb_bench_entry_insert_rate", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "Inserts", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "merkledb_bench_entry_update_rate", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "Updates", + "useBackend": false + } + ], + "title": "Rows Operations Rate", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "rate(merkledb_bench_deletes_counter[10m])", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "Deletes", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "rate(merkledb_bench_insert_counter[10m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "Inserts", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(merkledb_bench_updates_counter[10m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "Updates", + "useBackend": false + } + ], + "title": "Rows Operations - Counter Interval", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "merkledb_bench_batch_write_rate", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "Batch Write Operation", + "useBackend": false + } + ], + "title": "Batch Rate", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(merkledb_bench_batch[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "Effective Rate", + "useBackend": false + } + ], + "title": "Batch Operation - Counter Interval", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 16 + }, + "id": 6, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "merkledb_bench_insert_counter", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Inserted Rows", + "type": "gauge" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 16 + }, + "id": 7, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "merkledb_bench_updates_counter", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Updated Rows", + "type": "gauge" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "merkledb_bench_deletes_counter", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Deleted Rows", + "type": "gauge" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "MerkleDB operations benchmark", + "uid": "edwwwnfme46ioa", + "version": 8, + "weekStart": "" +} \ No newline at end of file diff --git a/x/merkledb/benchmarks/directories_default.go b/x/merkledb/benchmarks/directories_default.go new file mode 100644 index 000000000000..976f02311072 --- /dev/null +++ b/x/merkledb/benchmarks/directories_default.go @@ -0,0 +1,121 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build linux || darwin || unix + +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" +) + +func CopyDirectory(scrDir, dest string) error { + entries, err := os.ReadDir(scrDir) + if err != nil { + return err + } + for _, entry := range entries { + sourcePath := filepath.Join(scrDir, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + fileInfo, err := os.Stat(sourcePath) + if err != nil { + return err + } + + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath) + } + + switch fileInfo.Mode() & os.ModeType { + case os.ModeDir: + if err := CreateIfNotExists(destPath, 0o755); err != nil { + return err + } + if err := CopyDirectory(sourcePath, destPath); err != nil { + return err + } + case os.ModeSymlink: + if err := CopySymLink(sourcePath, destPath); err != nil { + return err + } + default: + if err := Copy(sourcePath, destPath); err != nil { + return err + } + } + + if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + + fInfo, err := entry.Info() + if err != nil { + return err + } + + isSymlink := fInfo.Mode()&os.ModeSymlink != 0 + if !isSymlink { + if err := os.Chmod(destPath, fInfo.Mode()); err != nil { + return err + } + } + } + return nil +} + +func Copy(srcFile, dstFile string) error { + out, err := os.Create(dstFile) + if err != nil { + return err + } + + defer out.Close() + + in, err := os.Open(srcFile) + if err != nil { + return err + } + + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return nil +} + +func Exists(filePath string) bool { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return false + } + + return true +} + +func CreateIfNotExists(dir string, perm os.FileMode) error { + if Exists(dir) { + return nil + } + + if err := os.MkdirAll(dir, perm); err != nil { + return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) + } + + return nil +} + +func CopySymLink(source, dest string) error { + link, err := os.Readlink(source) + if err != nil { + return err + } + return os.Symlink(link, dest) +} diff --git a/x/merkledb/benchmarks/directories_windows.go b/x/merkledb/benchmarks/directories_windows.go new file mode 100644 index 000000000000..d3fc82b445ca --- /dev/null +++ b/x/merkledb/benchmarks/directories_windows.go @@ -0,0 +1,107 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build windows + +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +func CopyDirectory(scrDir, dest string) error { + entries, err := os.ReadDir(scrDir) + if err != nil { + return err + } + for _, entry := range entries { + sourcePath := filepath.Join(scrDir, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + fileInfo, err := os.Stat(sourcePath) + if err != nil { + return err + } + + switch fileInfo.Mode() & os.ModeType { + case os.ModeDir: + if err := CreateIfNotExists(destPath, 0o755); err != nil { + return err + } + if err := CopyDirectory(sourcePath, destPath); err != nil { + return err + } + default: + if err := Copy(sourcePath, destPath); err != nil { + return err + } + } + + fInfo, err := entry.Info() + if err != nil { + return err + } + + isSymlink := fInfo.Mode()&os.ModeSymlink != 0 + if !isSymlink { + if err := os.Chmod(destPath, fInfo.Mode()); err != nil { + return err + } + } + } + return nil +} + +func Copy(srcFile, dstFile string) error { + out, err := os.Create(dstFile) + if err != nil { + return err + } + + defer out.Close() + + in, err := os.Open(srcFile) + if err != nil { + return err + } + + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return nil +} + +func Exists(filePath string) bool { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return false + } + + return true +} + +func CreateIfNotExists(dir string, perm os.FileMode) error { + if Exists(dir) { + return nil + } + + if err := os.MkdirAll(dir, perm); err != nil { + return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) + } + + return nil +} + +func CopySymLink(source, dest string) error { + link, err := os.Readlink(source) + if err != nil { + return err + } + return os.Symlink(link, dest) +} diff --git a/x/merkledb/benchmarks/levelDBConfig.go b/x/merkledb/benchmarks/levelDBConfig.go new file mode 100644 index 000000000000..3a2a2d3115b1 --- /dev/null +++ b/x/merkledb/benchmarks/levelDBConfig.go @@ -0,0 +1,125 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "encoding/json" + "time" + + "github.com/syndtr/goleveldb/leveldb/opt" + + "github.com/ava-labs/avalanchego/database/leveldb" +) + +type config struct { + // BlockCacheCapacity defines the capacity of the 'sorted table' block caching. + // Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher. + // + // The default value is 12MiB. + BlockCacheCapacity int `json:"blockCacheCapacity"` + // BlockSize is the minimum uncompressed size in bytes of each 'sorted table' + // block. + // + // The default value is 4KiB. + BlockSize int `json:"blockSize"` + // CompactionExpandLimitFactor limits compaction size after expanded. + // This will be multiplied by table size limit at compaction target level. + // + // The default value is 25. + CompactionExpandLimitFactor int `json:"compactionExpandLimitFactor"` + // CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2) + // that a single 'sorted table' generates. This will be multiplied by + // table size limit at grandparent level. + // + // The default value is 10. + CompactionGPOverlapsFactor int `json:"compactionGPOverlapsFactor"` + // CompactionL0Trigger defines number of 'sorted table' at level-0 that will + // trigger compaction. + // + // The default value is 4. + CompactionL0Trigger int `json:"compactionL0Trigger"` + // CompactionSourceLimitFactor limits compaction source size. This doesn't apply to + // level-0. + // This will be multiplied by table size limit at compaction target level. + // + // The default value is 1. + CompactionSourceLimitFactor int `json:"compactionSourceLimitFactor"` + // CompactionTableSize limits size of 'sorted table' that compaction generates. + // The limits for each level will be calculated as: + // CompactionTableSize * (CompactionTableSizeMultiplier ^ Level) + // The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel. + // + // The default value is 2MiB. + CompactionTableSize int `json:"compactionTableSize"` + // CompactionTableSizeMultiplier defines multiplier for CompactionTableSize. + // + // The default value is 1. + CompactionTableSizeMultiplier float64 `json:"compactionTableSizeMultiplier"` + // CompactionTableSizeMultiplierPerLevel defines per-level multiplier for + // CompactionTableSize. + // Use zero to skip a level. + // + // The default value is nil. + CompactionTableSizeMultiplierPerLevel []float64 `json:"compactionTableSizeMultiplierPerLevel"` + // CompactionTotalSize limits total size of 'sorted table' for each level. + // The limits for each level will be calculated as: + // CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level) + // The multiplier for each level can also fine-tuned using + // CompactionTotalSizeMultiplierPerLevel. + // + // The default value is 10MiB. + CompactionTotalSize int `json:"compactionTotalSize"` + // CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize. + // + // The default value is 10. + CompactionTotalSizeMultiplier float64 `json:"compactionTotalSizeMultiplier"` + // DisableSeeksCompaction allows disabling 'seeks triggered compaction'. + // The purpose of 'seeks triggered compaction' is to optimize database so + // that 'level seeks' can be minimized, however this might generate many + // small compaction which may not preferable. + // + // The default is true. + DisableSeeksCompaction bool `json:"disableSeeksCompaction"` + // OpenFilesCacheCapacity defines the capacity of the open files caching. + // Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher. + // + // The default value is 1024. + OpenFilesCacheCapacity int `json:"openFilesCacheCapacity"` + // WriteBuffer defines maximum size of a 'memdb' before flushed to + // 'sorted table'. 'memdb' is an in-memory DB backed by an on-disk + // unsorted journal. + // + // LevelDB may held up to two 'memdb' at the same time. + // + // The default value is 6MiB. + WriteBuffer int `json:"writeBuffer"` + FilterBitsPerKey int `json:"filterBitsPerKey"` + + // MaxManifestFileSize is the maximum size limit of the MANIFEST-****** file. + // When the MANIFEST-****** file grows beyond this size, LevelDB will create + // a new MANIFEST file. + // + // The default value is infinity. + MaxManifestFileSize int64 `json:"maxManifestFileSize"` + + // MetricUpdateFrequency is the frequency to poll LevelDB metrics. + // If <= 0, LevelDB metrics aren't polled. + MetricUpdateFrequency time.Duration `json:"metricUpdateFrequency"` +} + +func getLevelDBConfig() []byte { + // align with geth test and use 6gb as cache. + cacheSize := int(6000) + parsedConfig := config{ + BlockCacheCapacity: (cacheSize / 2) * opt.MiB, + DisableSeeksCompaction: true, + OpenFilesCacheCapacity: 64, + WriteBuffer: (cacheSize / 4) * opt.MiB, + FilterBitsPerKey: leveldb.DefaultBitsPerKey, + MaxManifestFileSize: leveldb.DefaultMaxManifestFileSize, + MetricUpdateFrequency: leveldb.DefaultMetricUpdateFrequency, + } + bytes, _ := json.Marshal(parsedConfig) + return bytes +} diff --git a/x/merkledb/benchmarks_eth/benchmark.go b/x/merkledb/benchmarks_eth/benchmark.go new file mode 100644 index 000000000000..52d71de923cb --- /dev/null +++ b/x/merkledb/benchmarks_eth/benchmark.go @@ -0,0 +1,472 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "net/http" + "os" + "path" + "time" + + "github.com/ava-labs/avalanchego/utils/units" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/spf13/cobra" + "github.com/syndtr/goleveldb/leveldb/opt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" +) + +const ( + defaultDatabaseEntries = 0 + databaseCreationBatchSize = 10_000 + databaseRunningBatchSize = databaseCreationBatchSize / 4 + databaseRunningUpdateSize = databaseCreationBatchSize / 2 + defaultMetricsPort = 3_000 + benchmarkRevisionHistorySize = 128 + defaultZipfS = float64(1.2) + defaultZipfV = float64(1.0) +) + +// TODO: Adjust these cache sizes for maximum performance +const ( + cleanCacheSizeBytes = 1 * units.GiB + levelDBCacheSizeMB = 3 * units.GiB / (2 * units.MiB) + + // TODO: Why 200? The default is 500 + // see https://pkg.go.dev/github.com/syndtr/goleveldb@v1.0.0/leveldb/opt#Options + openFilesCacheCapacity = 200 +) + +var pathDBConfig = pathdb.Config{ + StateHistory: benchmarkRevisionHistorySize, + CleanCacheSize: cleanCacheSizeBytes, + DirtyCacheSize: 0, + ReadOnly: false, +} + +var stats = struct { + inserts prometheus.Counter + deletes prometheus.Counter + updates prometheus.Counter + batches prometheus.Counter + deleteRate prometheus.Gauge + updateRate prometheus.Gauge + insertRate prometheus.Gauge + batchWriteRate prometheus.Gauge + registry *prometheus.Registry +}{ + inserts: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "insert_counter", + Help: "Total number of inserts", + }), + deletes: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "deletes_counter", + Help: "Total number of deletes", + }), + updates: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "updates_counter", + Help: "Total number of updates", + }), + batches: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "merkledb_bench", + Name: "batch", + Help: "Total number of batches written", + }), + deleteRate: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "entry_delete_rate", + Help: "The rate at which elements are deleted", + }), + updateRate: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "entry_update_rate", + Help: "The rate at which elements are updated", + }), + insertRate: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "entry_insert_rate", + Help: "The rate at which elements are inserted", + }), + batchWriteRate: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "merkledb_bench", + Name: "batch_write_rate", + Help: "The rate at which the batch was written", + }), + + registry: prometheus.NewRegistry(), +} + +// command line arguments parameters +var ( + databaseEntries *uint64 + httpMetricPort *uint64 + verbose *bool + sZipf *float64 + vZipf *float64 +) + +func Must[T any](obj T, err error) T { + if err != nil { + panic(err) + } + return obj +} + +func getGoldenStagingDatabaseDirectory(databaseEntries uint64) string { + return path.Join(Must(os.Getwd()), fmt.Sprintf("db-bench-test-golden-staging-%d", databaseEntries)) +} + +func getGoldenDatabaseDirectory(databaseEntries uint64) string { + return path.Join(Must(os.Getwd()), fmt.Sprintf("db-bench-test-golden-%d", databaseEntries)) +} + +func getRunningDatabaseDirectory(databaseEntries uint64) string { + return path.Join(Must(os.Getwd()), fmt.Sprintf("db-bench-test-running-%d", databaseEntries)) +} + +func calculateIndexEncoding(idx uint64) []byte { + var entryEncoding [8]byte + binary.NativeEndian.PutUint64(entryEncoding[:], idx) + entryHash := sha256.Sum256(entryEncoding[:]) + return entryHash[:] +} + +func createGoldenDatabase(databaseEntries uint64) error { + stagingDirectory := getGoldenStagingDatabaseDirectory(databaseEntries) + err := os.RemoveAll(stagingDirectory) + if err != nil { + return fmt.Errorf("unable to remove running directory : %v", err) + } + err = os.Mkdir(stagingDirectory, 0o777) + if err != nil { + return fmt.Errorf("unable to create golden staging directory : %v", err) + } + + ldb, err := openRawDatabaseNoCompression(stagingDirectory, levelDBCacheSizeMB, openFilesCacheCapacity) + if err != nil { + return fmt.Errorf("unable to create level db database : %v", err) + } + + trieDb := triedb.NewDatabase(ldb, &triedb.Config{ + Preimages: false, + IsVerkle: false, + HashDB: nil, + PathDB: &pathDBConfig, + }) + tdb := trie.NewEmpty(trieDb) + + fmt.Print("Initializing database.") + ticksCh := make(chan interface{}) + go func() { + t := time.NewTicker(time.Second) + defer t.Stop() + for { + select { + case <-ticksCh: + return + case <-t.C: + fmt.Print(".") + } + } + }() + + var root common.Hash + parentHash := types.EmptyRootHash + blockHeight := uint64(1) + writeBatch := func() error { + var nodes *trienode.NodeSet + root, nodes = tdb.Commit(false) + err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/) + if err != nil { + return fmt.Errorf("unable to update trie : %v", err) + } + + tdb, err = trie.New(trie.TrieID(root), trieDb) + if err != nil { + return fmt.Errorf("unable to create new trie : %v", err) + } + parentHash = root + blockHeight++ + return nil + } + + startInsertTime := time.Now() + startInsertBatchTime := startInsertTime + for entryIdx := range databaseEntries { + entryHash := calculateIndexEncoding(entryIdx) + tdb.Update(entryHash, entryHash) + + if entryIdx%databaseCreationBatchSize == (databaseCreationBatchSize - 1) { + addDuration := time.Since(startInsertBatchTime) + stats.insertRate.Set(float64(databaseCreationBatchSize) * float64(time.Second) / float64(addDuration)) + stats.inserts.Add(databaseCreationBatchSize) + + batchWriteStartTime := time.Now() + + err = writeBatch() + if err != nil { + return fmt.Errorf("unable to write value in database : %v", err) + } + + batchWriteDuration := time.Since(batchWriteStartTime) + stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration)) + stats.batches.Inc() + + startInsertBatchTime = time.Now() + } + } + // write the last batch. In our default case, there won't be a last batch + if databaseEntries%databaseCreationBatchSize != 0 { + err = writeBatch() + if err != nil { + return fmt.Errorf("unable to write value in database : %v", err) + } + } + // Note that without this commit, the final hash is not available + // TODO: figure out why this is necessary + err = trieDb.Commit(root, false) + if err != nil { + return fmt.Errorf("unable to commit trie : %v", err) + } + close(ticksCh) + fmt.Print(" done!\n") + + if err = trieDb.Close(); err != nil { + return fmt.Errorf("unable to close trie database : %v", err) + } + + if err = ldb.Close(); err != nil { + return fmt.Errorf("unable to close levelDB database : %v", err) + } + + fmt.Printf("Generated and inserted %d batches of size %d in %v\n", + databaseEntries/databaseCreationBatchSize, databaseCreationBatchSize, time.Since(startInsertTime)) + + if err = os.WriteFile(path.Join(stagingDirectory, "root.txt"), root.Bytes(), 0o644); err != nil { + return fmt.Errorf("unable to save root : %v", err) + } + + err = os.Rename(getGoldenStagingDatabaseDirectory(databaseEntries), getGoldenDatabaseDirectory(databaseEntries)) + if err != nil { + return fmt.Errorf("unable to rename golden staging directory : %v", err) + } + fmt.Printf("Completed initialization with hash of %v\n", root.Hex()) + return nil +} + +func resetRunningDatabaseDirectory(databaseEntries uint64) error { + runningDir := getRunningDatabaseDirectory(databaseEntries) + if _, err := os.Stat(runningDir); err == nil { + err := os.RemoveAll(runningDir) + if err != nil { + return fmt.Errorf("unable to remove running directory : %v", err) + } + } + err := os.Mkdir(runningDir, 0o777) + if err != nil { + return fmt.Errorf("unable to create running directory : %v", err) + } + err = CopyDirectory(getGoldenDatabaseDirectory(databaseEntries), runningDir) + if err != nil { + return fmt.Errorf("unable to duplicate golden directory : %v", err) + } + return nil +} + +func setupMetrics() error { + if err := prometheus.Register(stats.registry); err != nil { + return err + } + stats.registry.MustRegister(stats.inserts) + stats.registry.MustRegister(stats.deletes) + stats.registry.MustRegister(stats.updates) + stats.registry.MustRegister(stats.batches) + stats.registry.MustRegister(stats.deleteRate) + stats.registry.MustRegister(stats.updateRate) + stats.registry.MustRegister(stats.insertRate) + stats.registry.MustRegister(stats.batchWriteRate) + + http.Handle("/metrics", promhttp.Handler()) + + prometheusServer := &http.Server{ + Addr: fmt.Sprintf(":%d", *httpMetricPort), + ReadHeaderTimeout: 3 * time.Second, + } + go func() { + err := prometheusServer.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + panic(fmt.Sprintf("unable to listen and serve : %v\n", err)) + } + }() + return nil +} + +func main() { + rootCmd := &cobra.Command{ + Use: "benchmarks_eth", + Short: "benchmarks_eth", + RunE: func(*cobra.Command, []string) error { + fmt.Printf("Please specify which type of benchmark you'd like to run : [tenkrandom, zipf, single]\n") + return nil + }, + } + databaseEntries = rootCmd.PersistentFlags().Uint64("n", defaultDatabaseEntries, "Number of database entries in golden database") + httpMetricPort = rootCmd.PersistentFlags().Uint64("p", defaultMetricsPort, "HTTP metrics port") + verbose = rootCmd.PersistentFlags().Bool("verbose", false, "Verbose mode") + rootCmd.MarkPersistentFlagRequired("n") + + tenkrandomCmd := &cobra.Command{ + Use: "tenkrandom", + Short: "Run the tenkrandom benchmark", + RunE: func(*cobra.Command, []string) error { + tenkrandomBenchmark() + return nil + }, + } + zipfCmd := &cobra.Command{ + Use: "zipf", + Short: "Run the zipf benchmark", + PreRun: func(cmd *cobra.Command, args []string) { + if *sZipf <= 1.0 { + fmt.Fprintf(os.Stderr, "The s value for Zipf distribution must be greater than 1\n") + os.Exit(1) + } + if *vZipf < 1.0 { + fmt.Fprintf(os.Stderr, "The v value for Zipf distribution must be greater or equal to 1\n") + os.Exit(1) + } + }, + RunE: func(*cobra.Command, []string) error { + zipfBenchmark() + return nil + }, + } + sZipf = zipfCmd.PersistentFlags().Float64("s", defaultZipfS, "s (Zipf distribution = [(v+k)^(-s)], Default = 1.00)") + vZipf = zipfCmd.PersistentFlags().Float64("v", defaultZipfV, "v (Zipf distribution = [(v+k)^(-s)], Default = 2.7)") + singleCmd := &cobra.Command{ + Use: "single", + Short: "Run the single benchmark", + RunE: func(*cobra.Command, []string) error { + singleBenchmark() + return nil + }, + } + + rootCmd.AddCommand(tenkrandomCmd, zipfCmd, singleCmd) + + if err := setupMetrics(); err != nil { + fmt.Fprintf(os.Stderr, "unable to setup metrics : %v\n", err) + os.Exit(1) + } + + if err := rootCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "tmpnetctl failed: %v\n", err) + os.Exit(1) + } + os.Exit(0) +} + +func commonBenchmarkBootstrap() { + if *databaseEntries == 0 { + fmt.Fprintf(os.Stderr, "Please specify number of database entries by using the --n flag\n") + os.Exit(1) + } + goldenDir := getGoldenDatabaseDirectory(*databaseEntries) + if _, err := os.Stat(goldenDir); os.IsNotExist(err) { + // create golden image. + if err := createGoldenDatabase(*databaseEntries); err != nil { + fmt.Fprintf(os.Stderr, "unable to create golden database : %v\n", err) + os.Exit(1) + } + } + if err := resetRunningDatabaseDirectory(*databaseEntries); err != nil { + fmt.Fprintf(os.Stderr, "Unable to reset running database directory: %v\n", err) + os.Exit(1) + } +} + +func tenkrandomBenchmark() { + commonBenchmarkBootstrap() + + if err := runTenkrandomBenchmark(*databaseEntries); err != nil { + fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err) + os.Exit(1) + } +} + +func zipfBenchmark() { + commonBenchmarkBootstrap() + + if err := runZipfBenchmark(*databaseEntries, *sZipf, *vZipf); err != nil { + fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err) + os.Exit(1) + } +} + +func singleBenchmark() { + commonBenchmarkBootstrap() + + if err := runSingleBenchmark(*databaseEntries); err != nil { + fmt.Fprintf(os.Stderr, "Unable to run benchmark: %v\n", err) + os.Exit(1) + } +} + +func openRawDatabaseNoCompression(stagingDirectory string, cache int, handles int) (ethdb.Database, error) { + udb, err := leveldb.NewCustom(stagingDirectory, "metrics_prefix", func(options *opt.Options) { + // cache := levelDBCacheSizeMB + // handles := openFilesCacheCapacity + const minCache int = 16 + const minHandles int = 16 + // Ensure we have some minimal caching and file guarantees + if cache < minCache { + cache = minCache + } + if handles < minHandles { + handles = minHandles + } + // Set default options + options.OpenFilesCacheCapacity = handles + options.BlockCacheCapacity = cache / 2 * opt.MiB + options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally + // disable compression + options.Compression = opt.NoCompression + }) + if err != nil { + return nil, fmt.Errorf("unable to create level db database : %v", err) + } + + return rawdb.NewDatabase(udb), nil +} + +func openLevelDBDatabaseNoCompression(opt rawdb.OpenOptions) (ethdb.Database, error) { + ldb, err := openRawDatabaseNoCompression(opt.Directory, opt.Cache, opt.Handles) + if err != nil { + return nil, fmt.Errorf("unable to open level db database : %v", err) + } + if len(opt.AncientsDirectory) == 0 { + return ldb, nil + } + frdb, err := rawdb.NewDatabaseWithFreezer(ldb, opt.AncientsDirectory, opt.Namespace, opt.ReadOnly) + if err != nil { + ldb.Close() + return nil, err + } + return frdb, nil +} diff --git a/x/merkledb/benchmarks_eth/benchmark_test.go b/x/merkledb/benchmarks_eth/benchmark_test.go new file mode 100644 index 000000000000..e8e658a98a05 --- /dev/null +++ b/x/merkledb/benchmarks_eth/benchmark_test.go @@ -0,0 +1,196 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "os" + "path" + "testing" + + "github.com/ava-labs/avalanchego/utils/units" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/pathdb" + "github.com/stretchr/testify/require" +) + +var testPathDbConfig = pathdb.Config{ + CleanCacheSize: units.MiB, // use a small cache for testing + DirtyCacheSize: 0, // force changes to disk + ReadOnly: false, // allow writes +} + +func TestGoldenDatabaseCreation(t *testing.T) { + for _, size := range []uint64{9500, 20000, 50000, 103011} { + testGoldenDatabaseCreation(t, size) + } +} + +func testGoldenDatabaseCreation(t *testing.T, size uint64) { + path := getGoldenDatabaseDirectory(size) + if Exists(getGoldenDatabaseDirectory(size)) { + require.NoError(t, os.RemoveAll(path)) + } + require.NoError(t, createGoldenDatabase(size)) + resetRunningDatabaseDirectory(size) + testGoldenDatabaseContent(t, size) + require.NoError(t, os.RemoveAll(path)) + require.NoError(t, os.RemoveAll(getRunningDatabaseDirectory(size))) +} + +func testGoldenDatabaseContent(t *testing.T, size uint64) { + rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(size), "root.txt")) + require.NoError(t, err) + + ldb, err := rawdb.Open(rawdb.OpenOptions{ + Type: "leveldb", + Directory: getRunningDatabaseDirectory(size), + AncientsDirectory: "", + Namespace: "metrics_prefix", + Cache: levelDBCacheSizeMB, + Handles: 200, + ReadOnly: false, + Ephemeral: false, + }) + require.NoError(t, err) + + trieDb := triedb.NewDatabase(ldb, &triedb.Config{ + Preimages: false, + IsVerkle: false, + HashDB: nil, + PathDB: &testPathDbConfig, + }) + + parentHash := common.BytesToHash(rootBytes) + tdb, err := trie.New(trie.TrieID(parentHash), trieDb) + require.NoError(t, err) + + // read values and make sure they are what we expect them to be. + for keyIndex := uint64(0); keyIndex < size; keyIndex++ { + entryHash := calculateIndexEncoding(keyIndex) + entryValue, err := tdb.Get(entryHash) + require.NoError(t, err) + require.Equal(t, entryHash, entryValue) + } + + // try an entry beyond. + entryHash := calculateIndexEncoding(size + 1) + entryValue, err := tdb.Get(entryHash) + require.NoError(t, err) // for geth, we have no error in case of a missing key. + require.Equal(t, []byte(nil), entryValue) + + require.NoError(t, trieDb.Close()) + require.NoError(t, ldb.Close()) +} + +func TestRevisions(t *testing.T) { + require := require.New(t) + + // Clean up the database directory + wd, _ := os.Getwd() + dbPath := path.Join(wd, "db-test-revisions") + require.NoError(os.RemoveAll(dbPath)) + require.NoError(os.Mkdir(dbPath, 0o777)) + + // Create a new leveldb database + ldb, err := rawdb.NewLevelDBDatabase(dbPath, levelDBCacheSizeMB, 200, "metrics_prefix", false) + require.NoError(err) + + // Create a new trie database + const maxDiffs = 128 // triedb will store 128 diffs. + trieDB := triedb.NewDatabase( + ldb, + &triedb.Config{ + Preimages: false, + IsVerkle: false, + HashDB: nil, + PathDB: &testPathDbConfig, + }, + ) + + // Initially populate the trie + const diffSize = 100 + var ( + index uint64 + tdb = trie.NewEmpty(trieDB) + ) + for i := 0; i < diffSize; i++ { + entryHash := calculateIndexEncoding(index) + require.NoError(tdb.Update(entryHash, entryHash)) + index++ + } + + // Commit the initial state + root, nodes := tdb.Commit(false) + require.NoError(trieDB.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil /*states*/)) + + const numDiffs = 150 + rootHashes := []common.Hash{root} + for height := uint64(1); height < numDiffs; height++ { + prevRoot := root + tdb, err = trie.New(trie.TrieID(prevRoot), trieDB) + require.NoError(err) + + // update the tree based on the current revision. + for i := 0; i < diffSize; i++ { + entryHash := calculateIndexEncoding(index) + require.NoError(tdb.Update(entryHash, entryHash)) + index++ + } + + // Commit the changes + root, nodes = tdb.Commit(false) + require.NoError(trieDB.Update(root, prevRoot, height, trienode.NewWithNodeSet(nodes), nil /*states*/)) + + rootHashes = append(rootHashes, root) + } + + // Verify that old revisions were pruned. + numPruned := max(len(rootHashes)-maxDiffs-1, 0) + for _, root := range rootHashes[:numPruned] { + _, err = trie.New(trie.TrieID(root), trieDB) + require.Error(err) + } + + // Verify that all 128 revisions are queryable and return the expected data. + for height, root := range rootHashes[numPruned:] { + height += numPruned + + tdb, err = trie.New(trie.TrieID(root), trieDB) + require.NoError(err) + + maxIndex := diffSize * uint64(height+1) + for i := uint64(0); i < maxIndex; i++ { + entryHash := calculateIndexEncoding(i) + entryValue, err := tdb.Get(entryHash) + require.NoError(err) + require.Equal(entryHash, entryValue) + } + + entryHash := calculateIndexEncoding(maxIndex) + entryValue, err := tdb.Get(entryHash) + require.NoError(err) + require.Empty(entryValue) + } + + trieDB = triedb.NewDatabase( + ldb, + &triedb.Config{ + Preimages: false, + IsVerkle: false, + HashDB: nil, + PathDB: &pathdb.Config{ + CleanCacheSize: 1024 * 1024, + DirtyCacheSize: 0, // Disable dirty cache to force trieDB to write to disk. + }, + }, + ) + + _, err = trieDB.Reader(rootHashes[numPruned]) + require.NoError(err) +} diff --git a/x/merkledb/benchmarks_eth/directories_default.go b/x/merkledb/benchmarks_eth/directories_default.go new file mode 100644 index 000000000000..976f02311072 --- /dev/null +++ b/x/merkledb/benchmarks_eth/directories_default.go @@ -0,0 +1,121 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build linux || darwin || unix + +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" +) + +func CopyDirectory(scrDir, dest string) error { + entries, err := os.ReadDir(scrDir) + if err != nil { + return err + } + for _, entry := range entries { + sourcePath := filepath.Join(scrDir, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + fileInfo, err := os.Stat(sourcePath) + if err != nil { + return err + } + + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath) + } + + switch fileInfo.Mode() & os.ModeType { + case os.ModeDir: + if err := CreateIfNotExists(destPath, 0o755); err != nil { + return err + } + if err := CopyDirectory(sourcePath, destPath); err != nil { + return err + } + case os.ModeSymlink: + if err := CopySymLink(sourcePath, destPath); err != nil { + return err + } + default: + if err := Copy(sourcePath, destPath); err != nil { + return err + } + } + + if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + + fInfo, err := entry.Info() + if err != nil { + return err + } + + isSymlink := fInfo.Mode()&os.ModeSymlink != 0 + if !isSymlink { + if err := os.Chmod(destPath, fInfo.Mode()); err != nil { + return err + } + } + } + return nil +} + +func Copy(srcFile, dstFile string) error { + out, err := os.Create(dstFile) + if err != nil { + return err + } + + defer out.Close() + + in, err := os.Open(srcFile) + if err != nil { + return err + } + + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return nil +} + +func Exists(filePath string) bool { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return false + } + + return true +} + +func CreateIfNotExists(dir string, perm os.FileMode) error { + if Exists(dir) { + return nil + } + + if err := os.MkdirAll(dir, perm); err != nil { + return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) + } + + return nil +} + +func CopySymLink(source, dest string) error { + link, err := os.Readlink(source) + if err != nil { + return err + } + return os.Symlink(link, dest) +} diff --git a/x/merkledb/benchmarks_eth/directories_windows.go b/x/merkledb/benchmarks_eth/directories_windows.go new file mode 100644 index 000000000000..d3fc82b445ca --- /dev/null +++ b/x/merkledb/benchmarks_eth/directories_windows.go @@ -0,0 +1,107 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build windows + +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +func CopyDirectory(scrDir, dest string) error { + entries, err := os.ReadDir(scrDir) + if err != nil { + return err + } + for _, entry := range entries { + sourcePath := filepath.Join(scrDir, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + fileInfo, err := os.Stat(sourcePath) + if err != nil { + return err + } + + switch fileInfo.Mode() & os.ModeType { + case os.ModeDir: + if err := CreateIfNotExists(destPath, 0o755); err != nil { + return err + } + if err := CopyDirectory(sourcePath, destPath); err != nil { + return err + } + default: + if err := Copy(sourcePath, destPath); err != nil { + return err + } + } + + fInfo, err := entry.Info() + if err != nil { + return err + } + + isSymlink := fInfo.Mode()&os.ModeSymlink != 0 + if !isSymlink { + if err := os.Chmod(destPath, fInfo.Mode()); err != nil { + return err + } + } + } + return nil +} + +func Copy(srcFile, dstFile string) error { + out, err := os.Create(dstFile) + if err != nil { + return err + } + + defer out.Close() + + in, err := os.Open(srcFile) + if err != nil { + return err + } + + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return nil +} + +func Exists(filePath string) bool { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return false + } + + return true +} + +func CreateIfNotExists(dir string, perm os.FileMode) error { + if Exists(dir) { + return nil + } + + if err := os.MkdirAll(dir, perm); err != nil { + return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) + } + + return nil +} + +func CopySymLink(source, dest string) error { + link, err := os.Readlink(source) + if err != nil { + return err + } + return os.Symlink(link, dest) +} diff --git a/x/merkledb/benchmarks_eth/go.mod b/x/merkledb/benchmarks_eth/go.mod new file mode 100644 index 000000000000..ee72b7fcf35d --- /dev/null +++ b/x/merkledb/benchmarks_eth/go.mod @@ -0,0 +1,73 @@ +module github.com/ava-labs/avalanchego/x/merkledb/benchmarks_eth + +go 1.22.3 + +require ( + github.com/prometheus/client_golang v1.20.3 + github.com/spf13/cobra v1.5.0 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/DataDog/zstd v1.5.2 // indirect + github.com/VictoriaMetrics/fastcache v1.12.2 // indirect + github.com/bits-and-blooms/bitset v1.10.0 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v1.1.1 // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/holiman/uint256 v1.3.1 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/supranational/blst v0.3.11 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + golang.org/x/sync v0.7.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) + +require ( + github.com/ava-labs/avalanchego v1.11.11 + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/ethereum/go-ethereum v1.14.8 + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) diff --git a/x/merkledb/benchmarks_eth/go.sum b/x/merkledb/benchmarks_eth/go.sum new file mode 100644 index 000000000000..0f99057278f4 --- /dev/null +++ b/x/merkledb/benchmarks_eth/go.sum @@ -0,0 +1,275 @@ +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/ava-labs/avalanchego v1.11.11 h1:MIQq8xRavRj4ZXHA4G+aMiymig7SOScGOG1SApmMvBc= +github.com/ava-labs/avalanchego v1.11.11/go.mod h1:yFx3V31Jy9NFa8GZlgGnwiVf8KGjeF2+Uc99l9Scd/8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= +github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= +github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ1VTIig= +github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs= +github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= +github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= +github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= +github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= +golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/x/merkledb/benchmarks_eth/setup.sh b/x/merkledb/benchmarks_eth/setup.sh new file mode 100644 index 000000000000..e2f04cb87adf --- /dev/null +++ b/x/merkledb/benchmarks_eth/setup.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +#### run these commands from the root user #### +mkdir -p /etc/apt/keyrings/ +wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null +echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list +apt-get update +apt-get upgrade -y + +mkdir -p /etc/systemd/system/grafana-server.service.d +cat > /etc/systemd/system/grafana-server.service.d/override.conf <> /etc/prometheus/prometheus.yml <> /etc/default/prometheus-node-exporter <> /etc/fstab + mkdir -p "$NVME_MOUNT/ubuntu/firewood" + chown ubuntu:ubuntu "$NVME_MOUNT/ubuntu" "$NVME_MOUNT/ubuntu/firewood" + ln -s "$NVME_MOUNT/ubuntu/firewood" /home/ubuntu/avalanchego +fi + +## install go +export GO_INSTALLATION_FILE=go1.23.1.linux-amd64.tar.gz +curl -OL https://go.dev/dl/${GO_INSTALLATION_FILE} +rm -rf /usr/local/go && tar -C /usr/local -xzf ${GO_INSTALLATION_FILE} +rm ${GO_INSTALLATION_FILE} + +#### you can switch to the ubuntu user here #### +## ----------------------------------------- ### + +git clone https://github.com/ava-labs/avalanchego.git +cd avalanchego || exit +git checkout tsachi/bench_merkledb +cd ./x/merkledb/benchmarks_eth && PATH=$PATH:/usr/local/go/bin go build . + +#### stop here, these commands are run by hand #### + +# 10M rows: +nohup ./benchmarks_eth tenkrandom --n 10000000 > output.txt & +nohup ./benchmarks_eth zipf --n 10000000 > output.txt & +nohup ./benchmarks_eth single --n 10000000 > output.txt & + +# 50M rows: +nohup ./benchmarks_eth tenkrandom --n 50000000 > output.txt & +nohup ./benchmarks_eth zipf --n 50000000 > output.txt & +nohup ./benchmarks_eth single --n 50000000 > output.txt & + +# 100M rows: +nohup ./benchmarks_eth tenkrandom --n 100000000 > output.txt & +nohup ./benchmarks_eth zipf --n 100000000 > output.txt & +nohup ./benchmarks_eth single --n 100000000 > output.txt & + +cd ~/avalanchego/x/merkledb/benchmarks_eth && git pull && PATH=$PATH:/usr/local/go/bin go build . \ No newline at end of file diff --git a/x/merkledb/benchmarks_eth/single.go b/x/merkledb/benchmarks_eth/single.go new file mode 100644 index 000000000000..99f5b5ae853b --- /dev/null +++ b/x/merkledb/benchmarks_eth/single.go @@ -0,0 +1,121 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "encoding/binary" + "fmt" + "os" + "path" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" +) + +func runSingleBenchmark(databaseEntries uint64) error { + rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt")) + if err != nil { + return fmt.Errorf("unable to read root : %v", err) + } + + ldb, err := openLevelDBDatabaseNoCompression(rawdb.OpenOptions{ + Type: "leveldb", + Directory: getRunningDatabaseDirectory(databaseEntries), + AncientsDirectory: "", + Namespace: "metrics_prefix", + Cache: levelDBCacheSizeMB, + Handles: 200, + ReadOnly: false, + Ephemeral: false, + }) + if err != nil { + return fmt.Errorf("unable to create level db database : %v", err) + } + + trieDb := triedb.NewDatabase(ldb, &triedb.Config{ + Preimages: false, + IsVerkle: false, + HashDB: nil, + PathDB: &pathDBConfig, + }) + + parentHash := common.BytesToHash(rootBytes) + tdb, err := trie.New(trie.TrieID(parentHash), trieDb) + if err != nil { + return fmt.Errorf("unable to create trie database : %v", err) + } + var root common.Hash + blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize + writeBatch := func() error { + var nodes *trienode.NodeSet + root, nodes = tdb.Commit(false) + err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/) + if err != nil { + return fmt.Errorf("unable to update trie : %v", err) + } + tdb, err = trie.New(trie.TrieID(root), trieDb) + if err != nil { + return fmt.Errorf("unable to create new trie : %v", err) + } + parentHash = root + blockHeight++ + return nil + } + + batchIdx := uint64(0) + entriesKeys := make([][]byte, 10000) + for i := range entriesKeys { + entriesKeys[i] = calculateIndexEncoding(uint64(i)) + } + var updateDuration, batchDuration time.Duration + entriesToUpdate := uint64(1) + lastChangeEntriesCount := time.Now() + for { + startBatchTime := time.Now() + + // update a single entry, at random. + startUpdateTime := time.Now() + for i := uint64(0); i < entriesToUpdate; i++ { + err = tdb.Update(entriesKeys[i], binary.BigEndian.AppendUint64(nil, batchIdx*10000+i)) + if err != nil { + return fmt.Errorf("unable to update trie entry : %v", err) + } + } + + updateDuration = time.Since(startUpdateTime) + stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration)) + stats.updates.Add(databaseRunningUpdateSize) + + batchWriteStartTime := time.Now() + err = writeBatch() + if err != nil { + return fmt.Errorf("unable to write batch : %v", err) + } + batchDuration = time.Since(startBatchTime) + batchWriteDuration := time.Since(batchWriteStartTime) + stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration)) + + if *verbose { + fmt.Printf("update rate [%d]\tbatch rate [%d]\tbatch size [%d]\n", + time.Second/updateDuration, + time.Second/batchDuration, + entriesToUpdate) + } + + stats.batches.Inc() + batchIdx++ + + if time.Since(lastChangeEntriesCount) > 5*time.Minute { + lastChangeEntriesCount = time.Now() + entriesToUpdate *= 10 + if entriesToUpdate > 10000 { + entriesToUpdate = 1 + } + } + } +} diff --git a/x/merkledb/benchmarks_eth/tenkrandom.go b/x/merkledb/benchmarks_eth/tenkrandom.go new file mode 100644 index 000000000000..bf1f07ba0d1c --- /dev/null +++ b/x/merkledb/benchmarks_eth/tenkrandom.go @@ -0,0 +1,134 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "fmt" + "os" + "path" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" +) + +func runTenkrandomBenchmark(databaseEntries uint64) error { + rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt")) + if err != nil { + return fmt.Errorf("unable to read root : %v", err) + } + + ldb, err := openLevelDBDatabaseNoCompression(rawdb.OpenOptions{ + Type: "leveldb", + Directory: getRunningDatabaseDirectory(databaseEntries), + AncientsDirectory: "", + Namespace: "metrics_prefix", + Cache: levelDBCacheSizeMB, + Handles: 200, + ReadOnly: false, + Ephemeral: false, + }) + if err != nil { + return fmt.Errorf("unable to create level db database : %v", err) + } + + trieDb := triedb.NewDatabase(ldb, &triedb.Config{ + Preimages: false, + IsVerkle: false, + HashDB: nil, + PathDB: &pathDBConfig, + }) + + parentHash := common.BytesToHash(rootBytes) + tdb, err := trie.New(trie.TrieID(parentHash), trieDb) + if err != nil { + return fmt.Errorf("unable to create trie database : %v", err) + } + var root common.Hash + blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize + writeBatch := func() error { + var nodes *trienode.NodeSet + root, nodes = tdb.Commit(false) + err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/) + if err != nil { + return fmt.Errorf("unable to update trie : %v", err) + } + tdb, err = trie.New(trie.TrieID(root), trieDb) + if err != nil { + return fmt.Errorf("unable to create new trie : %v", err) + } + parentHash = root + blockHeight++ + return nil + } + + low := uint64(0) + var deleteDuration, addDuration, updateDuration, batchDuration time.Duration + for { + startBatchTime := time.Now() + + // delete first 2.5k keys from the beginning + startDeleteTime := time.Now() + for keyToDeleteIdx := low; keyToDeleteIdx < low+databaseRunningBatchSize; keyToDeleteIdx++ { + entryHash := calculateIndexEncoding(keyToDeleteIdx) + err = tdb.Delete(entryHash) + if err != nil { + return fmt.Errorf("unable to delete trie entry : %v", err) + } + } + deleteDuration = time.Since(startDeleteTime) + stats.deleteRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(deleteDuration)) + stats.deletes.Add(databaseRunningBatchSize) + + // add 2.5k past end. + startInsertTime := time.Now() + for keyToAddIdx := low + databaseEntries; keyToAddIdx < low+databaseEntries+databaseRunningBatchSize; keyToAddIdx++ { + entryHash := calculateIndexEncoding(keyToAddIdx) + err = tdb.Update(entryHash, entryHash) + if err != nil { + return fmt.Errorf("unable to insert trie entry : %v", err) + } + } + addDuration = time.Since(startInsertTime) + stats.insertRate.Set(float64(databaseRunningBatchSize) * float64(time.Second) / float64(addDuration)) + stats.inserts.Add(databaseRunningBatchSize) + + // update middle 5k entries + startUpdateTime := time.Now() + updateEntryValue := calculateIndexEncoding(low) + for keyToUpdateIdx := low + (databaseEntries / 2); keyToUpdateIdx < low+(databaseEntries/2)+databaseRunningUpdateSize; keyToUpdateIdx++ { + updateEntryKey := calculateIndexEncoding(keyToUpdateIdx) + err = tdb.Update(updateEntryKey, updateEntryValue) + if err != nil { + return fmt.Errorf("unable to update trie entry : %v", err) + } + } + updateDuration = time.Since(startUpdateTime) + stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration)) + stats.updates.Add(databaseRunningUpdateSize) + + batchWriteStartTime := time.Now() + err = writeBatch() + if err != nil { + return fmt.Errorf("unable to write batch : %v", err) + } + batchDuration = time.Since(startBatchTime) + batchWriteDuration := time.Since(batchWriteStartTime) + stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration)) + + if *verbose { + fmt.Printf("delete rate [%d] update rate [%d] insert rate [%d] batch rate [%d]\n", + time.Second/deleteDuration, + time.Second/updateDuration, + time.Second/addDuration, + time.Second/batchDuration) + } + + stats.batches.Inc() + low += databaseRunningBatchSize + } +} diff --git a/x/merkledb/benchmarks_eth/zipf.go b/x/merkledb/benchmarks_eth/zipf.go new file mode 100644 index 000000000000..4f86f28ebef7 --- /dev/null +++ b/x/merkledb/benchmarks_eth/zipf.go @@ -0,0 +1,110 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "fmt" + "math/rand" + "os" + "path" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb" +) + +func runZipfBenchmark(databaseEntries uint64, sZipf float64, vZipf float64) error { + rootBytes, err := os.ReadFile(path.Join(getRunningDatabaseDirectory(databaseEntries), "root.txt")) + if err != nil { + return fmt.Errorf("unable to read root : %v", err) + } + + ldb, err := openLevelDBDatabaseNoCompression(rawdb.OpenOptions{ + Type: "leveldb", + Directory: getRunningDatabaseDirectory(databaseEntries), + AncientsDirectory: "", + Namespace: "metrics_prefix", + Cache: levelDBCacheSizeMB, + Handles: 200, + ReadOnly: false, + Ephemeral: false, + }) + if err != nil { + return fmt.Errorf("unable to create level db database : %v", err) + } + + trieDb := triedb.NewDatabase(ldb, &triedb.Config{ + Preimages: false, + IsVerkle: false, + HashDB: nil, + PathDB: &pathDBConfig, + }) + + parentHash := common.BytesToHash(rootBytes) + tdb, err := trie.New(trie.TrieID(parentHash), trieDb) + if err != nil { + return fmt.Errorf("unable to create trie database : %v", err) + } + var root common.Hash + blockHeight := (databaseEntries + databaseCreationBatchSize - 1) / databaseCreationBatchSize + writeBatch := func() error { + var nodes *trienode.NodeSet + root, nodes = tdb.Commit(false) + err = trieDb.Update(root, parentHash, blockHeight, trienode.NewWithNodeSet(nodes), nil /*states*/) + if err != nil { + return fmt.Errorf("unable to update trie : %v", err) + } + tdb, err = trie.New(trie.TrieID(root), trieDb) + if err != nil { + return fmt.Errorf("unable to create new trie : %v", err) + } + parentHash = root + blockHeight++ + return nil + } + + batchIndex := uint64(0) + var updateDuration, batchDuration time.Duration + + zipf := rand.NewZipf(rand.New(rand.NewSource(0)), sZipf, vZipf, databaseEntries) + for { + startBatchTime := time.Now() + + // update 10k entries + startUpdateTime := time.Now() + updateEntryValue := calculateIndexEncoding(batchIndex) + + for keyToUpdateIdx := 0; keyToUpdateIdx < databaseCreationBatchSize; keyToUpdateIdx++ { + updateEntryKey := calculateIndexEncoding(zipf.Uint64()) + err = tdb.Update(updateEntryKey, updateEntryValue) + if err != nil { + return fmt.Errorf("unable to update trie entry : %v", err) + } + } + updateDuration = time.Since(startUpdateTime) + stats.updateRate.Set(float64(databaseRunningUpdateSize) * float64(time.Second) / float64(updateDuration)) + stats.updates.Add(databaseRunningUpdateSize) + + batchWriteStartTime := time.Now() + err = writeBatch() + if err != nil { + return fmt.Errorf("unable to write batch : %v", err) + } + batchDuration = time.Since(startBatchTime) + batchWriteDuration := time.Since(batchWriteStartTime) + stats.batchWriteRate.Set(float64(time.Second) / float64(batchWriteDuration)) + + if *verbose { + fmt.Printf("update rate [%d] batch rate [%d]\n", + time.Second/updateDuration, + time.Second/batchDuration) + } + + stats.batches.Inc() + batchIndex++ + } +} diff --git a/x/merkledb/benchmarks_eth/zipf_test.go b/x/merkledb/benchmarks_eth/zipf_test.go new file mode 100644 index 000000000000..e18ec167c5a8 --- /dev/null +++ b/x/merkledb/benchmarks_eth/zipf_test.go @@ -0,0 +1,24 @@ +// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import ( + "math/rand" + "testing" +) + +func TestZipfDist(t *testing.T) { + dbSize := uint64(1000000) + chunkSize := 10000 + for _, sVal := range []float64{1.01, 1.1, 1.2, 1.3, 1.4} { + for _, vVal := range []float64{1.0} { + zipf := rand.NewZipf(rand.New(rand.NewSource(0)), sVal, vVal, dbSize) + samples := make(map[uint64]uint64) + for i := 0; i < chunkSize; i++ { + samples[zipf.Uint64()]++ + } + t.Logf("number of unique keys for sVal = %v and vVal = %v is %d when sampling %d out of %d\n", sVal, vVal, len(samples), chunkSize, dbSize) + } + } +}