From fe6a2dafc06eb2a8dc44467811bcacdc7b773845 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Mon, 2 Dec 2024 10:31:51 +0100 Subject: [PATCH 01/14] Caplin: fix broken snapshots (#12954) --- erigon-lib/downloader/snaptype/files.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/erigon-lib/downloader/snaptype/files.go b/erigon-lib/downloader/snaptype/files.go index 22b61d09cd2..e556bfaf880 100644 --- a/erigon-lib/downloader/snaptype/files.go +++ b/erigon-lib/downloader/snaptype/files.go @@ -141,10 +141,8 @@ func parseFileName(dir, fileName string) (res FileInfo, ok bool) { } res.Type, ok = ParseFileType(parts[len(parts)-1]) - - if !ok { - res.Type, ok = ParseFileType(parts[0]) - } + // This is a caplin hack - it is because with caplin state snapshots ok is always false + res.TypeString = parts[len(parts)-1] if ok { res.TypeString = res.Type.Name() From 392c8e1cc68f6ba850ef254f27fa56d9f70c1f9a Mon Sep 17 00:00:00 2001 From: antonis19 Date: Mon, 2 Dec 2024 11:00:17 +0100 Subject: [PATCH 02/14] move core/types/accounts to erigon-lib/types/accounts (#12955) Follow-up to https://github.com/erigontech/erigon/pull/12915 --------- Co-authored-by: antonis19 --- cmd/verkle/main.go | 2 +- cmd/verkle/verkletrie/incrementAccount.go | 2 +- cmd/verkle/verkletrie/pedersen_hashstate.go | 2 +- cmd/verkle/verkletrie/verkle_tree_writer.go | 2 +- cmd/verkle/verkletrie/workers.go | 2 +- consensus/clique/clique.go | 2 +- core/chain_makers.go | 2 +- core/state/cached_reader.go | 2 +- core/state/cached_reader3.go | 2 +- core/state/cached_writer.go | 2 +- core/state/database.go | 2 +- core/state/dump.go | 2 +- core/state/history_reader_v3.go | 2 +- core/state/intra_block_state.go | 2 +- core/state/rw_v3.go | 2 +- core/state/state_object.go | 2 +- core/state/state_test.go | 2 +- core/state/state_writer_v4.go | 2 +- core/state/txtask.go | 2 +- core/test/domains_restart_test.go | 2 +- {core => erigon-lib}/types/accounts/account.go | 10 ++++------ .../types/accounts/account_benchmark_test.go | 0 {core => erigon-lib}/types/accounts/account_proof.go | 0 {core => erigon-lib}/types/accounts/account_test.go | 2 ++ eth/stagedsync/exec3.go | 2 +- eth/stagedsync/stage_mining_exec.go | 2 +- eth/stagedsync/testutil.go | 2 +- polygon/bor/bor.go | 2 +- turbo/adapter/ethapi/get_proof.go | 2 +- turbo/jsonrpc/debug_api.go | 2 +- turbo/jsonrpc/erigon_block.go | 2 +- turbo/jsonrpc/eth_api.go | 2 +- turbo/jsonrpc/eth_call.go | 2 +- turbo/jsonrpc/otterscan_contract_creator.go | 2 +- .../otterscan_transaction_by_sender_and_nonce.go | 2 +- turbo/jsonrpc/trace_adhoc.go | 2 +- turbo/shards/state_cache.go | 2 +- turbo/shards/state_cache_test.go | 2 +- turbo/shards/trie_cache.go | 2 +- turbo/trie/account_node_test.go | 2 +- turbo/trie/delete_subrtee_test.go | 2 +- turbo/trie/flatdb_sub_trie_loader_test.go | 2 +- turbo/trie/hashbuilder.go | 2 +- turbo/trie/node.go | 2 +- turbo/trie/proof.go | 2 +- turbo/trie/retain_list.go | 2 +- turbo/trie/retain_list_test.go | 2 +- turbo/trie/stream.go | 2 +- turbo/trie/stream_test.go | 2 +- turbo/trie/trie.go | 2 +- turbo/trie/trie_root.go | 2 +- turbo/trie/trie_test.go | 2 +- turbo/trie/trie_transform.go | 2 +- 53 files changed, 55 insertions(+), 55 deletions(-) rename {core => erigon-lib}/types/accounts/account.go (99%) rename {core => erigon-lib}/types/accounts/account_benchmark_test.go (100%) rename {core => erigon-lib}/types/accounts/account_proof.go (100%) rename {core => erigon-lib}/types/accounts/account_test.go (99%) diff --git a/cmd/verkle/main.go b/cmd/verkle/main.go index c9f2aa49b85..c4030d80ff4 100644 --- a/cmd/verkle/main.go +++ b/cmd/verkle/main.go @@ -36,8 +36,8 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/cmd/verkle/verkletrie" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/eth/stagedsync/stages" ) diff --git a/cmd/verkle/verkletrie/incrementAccount.go b/cmd/verkle/verkletrie/incrementAccount.go index 6618691dc9c..d714a9a5420 100644 --- a/cmd/verkle/verkletrie/incrementAccount.go +++ b/cmd/verkle/verkletrie/incrementAccount.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/common/debug" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func IncrementAccount(vTx kv.RwTx, tx kv.Tx, workers uint64, verkleWriter *VerkleTreeWriter, from, to uint64, tmpdir string) error { diff --git a/cmd/verkle/verkletrie/pedersen_hashstate.go b/cmd/verkle/verkletrie/pedersen_hashstate.go index edfbebba9ee..00569d2f660 100644 --- a/cmd/verkle/verkletrie/pedersen_hashstate.go +++ b/cmd/verkle/verkletrie/pedersen_hashstate.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/debug" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func RegeneratePedersenAccounts(outTx kv.RwTx, readTx kv.Tx, workers uint64, verkleWriter *VerkleTreeWriter) error { diff --git a/cmd/verkle/verkletrie/verkle_tree_writer.go b/cmd/verkle/verkletrie/verkle_tree_writer.go index 40f74b8e2b2..f0a302c0119 100644 --- a/cmd/verkle/verkletrie/verkle_tree_writer.go +++ b/cmd/verkle/verkletrie/verkle_tree_writer.go @@ -31,8 +31,8 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/rawdb" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/turbo/trie/vtree" ) diff --git a/cmd/verkle/verkletrie/workers.go b/cmd/verkle/verkletrie/workers.go index 0bd66244225..0ee88a7c7a7 100644 --- a/cmd/verkle/verkletrie/workers.go +++ b/cmd/verkle/verkletrie/workers.go @@ -22,7 +22,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/holiman/uint256" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/trie/vtree" ) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 413242b2415..c72202bb31b 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -44,11 +44,11 @@ import ( "github.com/erigontech/erigon-lib/kv/dbutils" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/consensus" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/tracing" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm/evmtypes" "github.com/erigontech/erigon/params" "github.com/erigontech/erigon/rpc" diff --git a/core/chain_makers.go b/core/chain_makers.go index 4ce7ea744cc..d3b17010a21 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -35,12 +35,12 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/rlp" libstate "github.com/erigontech/erigon-lib/state" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/consensus" "github.com/erigontech/erigon/consensus/merge" "github.com/erigontech/erigon/consensus/misc" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm" "github.com/erigontech/erigon/params" "github.com/erigontech/erigon/polygon/heimdall" diff --git a/core/state/cached_reader.go b/core/state/cached_reader.go index 629c1d0da09..fba1496474b 100644 --- a/core/state/cached_reader.go +++ b/core/state/cached_reader.go @@ -19,7 +19,7 @@ package state import ( "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/shards" ) diff --git a/core/state/cached_reader3.go b/core/state/cached_reader3.go index 81f2e18a7d6..b8ceec09f76 100644 --- a/core/state/cached_reader3.go +++ b/core/state/cached_reader3.go @@ -23,7 +23,7 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/kvcache" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) // CachedReader3 is a wrapper for an instance of type StateReader diff --git a/core/state/cached_writer.go b/core/state/cached_writer.go index db832e4475f..b3b6850080d 100644 --- a/core/state/cached_writer.go +++ b/core/state/cached_writer.go @@ -21,7 +21,7 @@ import ( "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/shards" ) diff --git a/core/state/database.go b/core/state/database.go index f99f7516451..eccdc195b2d 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -25,7 +25,7 @@ import ( "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) const ( diff --git a/core/state/dump.go b/core/state/dump.go index 884a64d5f92..897d3153414 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/order" "github.com/erigontech/erigon-lib/kv/rawdbv3" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/trie" ) diff --git a/core/state/history_reader_v3.go b/core/state/history_reader_v3.go index d404d1ba82d..05db794cb5f 100644 --- a/core/state/history_reader_v3.go +++ b/core/state/history_reader_v3.go @@ -22,7 +22,7 @@ import ( "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/state" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) // HistoryReaderV3 Implements StateReader and StateWriter diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index 854d1a733d7..4c64fa354d7 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -31,9 +31,9 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/u256" "github.com/erigontech/erigon-lib/crypto" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/tracing" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm/evmtypes" "github.com/erigontech/erigon/turbo/trie" ) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index f9ff517316b..baf67f3d5cf 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -35,7 +35,7 @@ import ( "github.com/erigontech/erigon-lib/metrics" "github.com/erigontech/erigon-lib/state" libstate "github.com/erigontech/erigon-lib/state" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/shards" ) diff --git a/core/state/state_object.go b/core/state/state_object.go index c3745e63354..469abd5541d 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -30,8 +30,8 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/tracing" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/turbo/trie" ) diff --git a/core/state/state_test.go b/core/state/state_test.go index cacb01edd41..6ca1639a8f2 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -39,8 +39,8 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/state" stateLib "github.com/erigontech/erigon-lib/state" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/tracing" - "github.com/erigontech/erigon/core/types/accounts" ) var toAddr = common.BytesToAddress diff --git a/core/state/state_writer_v4.go b/core/state/state_writer_v4.go index 12654ea5298..a6d11763e24 100644 --- a/core/state/state_writer_v4.go +++ b/core/state/state_writer_v4.go @@ -23,7 +23,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) var _ StateWriter = (*WriterV4)(nil) diff --git a/core/state/txtask.go b/core/state/txtask.go index 4fbb0819bbe..6ea205cbeff 100644 --- a/core/state/txtask.go +++ b/core/state/txtask.go @@ -29,8 +29,8 @@ import ( "github.com/erigontech/erigon-lib/chain" libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/state" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm/evmtypes" ) diff --git a/core/test/domains_restart_test.go b/core/test/domains_restart_test.go index 10e3acf5226..5fd7ea02572 100644 --- a/core/test/domains_restart_test.go +++ b/core/test/domains_restart_test.go @@ -44,11 +44,11 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/state" types2 "github.com/erigontech/erigon-lib/types" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core" reset2 "github.com/erigontech/erigon/core/rawdb/rawdbreset" state2 "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/params" ) diff --git a/core/types/accounts/account.go b/erigon-lib/types/accounts/account.go similarity index 99% rename from core/types/accounts/account.go rename to erigon-lib/types/accounts/account.go index 2c14bbb2b79..4bf92881a27 100644 --- a/core/types/accounts/account.go +++ b/erigon-lib/types/accounts/account.go @@ -157,21 +157,19 @@ func decodeLengthForHashing(buffer []byte, pos int) (length int, structure bool, case firstByte < 192: // Next byte is the length of the length + 183 lenEnd := pos + 1 + firstByte - 183 - len := 0 for i := pos + 1; i < lenEnd; i++ { - len = (len << 8) + int(buffer[i]) + length = (length << 8) + int(buffer[i]) } - return len, false, lenEnd + return length, false, lenEnd case firstByte < 248: return firstByte - 192, true, pos + 1 default: // Next byte is the length of the length + 247 lenEnd := pos + 1 + firstByte - 247 - len := 0 for i := pos + 1; i < lenEnd; i++ { - len = (len << 8) + int(buffer[i]) + length = (length << 8) + int(buffer[i]) } - return len, true, lenEnd + return length, true, lenEnd } } diff --git a/core/types/accounts/account_benchmark_test.go b/erigon-lib/types/accounts/account_benchmark_test.go similarity index 100% rename from core/types/accounts/account_benchmark_test.go rename to erigon-lib/types/accounts/account_benchmark_test.go diff --git a/core/types/accounts/account_proof.go b/erigon-lib/types/accounts/account_proof.go similarity index 100% rename from core/types/accounts/account_proof.go rename to erigon-lib/types/accounts/account_proof.go diff --git a/core/types/accounts/account_test.go b/erigon-lib/types/accounts/account_test.go similarity index 99% rename from core/types/accounts/account_test.go rename to erigon-lib/types/accounts/account_test.go index 25226d0d32a..4f8d5087583 100644 --- a/core/types/accounts/account_test.go +++ b/erigon-lib/types/accounts/account_test.go @@ -194,6 +194,7 @@ func TestEncodeAccountWithEmptyBalanceAndNotZeroIncarnation(t *testing.T) { } func isAccountsEqual(t *testing.T, src, dst Account) { + t.Helper() if dst.Initialised != src.Initialised { t.Fatal("cant decode the account Initialised", src.Initialised, dst.Initialised) } @@ -325,6 +326,7 @@ func TestIncarnationWithInvalidEncodedAccount(t *testing.T) { } func isIncarnationEqual(t *testing.T, initialIncarnation uint64, decodedIncarnation uint64) { + t.Helper() if initialIncarnation != decodedIncarnation { t.Fatal("Can't decode the incarnation", initialIncarnation, decodedIncarnation) } diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 2303490f3f4..d148f3a249d 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -37,6 +37,7 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/metrics" state2 "github.com/erigontech/erigon-lib/state" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon-lib/wrap" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/rawdb" @@ -44,7 +45,6 @@ import ( "github.com/erigontech/erigon/core/rawdb/rawtemporaldb" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/eth/ethconfig/estimate" "github.com/erigontech/erigon/eth/stagedsync/stages" "github.com/erigontech/erigon/turbo/services" diff --git a/eth/stagedsync/stage_mining_exec.go b/eth/stagedsync/stage_mining_exec.go index f9a07141143..db03cb9fe53 100644 --- a/eth/stagedsync/stage_mining_exec.go +++ b/eth/stagedsync/stage_mining_exec.go @@ -33,13 +33,13 @@ import ( "github.com/erigontech/erigon-lib/kv/membatchwithdb" "github.com/erigontech/erigon-lib/log/v3" state2 "github.com/erigontech/erigon-lib/state" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon-lib/wrap" "github.com/erigontech/erigon/consensus" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/rawdb" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm" "github.com/erigontech/erigon/eth/stagedsync/stages" "github.com/erigontech/erigon/params" diff --git a/eth/stagedsync/testutil.go b/eth/stagedsync/testutil.go index 5bb4a71ef13..802a9a8aec1 100644 --- a/eth/stagedsync/testutil.go +++ b/eth/stagedsync/testutil.go @@ -24,8 +24,8 @@ import ( "github.com/holiman/uint256" libcommon "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/state" - "github.com/erigontech/erigon/core/types/accounts" ) const ( diff --git a/polygon/bor/bor.go b/polygon/bor/bor.go index 3b5c8c9401e..1bba50d515d 100644 --- a/polygon/bor/bor.go +++ b/polygon/bor/bor.go @@ -47,13 +47,13 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/consensus" "github.com/erigontech/erigon/consensus/misc" "github.com/erigontech/erigon/core/rawdb" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/tracing" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm/evmtypes" "github.com/erigontech/erigon/eth/ethconfig/estimate" "github.com/erigontech/erigon/params" diff --git a/turbo/adapter/ethapi/get_proof.go b/turbo/adapter/ethapi/get_proof.go index 3bc2f606fe9..aed1cb7ec1d 100644 --- a/turbo/adapter/ethapi/get_proof.go +++ b/turbo/adapter/ethapi/get_proof.go @@ -24,7 +24,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/length" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/trie" ) diff --git a/turbo/jsonrpc/debug_api.go b/turbo/jsonrpc/debug_api.go index 139c62485f1..013dcfd62cb 100644 --- a/turbo/jsonrpc/debug_api.go +++ b/turbo/jsonrpc/debug_api.go @@ -30,8 +30,8 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/state" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/eth/stagedsync/stages" tracersConfig "github.com/erigontech/erigon/eth/tracers/config" "github.com/erigontech/erigon/rpc" diff --git a/turbo/jsonrpc/erigon_block.go b/turbo/jsonrpc/erigon_block.go index e78381ab7ff..8b66cec0032 100644 --- a/turbo/jsonrpc/erigon_block.go +++ b/turbo/jsonrpc/erigon_block.go @@ -32,9 +32,9 @@ import ( "github.com/erigontech/erigon/turbo/services" "github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/rawdb" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/rpc" "github.com/erigontech/erigon/turbo/adapter/ethapi" "github.com/erigontech/erigon/turbo/rpchelper" diff --git a/turbo/jsonrpc/eth_api.go b/turbo/jsonrpc/eth_api.go index 94a5e53ddb2..3db26e672e0 100644 --- a/turbo/jsonrpc/eth_api.go +++ b/turbo/jsonrpc/eth_api.go @@ -39,12 +39,12 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/kvcache" "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/consensus" "github.com/erigontech/erigon/consensus/misc" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/rawdb" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" ethFilters "github.com/erigontech/erigon/eth/filters" "github.com/erigontech/erigon/ethdb/prune" "github.com/erigontech/erigon/polygon/bor/borcfg" diff --git a/turbo/jsonrpc/eth_call.go b/turbo/jsonrpc/eth_call.go index eb88d667f13..2f9cd154068 100644 --- a/turbo/jsonrpc/eth_call.go +++ b/turbo/jsonrpc/eth_call.go @@ -34,10 +34,10 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/rawdbv3" "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm" "github.com/erigontech/erigon/core/vm/evmtypes" "github.com/erigontech/erigon/eth/tracers/logger" diff --git a/turbo/jsonrpc/otterscan_contract_creator.go b/turbo/jsonrpc/otterscan_contract_creator.go index 3d1d13ec484..d26e3b3aa81 100644 --- a/turbo/jsonrpc/otterscan_contract_creator.go +++ b/turbo/jsonrpc/otterscan_contract_creator.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon-lib/kv/order" "github.com/erigontech/erigon-lib/kv/rawdbv3" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/rpchelper" "github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks" ) diff --git a/turbo/jsonrpc/otterscan_transaction_by_sender_and_nonce.go b/turbo/jsonrpc/otterscan_transaction_by_sender_and_nonce.go index 2411bb10c5c..62b52ba945a 100644 --- a/turbo/jsonrpc/otterscan_transaction_by_sender_and_nonce.go +++ b/turbo/jsonrpc/otterscan_transaction_by_sender_and_nonce.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon-lib/kv/order" "github.com/erigontech/erigon-lib/kv/rawdbv3" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks" ) diff --git a/turbo/jsonrpc/trace_adhoc.go b/turbo/jsonrpc/trace_adhoc.go index fdc5e8be790..0a24f0beca4 100644 --- a/turbo/jsonrpc/trace_adhoc.go +++ b/turbo/jsonrpc/trace_adhoc.go @@ -33,10 +33,10 @@ import ( math2 "github.com/erigontech/erigon-lib/common/math" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/core/types/accounts" "github.com/erigontech/erigon/core/vm" "github.com/erigontech/erigon/core/vm/evmtypes" "github.com/erigontech/erigon/eth/tracers/config" diff --git a/turbo/shards/state_cache.go b/turbo/shards/state_cache.go index 2e7a4e59482..e42a8f0e98f 100644 --- a/turbo/shards/state_cache.go +++ b/turbo/shards/state_cache.go @@ -29,7 +29,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/metrics" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) // LRU state cache consists of two structures - B-Tree and binary heap diff --git a/turbo/shards/state_cache_test.go b/turbo/shards/state_cache_test.go index 4ad54f3c6da..da425e68074 100644 --- a/turbo/shards/state_cache_test.go +++ b/turbo/shards/state_cache_test.go @@ -26,7 +26,7 @@ import ( "github.com/stretchr/testify/assert" "golang.org/x/crypto/sha3" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func TestCacheBtreeOrderAccountStorage2(t *testing.T) { diff --git a/turbo/shards/trie_cache.go b/turbo/shards/trie_cache.go index 28c1d29331d..e9002f882c1 100644 --- a/turbo/shards/trie_cache.go +++ b/turbo/shards/trie_cache.go @@ -27,7 +27,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv/dbutils" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) // An optional addition to the state cache, helping to calculate state root diff --git a/turbo/trie/account_node_test.go b/turbo/trie/account_node_test.go index d36cfa47bd9..04c50eb0208 100644 --- a/turbo/trie/account_node_test.go +++ b/turbo/trie/account_node_test.go @@ -27,7 +27,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/kv/dbutils" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func TestGetAccount(t *testing.T) { diff --git a/turbo/trie/delete_subrtee_test.go b/turbo/trie/delete_subrtee_test.go index ea226f49843..b29dccf8e90 100644 --- a/turbo/trie/delete_subrtee_test.go +++ b/turbo/trie/delete_subrtee_test.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/crypto" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func TestTrieDeleteSubtree_ShortNode(t *testing.T) { diff --git a/turbo/trie/flatdb_sub_trie_loader_test.go b/turbo/trie/flatdb_sub_trie_loader_test.go index a9b8a399b73..bb672135f16 100644 --- a/turbo/trie/flatdb_sub_trie_loader_test.go +++ b/turbo/trie/flatdb_sub_trie_loader_test.go @@ -26,7 +26,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func TestCreateLoadingPrefixes(t *testing.T) { diff --git a/turbo/trie/hashbuilder.go b/turbo/trie/hashbuilder.go index d8e3c53126d..2ee2d9266d4 100644 --- a/turbo/trie/hashbuilder.go +++ b/turbo/trie/hashbuilder.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/rlphacks" ) diff --git a/turbo/trie/node.go b/turbo/trie/node.go index 1423a8aeba4..5fbdfb187df 100644 --- a/turbo/trie/node.go +++ b/turbo/trie/node.go @@ -25,7 +25,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon-lib/rlp" ) diff --git a/turbo/trie/proof.go b/turbo/trie/proof.go index 5c3eb271418..9fcd8770aea 100644 --- a/turbo/trie/proof.go +++ b/turbo/trie/proof.go @@ -26,7 +26,7 @@ import ( "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) // Prove constructs a merkle proof for key. The result contains all encoded nodes diff --git a/turbo/trie/retain_list.go b/turbo/trie/retain_list.go index 5d43c95b451..fb9215c3c4f 100644 --- a/turbo/trie/retain_list.go +++ b/turbo/trie/retain_list.go @@ -34,7 +34,7 @@ import ( "github.com/erigontech/erigon-lib/common/hexutility" "github.com/erigontech/erigon-lib/common/length" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) type RetainDecider interface { diff --git a/turbo/trie/retain_list_test.go b/turbo/trie/retain_list_test.go index 96e1381544a..563d4c52fc2 100644 --- a/turbo/trie/retain_list_test.go +++ b/turbo/trie/retain_list_test.go @@ -24,7 +24,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/hexutil" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func FakePreimage(hash libcommon.Hash) libcommon.Hash { diff --git a/turbo/trie/stream.go b/turbo/trie/stream.go index 19e81503d05..7d53c84abab 100644 --- a/turbo/trie/stream.go +++ b/turbo/trie/stream.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/rlphacks" ) diff --git a/turbo/trie/stream_test.go b/turbo/trie/stream_test.go index 2c4ea1ee298..213859c4d07 100644 --- a/turbo/trie/stream_test.go +++ b/turbo/trie/stream_test.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func TestHashWithModificationsEmpty(t *testing.T) { diff --git a/turbo/trie/trie.go b/turbo/trie/trie.go index 09686362c77..6254464afe8 100644 --- a/turbo/trie/trie.go +++ b/turbo/trie/trie.go @@ -28,7 +28,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/crypto" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/ethdb" ) diff --git a/turbo/trie/trie_root.go b/turbo/trie/trie_root.go index fe89780a437..67e3ae3105d 100644 --- a/turbo/trie/trie_root.go +++ b/turbo/trie/trie_root.go @@ -32,7 +32,7 @@ import ( "github.com/erigontech/erigon-lib/kv" dbutils2 "github.com/erigontech/erigon-lib/kv/dbutils" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/turbo/rlphacks" ) diff --git a/turbo/trie/trie_test.go b/turbo/trie/trie_test.go index c7edcb84eb3..42b32784f11 100644 --- a/turbo/trie/trie_test.go +++ b/turbo/trie/trie_test.go @@ -35,7 +35,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) func init() { diff --git a/turbo/trie/trie_transform.go b/turbo/trie/trie_transform.go index ff1d15a2324..9bdd8679c2c 100644 --- a/turbo/trie/trie_transform.go +++ b/turbo/trie/trie_transform.go @@ -18,7 +18,7 @@ package trie import ( "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/core/types/accounts" + "github.com/erigontech/erigon-lib/types/accounts" ) type keyTransformFunc func([]byte) []byte From 067f3b0dc87214584a5f183429eb7ba631be823a Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 18:08:19 +0700 Subject: [PATCH 03/14] use `bytes.Clone` from stdlib (#12947) --- erigon-lib/common/bytes.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/erigon-lib/common/bytes.go b/erigon-lib/common/bytes.go index 1e5d4cce608..cfc7fc6378c 100644 --- a/erigon-lib/common/bytes.go +++ b/erigon-lib/common/bytes.go @@ -46,14 +46,7 @@ func MBToGB(b uint64) (float64, int) { return float64(b) / float64(div), exp } -func Copy(b []byte) []byte { - if b == nil { - return nil - } - c := make([]byte, len(b)) - copy(c, b) - return c -} +var Copy = bytes.Clone func Append(data ...[]byte) []byte { s := new(bytes.Buffer) From 73b12da58ab2fa946b68b836671272c0e85e453c Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 18:14:48 +0700 Subject: [PATCH 04/14] StateReader: remove CodeHash parameter (#12936) - moved empty-code-hash check to callers: `ibs` and `stateObject` - removed CodeHash param from state reader interface (E3 does store code by acc address. E2 did by code hash) --- cmd/integration/commands/state_domains.go | 2 +- core/state/cached_reader.go | 11 ++++------- core/state/cached_reader3.go | 11 +++-------- core/state/database.go | 4 ++-- core/state/database_test.go | 15 ++++++--------- core/state/history_reader_v3.go | 9 +++------ core/state/intra_block_state.go | 5 ++++- core/state/rw_v3.go | 11 ++++------- core/state/state_object.go | 2 +- eth/stagedsync/stage_mining_exec.go | 2 +- turbo/jsonrpc/eth_accounts.go | 2 +- 11 files changed, 30 insertions(+), 44 deletions(-) diff --git a/cmd/integration/commands/state_domains.go b/cmd/integration/commands/state_domains.go index 66098eea36b..6ca2ecfbea1 100644 --- a/cmd/integration/commands/state_domains.go +++ b/cmd/integration/commands/state_domains.go @@ -171,7 +171,7 @@ func requestDomains(chainDb, stateDb kv.RwDB, ctx context.Context, readDomain st } case "code": for _, addr := range addrs { - code, err := r.ReadAccountCode(libcommon.BytesToAddress(addr), 0, libcommon.Hash{}) + code, err := r.ReadAccountCode(libcommon.BytesToAddress(addr), 0) if err != nil { logger.Error("failed to read code", "addr", addr, "err", err) continue diff --git a/core/state/cached_reader.go b/core/state/cached_reader.go index fba1496474b..2020fd8d285 100644 --- a/core/state/cached_reader.go +++ b/core/state/cached_reader.go @@ -78,14 +78,11 @@ func (cr *CachedReader) ReadAccountStorage(address common.Address, incarnation u // ReadAccountCode is called when code of an account needs to be fetched from the state // Usually, one of (address;incarnation) or codeHash is enough to uniquely identify the code -func (cr *CachedReader) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { - if codeHash == emptyCodeHashH { - return nil, nil - } +func (cr *CachedReader) ReadAccountCode(address common.Address, incarnation uint64) ([]byte, error) { if c, ok := cr.cache.GetCode(address.Bytes(), incarnation); ok { return c, nil } - c, err := cr.r.ReadAccountCode(address, incarnation, codeHash) + c, err := cr.r.ReadAccountCode(address, incarnation) if err != nil { return nil, err } @@ -95,8 +92,8 @@ func (cr *CachedReader) ReadAccountCode(address common.Address, incarnation uint return c, nil } -func (cr *CachedReader) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { - c, err := cr.ReadAccountCode(address, incarnation, codeHash) +func (cr *CachedReader) ReadAccountCodeSize(address common.Address, incarnation uint64) (int, error) { + c, err := cr.ReadAccountCode(address, incarnation) return len(c), err } diff --git a/core/state/cached_reader3.go b/core/state/cached_reader3.go index b8ceec09f76..2d8d93450ae 100644 --- a/core/state/cached_reader3.go +++ b/core/state/cached_reader3.go @@ -17,8 +17,6 @@ package state import ( - "bytes" - "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/kvcache" @@ -83,10 +81,7 @@ func (r *CachedReader3) ReadAccountStorage(address common.Address, incarnation u return enc, nil } -func (r *CachedReader3) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { - if bytes.Equal(codeHash.Bytes(), emptyCodeHash) { - return nil, nil - } +func (r *CachedReader3) ReadAccountCode(address common.Address, incarnation uint64) ([]byte, error) { code, err := r.cache.GetCode(address[:]) if len(code) == 0 { return nil, nil @@ -94,8 +89,8 @@ func (r *CachedReader3) ReadAccountCode(address common.Address, incarnation uint return code, err } -func (r *CachedReader3) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { - code, err := r.ReadAccountCode(address, incarnation, codeHash) +func (r *CachedReader3) ReadAccountCodeSize(address common.Address, incarnation uint64) (int, error) { + code, err := r.ReadAccountCode(address, incarnation) return len(code), err } diff --git a/core/state/database.go b/core/state/database.go index eccdc195b2d..20a4d6e2109 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -39,8 +39,8 @@ type StateReader interface { ReadAccountData(address common.Address) (*accounts.Account, error) ReadAccountDataForDebug(address common.Address) (*accounts.Account, error) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) - ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) - ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) + ReadAccountCode(address common.Address, incarnation uint64) ([]byte, error) + ReadAccountCodeSize(address common.Address, incarnation uint64) (int, error) ReadAccountIncarnation(address common.Address) (uint64, error) } diff --git a/core/state/database_test.go b/core/state/database_test.go index 0e2b776c8bd..b173200b8a9 100644 --- a/core/state/database_test.go +++ b/core/state/database_test.go @@ -1367,8 +1367,7 @@ func TestChangeAccountCodeBetweenBlocks(t *testing.T) { sd.SetTxNum(2) sd.SetBlockNum(1) - oldCodeHash := libcommon.BytesToHash(crypto.Keccak256(oldCode)) - trieCode, tcErr := r.ReadAccountCode(contract, 1, oldCodeHash) + trieCode, tcErr := r.ReadAccountCode(contract, 1) assert.NoError(t, tcErr, "you can receive the new code") assert.Equal(t, oldCode, trieCode, "new code should be received") @@ -1379,8 +1378,7 @@ func TestChangeAccountCodeBetweenBlocks(t *testing.T) { t.Errorf("error finalising 1st tx: %v", err) } - newCodeHash := libcommon.BytesToHash(crypto.Keccak256(newCode)) - trieCode, tcErr = r.ReadAccountCode(contract, 1, newCodeHash) + trieCode, tcErr = r.ReadAccountCode(contract, 1) assert.NoError(t, tcErr, "you can receive the new code") assert.Equal(t, newCode, trieCode, "new code should be received") @@ -1417,12 +1415,11 @@ func TestCacheCodeSizeSeparately(t *testing.T) { t.Errorf("error committing block: %v", err) } - codeHash := libcommon.BytesToHash(crypto.Keccak256(code)) - codeSize, err := r.ReadAccountCodeSize(contract, 1, codeHash) + codeSize, err := r.ReadAccountCodeSize(contract, 1) assert.NoError(t, err, "you can receive the new code") assert.Equal(t, len(code), codeSize, "new code should be received") - code2, err := r.ReadAccountCode(contract, 1, codeHash) + code2, err := r.ReadAccountCode(contract, 1) assert.NoError(t, err, "you can receive the new code") assert.Equal(t, code, code2, "new code should be received") } @@ -1461,13 +1458,13 @@ func TestCacheCodeSizeInTrie(t *testing.T) { require.EqualValues(t, root, libcommon.CastToHash(r2)) codeHash := libcommon.BytesToHash(crypto.Keccak256(code)) - codeSize, err := r.ReadAccountCodeSize(contract, 1, codeHash) + codeSize, err := r.ReadAccountCodeSize(contract, 1) assert.NoError(t, err, "you can receive the code size ") assert.Equal(t, len(code), codeSize, "you can receive the code size") assert.NoError(t, tx.Delete(kv.Code, codeHash[:]), nil) - codeSize2, err := r.ReadAccountCodeSize(contract, 1, codeHash) + codeSize2, err := r.ReadAccountCodeSize(contract, 1) assert.NoError(t, err, "you can still receive code size even with empty DB") assert.Equal(t, len(code), codeSize2, "code size should be received even with empty DB") diff --git a/core/state/history_reader_v3.go b/core/state/history_reader_v3.go index 05db794cb5f..d6fd7af3d2d 100644 --- a/core/state/history_reader_v3.go +++ b/core/state/history_reader_v3.go @@ -87,20 +87,17 @@ func (hr *HistoryReaderV3) ReadAccountStorage(address common.Address, incarnatio return enc, err } -func (hr *HistoryReaderV3) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { - if codeHash == emptyCodeHashH { - return nil, nil - } +func (hr *HistoryReaderV3) ReadAccountCode(address common.Address, incarnation uint64) ([]byte, error) { // must pass key2=Nil here: because Erigon4 does concatinate key1+key2 under the hood //code, _, err := hr.ttx.GetAsOf(kv.CodeDomain, address.Bytes(), codeHash.Bytes(), hr.txNum) code, _, err := hr.ttx.GetAsOf(kv.CodeDomain, address[:], nil, hr.txNum) if hr.trace { - fmt.Printf("ReadAccountCode [%x %x] => [%x]\n", address, codeHash, code) + fmt.Printf("ReadAccountCode [%x] => [%x]\n", address, code) } return code, err } -func (hr *HistoryReaderV3) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { +func (hr *HistoryReaderV3) ReadAccountCodeSize(address common.Address, incarnation uint64) (int, error) { enc, _, err := hr.ttx.GetAsOf(kv.CodeDomain, address[:], nil, hr.txNum) return len(enc), err } diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index 4c64fa354d7..0a2ae087ee0 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -310,7 +310,10 @@ func (sdb *IntraBlockState) GetCodeSize(addr libcommon.Address) (int, error) { if stateObject.code != nil { return len(stateObject.code), nil } - l, err := sdb.stateReader.ReadAccountCodeSize(addr, stateObject.data.Incarnation, stateObject.data.CodeHash) + if stateObject.data.CodeHash == emptyCodeHashH { + return 0, nil + } + l, err := sdb.stateReader.ReadAccountCodeSize(addr, stateObject.data.Incarnation) if err != nil { sdb.setErrorUnsafe(err) return l, err diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index baf67f3d5cf..0c01eaf4f15 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -629,10 +629,7 @@ func (r *ReaderV3) ReadAccountStorage(address common.Address, incarnation uint64 return enc, nil } -func (r *ReaderV3) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { - //if codeHash == emptyCodeHashH { // TODO: how often do we have this case on mainnet/bor-mainnet? - // return nil, nil - //} +func (r *ReaderV3) ReadAccountCode(address common.Address, incarnation uint64) ([]byte, error) { enc, _, err := r.tx.GetLatest(kv.CodeDomain, address[:], nil) if err != nil { return nil, err @@ -643,7 +640,7 @@ func (r *ReaderV3) ReadAccountCode(address common.Address, incarnation uint64, c return enc, nil } -func (r *ReaderV3) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { +func (r *ReaderV3) ReadAccountCodeSize(address common.Address, incarnation uint64) (int, error) { enc, _, err := r.tx.GetLatest(kv.CodeDomain, address[:], nil) if err != nil { return 0, err @@ -754,7 +751,7 @@ func (r *ReaderParallelV3) ReadAccountStorage(address common.Address, incarnatio return enc, nil } -func (r *ReaderParallelV3) ReadAccountCode(address common.Address, incarnation uint64, codeHash common.Hash) ([]byte, error) { +func (r *ReaderParallelV3) ReadAccountCode(address common.Address, incarnation uint64) ([]byte, error) { enc, _, err := r.sd.GetLatest(kv.CodeDomain, address[:], nil) if err != nil { return nil, err @@ -769,7 +766,7 @@ func (r *ReaderParallelV3) ReadAccountCode(address common.Address, incarnation u return enc, nil } -func (r *ReaderParallelV3) ReadAccountCodeSize(address common.Address, incarnation uint64, codeHash common.Hash) (int, error) { +func (r *ReaderParallelV3) ReadAccountCodeSize(address common.Address, incarnation uint64) (int, error) { enc, _, err := r.sd.GetLatest(kv.CodeDomain, address[:], nil) if err != nil { return 0, err diff --git a/core/state/state_object.go b/core/state/state_object.go index 469abd5541d..4b29e75caff 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -332,7 +332,7 @@ func (so *stateObject) Code() []byte { if so.data.CodeHash == emptyCodeHashH { return nil } - code, err := so.db.stateReader.ReadAccountCode(so.Address(), so.data.Incarnation, so.data.CodeHash) + code, err := so.db.stateReader.ReadAccountCode(so.Address(), so.data.Incarnation) if err != nil { so.setError(fmt.Errorf("can't load code hash %x: %w", so.data.CodeHash, err)) } diff --git a/eth/stagedsync/stage_mining_exec.go b/eth/stagedsync/stage_mining_exec.go index db03cb9fe53..cc106810bda 100644 --- a/eth/stagedsync/stage_mining_exec.go +++ b/eth/stagedsync/stage_mining_exec.go @@ -345,7 +345,7 @@ func filterBadTransactions(transactions []types.Transaction, chainID *uint256.In if !account.IsEmptyCodeHash() { isEoaCodeAllowed := false if config.IsPrague(header.Time) { - code, err := simStateReader.ReadAccountCode(sender, account.Incarnation, account.CodeHash) + code, err := simStateReader.ReadAccountCode(sender, account.Incarnation) if err != nil { return nil, err } diff --git a/turbo/jsonrpc/eth_accounts.go b/turbo/jsonrpc/eth_accounts.go index fb2c8587e3b..8e4111136a2 100644 --- a/turbo/jsonrpc/eth_accounts.go +++ b/turbo/jsonrpc/eth_accounts.go @@ -111,7 +111,7 @@ func (api *APIImpl) GetCode(ctx context.Context, address libcommon.Address, bloc if acc == nil || err != nil { return hexutility.Bytes(""), nil } - res, _ := reader.ReadAccountCode(address, acc.Incarnation, acc.CodeHash) + res, _ := reader.ReadAccountCode(address, acc.Incarnation) if res == nil { return hexutility.Bytes(""), nil } From 33732fb5035a365dbc4e3a230b07554b8becb937 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 20:48:56 +0700 Subject: [PATCH 05/14] d_lru: to store decompressed values (#12951) - decompressing CodeDomain values: taking ~7% of execution speed --- erigon-lib/state/cache.go | 1 + erigon-lib/state/domain.go | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/erigon-lib/state/cache.go b/erigon-lib/state/cache.go index e3efef88227..3a718284ae7 100644 --- a/erigon-lib/state/cache.go +++ b/erigon-lib/state/cache.go @@ -27,6 +27,7 @@ type domainGetFromFileCacheItem struct { lvl uint8 exists bool offset uint64 + v []byte } var ( diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 4eea0477597..51b45eb1256 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -1541,11 +1541,11 @@ func (dt *DomainRoTx) getFromFiles(filekey []byte, maxTxNum uint64) (v []byte, f if !cv.exists { return nil, true, dt.files[cv.lvl].startTxNum, dt.files[cv.lvl].endTxNum, nil } - g := dt.statelessGetter(int(cv.lvl)) - g.Reset(cv.offset) - g.Skip() - v, _ = g.Next(nil) // can be compressed - return v, true, dt.files[cv.lvl].startTxNum, dt.files[cv.lvl].endTxNum, nil + //g := dt.statelessGetter(int(cv.lvl)) + //g.Reset(cv.offset) + //g.Skip() + //v, _ = g.Next(nil) // can be compressed + return cv.v, true, dt.files[cv.lvl].startTxNum, dt.files[cv.lvl].endTxNum, nil } } @@ -1589,7 +1589,7 @@ func (dt *DomainRoTx) getFromFiles(filekey []byte, maxTxNum uint64) (v []byte, f } if dt.getFromFileCache != nil { - dt.getFromFileCache.Add(hi, domainGetFromFileCacheItem{lvl: uint8(i), offset: offset, exists: true}) + dt.getFromFileCache.Add(hi, domainGetFromFileCacheItem{lvl: uint8(i), offset: offset, exists: true, v: v}) } return v, true, dt.files[i].startTxNum, dt.files[i].endTxNum, nil } @@ -1598,7 +1598,7 @@ func (dt *DomainRoTx) getFromFiles(filekey []byte, maxTxNum uint64) (v []byte, f } if dt.getFromFileCache != nil { - dt.getFromFileCache.Add(hi, domainGetFromFileCacheItem{lvl: 0, offset: 0, exists: false}) + dt.getFromFileCache.Add(hi, domainGetFromFileCacheItem{lvl: 0, offset: 0, exists: false, v: nil}) } return nil, false, 0, 0, nil } From f6205c24b7b1ea1f8813f330043f013a746c1f49 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 20:49:25 +0700 Subject: [PATCH 06/14] sdc.Account: simplify new Update object creation (#12934) --- erigon-lib/commitment/commitment.go | 2 +- erigon-lib/state/domain_shared.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/erigon-lib/commitment/commitment.go b/erigon-lib/commitment/commitment.go index db7464ed17f..56368775d88 100644 --- a/erigon-lib/commitment/commitment.go +++ b/erigon-lib/commitment/commitment.go @@ -1218,7 +1218,7 @@ func (u *Update) Reset() { u.Balance.Clear() u.Nonce = 0 u.StorageLen = 0 - copy(u.CodeHash[:], EmptyCodeHash) + u.CodeHash = EmptyCodeHashArray } func (u *Update) Merge(b *Update) { diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 9af26644f0c..00762a629c4 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -1201,8 +1201,7 @@ func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitme } } - u = new(commitment.Update) - u.Reset() + u = &commitment.Update{CodeHash: commitment.EmptyCodeHashArray} if len(encAccount) > 0 { nonce, balance, chash := types.DecodeAccountBytesV3(encAccount) @@ -1237,9 +1236,8 @@ func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitme sdc.keccak.Write(code) sdc.keccak.Read(u.CodeHash[:]) u.Flags |= commitment.CodeUpdate - } else { - copy(u.CodeHash[:], commitment.EmptyCodeHashArray[:]) + u.CodeHash = commitment.EmptyCodeHashArray } if len(encAccount) == 0 && len(code) == 0 { From 38f9253a61cf951341f7ae44b57791aac576e425 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 21:50:20 +0700 Subject: [PATCH 07/14] sd: more cast (#12942) --- erigon-lib/state/domain_shared.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 00762a629c4..c45f1967bca 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -761,11 +761,10 @@ func (sd *SharedDomains) IterateStoragePrefix(prefix []byte, it func(k []byte, v iter := sd.storage.Iter() if iter.Seek(string(prefix)) { - kx := iter.Key() + k := toBytesZeroCopy(iter.Key()) v = iter.Value().data - k = []byte(kx) - if len(kx) > 0 && bytes.HasPrefix(k, prefix) { + if len(k) > 0 && bytes.HasPrefix(k, prefix) { heap.Push(cpPtr, &CursorItem{t: RAM_CURSOR, key: common.Copy(k), val: common.Copy(v), step: 0, iter: iter, endTxNum: sd.txNum, reverse: true}) } } @@ -1027,13 +1026,13 @@ func (sd *SharedDomains) DomainPut(domain kv.Domain, k1, k2 []byte, val, prevVal } return sd.updateAccountCode(k1, val, prevVal, prevStep) case kv.CommitmentDomain: - sd.put(domain, string(append(k1, k2...)), val) + sd.put(domain, toStringZeroCopy(append(k1, k2...)), val) return sd.domainWriters[domain].PutWithPrev(k1, k2, val, prevVal, prevStep) default: if bytes.Equal(prevVal, val) { return nil } - sd.put(domain, string(append(k1, k2...)), val) + sd.put(domain, toStringZeroCopy(append(k1, k2...)), val) return sd.domainWriters[domain].PutWithPrev(k1, k2, val, prevVal, prevStep) } } @@ -1065,7 +1064,7 @@ func (sd *SharedDomains) DomainDel(domain kv.Domain, k1, k2 []byte, prevVal []by case kv.CommitmentDomain: return sd.updateCommitmentData(toStringZeroCopy(k1), nil, prevVal, prevStep) default: - sd.put(domain, string(append(k1, k2...)), nil) + sd.put(domain, toStringZeroCopy(append(k1, k2...)), nil) return sd.domainWriters[domain].DeleteWithPrev(k1, k2, prevVal, prevStep) } } @@ -1152,8 +1151,7 @@ func (sdc *SharedDomainsCommitmentContext) ResetBranchCache() { } func (sdc *SharedDomainsCommitmentContext) Branch(pref []byte) ([]byte, uint64, error) { - prefixS := string(pref) - cached, ok := sdc.branches[prefixS] + cached, ok := sdc.branches[toStringZeroCopy(pref)] if ok { // cached value is already transformed/clean to read. // Cache should ResetBranchCache after each commitment computation @@ -1162,14 +1160,14 @@ func (sdc *SharedDomainsCommitmentContext) Branch(pref []byte) ([]byte, uint64, v, step, err := sdc.sharedDomains.LatestCommitment(pref) if err != nil { - return nil, 0, fmt.Errorf("Branch failed: %w", err) + return nil, 0, fmt.Errorf("branch failed: %w", err) } if sdc.sharedDomains.trace { fmt.Printf("[SDC] Branch: %x: %x\n", pref, v) } // Trie reads prefix during unfold and after everything is ready reads it again to Merge update, if any, so // cache branch until ResetBranchCache called - sdc.branches[prefixS] = cachedBranch{data: v, step: step} + sdc.branches[string(pref)] = cachedBranch{data: v, step: step} if len(v) == 0 { return nil, 0, nil @@ -1368,7 +1366,7 @@ func (sdc *SharedDomainsCommitmentContext) storeCommitmentState(blockNum uint64, if sdc.sharedDomains.trace { fmt.Printf("[commitment] store txn %d block %d rootHash %x\n", sdc.sharedDomains.txNum, blockNum, rootHash) } - sdc.sharedDomains.put(kv.CommitmentDomain, string(keyCommitmentState), encodedState) + sdc.sharedDomains.put(kv.CommitmentDomain, keyCommitmentStateS, encodedState) return sdc.sharedDomains.domainWriters[kv.CommitmentDomain].PutWithPrev(keyCommitmentState, nil, encodedState, prevState, prevStep) } @@ -1395,7 +1393,9 @@ func (sdc *SharedDomainsCommitmentContext) encodeCommitmentState(blockNum, txNum } // by that key stored latest root hash and tree state -var keyCommitmentState = []byte("state") +const keyCommitmentStateS = "state" + +var keyCommitmentState = []byte(keyCommitmentStateS) func (sd *SharedDomains) LatestCommitmentState(tx kv.Tx, sinceTx, untilTx uint64) (blockNum, txNum uint64, state []byte, err error) { return sd.sdCtx.LatestCommitmentState() From 0b31e6d3d10cc808f9d99c68a82dfefffa05bdc2 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 21:50:48 +0700 Subject: [PATCH 08/14] diffset: serialize buf (#12949) depends on https://github.com/erigontech/erigon/pull/12945 --- erigon-lib/state/state_changeset.go | 52 ++++++++++++++++------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/erigon-lib/state/state_changeset.go b/erigon-lib/state/state_changeset.go index f3d9877a0c6..925aa6e5d7d 100644 --- a/erigon-lib/state/state_changeset.go +++ b/erigon-lib/state/state_changeset.go @@ -23,6 +23,7 @@ import ( "math" "sort" "strings" + "sync" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" @@ -53,6 +54,8 @@ type StateDiffDomain struct { keys map[string][]byte prevValues map[string][]byte prevValsSlice []DomainEntryDiff + + prevStepBuf, keyBuf []byte } func (d *StateDiffDomain) Copy() *StateDiffDomain { @@ -62,32 +65,25 @@ func (d *StateDiffDomain) Copy() *StateDiffDomain { // RecordDelta records a state change. func (d *StateDiffDomain) DomainUpdate(key1, key2, prevValue, stepBytes []byte, prevStep uint64) { if d.keys == nil { - d.keys = make(map[string][]byte) - } - if d.prevValues == nil { - d.prevValues = make(map[string][]byte) + d.keys = make(map[string][]byte, 16) + d.prevValues = make(map[string][]byte, 16) + d.prevStepBuf = make([]byte, 8) } - prevStepBytes := make([]byte, 8) - binary.BigEndian.PutUint64(prevStepBytes, ^prevStep) + binary.BigEndian.PutUint64(d.prevStepBuf, ^prevStep) - key := make([]byte, len(key1)+len(key2)) - copy(key, key1) - copy(key[len(key1):], key2) - - keyS := toStringZeroCopy(key) - if _, ok := d.keys[keyS]; !ok { - d.keys[keyS] = prevStepBytes + d.keyBuf = append(append(append(d.keyBuf[:0], key1...), key2...), stepBytes...) + key := toStringZeroCopy(d.keyBuf[:len(key1)+len(key2)]) + if _, ok := d.keys[key]; !ok { + d.keys[strings.Clone(key)] = common.Copy(d.prevStepBuf) } - valsKey := make([]byte, len(key)+len(stepBytes)) - copy(valsKey, key) - copy(valsKey[len(key):], stepBytes) - valsKeyS := toStringZeroCopy(valsKey) - if _, ok := d.prevValues[valsKeyS]; !ok { - if bytes.Equal(stepBytes, prevStepBytes) { - d.prevValues[valsKeyS] = common.Copy(prevValue) + valsKey := toStringZeroCopy(d.keyBuf) + if _, ok := d.prevValues[valsKey]; !ok { + valsKeySCopy := strings.Clone(valsKey) + if bytes.Equal(stepBytes, d.prevStepBuf) { + d.prevValues[valsKeySCopy] = common.Copy(prevValue) } else { - d.prevValues[valsKeyS] = []byte{} // We need to delete the current step but restore the previous one + d.prevValues[valsKeySCopy] = []byte{} // We need to delete the current step but restore the previous one } d.prevValsSlice = nil } @@ -268,9 +264,19 @@ func DeserializeKeys(in []byte) [kv.DomainLen][]DomainEntryDiff { const diffChunkKeyLen = 48 const diffChunkLen = 4*1024 - 32 +type threadSafeBuf struct { + b []byte + sync.Mutex +} + +var writeDiffsetBuf = &threadSafeBuf{} + func WriteDiffSet(tx kv.RwTx, blockNumber uint64, blockHash common.Hash, diffSet *StateChangeSet) error { - // Write the diffSet to the database - keys := diffSet.SerializeKeys(nil) + writeDiffsetBuf.Lock() + defer writeDiffsetBuf.Unlock() + writeDiffsetBuf.b = diffSet.SerializeKeys(writeDiffsetBuf.b[:0]) + keys := writeDiffsetBuf.b + chunkCount := (len(keys) + diffChunkLen - 1) / diffChunkLen // Data Format // dbutils.BlockBodyKey(blockNumber, blockHash) -> chunkCount From 1b967b09f211207ac657d45f2489d4808baf5162 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 21:51:50 +0700 Subject: [PATCH 09/14] agg: tables list method (#12939) --- cmd/integration/commands/flags.go | 3 +- cmd/integration/commands/reset_state.go | 2 +- cmd/integration/commands/stages.go | 18 ++---------- core/rawdb/rawdbreset/reset_stages.go | 39 +++++-------------------- core/test/domains_restart_test.go | 4 +-- erigon-lib/state/aggregator.go | 13 +++++++++ erigon-lib/state/domain.go | 4 +++ erigon-lib/state/history.go | 4 +++ erigon-lib/state/inverted_index.go | 2 ++ 9 files changed, 37 insertions(+), 52 deletions(-) diff --git a/cmd/integration/commands/flags.go b/cmd/integration/commands/flags.go index 615281461af..a61fcaab2d3 100644 --- a/cmd/integration/commands/flags.go +++ b/cmd/integration/commands/flags.go @@ -32,7 +32,7 @@ var ( block, pruneTo, unwind uint64 unwindEvery uint64 batchSizeStr string - reset, warmup, noCommit bool + reset, noCommit bool resetPruneAt bool bucket string datadirCli, toChaindata string @@ -116,7 +116,6 @@ func withUnwindEvery(cmd *cobra.Command) { func withReset(cmd *cobra.Command) { cmd.Flags().BoolVar(&reset, "reset", false, "reset given stage") - cmd.Flags().BoolVar(&warmup, "warmup", false, "warmup relevant tables by parallel random reads") } func withResetPruneAt(cmd *cobra.Command) { diff --git a/cmd/integration/commands/reset_state.go b/cmd/integration/commands/reset_state.go index 4a4a61a5d65..f5b7785ba07 100644 --- a/cmd/integration/commands/reset_state.go +++ b/cmd/integration/commands/reset_state.go @@ -67,7 +67,7 @@ var cmdResetState = &cobra.Command{ return } - if err = reset2.ResetState(db, ctx, chain, "", log.Root()); err != nil { + if err = reset2.ResetState(db, agg, ctx, chain, "", log.Root()); err != nil { if !errors.Is(err, context.Canceled) { logger.Error(err.Error()) } diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index d94f33d7f24..a9326f52d25 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -646,7 +646,6 @@ func stageSnapshots(db kv.RwDB, ctx context.Context, logger log.Logger) error { br, bw := blocksIO(db, logger) _, _, _, _, _ = newSync(ctx, db, nil /* miningConfig */, logger) - chainConfig, _ := fromdb.ChainConfig(db), fromdb.PruneMode(db) return db.Update(ctx, func(tx kv.RwTx) error { if reset { @@ -655,7 +654,7 @@ func stageSnapshots(db kv.RwDB, ctx context.Context, logger log.Logger) error { } } dirs := datadir.New(datadirCli) - if err := reset2.ResetBlocks(tx, db, agg, br, bw, dirs, *chainConfig, logger); err != nil { + if err := reset2.ResetBlocks(tx, db, agg, br, bw, dirs, logger); err != nil { return fmt.Errorf("resetting blocks: %w", err) } ac := agg.BeginFilesRo() @@ -701,7 +700,6 @@ func stageHeaders(db kv.RwDB, ctx context.Context, logger log.Logger) error { defer agg.Close() br, bw := blocksIO(db, logger) _, _, _, _, _ = newSync(ctx, db, nil /* miningConfig */, logger) - chainConfig, _ := fromdb.ChainConfig(db), fromdb.PruneMode(db) if integritySlow { if err := db.View(ctx, func(tx kv.Tx) error { @@ -721,7 +719,7 @@ func stageHeaders(db kv.RwDB, ctx context.Context, logger log.Logger) error { return db.Update(ctx, func(tx kv.RwTx) error { if reset { - if err := reset2.ResetBlocks(tx, db, agg, br, bw, dirs, *chainConfig, logger); err != nil { + if err := reset2.ResetBlocks(tx, db, agg, br, bw, dirs, logger); err != nil { return err } return nil @@ -1031,11 +1029,8 @@ func stageExec(db kv.RwDB, ctx context.Context, logger log.Logger) error { defer sn.Close() defer borSn.Close() defer agg.Close() - if warmup { - return reset2.WarmupExec(ctx, db) - } if reset { - if err := reset2.ResetExec(ctx, db, chain, "", logger); err != nil { + if err := reset2.ResetExec(ctx, db, agg, chain, "", logger); err != nil { return err } return nil @@ -1177,10 +1172,6 @@ func stageCustomTrace(db kv.RwDB, ctx context.Context, logger log.Logger) error defer sn.Close() defer borSn.Close() defer agg.Close() - if warmup { - panic("not implemented") - //return reset2.WarmupExec(ctx, db) - } if reset { if err := reset2.Reset(ctx, db, stages.CustomTrace); err != nil { return err @@ -1218,9 +1209,6 @@ func stagePatriciaTrie(db kv.RwDB, ctx context.Context, logger log.Logger) error defer agg.Close() _, _, _, _, _ = newSync(ctx, db, nil /* miningConfig */, logger) - if warmup { - return reset2.Warmup(ctx, db, log.LvlInfo, stages.Execution) - } if reset { return reset2.Reset(ctx, db, stages.Execution) } diff --git a/core/rawdb/rawdbreset/reset_stages.go b/core/rawdb/rawdbreset/reset_stages.go index dcc4886027b..c7265c1a639 100644 --- a/core/rawdb/rawdbreset/reset_stages.go +++ b/core/rawdb/rawdbreset/reset_stages.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/turbo/services" ) -func ResetState(db kv.RwDB, ctx context.Context, chain string, tmpDir string, logger log.Logger) error { +func ResetState(db kv.RwDB, agg *state.Aggregator, ctx context.Context, chain string, tmpDir string, logger log.Logger) error { // don't reset senders here if err := db.Update(ctx, ResetTxLookup); err != nil { return err @@ -45,13 +45,13 @@ func ResetState(db kv.RwDB, ctx context.Context, chain string, tmpDir string, lo return err } - if err := ResetExec(ctx, db, chain, tmpDir, logger); err != nil { + if err := ResetExec(ctx, db, agg, chain, tmpDir, logger); err != nil { return err } return nil } -func ResetBlocks(tx kv.RwTx, db kv.RoDB, agg *state.Aggregator, br services.FullBlockReader, bw *blockio.BlockWriter, dirs datadir.Dirs, cc chain.Config, logger log.Logger) error { +func ResetBlocks(tx kv.RwTx, db kv.RoDB, agg *state.Aggregator, br services.FullBlockReader, bw *blockio.BlockWriter, dirs datadir.Dirs, logger log.Logger) error { // keep Genesis if err := rawdb.TruncateBlocks(context.Background(), tx, 1); err != nil { return err @@ -144,7 +144,7 @@ func ResetPolygonSync(tx kv.RwTx, db kv.RoDB, agg *state.Aggregator, br services } } - if err := ResetBlocks(tx, db, agg, br, bw, dirs, cc, logger); err != nil { + if err := ResetBlocks(tx, db, agg, br, bw, dirs, logger); err != nil { return err } @@ -158,22 +158,12 @@ func ResetSenders(ctx context.Context, db kv.RwDB, tx kv.RwTx) error { return clearStageProgress(tx, stages.Senders) } -func WarmupExec(ctx context.Context, db kv.RwDB) (err error) { - for _, tbl := range stateBuckets { - backup.WarmupTable(ctx, db, tbl, log.LvlInfo, backup.ReadAheadThreads) - } - for _, tbl := range stateHistoryV3Buckets { - backup.WarmupTable(ctx, db, tbl, log.LvlInfo, backup.ReadAheadThreads) - } - return -} - -func ResetExec(ctx context.Context, db kv.RwDB, chain string, tmpDir string, logger log.Logger) (err error) { +func ResetExec(ctx context.Context, db kv.RwDB, agg *state.Aggregator, chain string, tmpDir string, logger log.Logger) (err error) { cleanupList := make([]string, 0) cleanupList = append(cleanupList, stateBuckets...) cleanupList = append(cleanupList, stateHistoryBuckets...) - cleanupList = append(cleanupList, stateHistoryV3Buckets...) - cleanupList = append(cleanupList, stateV3Buckets...) + cleanupList = append(cleanupList, agg.DomainTables(kv.AccountsDomain, kv.StorageDomain, kv.CodeDomain, kv.CommitmentDomain, kv.ReceiptDomain)...) + cleanupList = append(cleanupList, agg.InvertedIndexTables(kv.LogAddrIdxPos, kv.LogTopicIdxPos, kv.TracesFromIdxPos, kv.TracesToIdxPos)...) return db.Update(ctx, func(tx kv.RwTx) error { if err := clearStageProgress(tx, stages.Execution); err != nil { @@ -212,21 +202,6 @@ var stateBuckets = []string{ kv.PlainContractCode, kv.ContractCode, kv.IncarnationMap, } var stateHistoryBuckets = []string{ - kv.Receipts, -} -var stateHistoryV3Buckets = []string{ - kv.TblAccountHistoryKeys, kv.TblAccountHistoryVals, kv.TblAccountIdx, - kv.TblStorageHistoryKeys, kv.TblStorageHistoryVals, kv.TblStorageIdx, - kv.TblCodeHistoryKeys, kv.TblCodeHistoryVals, kv.TblCodeIdx, - kv.TblLogAddressKeys, kv.TblLogAddressIdx, - kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, - kv.TblTracesFromKeys, kv.TblTracesFromIdx, - kv.TblTracesToKeys, kv.TblTracesToIdx, -} -var stateV3Buckets = []string{ - kv.TblAccountVals, kv.TblStorageVals, kv.TblCodeVals, kv.TblCommitmentVals, kv.TblReceiptVals, - kv.TblCommitmentHistoryKeys, kv.TblCommitmentHistoryVals, kv.TblCommitmentIdx, - kv.TblReceiptHistoryKeys, kv.TblReceiptHistoryVals, kv.TblReceiptIdx, kv.TblPruningProgress, kv.ChangeSets3, } diff --git a/core/test/domains_restart_test.go b/core/test/domains_restart_test.go index 5fd7ea02572..952906fbb6b 100644 --- a/core/test/domains_restart_test.go +++ b/core/test/domains_restart_test.go @@ -241,7 +241,7 @@ func Test_AggregatorV3_RestartOnDatadir_WithoutDB(t *testing.T) { domCtx.Close() domains.Close() - err = reset2.ResetExec(ctx, db, networkname.Test, "", log.New()) + err = reset2.ResetExec(ctx, db, agg, networkname.Test, "", log.New()) require.NoError(t, err) // ======== reset domains end ======== @@ -411,7 +411,7 @@ func Test_AggregatorV3_RestartOnDatadir_WithoutAnything(t *testing.T) { domCtx.Close() domains.Close() - err = reset2.ResetExec(ctx, db, networkname.Test, "", log.New()) + err = reset2.ResetExec(ctx, db, agg, networkname.Test, "", log.New()) require.NoError(t, err) // ======== reset domains end ======== diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 482ebbbbe45..28dca3ee8e6 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -857,6 +857,19 @@ func (a *Aggregator) integrateDirtyFiles(sf AggV3StaticFiles, txNumFrom, txNumTo } } +func (a *Aggregator) DomainTables(domains ...kv.Domain) (tables []string) { + for _, domain := range domains { + tables = append(tables, a.d[domain].Tables()...) + } + return tables +} +func (a *Aggregator) InvertedIndexTables(indices ...kv.InvertedIdxPos) (tables []string) { + for _, idx := range indices { + tables = append(tables, a.iis[idx].Tables()...) + } + return tables +} + type flusher interface { Flush(ctx context.Context, tx kv.RwTx) error } diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 51b45eb1256..2d134eb89f9 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -450,6 +450,10 @@ func (d *Domain) reCalcVisibleFiles(toTxNum uint64) { d.History.reCalcVisibleFiles(toTxNum) } +func (d *Domain) Tables() []string { + return append([]string{d.keysTable, d.valuesTable}, d.History.Tables()...) +} + func (d *Domain) Close() { if d == nil { return diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 2ab7c30d3ba..78ddeb764c2 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -284,6 +284,10 @@ func (h *History) closeWhatNotInList(fNames []string) { } } +func (h *History) Tables() []string { + return append([]string{h.keysTable, h.valuesTable}, h.InvertedIndex.Tables()...) +} + func (h *History) Close() { if h == nil { return diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 7ce941d7510..83a96072511 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -332,6 +332,8 @@ func (ii *InvertedIndex) closeWhatNotInList(fNames []string) { } } +func (ii *InvertedIndex) Tables() []string { return []string{ii.keysTable, ii.valuesTable} } + func (ii *InvertedIndex) Close() { if ii == nil { return From 0ac3978f9847fdb6bcdd813a70f1626e68e1c2fb Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Dec 2024 21:52:57 +0700 Subject: [PATCH 10/14] sdc.Account: separate reads by domain on pprof (#12933) now `.Account()` method does 2 reads - but on pprof i can't separate "which of them impacts performance. created named funcs for them. Pic after change: Screenshot 2024-12-01 at 14 57 31 --- erigon-lib/state/domain_shared.go | 66 +++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index c45f1967bca..f5518f9e0a4 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -1185,18 +1185,54 @@ func (sdc *SharedDomainsCommitmentContext) PutBranch(prefix []byte, data []byte, return sdc.sharedDomains.updateCommitmentData(prefixS, data, prevData, prevStep) } -func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitment.Update, err error) { - var encAccount []byte - if sdc.limitReadAsOfTxNum == 0 { - encAccount, _, err = sdc.sharedDomains.GetLatest(kv.AccountsDomain, plainKey, nil) +func (sdc *SharedDomainsCommitmentContext) readAccount(plainKey []byte) (encAccount []byte, err error) { + if sdc.limitReadAsOfTxNum > 0 { + encAccount, _, err = sdc.sharedDomains.getAsOfFile(kv.AccountsDomain, plainKey, nil, sdc.limitReadAsOfTxNum) if err != nil { return nil, fmt.Errorf("GetAccount failed: %w", err) } - } else { - encAccount, _, err = sdc.sharedDomains.getAsOfFile(kv.AccountsDomain, plainKey, nil, sdc.limitReadAsOfTxNum) + return encAccount, nil + } + encAccount, _, err = sdc.sharedDomains.GetLatest(kv.AccountsDomain, plainKey, nil) + if err != nil { + return nil, fmt.Errorf("GetAccount failed: %w", err) + } + return encAccount, nil +} + +func (sdc *SharedDomainsCommitmentContext) readCode(plainKey []byte) (code []byte, err error) { + if sdc.limitReadAsOfTxNum > 0 { + code, _, err = sdc.sharedDomains.getAsOfFile(kv.CodeDomain, plainKey, nil, sdc.limitReadAsOfTxNum) if err != nil { - return nil, fmt.Errorf("GetAccount failed: %w", err) + return nil, fmt.Errorf("GetAccount/Code: failed to read latest code: %w", err) + } + return code, nil + } + code, _, err = sdc.sharedDomains.GetLatest(kv.CodeDomain, plainKey, nil) + if err != nil { + return nil, fmt.Errorf("GetAccount/Code: failed to read latest code: %w", err) + } + return code, nil +} +func (sdc *SharedDomainsCommitmentContext) readStorage(plainKey []byte) (enc []byte, err error) { + if sdc.limitReadAsOfTxNum > 0 { + enc, _, err = sdc.sharedDomains.getAsOfFile(kv.StorageDomain, plainKey, nil, sdc.limitReadAsOfTxNum) + if err != nil { + return nil, fmt.Errorf("GetAccount/Code: failed to read latest code: %w", err) } + return enc, nil + } + enc, _, err = sdc.sharedDomains.GetLatest(kv.StorageDomain, plainKey, nil) + if err != nil { + return nil, fmt.Errorf("GetAccount/Code: failed to read latest code: %w", err) + } + return enc, nil +} + +func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitment.Update, err error) { + encAccount, err := sdc.readAccount(plainKey) + if err != nil { + return nil, err } u = &commitment.Update{CodeHash: commitment.EmptyCodeHashArray} @@ -1219,14 +1255,9 @@ func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitme return u, nil } - var code []byte - if sdc.limitReadAsOfTxNum == 0 { - code, _, err = sdc.sharedDomains.GetLatest(kv.CodeDomain, plainKey, nil) - } else { - code, _, err = sdc.sharedDomains.getAsOfFile(kv.CodeDomain, plainKey, nil, sdc.limitReadAsOfTxNum) - } + code, err := sdc.readCode(plainKey) if err != nil { - return nil, fmt.Errorf("GetAccount/Code: failed to read latest code: %w", err) + return nil, err } if len(code) > 0 { @@ -1246,12 +1277,7 @@ func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitme func (sdc *SharedDomainsCommitmentContext) Storage(plainKey []byte) (u *commitment.Update, err error) { // Look in the summary table first - var enc []byte - if sdc.limitReadAsOfTxNum == 0 { - enc, _, err = sdc.sharedDomains.GetLatest(kv.StorageDomain, plainKey, nil) - } else { - enc, _, err = sdc.sharedDomains.getAsOfFile(kv.StorageDomain, plainKey, nil, sdc.limitReadAsOfTxNum) - } + enc, err := sdc.readStorage(plainKey) if err != nil { return nil, err } From 2db72340f843ad55ef2009afc55b10b8d1434f06 Mon Sep 17 00:00:00 2001 From: antonis19 Date: Mon, 2 Dec 2024 16:52:39 +0100 Subject: [PATCH 11/14] move turbo/rlphacks to erigon-lib/rlphacks (#12956) Co-authored-by: antonis19 --- core/types/hashing.go | 2 +- {turbo => erigon-lib}/rlphacks/bytes.go | 0 {turbo => erigon-lib}/rlphacks/bytes_test.go | 0 {turbo => erigon-lib}/rlphacks/serializable.go | 0 {turbo => erigon-lib}/rlphacks/struct.go | 0 {turbo => erigon-lib}/rlphacks/utils_bytes.go | 0 turbo/trie/gen_struct_step.go | 2 +- turbo/trie/hashbuilder.go | 2 +- turbo/trie/hasher.go | 2 +- turbo/trie/stream.go | 2 +- turbo/trie/structural_test.go | 2 +- turbo/trie/trie_root.go | 3 +-- 12 files changed, 7 insertions(+), 8 deletions(-) rename {turbo => erigon-lib}/rlphacks/bytes.go (100%) rename {turbo => erigon-lib}/rlphacks/bytes_test.go (100%) rename {turbo => erigon-lib}/rlphacks/serializable.go (100%) rename {turbo => erigon-lib}/rlphacks/struct.go (100%) rename {turbo => erigon-lib}/rlphacks/utils_bytes.go (100%) diff --git a/core/types/hashing.go b/core/types/hashing.go index 3ad4261e8af..08e3d921cce 100644 --- a/core/types/hashing.go +++ b/core/types/hashing.go @@ -30,7 +30,7 @@ import ( "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/crypto/cryptopool" "github.com/erigontech/erigon-lib/rlp" - "github.com/erigontech/erigon/turbo/rlphacks" + "github.com/erigontech/erigon-lib/rlphacks" "github.com/erigontech/erigon/turbo/trie" ) diff --git a/turbo/rlphacks/bytes.go b/erigon-lib/rlphacks/bytes.go similarity index 100% rename from turbo/rlphacks/bytes.go rename to erigon-lib/rlphacks/bytes.go diff --git a/turbo/rlphacks/bytes_test.go b/erigon-lib/rlphacks/bytes_test.go similarity index 100% rename from turbo/rlphacks/bytes_test.go rename to erigon-lib/rlphacks/bytes_test.go diff --git a/turbo/rlphacks/serializable.go b/erigon-lib/rlphacks/serializable.go similarity index 100% rename from turbo/rlphacks/serializable.go rename to erigon-lib/rlphacks/serializable.go diff --git a/turbo/rlphacks/struct.go b/erigon-lib/rlphacks/struct.go similarity index 100% rename from turbo/rlphacks/struct.go rename to erigon-lib/rlphacks/struct.go diff --git a/turbo/rlphacks/utils_bytes.go b/erigon-lib/rlphacks/utils_bytes.go similarity index 100% rename from turbo/rlphacks/utils_bytes.go rename to erigon-lib/rlphacks/utils_bytes.go diff --git a/turbo/trie/gen_struct_step.go b/turbo/trie/gen_struct_step.go index 6a751c84b9c..93eda075aaf 100644 --- a/turbo/trie/gen_struct_step.go +++ b/turbo/trie/gen_struct_step.go @@ -26,7 +26,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/turbo/rlphacks" + "github.com/erigontech/erigon-lib/rlphacks" ) // Experimental code for separating data and structural information diff --git a/turbo/trie/hashbuilder.go b/turbo/trie/hashbuilder.go index 2ee2d9266d4..353d2c401b1 100644 --- a/turbo/trie/hashbuilder.go +++ b/turbo/trie/hashbuilder.go @@ -31,8 +31,8 @@ import ( "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon-lib/rlphacks" "github.com/erigontech/erigon-lib/types/accounts" - "github.com/erigontech/erigon/turbo/rlphacks" ) const hashStackStride = length2.Hash + 1 // + 1 byte for RLP encoding diff --git a/turbo/trie/hasher.go b/turbo/trie/hasher.go index 27e41816c24..e191ca32044 100644 --- a/turbo/trie/hasher.go +++ b/turbo/trie/hasher.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" - "github.com/erigontech/erigon/turbo/rlphacks" + "github.com/erigontech/erigon-lib/rlphacks" ) type hasher struct { diff --git a/turbo/trie/stream.go b/turbo/trie/stream.go index 7d53c84abab..9d106f62c78 100644 --- a/turbo/trie/stream.go +++ b/turbo/trie/stream.go @@ -29,8 +29,8 @@ import ( "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/rlphacks" "github.com/erigontech/erigon-lib/types/accounts" - "github.com/erigontech/erigon/turbo/rlphacks" ) // StreamItem is an enum type for values that help distinguish different diff --git a/turbo/trie/structural_test.go b/turbo/trie/structural_test.go index 83219343804..883dccda148 100644 --- a/turbo/trie/structural_test.go +++ b/turbo/trie/structural_test.go @@ -36,7 +36,7 @@ import ( "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon/turbo/rlphacks" + "github.com/erigontech/erigon-lib/rlphacks" ) func TestV2HashBuilding(t *testing.T) { diff --git a/turbo/trie/trie_root.go b/turbo/trie/trie_root.go index 67e3ae3105d..b060a95cce8 100644 --- a/turbo/trie/trie_root.go +++ b/turbo/trie/trie_root.go @@ -31,9 +31,8 @@ import ( length2 "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/kv" dbutils2 "github.com/erigontech/erigon-lib/kv/dbutils" - + "github.com/erigontech/erigon-lib/rlphacks" "github.com/erigontech/erigon-lib/types/accounts" - "github.com/erigontech/erigon/turbo/rlphacks" ) /* From f3e31e81dc8bbc4d6712332506099727432e2fe4 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Mon, 2 Dec 2024 17:14:26 +0100 Subject: [PATCH 12/14] Downloader: fixed salt files (#12960) --- erigon-lib/downloader/snaptype/files.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/erigon-lib/downloader/snaptype/files.go b/erigon-lib/downloader/snaptype/files.go index e556bfaf880..b07e1e9f632 100644 --- a/erigon-lib/downloader/snaptype/files.go +++ b/erigon-lib/downloader/snaptype/files.go @@ -130,6 +130,10 @@ func ParseFileName(dir, fileName string) (res FileInfo, isE3Seedable bool, ok bo return res, isStateFile, isStateFile } +func isSaltFile(name string) bool { + return strings.HasPrefix(name, "salt") +} + func parseFileName(dir, fileName string) (res FileInfo, ok bool) { ext := filepath.Ext(fileName) onlyName := fileName[:len(fileName)-len(ext)] @@ -139,10 +143,15 @@ func parseFileName(dir, fileName string) (res FileInfo, ok bool) { if len(parts) < 2 { return res, ok } - - res.Type, ok = ParseFileType(parts[len(parts)-1]) - // This is a caplin hack - it is because with caplin state snapshots ok is always false - res.TypeString = parts[len(parts)-1] + if isSaltFile(fileName) { + // format for salt files is different: salt-.txt + res.Type, ok = ParseFileType(parts[0]) + res.TypeString = parts[0] + } else { + res.Type, ok = ParseFileType(parts[len(parts)-1]) + // This is a caplin hack - it is because with caplin state snapshots ok is always false + res.TypeString = parts[len(parts)-1] + } if ok { res.TypeString = res.Type.Name() From bfffa11da9708c11bc32fdb3faaa7f0c3bee112b Mon Sep 17 00:00:00 2001 From: antonis19 Date: Mon, 2 Dec 2024 19:27:22 +0100 Subject: [PATCH 13/14] move `turbo/trie` to` erigon-lib/trie` (#12961) Last follow-up to https://github.com/erigontech/erigon/pull/12915 Co-authored-by: antonis19 --- cmd/devnet/services/polygon/proofgenerator.go | 2 +- cmd/pics/state.go | 2 +- cmd/verkle/verkletrie/verkle_tree_writer.go | 2 +- cmd/verkle/verkletrie/workers.go | 2 +- consensus/aura/aura_test.go | 2 +- core/state/dump.go | 2 +- core/state/intra_block_state.go | 2 +- core/state/state_object.go | 2 +- core/types/hashing.go | 2 +- core/types/hashing_test.go | 2 +- core/vm/evm.go | 2 +- erigon-lib/go.mod | 4 +- erigon-lib/go.sum | 5 ++ {turbo => erigon-lib}/trie/.gitignore | 0 .../trie/account_node_test.go | 0 {turbo => erigon-lib}/trie/debug.go | 0 .../trie/delete_subrtee_test.go | 0 {turbo => erigon-lib}/trie/encoding.go | 0 {turbo => erigon-lib}/trie/encoding_test.go | 0 {turbo => erigon-lib}/trie/errors.go | 0 .../trie/flatdb_sub_trie_loader_test.go | 0 {turbo => erigon-lib}/trie/gen_struct_step.go | 0 {turbo => erigon-lib}/trie/hack.go | 0 {turbo => erigon-lib}/trie/hashbuilder.go | 19 +---- {turbo => erigon-lib}/trie/hasher.go | 0 {turbo => erigon-lib}/trie/hasher_test.go | 0 .../trie/intermediate_hashes_test.go | 0 {turbo => erigon-lib}/trie/node.go | 0 {turbo => erigon-lib}/trie/proof.go | 0 {turbo => erigon-lib}/trie/retain_list.go | 0 .../trie/retain_list_test.go | 1 + {turbo => erigon-lib}/trie/stream.go | 0 {turbo => erigon-lib}/trie/stream_test.go | 0 {turbo => erigon-lib}/trie/structural_test.go | 0 {turbo => erigon-lib}/trie/sub_trie_loader.go | 0 {turbo => erigon-lib}/trie/trie.go | 5 +- {turbo => erigon-lib}/trie/trie_root.go | 1 - {turbo => erigon-lib}/trie/trie_test.go | 60 --------------- {turbo => erigon-lib}/trie/utils.go | 0 .../trie/vtree/verkle_utils.go | 0 .../trie/vtree/verkle_utils_test.go | 0 eth/stagedsync/stage_commit_rebuild.go | 3 +- go.mod | 2 +- turbo/adapter/ethapi/get_proof.go | 2 +- turbo/jsonrpc/eth_call_test.go | 2 +- turbo/trie/trie_transform.go | 75 ------------------- 46 files changed, 31 insertions(+), 170 deletions(-) rename {turbo => erigon-lib}/trie/.gitignore (100%) rename {turbo => erigon-lib}/trie/account_node_test.go (100%) rename {turbo => erigon-lib}/trie/debug.go (100%) rename {turbo => erigon-lib}/trie/delete_subrtee_test.go (100%) rename {turbo => erigon-lib}/trie/encoding.go (100%) rename {turbo => erigon-lib}/trie/encoding_test.go (100%) rename {turbo => erigon-lib}/trie/errors.go (100%) rename {turbo => erigon-lib}/trie/flatdb_sub_trie_loader_test.go (100%) rename {turbo => erigon-lib}/trie/gen_struct_step.go (100%) rename {turbo => erigon-lib}/trie/hack.go (100%) rename {turbo => erigon-lib}/trie/hashbuilder.go (97%) rename {turbo => erigon-lib}/trie/hasher.go (100%) rename {turbo => erigon-lib}/trie/hasher_test.go (100%) rename {turbo => erigon-lib}/trie/intermediate_hashes_test.go (100%) rename {turbo => erigon-lib}/trie/node.go (100%) rename {turbo => erigon-lib}/trie/proof.go (100%) rename {turbo => erigon-lib}/trie/retain_list.go (100%) rename {turbo => erigon-lib}/trie/retain_list_test.go (99%) rename {turbo => erigon-lib}/trie/stream.go (100%) rename {turbo => erigon-lib}/trie/stream_test.go (100%) rename {turbo => erigon-lib}/trie/structural_test.go (100%) rename {turbo => erigon-lib}/trie/sub_trie_loader.go (100%) rename {turbo => erigon-lib}/trie/trie.go (99%) rename {turbo => erigon-lib}/trie/trie_root.go (99%) rename {turbo => erigon-lib}/trie/trie_test.go (91%) rename {turbo => erigon-lib}/trie/utils.go (100%) rename {turbo => erigon-lib}/trie/vtree/verkle_utils.go (100%) rename {turbo => erigon-lib}/trie/vtree/verkle_utils_test.go (100%) delete mode 100644 turbo/trie/trie_transform.go diff --git a/cmd/devnet/services/polygon/proofgenerator.go b/cmd/devnet/services/polygon/proofgenerator.go index 15d64c5ae56..1ce9317b941 100644 --- a/cmd/devnet/services/polygon/proofgenerator.go +++ b/cmd/devnet/services/polygon/proofgenerator.go @@ -33,6 +33,7 @@ import ( "github.com/erigontech/erigon-lib/common/hexutility" "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon/accounts/abi/bind" "github.com/erigontech/erigon/cl/merkle_tree" "github.com/erigontech/erigon/cmd/devnet/devnet" @@ -41,7 +42,6 @@ import ( bortypes "github.com/erigontech/erigon/polygon/bor/types" "github.com/erigontech/erigon/rpc" "github.com/erigontech/erigon/turbo/jsonrpc" - "github.com/erigontech/erigon/turbo/trie" ) var ErrTokenIndexOutOfRange = errors.New("index is grater than the number of tokens in transaction") diff --git a/cmd/pics/state.go b/cmd/pics/state.go index 99364e97e6b..e3c75c30275 100644 --- a/cmd/pics/state.go +++ b/cmd/pics/state.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/memdb" "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon/accounts/abi/bind" "github.com/erigontech/erigon/accounts/abi/bind/backends" "github.com/erigontech/erigon/cmd/pics/contracts" @@ -43,7 +44,6 @@ import ( "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/params" "github.com/erigontech/erigon/turbo/stages/mock" - "github.com/erigontech/erigon/turbo/trie" ) /*func statePicture(t *trie.Trie, number int, keyCompression int, codeCompressed bool, valCompressed bool, diff --git a/cmd/verkle/verkletrie/verkle_tree_writer.go b/cmd/verkle/verkletrie/verkle_tree_writer.go index f0a302c0119..f93c58db382 100644 --- a/cmd/verkle/verkletrie/verkle_tree_writer.go +++ b/cmd/verkle/verkletrie/verkle_tree_writer.go @@ -31,9 +31,9 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/trie/vtree" "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/rawdb" - "github.com/erigontech/erigon/turbo/trie/vtree" ) func int256ToVerkleFormat(x *uint256.Int, buffer []byte) { diff --git a/cmd/verkle/verkletrie/workers.go b/cmd/verkle/verkletrie/workers.go index 0ee88a7c7a7..ed572faa36e 100644 --- a/cmd/verkle/verkletrie/workers.go +++ b/cmd/verkle/verkletrie/workers.go @@ -22,8 +22,8 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/holiman/uint256" + "github.com/erigontech/erigon-lib/trie/vtree" "github.com/erigontech/erigon-lib/types/accounts" - "github.com/erigontech/erigon/turbo/trie/vtree" ) type regeneratePedersenAccountsJob struct { diff --git a/consensus/aura/aura_test.go b/consensus/aura/aura_test.go index b26c59a0c49..f95e5815693 100644 --- a/consensus/aura/aura_test.go +++ b/consensus/aura/aura_test.go @@ -30,13 +30,13 @@ import ( "github.com/erigontech/erigon-lib/kv/memdb" "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon/accounts/abi" "github.com/erigontech/erigon/consensus/aura" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/state" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/turbo/stages/mock" - "github.com/erigontech/erigon/turbo/trie" ) // Check that the first block of Gnosis Chain, which doesn't have any transactions, diff --git a/core/state/dump.go b/core/state/dump.go index 897d3153414..aced41f6052 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -31,8 +31,8 @@ import ( "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/order" "github.com/erigontech/erigon-lib/kv/rawdbv3" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon-lib/types/accounts" - "github.com/erigontech/erigon/turbo/trie" ) type Dumper struct { diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index 0a2ae087ee0..dbd3ee0fbfd 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -31,11 +31,11 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/u256" "github.com/erigontech/erigon-lib/crypto" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/tracing" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/core/vm/evmtypes" - "github.com/erigontech/erigon/turbo/trie" ) var _ evmtypes.IntraBlockState = new(IntraBlockState) // compile-time interface-check diff --git a/core/state/state_object.go b/core/state/state_object.go index 4b29e75caff..30cf1d110e6 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -30,9 +30,9 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon-lib/types/accounts" "github.com/erigontech/erigon/core/tracing" - "github.com/erigontech/erigon/turbo/trie" ) var emptyCodeHash = crypto.Keccak256(nil) diff --git a/core/types/hashing.go b/core/types/hashing.go index 08e3d921cce..6a632b2907d 100644 --- a/core/types/hashing.go +++ b/core/types/hashing.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon-lib/crypto/cryptopool" "github.com/erigontech/erigon-lib/rlp" "github.com/erigontech/erigon-lib/rlphacks" - "github.com/erigontech/erigon/turbo/trie" + "github.com/erigontech/erigon-lib/trie" ) type DerivableList interface { diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index f40909cef43..dcd2e37c798 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -26,7 +26,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/rlp" - "github.com/erigontech/erigon/turbo/trie" + "github.com/erigontech/erigon-lib/trie" ) func genTransactions(n uint64) Transactions { diff --git a/core/vm/evm.go b/core/vm/evm.go index 7b09c92f98f..4b85d8c70f1 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -29,10 +29,10 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/u256" "github.com/erigontech/erigon-lib/crypto" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon/core/tracing" "github.com/erigontech/erigon/core/vm/evmtypes" "github.com/erigontech/erigon/params" - "github.com/erigontech/erigon/turbo/trie" ) var emptyHash = libcommon.Hash{} diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 4ff738b58b2..cb61837a06e 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -19,6 +19,7 @@ require ( github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/containerd/cgroups/v3 v3.0.3 + github.com/crate-crypto/go-ipa v0.0.0-20221111143132-9aa5d42120bc github.com/crate-crypto/go-kzg-4844 v0.7.0 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 github.com/edsrzf/mmap-go v1.1.0 @@ -94,9 +95,10 @@ require ( github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35 github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect github.com/go-llsqlite/crawshaw v0.4.0 // indirect github.com/go-logr/logr v1.2.3 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 99a9c905fd4..63af7e40d9e 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -120,6 +120,8 @@ github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGD github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/crate-crypto/go-ipa v0.0.0-20221111143132-9aa5d42120bc h1:mtR7MuscVeP/s0/ERWA2uSr5QOrRYy1pdvZqG1USfXI= +github.com/crate-crypto/go-ipa v0.0.0-20221111143132-9aa5d42120bc/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -167,6 +169,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35 h1:I8QswD9gf3VEpr7bpepKKOm7ChxFITIG+oc1I5/S0no= +github.com/gballet/go-verkle v0.0.0-20221121182333-31427a1f2d35/go.mod h1:DMDd04jjQgdynaAwbEgiRERIGpC8fDjx0+y06an7Psg= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -586,6 +590,7 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/turbo/trie/.gitignore b/erigon-lib/trie/.gitignore similarity index 100% rename from turbo/trie/.gitignore rename to erigon-lib/trie/.gitignore diff --git a/turbo/trie/account_node_test.go b/erigon-lib/trie/account_node_test.go similarity index 100% rename from turbo/trie/account_node_test.go rename to erigon-lib/trie/account_node_test.go diff --git a/turbo/trie/debug.go b/erigon-lib/trie/debug.go similarity index 100% rename from turbo/trie/debug.go rename to erigon-lib/trie/debug.go diff --git a/turbo/trie/delete_subrtee_test.go b/erigon-lib/trie/delete_subrtee_test.go similarity index 100% rename from turbo/trie/delete_subrtee_test.go rename to erigon-lib/trie/delete_subrtee_test.go diff --git a/turbo/trie/encoding.go b/erigon-lib/trie/encoding.go similarity index 100% rename from turbo/trie/encoding.go rename to erigon-lib/trie/encoding.go diff --git a/turbo/trie/encoding_test.go b/erigon-lib/trie/encoding_test.go similarity index 100% rename from turbo/trie/encoding_test.go rename to erigon-lib/trie/encoding_test.go diff --git a/turbo/trie/errors.go b/erigon-lib/trie/errors.go similarity index 100% rename from turbo/trie/errors.go rename to erigon-lib/trie/errors.go diff --git a/turbo/trie/flatdb_sub_trie_loader_test.go b/erigon-lib/trie/flatdb_sub_trie_loader_test.go similarity index 100% rename from turbo/trie/flatdb_sub_trie_loader_test.go rename to erigon-lib/trie/flatdb_sub_trie_loader_test.go diff --git a/turbo/trie/gen_struct_step.go b/erigon-lib/trie/gen_struct_step.go similarity index 100% rename from turbo/trie/gen_struct_step.go rename to erigon-lib/trie/gen_struct_step.go diff --git a/turbo/trie/hack.go b/erigon-lib/trie/hack.go similarity index 100% rename from turbo/trie/hack.go rename to erigon-lib/trie/hack.go diff --git a/turbo/trie/hashbuilder.go b/erigon-lib/trie/hashbuilder.go similarity index 97% rename from turbo/trie/hashbuilder.go rename to erigon-lib/trie/hashbuilder.go index 353d2c401b1..a38d922aade 100644 --- a/turbo/trie/hashbuilder.go +++ b/erigon-lib/trie/hashbuilder.go @@ -658,17 +658,6 @@ func (hb *HashBuilder) code(code []byte) error { return nil } -func (hb *HashBuilder) emptyRoot() { - if hb.trace { - fmt.Printf("EMPTYROOT\n") - } - hb.nodeStack = append(hb.nodeStack, nil) - var hash [hashStackStride]byte // RLP representation of hash (or un-hashes value) - hash[0] = 0x80 + length2.Hash - copy(hash[1:], EmptyRoot[:]) - hb.hashStack = append(hb.hashStack, hash[:]...) -} - func (hb *HashBuilder) RootHash() (libcommon.Hash, error) { if !hb.hasRoot() { return libcommon.Hash{}, errors.New("no root in the tree") @@ -695,12 +684,12 @@ func (hb *HashBuilder) rootHash() libcommon.Hash { func (hb *HashBuilder) topHash() []byte { pos := len(hb.hashStack) - hashStackStride - len := hb.hashStack[pos] - 0x80 - if len > 32 { + length := hb.hashStack[pos] - 0x80 + if length > 32 { // node itself (RLP list), not its hash - len = hb.hashStack[pos] - 0xc0 + length = hb.hashStack[pos] - 0xc0 } - return hb.hashStack[pos : pos+1+int(len)] + return hb.hashStack[pos : pos+1+int(length)] } func (hb *HashBuilder) printTopHashes(prefix []byte, _, children uint16) { diff --git a/turbo/trie/hasher.go b/erigon-lib/trie/hasher.go similarity index 100% rename from turbo/trie/hasher.go rename to erigon-lib/trie/hasher.go diff --git a/turbo/trie/hasher_test.go b/erigon-lib/trie/hasher_test.go similarity index 100% rename from turbo/trie/hasher_test.go rename to erigon-lib/trie/hasher_test.go diff --git a/turbo/trie/intermediate_hashes_test.go b/erigon-lib/trie/intermediate_hashes_test.go similarity index 100% rename from turbo/trie/intermediate_hashes_test.go rename to erigon-lib/trie/intermediate_hashes_test.go diff --git a/turbo/trie/node.go b/erigon-lib/trie/node.go similarity index 100% rename from turbo/trie/node.go rename to erigon-lib/trie/node.go diff --git a/turbo/trie/proof.go b/erigon-lib/trie/proof.go similarity index 100% rename from turbo/trie/proof.go rename to erigon-lib/trie/proof.go diff --git a/turbo/trie/retain_list.go b/erigon-lib/trie/retain_list.go similarity index 100% rename from turbo/trie/retain_list.go rename to erigon-lib/trie/retain_list.go diff --git a/turbo/trie/retain_list_test.go b/erigon-lib/trie/retain_list_test.go similarity index 99% rename from turbo/trie/retain_list_test.go rename to erigon-lib/trie/retain_list_test.go index 563d4c52fc2..8ea96798208 100644 --- a/turbo/trie/retain_list_test.go +++ b/erigon-lib/trie/retain_list_test.go @@ -40,6 +40,7 @@ func FakePreimage(hash libcommon.Hash) libcommon.Hash { // especially useful for tests which want to manually manipulate the hash // databases without worrying about generating and tracking pre-images. func NewManualProofRetainer(t *testing.T, acc *accounts.Account, rl *RetainList, keys [][]byte) *ProofRetainer { + t.Helper() var accHexKey []byte var storageKeys []libcommon.Hash var storageHexKeys [][]byte diff --git a/turbo/trie/stream.go b/erigon-lib/trie/stream.go similarity index 100% rename from turbo/trie/stream.go rename to erigon-lib/trie/stream.go diff --git a/turbo/trie/stream_test.go b/erigon-lib/trie/stream_test.go similarity index 100% rename from turbo/trie/stream_test.go rename to erigon-lib/trie/stream_test.go diff --git a/turbo/trie/structural_test.go b/erigon-lib/trie/structural_test.go similarity index 100% rename from turbo/trie/structural_test.go rename to erigon-lib/trie/structural_test.go diff --git a/turbo/trie/sub_trie_loader.go b/erigon-lib/trie/sub_trie_loader.go similarity index 100% rename from turbo/trie/sub_trie_loader.go rename to erigon-lib/trie/sub_trie_loader.go diff --git a/turbo/trie/trie.go b/erigon-lib/trie/trie.go similarity index 99% rename from turbo/trie/trie.go rename to erigon-lib/trie/trie.go index 6254464afe8..5fb5c9e7dd9 100644 --- a/turbo/trie/trie.go +++ b/erigon-lib/trie/trie.go @@ -29,7 +29,6 @@ import ( "github.com/erigontech/erigon-lib/crypto" "github.com/erigontech/erigon-lib/types/accounts" - "github.com/erigontech/erigon/ethdb" ) var ( @@ -334,7 +333,7 @@ func (t *Trie) UpdateAccountCode(key []byte, code codeNode) error { accNode, gotValue := t.getAccount(t.root, hex, 0) if accNode == nil || !gotValue { - return fmt.Errorf("account not found with key: %x, %w", key, ethdb.ErrKeyNotFound) + return fmt.Errorf("account not found with key: %x", key) } actualCodeHash := crypto.Keccak256(code) @@ -360,7 +359,7 @@ func (t *Trie) UpdateAccountCodeSize(key []byte, codeSize int) error { accNode, gotValue := t.getAccount(t.root, hex, 0) if accNode == nil || !gotValue { - return fmt.Errorf("account not found with key: %x, %w", key, ethdb.ErrKeyNotFound) + return fmt.Errorf("account not found with key: %x", key) } accNode.codeSize = codeSize diff --git a/turbo/trie/trie_root.go b/erigon-lib/trie/trie_root.go similarity index 99% rename from turbo/trie/trie_root.go rename to erigon-lib/trie/trie_root.go index b060a95cce8..0b598f21c75 100644 --- a/turbo/trie/trie_root.go +++ b/erigon-lib/trie/trie_root.go @@ -126,7 +126,6 @@ type RootHashAggregator struct { curr bytes.Buffer // Current key for the structure generation algorithm, as well as the input tape for the hash builder succ bytes.Buffer currAccK []byte - value []byte // Current value to be used as the value tape for the hash builder hadTreeAcc bool groups []uint16 // `groups` parameter is the map of the stack. each element of the `groups` slice is a bitmask, one bit per element currently on the stack. See `GenStructStep` docs hasTree []uint16 diff --git a/turbo/trie/trie_test.go b/erigon-lib/trie/trie_test.go similarity index 91% rename from turbo/trie/trie_test.go rename to erigon-lib/trie/trie_test.go index 42b32784f11..4fe3b5d83e2 100644 --- a/turbo/trie/trie_test.go +++ b/erigon-lib/trie/trie_test.go @@ -636,63 +636,3 @@ func TestNextSubtreeHex(t *testing.T) { assert.Equal(tc.expect, res, "%s, %s", tc.prev, tc.next) } } - -//func TestIHCursorCanUseNextParent(t *testing.T) { -// db, assert := ethdb.NewMemDatabase(), require.New(t) -// defer db.Close() -// hash := fmt.Sprintf("%064d", 0) -// -// ih := AccTrie(nil, nil, nil, nil) -// -// ih.k[1], ih.v[1], ih.hasTree[1] = common.FromHex("00"), common.FromHex(hash+hash), 0b0000000000000110 -// ih.k[2], ih.v[2], ih.hasTree[2] = common.FromHex("0001"), common.FromHex(hash), 0b1000000000000000 -// ih.lvl = 2 -// ih.hashID[2] = 1 -// ih.hashID[1] = 0 -// assert.True(ih._nextSiblingOfParentInMem()) -// assert.Equal(ih.k[ih.lvl], common.FromHex("00")) -// -// ih.k[1], ih.v[1], ih.hasTree[1] = common.FromHex("00"), common.FromHex(hash+hash), 0b0000000000000110 -// ih.k[3], ih.v[3], ih.hasTree[3] = common.FromHex("000101"), common.FromHex(hash), 0b1000000000000000 -// ih.lvl = 3 -// ih.hashID[3] = 1 -// ih.hashID[1] = 0 -// assert.True(ih._nextSiblingOfParentInMem()) -// assert.Equal(ih.k[ih.lvl], common.FromHex("00")) -// -//} -// -//func _TestEmptyRoot(t *testing.T) { -// sc := shards.NewStateCache(32, 64*1024) -// -// sc.SetAccountHashesRead(common.FromHex("00"), 0b111, 0b111, 0b111, []libcommon.Hash{{}, {}, {}}) -// sc.SetAccountHashesRead(common.FromHex("01"), 0b111, 0b111, 0b111, []libcommon.Hash{{}, {}, {}}) -// sc.SetAccountHashesRead(common.FromHex("02"), 0b111, 0b111, 0b111, []libcommon.Hash{{}, {}, {}}) -// -// rl := NewRetainList(0) -// rl.AddHex(common.FromHex("01")) -// rl.AddHex(common.FromHex("0101")) -// canUse := func(prefix []byte) bool { return !rl.Retain(prefix) } -// i := 0 -// if err := sc.AccountTree([]byte{}, func(ihK []byte, h libcommon.Hash, hasTree, skipState bool) (toChild bool, err error) { -// i++ -// switch i { -// case 1: -// assert.Equal(t, common.FromHex("0001"), ihK) -// case 2: -// assert.Equal(t, common.FromHex("0100"), ihK) -// case 3: -// assert.Equal(t, common.FromHex("0102"), ihK) -// case 4: -// assert.Equal(t, common.FromHex("0202"), ihK) -// } -// if ok := canUse(ihK); ok { -// return false, nil -// } -// return hasTree, nil -// }, func(cur []byte) { -// panic(fmt.Errorf("key %x not found in cache", cur)) -// }); err != nil { -// t.Fatal(err) -// } -//} diff --git a/turbo/trie/utils.go b/erigon-lib/trie/utils.go similarity index 100% rename from turbo/trie/utils.go rename to erigon-lib/trie/utils.go diff --git a/turbo/trie/vtree/verkle_utils.go b/erigon-lib/trie/vtree/verkle_utils.go similarity index 100% rename from turbo/trie/vtree/verkle_utils.go rename to erigon-lib/trie/vtree/verkle_utils.go diff --git a/turbo/trie/vtree/verkle_utils_test.go b/erigon-lib/trie/vtree/verkle_utils_test.go similarity index 100% rename from turbo/trie/vtree/verkle_utils_test.go rename to erigon-lib/trie/vtree/verkle_utils_test.go diff --git a/eth/stagedsync/stage_commit_rebuild.go b/eth/stagedsync/stage_commit_rebuild.go index 82ce0303181..2bb0d6696af 100644 --- a/eth/stagedsync/stage_commit_rebuild.go +++ b/eth/stagedsync/stage_commit_rebuild.go @@ -19,6 +19,7 @@ package stagedsync import ( "context" "errors" + "github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks" "github.com/erigontech/erigon/turbo/stages/headerdownload" @@ -28,7 +29,7 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/state" - "github.com/erigontech/erigon/turbo/trie" + "github.com/erigontech/erigon-lib/trie" ) type TrieCfg struct { diff --git a/go.mod b/go.mod index 571d2469e52..62ee556903d 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/cenkalti/backoff/v4 v4.2.1 github.com/consensys/gnark-crypto v0.12.1 - github.com/crate-crypto/go-ipa v0.0.0-20221111143132-9aa5d42120bc github.com/crate-crypto/go-kzg-4844 v0.7.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 @@ -107,6 +106,7 @@ require ( require ( github.com/alecthomas/atomic v0.1.0-alpha2 // indirect github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b // indirect + github.com/crate-crypto/go-ipa v0.0.0-20221111143132-9aa5d42120bc // indirect github.com/elastic/go-freelru v0.13.0 // indirect github.com/erigontech/speedtest v0.0.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect diff --git a/turbo/adapter/ethapi/get_proof.go b/turbo/adapter/ethapi/get_proof.go index aed1cb7ec1d..855e6ea3f27 100644 --- a/turbo/adapter/ethapi/get_proof.go +++ b/turbo/adapter/ethapi/get_proof.go @@ -24,8 +24,8 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/length" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon-lib/types/accounts" - "github.com/erigontech/erigon/turbo/trie" ) // Result structs for GetProof diff --git a/turbo/jsonrpc/eth_call_test.go b/turbo/jsonrpc/eth_call_test.go index b95a1228254..4c5d9298713 100644 --- a/turbo/jsonrpc/eth_call_test.go +++ b/turbo/jsonrpc/eth_call_test.go @@ -39,6 +39,7 @@ import ( "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/turbo/testlog" + "github.com/erigontech/erigon-lib/trie" "github.com/erigontech/erigon/cmd/rpcdaemon/rpcdaemontest" "github.com/erigontech/erigon/core" "github.com/erigontech/erigon/core/rawdb" @@ -50,7 +51,6 @@ import ( "github.com/erigontech/erigon/turbo/adapter/ethapi" "github.com/erigontech/erigon/turbo/rpchelper" "github.com/erigontech/erigon/turbo/stages/mock" - "github.com/erigontech/erigon/turbo/trie" ) func TestEstimateGas(t *testing.T) { diff --git a/turbo/trie/trie_transform.go b/turbo/trie/trie_transform.go deleted file mode 100644 index 9bdd8679c2c..00000000000 --- a/turbo/trie/trie_transform.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package trie - -import ( - "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon-lib/types/accounts" -) - -type keyTransformFunc func([]byte) []byte - -func transformSubTrie(nd node, hex []byte, newTrie *Trie, transformFunc keyTransformFunc) { - switch n := nd.(type) { - case nil: - return - case valueNode: - nCopy := make(valueNode, len(n)) - copy(nCopy, n) - _, newTrie.root = newTrie.insert(newTrie.root, transformFunc(hex), nCopy) - return - case *accountNode: - accountCopy := accounts.NewAccount() - accountCopy.Copy(&n.Account) - var code []byte = nil - if n.code != nil { - code = make([]byte, len(n.code)) - copy(code, n.code) - } - _, newTrie.root = newTrie.insert(newTrie.root, transformFunc(hex), &accountNode{accountCopy, nil, true, code, n.codeSize}) - aHex := hex - if aHex[len(aHex)-1] == 16 { - aHex = aHex[:len(aHex)-1] - } - transformSubTrie(n.storage, aHex, newTrie, transformFunc) - case hashNode: - _, newTrie.root = newTrie.insert(newTrie.root, transformFunc(hex), hashNode{hash: common.CopyBytes(n.hash)}) - return - case *shortNode: - var hexVal []byte - hexVal = concat(hex, n.Key...) - transformSubTrie(n.Val, hexVal, newTrie, transformFunc) - case *duoNode: - i1, i2 := n.childrenIdx() - hex1 := make([]byte, len(hex)+1) - copy(hex1, hex) - hex1[len(hex)] = i1 - hex2 := make([]byte, len(hex)+1) - copy(hex2, hex) - hex2[len(hex)] = i2 - transformSubTrie(n.child1, hex1, newTrie, transformFunc) - transformSubTrie(n.child2, hex2, newTrie, transformFunc) - case *fullNode: - for i, child := range n.Children { - if child != nil { - transformSubTrie(child, concat(hex, byte(i)), newTrie, transformFunc) - } - } - default: - panic("") - } -} From d78797ffcc0e38637f6d6530009985309555040a Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 3 Dec 2024 09:31:25 +0700 Subject: [PATCH 14/14] sd: update - use string type (#12943) --- erigon-lib/commitment/commitment.go | 23 +++++++++++-------- erigon-lib/commitment/commitment_test.go | 4 ++-- .../commitment/hex_patricia_hashed_test.go | 4 ++-- .../commitment/patricia_state_mock_test.go | 6 ++--- erigon-lib/state/domain_shared.go | 7 +++--- erigon-lib/state/squeeze_test.go | 2 +- 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/erigon-lib/commitment/commitment.go b/erigon-lib/commitment/commitment.go index 56368775d88..3ea27ad1ffe 100644 --- a/erigon-lib/commitment/commitment.go +++ b/erigon-lib/commitment/commitment.go @@ -25,6 +25,7 @@ import ( "math/bits" "sort" "strings" + "unsafe" "github.com/holiman/uint256" @@ -1025,29 +1026,30 @@ func (t *Updates) Size() (updates uint64) { // TouchPlainKey marks plainKey as updated and applies different fn for different key types // (different behaviour for Code, Account and Storage key modifications). -func (t *Updates) TouchPlainKey(key, val []byte, fn func(c *KeyUpdate, val []byte)) { +func (t *Updates) TouchPlainKey(key string, val []byte, fn func(c *KeyUpdate, val []byte)) { switch t.mode { case ModeUpdate: pivot, updated := &KeyUpdate{plainKey: key, update: new(Update)}, false t.tree.DescendLessOrEqual(pivot, func(item *KeyUpdate) bool { - if bytes.Equal(item.plainKey, pivot.plainKey) { + if item.plainKey == pivot.plainKey { fn(item, val) updated = true } return false }) if !updated { - pivot.hashedKey = t.hasher(pivot.plainKey) + pivot.hashedKey = t.hasher(toBytesZeroCopy(pivot.plainKey)) fn(pivot, val) t.tree.ReplaceOrInsert(pivot) } case ModeDirect: - if _, ok := t.keys[string(key)]; !ok { - if err := t.etl.Collect(t.hasher(key), key); err != nil { + if _, ok := t.keys[key]; !ok { + keyBytes := toBytesZeroCopy(key) + if err := t.etl.Collect(t.hasher(keyBytes), keyBytes); err != nil { log.Warn("failed to collect updated key", "key", key, "err", err) } - t.keys[string(key)] = struct{}{} + t.keys[key] = struct{}{} } default: } @@ -1139,7 +1141,7 @@ func (t *Updates) HashSort(ctx context.Context, fn func(hk, pk []byte, update *U default: } - if err := fn(item.hashedKey, item.plainKey, item.update); err != nil { + if err := fn(item.hashedKey, toBytesZeroCopy(item.plainKey), item.update); err != nil { return false } return true @@ -1165,13 +1167,13 @@ func (t *Updates) Reset() { } type KeyUpdate struct { - plainKey []byte + plainKey string hashedKey []byte update *Update } func keyUpdateLessFn(i, j *KeyUpdate) bool { - return bytes.Compare(i.plainKey, j.plainKey) < 0 + return i.plainKey < j.plainKey } type UpdateFlags uint8 @@ -1349,3 +1351,6 @@ func (u *Update) String() string { } return sb.String() } + +func toStringZeroCopy(v []byte) string { return unsafe.String(&v[0], len(v)) } //nolint +func toBytesZeroCopy(s string) []byte { return unsafe.Slice(unsafe.StringData(s), len(s)) } diff --git a/erigon-lib/commitment/commitment_test.go b/erigon-lib/commitment/commitment_test.go index 9c8ba0cf08f..abe9d0afee7 100644 --- a/erigon-lib/commitment/commitment_test.go +++ b/erigon-lib/commitment/commitment_test.go @@ -355,8 +355,8 @@ func TestUpdates_TouchPlainKey(t *testing.T) { {common.FromHex("97c780315e7820752006b7a918ce7ec023df263a87a715b64d5ab445e1782a760a974f8810551f81dfb7f1425f7d835838888885"), []byte("value1")}, } for i := 0; i < len(upds); i++ { - utUpdate.TouchPlainKey(upds[i].key, upds[i].val, utUpdate.TouchStorage) - utDirect.TouchPlainKey(upds[i].key, upds[i].val, utDirect.TouchStorage) + utUpdate.TouchPlainKey(string(upds[i].key), upds[i].val, utUpdate.TouchStorage) + utDirect.TouchPlainKey(string(upds[i].key), upds[i].val, utDirect.TouchStorage) } uniqUpds := make(map[string]tc) diff --git a/erigon-lib/commitment/hex_patricia_hashed_test.go b/erigon-lib/commitment/hex_patricia_hashed_test.go index c251f98ad3b..a8a129f818c 100644 --- a/erigon-lib/commitment/hex_patricia_hashed_test.go +++ b/erigon-lib/commitment/hex_patricia_hashed_test.go @@ -249,7 +249,7 @@ func sortUpdatesByHashIncrease(t *testing.T, hph *HexPatriciaHashed, plainKeys [ ku := make([]*KeyUpdate, len(plainKeys)) for i, pk := range plainKeys { - ku[i] = &KeyUpdate{plainKey: pk, hashedKey: hph.hashAndNibblizeKey(pk), update: &updates[i]} + ku[i] = &KeyUpdate{plainKey: string(pk), hashedKey: hph.hashAndNibblizeKey(pk), update: &updates[i]} } sort.Slice(updates, func(i, j int) bool { @@ -259,7 +259,7 @@ func sortUpdatesByHashIncrease(t *testing.T, hph *HexPatriciaHashed, plainKeys [ pks := make([][]byte, len(updates)) upds := make([]Update, len(updates)) for i, u := range ku { - pks[i] = u.plainKey + pks[i] = []byte(u.plainKey) upds[i] = *u.update } return pks, upds diff --git a/erigon-lib/commitment/patricia_state_mock_test.go b/erigon-lib/commitment/patricia_state_mock_test.go index b620de4a0a5..c1bb9712f88 100644 --- a/erigon-lib/commitment/patricia_state_mock_test.go +++ b/erigon-lib/commitment/patricia_state_mock_test.go @@ -414,8 +414,8 @@ func WrapKeyUpdates(tb testing.TB, mode Mode, hasher keyHasher, keys [][]byte, u upd := NewUpdates(mode, tb.TempDir(), hasher) for i, key := range keys { - upd.TouchPlainKey(key, nil, func(c *KeyUpdate, _ []byte) { - c.plainKey = key + upd.TouchPlainKey(string(key), nil, func(c *KeyUpdate, _ []byte) { + c.plainKey = string(key) c.hashedKey = hasher(key) c.update = &updates[i] }) @@ -427,7 +427,7 @@ func WrapKeyUpdates(tb testing.TB, mode Mode, hasher keyHasher, keys [][]byte, u func WrapKeyUpdatesInto(tb testing.TB, upd *Updates, keys [][]byte, updates []Update) { tb.Helper() for i, key := range keys { - upd.TouchPlainKey(key, nil, func(c *KeyUpdate, _ []byte) { + upd.TouchPlainKey(string(key), nil, func(c *KeyUpdate, _ []byte) { c.update = &updates[i] }) } diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index f5518f9e0a4..ea4f32a5105 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -1312,15 +1312,14 @@ func (sdc *SharedDomainsCommitmentContext) TouchKey(d kv.Domain, key string, val if sdc.discard { return } - ks := []byte(key) switch d { case kv.AccountsDomain: - sdc.updates.TouchPlainKey(ks, val, sdc.updates.TouchAccount) + sdc.updates.TouchPlainKey(key, val, sdc.updates.TouchAccount) case kv.CodeDomain: - sdc.updates.TouchPlainKey(ks, val, sdc.updates.TouchCode) + sdc.updates.TouchPlainKey(key, val, sdc.updates.TouchCode) case kv.StorageDomain: - sdc.updates.TouchPlainKey(ks, val, sdc.updates.TouchStorage) + sdc.updates.TouchPlainKey(key, val, sdc.updates.TouchStorage) default: panic(fmt.Errorf("TouchKey: unknown domain %s", d)) } diff --git a/erigon-lib/state/squeeze_test.go b/erigon-lib/state/squeeze_test.go index fa23a46d3e5..778679c8a5d 100644 --- a/erigon-lib/state/squeeze_test.go +++ b/erigon-lib/state/squeeze_test.go @@ -122,7 +122,7 @@ func TestAggregator_SqueezeCommitment(t *testing.T) { for acit.HasNext() { k, _, err := acit.Next() require.NoError(t, err) - domains.sdCtx.updates.TouchPlainKey(k, nil, domains.sdCtx.updates.TouchAccount) + domains.sdCtx.updates.TouchPlainKey(string(k), nil, domains.sdCtx.updates.TouchAccount) } // check if the commitment is the same