merge main

.github/workflows/qa-constrained-tip-tracking.yml

@@ -7,15 +7,26 @@ on:
 
 jobs:
   constrained-tip-tracking-test:
-    runs-on: [self-hosted, Erigon3]
     timeout-minutes: 600
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - chain: mainnet
+            backend: Erigon3
+            cgroup_name: constrained_res_32G
+          - chain: bor-mainnet
+            backend: Polygon
+            cgroup_name: constrained_res_64G
+    runs-on: [ self-hosted, "${{ matrix.backend }}" ]
     env:
       ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
       ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
       ERIGON_QA_PATH: /home/qarunner/erigon-qa
       TRACKING_TIME_SECONDS: 14400 # 4 hours
       TOTAL_TIME_SECONDS: 28800 # 8 hours
-      CHAIN: mainnet
+      CHAIN: ${{ matrix.chain }}
+      CGROUP_NAME: ${{ matrix.cgroup_name }}
 
     steps:
     - name: Check out repository
@@ -44,15 +55,15 @@ jobs:
         set +e # Disable exit on error
         
         # Run Erigon under memory constraints, wait sync and check ability to maintain sync
-        cgexec -g memory:constrained_res_32G python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
+        cgexec -g memory:$CGROUP_NAME python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py \
           ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3 $CHAIN standard_node statistics
   
         # Capture monitoring script exit status
         test_exit_status=$?
 
         # Save the subsection reached status
-        echo "::set-output name=test_executed::true"
-
+        echo "test_executed=true" >> $GITHUB_OUTPUT
+        
         # Clean up Erigon process if it's still running
         if kill -0 $ERIGON_PID 2> /dev/null; then
           echo "Terminating Erigon"
@@ -94,14 +105,14 @@ jobs:
       if: steps.test_step.outputs.test_executed == 'true'
       uses: actions/upload-artifact@v4
       with:
-        name: test-results
+        name: test-results-${{ env.CHAIN }}
         path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json
 
     - name: Upload Erigon full log
       if: steps.test_step.outputs.test_executed == 'true'
       uses: actions/upload-artifact@v4
       with:
-        name: erigon-log
+        name: erigon-log-${{ env.CHAIN }}
         path: |
           ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/erigon.log
           ${{ env.ERIGON_TESTBED_DATA_DIR }}/proc_stat.log

.github/workflows/scripts/run_rpc_tests.sh

@@ -19,6 +19,8 @@ disabled_tests=(
     eth_coinbase/test_01.json
     eth_createAccessList/test_16.json
     eth_getTransactionByHash/test_02.json
+    # Small prune issue that leads to wrong ReceiptDomain data at 16999999 (probably at every million) block: https://github.com/erigontech/erigon/issues/13050
+    ots_searchTransactionsBefore/test_04.tar
     eth_getWork/test_01.json
     eth_mining/test_01.json
     eth_protocolVersion/test_1.json

accounts/abi/bind/backends/simulated.go

@@ -207,7 +207,7 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract libcommon.Addres
 	}
 	defer tx.Rollback()
 	stateDB := b.stateByBlockNumber(tx, blockNumber)
-	return stateDB.GetCode(contract), nil
+	return stateDB.GetCode(contract)
 }
 
 // BalanceAt returns the wei balance of a certain account in the blockchain.
@@ -220,7 +220,7 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract libcommon.Add
 	}
 	defer tx.Rollback()
 	stateDB := b.stateByBlockNumber(tx, blockNumber)
-	return stateDB.GetBalance(contract), nil
+	return stateDB.GetBalance(contract)
 }
 
 // NonceAt returns the nonce of a certain account in the blockchain.
@@ -234,7 +234,7 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract libcommon.Addre
 	defer tx.Rollback()
 
 	stateDB := b.stateByBlockNumber(tx, blockNumber)
-	return stateDB.GetNonce(contract), nil
+	return stateDB.GetNonce(contract)
 }
 
 // StorageAt returns the value of key in the storage of an account in the blockchain.
@@ -265,7 +265,7 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash libcom
 	defer tx.Rollback()
 
 	// Retrieve the context of the receipt based on the transaction hash
-	blockNumber, err := rawdb.ReadTxLookupEntry(tx, txHash)
+	blockNumber, _, err := rawdb.ReadTxLookupEntry(tx, txHash)
 	if err != nil {
 		return nil, err
 	}
@@ -308,7 +308,7 @@ func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash libcomm
 	if txn != nil {
 		return txn, true, nil
 	}
-	blockNumber, ok, err := b.BlockReader().TxnLookup(ctx, tx, txHash)
+	blockNumber, _, ok, err := b.BlockReader().TxnLookup(ctx, tx, txHash)
 	if err != nil {
 		return nil, false, err
 	}
@@ -518,7 +518,7 @@ func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract libcommon
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	return b.pendingState.GetCode(contract), nil
+	return b.pendingState.GetCode(contract)
 }
 
 func newRevertError(result *evmtypes.ExecutionResult) *revertError {
@@ -600,7 +600,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account libcommon
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	return b.pendingState.GetNonce(account), nil
+	return b.pendingState.GetNonce(account)
 }
 
 // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated
@@ -628,7 +628,10 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 	}
 	// Recap the highest gas allowance with account's balance.
 	if call.GasPrice != nil && !call.GasPrice.IsZero() {
-		balance := b.pendingState.GetBalance(call.From) // from can't be nil
+		balance, err := b.pendingState.GetBalance(call.From) // from can't be nil
+		if err != nil {
+			return 0, err
+		}
 		available := balance.ToBig()
 		if call.Value != nil {
 			if call.Value.ToBig().Cmp(available) >= 0 {
@@ -724,7 +727,10 @@ func (b *SimulatedBackend) callContract(_ context.Context, call ethereum.CallMsg
 		call.Value = new(uint256.Int)
 	}
 	// Set infinite balance to the fake caller account.
-	from := statedb.GetOrNewStateObject(call.From)
+	from, err := statedb.GetOrNewStateObject(call.From)
+	if err != nil {
+		return nil, err
+	}
 	from.SetBalance(uint256.NewInt(0).SetAllOne(), tracing.BalanceChangeUnspecified)
 	// Execute the call.
 	msg := callMsg{call}
@@ -752,7 +758,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, txn types.Transa
 	if senderErr != nil {
 		return fmt.Errorf("invalid transaction: %w", senderErr)
 	}
-	nonce := b.pendingState.GetNonce(sender)
+	nonce, err := b.pendingState.GetNonce(sender)
+	if err != nil {
+		return err
+	}
 	if txn.GetNonce() != nonce {
 		return fmt.Errorf("invalid transaction nonce: got %d, want %d", txn.GetNonce(), nonce)
 	}

accounts/abi/bind/backends/simulated_test.go

@@ -89,7 +89,7 @@ func TestSimulatedBackend(t *testing.T) {
 	sim.Commit()
 	_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
 	if err != nil {
-		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+		t.Fatalf("error getting transaction with hash: %v %v", txHash.String(), err.Error())
 	}
 	if isPending {
 		t.Fatal("transaction should not have pending status")
@@ -152,7 +152,10 @@ func TestNewSimulatedBackend(t *testing.T) {
 	}
 
 	statedb := sim.stateByBlockNumber(tx, new(big.Int).SetUint64(num+1))
-	bal := statedb.GetBalance(testAddr)
+	bal, err := statedb.GetBalance(testAddr)
+	if err != nil {
+		t.Fatal(err)
+	}
 	if !bal.Eq(expectedBal) {
 		t.Errorf("expected balance for test address not received. expected: %v actual: %v", expectedBal, bal)
 	}

cl/antiquary/antiquary.go

@@ -32,6 +32,7 @@ import (
 	"github.com/erigontech/erigon-lib/downloader/snaptype"
 	proto_downloader "github.com/erigontech/erigon-lib/gointerfaces/downloaderproto"
 	"github.com/erigontech/erigon-lib/kv"
+	"github.com/erigontech/erigon/cl/beacon/synced_data"
 	"github.com/erigontech/erigon/cl/clparams"
 	"github.com/erigontech/erigon/cl/persistence/beacon_indicies"
 	"github.com/erigontech/erigon/cl/persistence/blob_storage"
@@ -62,12 +63,13 @@ type Antiquary struct {
 
 	validatorsTable *state_accessors.StaticValidatorTable
 	genesisState    *state.CachingBeaconState
+	syncedData      synced_data.SyncedData
 	// set to nil
 	currentState *state.CachingBeaconState
 	balances32   []byte
 }
 
-func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, stateSn *snapshotsync.CaplinStateSnapshots, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, logger log.Logger, states, blocks, blobs, snapgen bool, snBuildSema *semaphore.Weighted) *Antiquary {
+func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, stateSn *snapshotsync.CaplinStateSnapshots, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, syncedData synced_data.SyncedData, logger log.Logger, states, blocks, blobs, snapgen bool, snBuildSema *semaphore.Weighted) *Antiquary {
 	backfilled := &atomic.Bool{}
 	blobBackfilled := &atomic.Bool{}
 	backfilled.Store(false)
@@ -92,6 +94,7 @@ func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, gen
 		blobs:           blobs,
 		snapgen:         snapgen,
 		stateSn:         stateSn,
+		syncedData:      syncedData,
 	}
 }
 
@@ -214,6 +217,14 @@ func (a *Antiquary) Loop() error {
 		}
 	}
 
+	if a.stateSn != nil {
+		if err := a.stateSn.OpenFolder(); err != nil {
+			return err
+		}
+	}
+	log.Info("[Caplin] Stat", "blocks-static", a.sn.BlocksAvailable(), "states-static", a.stateSn.BlocksAvailable(), "blobs-static", a.sn.FrozenBlobs(),
+		"state-history-enabled", a.states, "block-history-enabled", a.blocks, "blob-history-enabled", a.blobs, "snapgen", a.snapgen)
+
 	frozenSlots := a.sn.BlocksAvailable()
 	if frozenSlots != 0 {
 		if err := beacon_indicies.PruneBlocks(a.ctx, tx, frozenSlots); err != nil {

cl/antiquary/state_antiquary.go

@@ -57,11 +57,11 @@ func (s *Antiquary) loopStates(ctx context.Context) {
 	// Execute this each second
 	reqRetryTimer := time.NewTicker(100 * time.Millisecond)
 	defer reqRetryTimer.Stop()
+
 	if !initial_state.IsGenesisStateSupported(clparams.NetworkType(s.cfg.DepositNetworkID)) {
 		s.logger.Warn("Genesis state is not supported for this network, no historical states data will be available")
 		return
 	}
-
 	_, beforeFinalized, err := s.readHistoricalProcessingProgress(ctx)
 	if err != nil {
 		s.logger.Error("Failed to read historical processing progress", "err", err)
@@ -81,11 +81,16 @@ func (s *Antiquary) loopStates(ctx context.Context) {
 				s.logger.Error("Failed to read historical processing progress", "err", err)
 				continue
 			}
+			if s.sn == nil || s.syncedData.Syncing() {
+				continue
+			}
+
 			// We wait for updated finality.
 			if finalized == beforeFinalized {
 				continue
 			}
 			beforeFinalized = finalized
+
 			if err := s.IncrementBeaconState(ctx, finalized); err != nil {
 				if s.currentState != nil {
 					s.logger.Warn("Could not to increment beacon state, trying again later", "err", err, "slot", s.currentState.Slot())
@@ -137,9 +142,16 @@ func FillStaticValidatorsTableIfNeeded(ctx context.Context, logger log.Logger, s
 	blocksAvaiable := stateSn.BlocksAvailable()
 	stateSnRoTx := stateSn.View()
 	defer stateSnRoTx.Close()
-
+	log.Info("[Caplin-Archive] filling validators table", "from", 0, "to", stateSn.BlocksAvailable())
+	logTicker := time.NewTicker(10 * time.Second)
+	defer logTicker.Stop()
 	start := time.Now()
 	for slot := uint64(0); slot <= stateSn.BlocksAvailable(); slot++ {
+		select {
+		case <-logTicker.C:
+			log.Info("[Caplin-Archive] Filled validators table", "progress", fmt.Sprintf("%d/%d", slot, stateSn.BlocksAvailable()))
+		default:
+		}
 		seg, ok := stateSnRoTx.VisibleSegment(slot, kv.StateEvents)
 		if !ok {
 			return false, fmt.Errorf("segment not found for slot %d", slot)
@@ -434,7 +446,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		// We now do some post-processing on the state.
 		select {
 		case <-progressTimer.C:
-			log.Log(logLvl, "State processing progress", "slot", slot, "blk/sec", fmt.Sprintf("%.2f", float64(slot-prevSlot)/60))
+			log.Log(logLvl, "[Caplin-Archive] Historical States reconstruction", "slot", slot, "blk/sec", fmt.Sprintf("%.2f", float64(slot-prevSlot)/60))
 			prevSlot = slot
 		default:
 		}
@@ -488,7 +500,7 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
 		return err
 	}
 
-	log.Info("Historical states antiquated", "slot", s.currentState.Slot(), "root", libcommon.Hash(stateRoot), "latency", endTime)
+	log.Info("[Caplin-Archive] Historical states antiquated", "slot", s.currentState.Slot(), "root", libcommon.Hash(stateRoot), "latency", endTime)
 	if s.stateSn != nil {
 		if err := s.stateSn.OpenFolder(); err != nil {
 			return err
@@ -574,7 +586,7 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
 	backoffStrides := uint64(10)
 	backoffStep := backoffStrides
 
-	historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.genesisState, s.stateSn)
+	historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.genesisState, s.stateSn, s.syncedData)
 
 	for {
 		attempt, err := computeSlotToBeRequested(tx, s.cfg, s.genesisState.Slot(), targetSlot, backoffStep)

cl/antiquary/state_antiquary_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/erigontech/erigon-lib/kv/memdb"
 	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon/cl/antiquary/tests"
+	"github.com/erigontech/erigon/cl/beacon/synced_data"
 	"github.com/erigontech/erigon/cl/clparams"
 	"github.com/erigontech/erigon/cl/cltypes"
 	state_accessors "github.com/erigontech/erigon/cl/persistence/state"
@@ -39,13 +40,19 @@ import (
 func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postState *state.CachingBeaconState) {
 	db := memdb.NewTestDB(t, kv.ChainDB)
 	reader := tests.LoadChain(blocks, postState, db, t)
-
+	sn := synced_data.NewSyncedDataManager(&clparams.MainnetBeaconConfig, true)
+	sn.OnHeadState(postState)
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, nil, reader, log.New(), true, true, true, false, nil)
+	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, nil, reader, sn, log.New(), true, true, true, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 }
 
+func TestStateAntiquaryElectra(t *testing.T) {
+	blocks, preState, postState := tests.GetElectraRandom()
+	runTest(t, blocks, preState, postState)
+}
+
 func TestStateAntiquaryCapella(t *testing.T) {
 	blocks, preState, postState := tests.GetCapellaRandom()
 	runTest(t, blocks, preState, postState)