TxLookup index per-txn-granularity

.github/workflows/scripts/run_rpc_tests.sh

@@ -19,6 +19,8 @@ disabled_tests=(
     eth_coinbase/test_01.json
     eth_createAccessList/test_16.json
     eth_getTransactionByHash/test_02.json
+    # Small prune issue that leads to wrong ReceiptDomain data at 16999999 (probably at every million) block: https://github.com/erigontech/erigon/issues/13050
+    eth_getTransactionReceipt/test_04.json
     eth_getWork/test_01.json
     eth_mining/test_01.json
     eth_protocolVersion/test_1.json

ChangeLog.md

@@ -36,6 +36,7 @@ ChangeLog
 ### New features:
 
 - Reduced `.idx` and `.efi` files size by 25% (require re-sync)
+- Support: `debug_getRawReceipts`
 - debian packages
 - `--externalcl` support
 - bor-mainnet can work on 32G machine

accounts/abi/bind/backends/simulated.go

@@ -265,7 +265,7 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash libcom
 	defer tx.Rollback()
 
 	// Retrieve the context of the receipt based on the transaction hash
-	blockNumber, err := rawdb.ReadTxLookupEntry(tx, txHash)
+	blockNumber, _, err := rawdb.ReadTxLookupEntry(tx, txHash)
 	if err != nil {
 		return nil, err
 	}
@@ -308,7 +308,7 @@ func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash libcomm
 	if txn != nil {
 		return txn, true, nil
 	}
-	blockNumber, ok, err := b.BlockReader().TxnLookup(ctx, tx, txHash)
+	blockNumber, _, ok, err := b.BlockReader().TxnLookup(ctx, tx, txHash)
 	if err != nil {
 		return nil, false, err
 	}

accounts/abi/bind/backends/simulated_test.go

@@ -89,7 +89,7 @@ func TestSimulatedBackend(t *testing.T) {
 	sim.Commit()
 	_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
 	if err != nil {
-		t.Fatalf("error getting transaction with hash: %v", txHash.String())
+		t.Fatalf("error getting transaction with hash: %v %v", txHash.String(), err.Error())
 	}
 	if isPending {
 		t.Fatal("transaction should not have pending status")

cmd/rpcdaemon/rpcservices/eth_backend.go

@@ -282,7 +282,7 @@ func (back *RemoteBackend) SubscribeLogs(ctx context.Context, onNewLogs func(rep
 	return nil
 }
 
-func (back *RemoteBackend) TxnLookup(ctx context.Context, tx kv.Getter, txnHash common.Hash) (uint64, bool, error) {
+func (back *RemoteBackend) TxnLookup(ctx context.Context, tx kv.Getter, txnHash common.Hash) (uint64, uint64, bool, error) {
 	return back.blockReader.TxnLookup(ctx, tx, txnHash)
 }
 func (back *RemoteBackend) HasSenders(ctx context.Context, tx kv.Getter, hash common.Hash, blockNum uint64) (bool, error) {

core/rawdb/accessors_indexes.go

@@ -20,8 +20,7 @@
 package rawdb
 
 import (
-	"math/big"
-
+	"encoding/binary"
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon-lib/log/v3"
@@ -39,23 +38,28 @@ type TxLookupEntry struct {
 
 // ReadTxLookupEntry retrieves the positional metadata associated with a transaction
 // hash to allow retrieving the transaction or receipt by hash.
-func ReadTxLookupEntry(db kv.Getter, txnHash libcommon.Hash) (*uint64, error) {
+func ReadTxLookupEntry(db kv.Getter, txnHash libcommon.Hash) (blockNumber *uint64, txNum *uint64, err error) {
 	data, err := db.GetOne(kv.TxLookup, txnHash.Bytes())
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	if len(data) == 0 {
-		return nil, nil
+	if len(data) != 16 {
+		return nil, nil, nil
 	}
-	number := new(big.Int).SetBytes(data).Uint64()
-	return &number, nil
+	numberBlockNum := binary.BigEndian.Uint64(data[:8])
+	numberTxNum := binary.BigEndian.Uint64(data[8:]) + 1
+
+	return &numberBlockNum, &numberTxNum, nil
 }
 
 // WriteTxLookupEntries stores a positional metadata for every transaction from
 // a block, enabling hash based transaction and receipt lookups.
-func WriteTxLookupEntries(db kv.Putter, block *types.Block) {
-	for _, txn := range block.Transactions() {
-		data := block.Number().Bytes()
+func WriteTxLookupEntries(db kv.Putter, block *types.Block, txNum uint64) {
+	data := make([]byte, 16)
+	for i, txn := range block.Transactions() {
+		binary.BigEndian.PutUint64(data[:8], block.NumberU64())
+		binary.BigEndian.PutUint64(data[8:], txNum+uint64(i)+1)
+
 		if err := db.Put(kv.TxLookup, txn.Hash().Bytes(), data); err != nil {
 			log.Crit("Failed to store transaction lookup entry", "err", err)
 		}

core/rawdb/accessors_indexes_test.go

@@ -29,6 +29,7 @@ import (
 
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/kv"
+	"github.com/erigontech/erigon-lib/kv/rawdbv3"
 	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon/core/rawdb"
 	"github.com/erigontech/erigon/core/types"
@@ -41,12 +42,12 @@ func TestLookupStorage(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
 		name                 string
-		writeTxLookupEntries func(kv.Putter, *types.Block)
+		writeTxLookupEntries func(kv.Putter, *types.Block, uint64)
 	}{
 		{
 			"DatabaseV6",
-			func(db kv.Putter, block *types.Block) {
-				rawdb.WriteTxLookupEntries(db, block)
+			func(db kv.Putter, block *types.Block, txNum uint64) {
+				rawdb.WriteTxLookupEntries(db, block, txNum)
 			},
 		},
 		// Erigon: older databases are removed, no backward compatibility
@@ -71,7 +72,7 @@ func TestLookupStorage(t *testing.T) {
 
 			// Check that no transactions entries are in a pristine database
 			for i, txn := range txs {
-				if txn2, _, _, _, _ := readTransactionByHash(tx, txn.Hash(), br); txn2 != nil {
+				if txn2, _, _, _, _, _ := readTransactionByHash(tx, txn.Hash(), br); txn2 != nil {
 					t.Fatalf("txn #%d [%x]: non existent transaction returned: %v", i, txn.Hash(), txn2)
 				}
 			}
@@ -85,26 +86,34 @@ func TestLookupStorage(t *testing.T) {
 			if err := rawdb.WriteSenders(tx, block.Hash(), block.NumberU64(), block.Body().SendersFromTxs()); err != nil {
 				t.Fatal(err)
 			}
-			tc.writeTxLookupEntries(tx, block)
+			txNumMin, err := rawdbv3.TxNums.Min(tx, block.NumberU64())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			tc.writeTxLookupEntries(tx, block, txNumMin)
 
 			for i, txn := range txs {
-				if txn2, hash, number, index, _ := readTransactionByHash(tx, txn.Hash(), br); txn2 == nil {
+				if txn2, hash, blockNumber, txNum, index, _ := readTransactionByHash(tx, txn.Hash(), br); txn2 == nil {
 					t.Fatalf("txn #%d [%x]: transaction not found", i, txn.Hash())
 				} else {
-					if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
-						t.Fatalf("txn #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, txn.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
+					if hash != block.Hash() || blockNumber != block.NumberU64() || index != uint64(i) {
+						t.Fatalf("txn #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, txn.Hash(), hash, blockNumber, index, block.Hash(), block.NumberU64(), i)
 					}
 					if txn.Hash() != txn2.Hash() {
 						t.Fatalf("txn #%d [%x]: transaction mismatch: have %v, want %v", i, txn.Hash(), txn, txn2)
 					}
+					if txNum != txNumMin+uint64(i)+2 {
+						t.Fatalf("txn #%d [%x]: txnum mismatch: have %d, want %d", i, txn.Hash(), txNum, txNumMin+uint64(i)+1)
+					}
 				}
 			}
 			// Delete the transactions and check purge
 			for i, txn := range txs {
 				if err := rawdb.DeleteTxLookupEntry(tx, txn.Hash()); err != nil {
 					t.Fatal(err)
 				}
-				if txn2, _, _, _, _ := readTransactionByHash(tx, txn.Hash(), br); txn2 != nil {
+				if txn2, _, _, _, _, _ := readTransactionByHash(tx, txn.Hash(), br); txn2 != nil {
 					t.Fatalf("txn #%d [%x]: deleted transaction returned: %v", i, txn.Hash(), txn2)
 				}
 			}
@@ -114,36 +123,41 @@ func TestLookupStorage(t *testing.T) {
 
 // ReadTransactionByHash retrieves a specific transaction from the database, along with
 // its added positional metadata.
-func readTransactionByHash(db kv.Tx, hash libcommon.Hash, br services.FullBlockReader) (types.Transaction, libcommon.Hash, uint64, uint64, error) {
-	blockNumber, err := rawdb.ReadTxLookupEntry(db, hash)
+func readTransactionByHash(db kv.Tx, hash libcommon.Hash, br services.FullBlockReader) (txn types.Transaction, blockHash libcommon.Hash, blockNumber uint64, txNum uint64, txIndex uint64, err error) {
+	blockNumberPtr, txNumPtr, err := rawdb.ReadTxLookupEntry(db, hash)
 	if err != nil {
-		return nil, libcommon.Hash{}, 0, 0, err
+		return nil, libcommon.Hash{}, 0, 0, 0, err
+	}
+	if blockNumberPtr == nil {
+		return nil, libcommon.Hash{}, 0, 0, 0, nil
 	}
-	if blockNumber == nil {
-		return nil, libcommon.Hash{}, 0, 0, nil
+	blockNumber = *blockNumberPtr
+	if txNumPtr == nil {
+		return nil, libcommon.Hash{}, 0, 0, 0, nil
 	}
-	blockHash, ok, err := br.CanonicalHash(context.Background(), db, *blockNumber)
+	txNum = *txNumPtr
+	blockHash, ok, err := br.CanonicalHash(context.Background(), db, blockNumber)
 	if err != nil {
-		return nil, libcommon.Hash{}, 0, 0, err
+		return nil, libcommon.Hash{}, 0, 0, 0, err
 	}
 	if !ok || blockHash == (libcommon.Hash{}) {
-		return nil, libcommon.Hash{}, 0, 0, nil
+		return nil, libcommon.Hash{}, 0, 0, 0, nil
 	}
-	body, _ := br.BodyWithTransactions(context.Background(), db, blockHash, *blockNumber)
+	body, _ := br.BodyWithTransactions(context.Background(), db, blockHash, blockNumber)
 	if body == nil {
 		log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash)
-		return nil, libcommon.Hash{}, 0, 0, nil
+		return nil, libcommon.Hash{}, 0, 0, 0, nil
 	}
-	senders, err1 := rawdb.ReadSenders(db, blockHash, *blockNumber)
+	senders, err1 := rawdb.ReadSenders(db, blockHash, blockNumber)
 	if err1 != nil {
-		return nil, libcommon.Hash{}, 0, 0, err1
+		return nil, libcommon.Hash{}, 0, 0, 0, err1
 	}
 	body.SendersToTxs(senders)
-	for txIndex, txn := range body.Transactions {
-		if txn.Hash() == hash {
-			return txn, blockHash, *blockNumber, uint64(txIndex), nil
+	for txInd, txnValue := range body.Transactions {
+		if txnValue.Hash() == hash {
+			return txnValue, blockHash, blockNumber, txNum, uint64(txInd), nil
 		}
 	}
 	log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
-	return nil, libcommon.Hash{}, 0, 0, nil
+	return nil, libcommon.Hash{}, 0, 0, 0, nil
 }

core/state/txtask.go

@@ -19,6 +19,8 @@ package state
 import (
 	"container/heap"
 	"context"
+	"fmt"
+	"github.com/erigontech/erigon-lib/common/dbg"
 	"sync"
 	"time"
 
@@ -106,6 +108,10 @@ func (t *TxTask) CreateReceipt(tx kv.Tx) {
 	}
 
 	cumulativeGasUsed += t.UsedGas
+	if t.UsedGas == 0 {
+		msg := fmt.Sprintf("no gas used stack: %s tx %+v", dbg.Stack(), t.Tx)
+		panic(msg)
+	}
 
 	r := t.createReceipt(cumulativeGasUsed)
 	r.FirstLogIndexWithinBlock = firstLogIndex

erigon-lib/go.mod

@@ -9,7 +9,7 @@ replace (
 
 require (
 	github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e
-	github.com/erigontech/interfaces v0.0.0-20241116035842-5d396f10468e
+	github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed
 	github.com/erigontech/mdbx-go v0.38.4
 	github.com/erigontech/secp256k1 v1.1.0
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417

erigon-lib/go.sum

@@ -152,8 +152,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:ZpIO6HeopuZPYDLldL6zR0qyRezj80kQrDOGEF779ts=
 github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M=
-github.com/erigontech/interfaces v0.0.0-20241116035842-5d396f10468e h1:KD/yuduvNJDKkdmEaaiDYb9RQstnkJn5ZcSZHjH/C/w=
-github.com/erigontech/interfaces v0.0.0-20241116035842-5d396f10468e/go.mod h1:N7OUkhkcagp9+7yb4ycHsG2VWCOmuJ1ONBecJshxtLE=
+github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed h1:un44S8Tuol4LBIC6R94t93GShM53BYjz7GsNPziDLQ8=
+github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed/go.mod h1:N7OUkhkcagp9+7yb4ycHsG2VWCOmuJ1ONBecJshxtLE=
 github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo=
 github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI=
 github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY=