increase receipt cache and make it configurable (#12862)

README.md

@@ -207,7 +207,7 @@ du -hsc /erigon/snapshots/*
 - **Store most of data in immutable files (segments/snapshots):**
     - can symlink/mount latest state to fast drive and history to cheap drive
     - `chaindata` is less than `15gb`. It's ok to `rm -rf chaindata`. (to prevent grow: recommend `--batchSize <= 1G`)
-- **`--prune` flags changed**: see `--prune.mode` (default: `archive`, full: `full`, EIP-4444: `minimal`)
+- **`--prune` flags changed**: see `--prune.mode` (default: `full`, archive: `archive`, EIP-4444: `minimal`)
 - **Other changes:**
     - ExecutionStage included many E2 stages: stage_hash_state, stage_trie, log_index, history_index, trace_index
     - Restart doesn't loose much partial progress: `--sync.loop.block.limit=5_000` enabled by default

cmd/rpcdaemon/test.http

@@ -105,7 +105,8 @@ Content-Type: application/json
 
 ###
 
-# curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_getBlockByNumber", "params": ["0x1b4", true], "id":1}' localhost:8545
+
+# curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_getBlockByNumber", "params": ["0x141F644", true], "id":1}' localhost:8545
 POST localhost:8545
 Content-Type: application/json
 

core/types/receipt.go

@@ -352,6 +352,14 @@ type Receipts []*Receipt
 // Len returns the number of receipts in this list.
 func (rs Receipts) Len() int { return len(rs) }
 
+func (rs Receipts) Copy() Receipts {
+	rsCopy := make(Receipts, 0, rs.Len())
+	for _, r := range rs {
+		rsCopy = append(rsCopy, r.Copy())
+	}
+	return rsCopy
+}
+
 // EncodeIndex encodes the i'th receipt to w.
 func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) {
 	r := rs[i]

eth/protocols/eth/handler_test.go

@@ -91,7 +91,7 @@ func TestGetBlockReceipts(t *testing.T) {
 	}
 	// Assemble the test environment
 	m := mockWithGenerator(t, 4, generator)
-	receiptsGetter := receipts.NewGenerator(32, m.BlockReader, m.Engine)
+	receiptsGetter := receipts.NewGenerator(m.BlockReader, m.Engine)
 	// Collect the hashes to request, and the response to expect
 	var (
 		hashes   []libcommon.Hash

p2p/sentry/sentry_multi_client/sentry_multi_client.go

@@ -226,7 +226,7 @@ func NewMultiClient(
 		disableBlockDownload:              disableBlockDownload,
 		logger:                            logger,
 		getReceiptsActiveGoroutineNumber:  semaphore.NewWeighted(1),
-		ethApiWrapper:                     receipts.NewGenerator(32, blockReader, engine),
+		ethApiWrapper:                     receipts.NewGenerator(blockReader, engine),
 	}
 
 	return cs, nil

turbo/jsonrpc/eth_api.go

@@ -149,13 +149,11 @@ type BaseAPI struct {
 
 func NewBaseApi(f *rpchelper.Filters, stateCache kvcache.Cache, blockReader services.FullBlockReader, singleNodeMode bool, evmCallTimeout time.Duration, engine consensus.EngineReader, dirs datadir.Dirs, bridgeReader bridgeReader) *BaseAPI {
 	var (
-		blocksLRUSize      = 128 // ~32Mb
-		receiptsCacheLimit = 32
+		blocksLRUSize = 128 // ~32Mb
 	)
 	// if RPCDaemon deployed as independent process: increase cache sizes
 	if !singleNodeMode {
 		blocksLRUSize *= 5
-		receiptsCacheLimit *= 5
 	}
 	blocksLRU, err := lru.New[common.Hash, *types.Block](blocksLRUSize)
 	if err != nil {
@@ -170,8 +168,8 @@ func NewBaseApi(f *rpchelper.Filters, stateCache kvcache.Cache, blockReader serv
 		_txnReader:          blockReader,
 		evmCallTimeout:      evmCallTimeout,
 		_engine:             engine,
-		receiptsGenerator:   receipts.NewGenerator(receiptsCacheLimit, blockReader, engine),
-		borReceiptGenerator: receipts.NewBorGenerator(receiptsCacheLimit, blockReader, engine),
+		receiptsGenerator:   receipts.NewGenerator(blockReader, engine),
+		borReceiptGenerator: receipts.NewBorGenerator(blockReader, engine),
 		dirs:                dirs,
 		useBridgeReader:     bridgeReader != nil && !reflect.ValueOf(bridgeReader).IsNil(), // needed for interface nil caveat
 		bridgeReader:        bridgeReader,

turbo/jsonrpc/eth_receipts.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 
 	"github.com/RoaringBitmap/roaring"
-
 	"github.com/erigontech/erigon-lib/chain"
 	"github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/kv"

turbo/jsonrpc/receipts/bor_receipts_generator.go

@@ -27,9 +27,9 @@ type BorGenerator struct {
 	engine       consensus.EngineReader
 }
 
-func NewBorGenerator(cacheSize int, blockReader services.FullBlockReader,
+func NewBorGenerator(blockReader services.FullBlockReader,
 	engine consensus.EngineReader) *BorGenerator {
-	receiptCache, err := lru.New[libcommon.Hash, *types.Receipt](cacheSize)
+	receiptCache, err := lru.New[libcommon.Hash, *types.Receipt](receiptsCacheLimit)
 	if err != nil {
 		panic(err)
 	}
@@ -62,7 +62,7 @@ func (g *BorGenerator) GenerateBorReceipt(ctx context.Context, tx kv.Tx, block *
 		return nil, err
 	}
 
-	g.receiptCache.Add(block.Hash(), receipt)
+	g.receiptCache.Add(block.Hash(), receipt.Copy())
 	return receipt, nil
 }
 

turbo/jsonrpc/receipts/receipts_generator.go

@@ -4,10 +4,9 @@ import (
 	"context"
 	"fmt"
 
-	lru "github.com/hashicorp/golang-lru/v2"
-
 	"github.com/erigontech/erigon-lib/chain"
 	"github.com/erigontech/erigon-lib/common"
+	"github.com/erigontech/erigon-lib/common/dbg"
 	"github.com/erigontech/erigon-lib/kv"
 	"github.com/erigontech/erigon-lib/kv/rawdbv3"
 	"github.com/erigontech/erigon-lib/log/v3"
@@ -19,12 +18,15 @@ import (
 	"github.com/erigontech/erigon/turbo/services"
 	"github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks"
 	"github.com/erigontech/erigon/turbo/transactions"
+	lru "github.com/hashicorp/golang-lru/v2"
 )
 
 type Generator struct {
-	receiptsCache *lru.Cache[common.Hash, []*types.Receipt]
-	blockReader   services.FullBlockReader
-	engine        consensus.EngineReader
+	receiptsCache      *lru.Cache[common.Hash, types.Receipts]
+	receiptsCacheTrace bool
+
+	blockReader services.FullBlockReader
+	engine      consensus.EngineReader
 }
 
 type ReceiptEnv struct {
@@ -37,20 +39,33 @@ type ReceiptEnv struct {
 	header      *types.Header
 }
 
-func NewGenerator(cacheSize int, blockReader services.FullBlockReader,
-	engine consensus.EngineReader) *Generator {
-	receiptsCache, err := lru.New[common.Hash, []*types.Receipt](cacheSize)
+var (
+	receiptsCacheLimit = dbg.EnvInt("R_LRU", 1024) //ethmainnet: 1K receipts is ~200mb RAM
+	receiptsCacheTrace = dbg.EnvBool("R_LRU_TRACE", false)
+)
+
+func NewGenerator(blockReader services.FullBlockReader, engine consensus.EngineReader) *Generator {
+	receiptsCache, err := lru.New[common.Hash, types.Receipts](receiptsCacheLimit)
 	if err != nil {
 		panic(err)
 	}
 
 	return &Generator{
-		receiptsCache: receiptsCache,
-		blockReader:   blockReader,
-		engine:        engine,
+		receiptsCache:      receiptsCache,
+		blockReader:        blockReader,
+		engine:             engine,
+		receiptsCacheTrace: receiptsCacheTrace,
 	}
 }
 
+func (g *Generator) LogStats() {
+	if g == nil || !g.receiptsCacheTrace {
+		return
+	}
+	//m := g.receiptsCache.Metrics()
+	//log.Warn("[dbg] ReceiptsCache", "hit", m.Hits, "total", m.Hits+m.Misses, "Collisions", m.Collisions, "Evictions", m.Evictions, "Inserts", m.Inserts, "limit", receiptsCacheLimit, "ratio", fmt.Sprintf("%.2f", float64(m.Hits)/float64(m.Hits+m.Misses)))
+}
+
 func (g *Generator) GetCachedReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, bool) {
 	return g.receiptsCache.Get(blockHash)
 }
@@ -87,7 +102,15 @@ func (g *Generator) PrepareEnv(ctx context.Context, block *types.Block, cfg *cha
 	}, nil
 }
 
+func (g *Generator) addToCache(header *types.Header, receipts types.Receipts) {
+	g.receiptsCache.Add(header.Hash(), receipts.Copy())
+}
+
 func (g *Generator) GetReceipt(ctx context.Context, cfg *chain.Config, tx kv.Tx, block *types.Block, index int, optimize bool) (*types.Receipt, error) {
+	if receipts, ok := g.receiptsCache.Get(block.Hash()); ok && len(receipts) > index {
+		return receipts[index], nil
+	}
+
 	var receipt *types.Receipt
 	if optimize {
 		genEnv, err := g.PrepareEnv(ctx, block, cfg, tx, index)
@@ -142,6 +165,6 @@ func (g *Generator) GetReceipts(ctx context.Context, cfg *chain.Config, tx kv.Tx
 		receipts[i] = receipt
 	}
 
-	g.receiptsCache.Add(block.Hash(), receipts)
+	g.addToCache(block.HeaderNoCopy(), receipts)
 	return receipts, nil
 }

turbo/stages/mock/mock_sentry.go

@@ -313,7 +313,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK
 		PeerId:         gointerfaces.ConvertHashToH512([64]byte{0x12, 0x34, 0x50}), // "12345"
 		BlockSnapshots: allSnapshots,
 		BlockReader:    br,
-		ReceiptsReader: receipts.NewGenerator(16, br, engine),
+		ReceiptsReader: receipts.NewGenerator(br, engine),
 		HistoryV3:      true,
 	}