e3: renamings (#12621)

- move streams to own files 
- use same `scanDirty` func 
- rename `DomainRange` to `RangeAsOf`
- rename `DomainGet` to `GetAsOf`
- `HistoryRange` and `HistorySeek` methods to accept `domain`. remove
concept of standalone history (and their names). Because we decided to
not expose `History` concept to users.

.github/workflows/scripts/run_rpc_tests.sh

@@ -20,12 +20,15 @@ disabled_tests=(
     eth_getBlockByHash/test_10.json
     eth_getBlockByNumber/test_12.json
     # Erigon bugs: https://github.com/erigontech/erigon/pull/12609
-    debug_accountRange,debug_storageRangeAt
+    debug_accountRange
+    debug_storageRangeAt
     # need update rpc-test - because Erigon is correct (@AskAlexSharov will do after https://github.com/erigontech/erigon/pull/12634)
     # remove this line after https://github.com/erigontech/rpc-tests/pull/273
-    debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber
+    debug_getModifiedAccountsByHash
+    debug_getModifiedAccountsByNumber
     # Erigon bug https://github.com/erigontech/erigon/issues/12603
-    erigon_getLatestLogs,erigon_getLogsByHash/test_04.json
+    erigon_getLatestLogs
+    erigon_getLogsByHash/test_04.json
     # Erigon bug https://github.com/erigontech/erigon/issues/12637
     debug_traceBlockByNumber/test_05.tar
     debug_traceBlockByNumber/test_08.tar
@@ -36,14 +39,15 @@ disabled_tests=(
     # remove this line after https://github.com/erigontech/rpc-tests/pull/281
     parity_getBlockReceipts
     parity_listStorageKeys/test_12.json
-    # to investigate
+    # created task https://github.com/erigontech/erigon/issues/12668
     debug_traceCallMany/test_02.tar
     debug_traceCallMany/test_04.tar
     debug_traceCallMany/test_05.tar
     debug_traceCallMany/test_06.tar
     debug_traceCallMany/test_07.tar
     debug_traceCallMany/test_09.json
     debug_traceCallMany/test_10.tar
+    # to investigate
     engine_exchangeCapabilities/test_1.json
     engine_exchangeTransitionConfigurationV1/test_01.json
     engine_getClientVersionV1/test_1.json

ChangeLog.md

@@ -3,6 +3,11 @@ ChangeLog
 
 ## v3.0.0-alpha5 (in development)
 
+- Mostly we did work on rpc-compatibility tests
+- Caplin eating 1Gb less RAM. And Erigon3 works on 16gb machine.
+- Hard-time-limit on chain-tip pruning: https://github.com/erigontech/erigon/pull/12535
+- 
+
 ### TODO
 
 Acknowledgements:

core/rawdb/rawtemporaldb/accessors_receipt_test.go

@@ -49,27 +49,27 @@ func TestAppendReceipt(t *testing.T) {
 	require.NoError(err)
 
 	ttx := tx.(kv.TemporalTx)
-	v, ok, err := ttx.HistorySeek(kv.ReceiptHistory, FirstLogIndexKey, 0)
+	v, ok, err := ttx.HistorySeek(kv.ReceiptDomain, FirstLogIndexKey, 0)
 	require.NoError(err)
 	require.True(ok)
 	require.Empty(v)
 
-	v, ok, err = ttx.HistorySeek(kv.ReceiptHistory, FirstLogIndexKey, 1)
+	v, ok, err = ttx.HistorySeek(kv.ReceiptDomain, FirstLogIndexKey, 1)
 	require.NoError(err)
 	require.True(ok)
 	require.Equal(uint64(0), uvarint(v))
 
-	v, ok, err = ttx.HistorySeek(kv.ReceiptHistory, FirstLogIndexKey, 2)
+	v, ok, err = ttx.HistorySeek(kv.ReceiptDomain, FirstLogIndexKey, 2)
 	require.NoError(err)
 	require.True(ok)
 	require.Equal(uint64(1), uvarint(v))
 
-	v, ok, err = ttx.HistorySeek(kv.ReceiptHistory, FirstLogIndexKey, 3)
+	v, ok, err = ttx.HistorySeek(kv.ReceiptDomain, FirstLogIndexKey, 3)
 	require.NoError(err)
 	require.True(ok)
 	require.Equal(uint64(1), uvarint(v))
 
-	v, ok, err = ttx.HistorySeek(kv.ReceiptHistory, FirstLogIndexKey, 4)
+	v, ok, err = ttx.HistorySeek(kv.ReceiptDomain, FirstLogIndexKey, 4)
 	require.NoError(err)
 	require.True(ok)
 	require.Equal(uint64(0), uvarint(v))

core/state/history_reader_v3.go

@@ -222,7 +222,7 @@ func (s *PlainState) ForEachStorage(addr common.Address, startLocation common.Ha
 	st := btree.New(16)
 	var k [length.Addr + length.Incarnation + length.Hash]byte
 	copy(k[:], addr[:])
-	accData, err := GetAsOf(s.tx, s.accHistoryC, s.accChangesC, false , addr[:], s.blockNr)
+	accData, err := DomainGetAsOf(s.tx, s.accHistoryC, s.accChangesC, false , addr[:], s.blockNr)
 	if err != nil {
 		return err
 	}

erigon-lib/kv/kv_interface.go

@@ -513,13 +513,10 @@ type TemporalTx interface {
 
 	// DomainGetAsOf - state as of given `ts`
 	// Example: GetAsOf(Account, key, txNum) - retuns account's value before `txNum` transaction changed it
-	// Means if you want re-execute `txNum` on historical state - do `GetAsOf(key, txNum)` to read state
+	// Means if you want re-execute `txNum` on historical state - do `DomainGetAsOf(key, txNum)` to read state
 	// `ok = false` means: key not found. or "future txNum" passed.
 	DomainGetAsOf(name Domain, k, k2 []byte, ts uint64) (v []byte, ok bool, err error)
-
-	// HistorySeek - like `DomainGetAsOf` but without latest state - only for `History`
-	// `ok == true && v != nil && len(v) == 0` means key-creation even
-	HistorySeek(name History, k []byte, ts uint64) (v []byte, ok bool, err error)
+	DomainRange(name Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int) (it stream.KV, err error)
 
 	// IndexRange - return iterator over range of inverted index for given key `k`
 	// Asc semantic:  [from, to) AND from > to
@@ -529,11 +526,14 @@ type TemporalTx interface {
 	// Example: IndexRange("IndexName", 10, 5, order.Desc, -1)
 	// Example: IndexRange("IndexName", -1, -1, order.Asc, 10)
 	IndexRange(name InvertedIdx, k []byte, fromTs, toTs int, asc order.By, limit int) (timestamps stream.U64, err error)
-	DomainRange(name Domain, fromKey, toKey []byte, ts uint64, asc order.By, limit int) (it stream.KV, err error)
+
+	// HistorySeek - like `DomainGetAsOf` but without latest state - only for `History`
+	// `ok == true && v != nil && len(v) == 0` means key-creation even
+	HistorySeek(name Domain, k []byte, ts uint64) (v []byte, ok bool, err error)
 
 	// HistoryRange - producing "state patch" - sorted list of keys updated at [fromTs,toTs) with their most-recent value.
 	//   no duplicates
-	HistoryRange(name History, fromTs, toTs int, asc order.By, limit int) (it stream.KV, err error)
+	HistoryRange(name Domain, fromTs, toTs int, asc order.By, limit int) (it stream.KV, err error)
 }
 
 type TemporalRwTx interface {

erigon-lib/kv/membatchwithdb/memory_mutation.go

@@ -717,7 +717,7 @@ func (m *MemoryMutation) DomainGet(name kv.Domain, k, k2 []byte) (v []byte, step
 	//return m.db.(kv.TemporalTx).DomainGet(name, k, k2)
 }
 
-func (m *MemoryMutation) DomainGetAsOf(name kv.Domain, k, k2 []byte, ts uint64) (v []byte, ok bool, err error) {
+func (m *MemoryMutation) GetAsOf(name kv.Domain, k, k2 []byte, ts uint64) (v []byte, ok bool, err error) {
 	panic("not supported")
 	//return m.db.(kv.TemporalTx).DomainGetAsOf(name, k, k2, ts)
 }

erigon-lib/kv/remotedb/kv_remote.go

@@ -650,16 +650,16 @@ func (tx *tx) DomainRange(name kv.Domain, fromKey, toKey []byte, ts uint64, asc
 		return reply.Keys, reply.Values, reply.NextPageToken, nil
 	}), nil
 }
-func (tx *tx) HistorySeek(name kv.History, k []byte, ts uint64) (v []byte, ok bool, err error) {
-	reply, err := tx.db.remoteKV.HistorySeek(tx.ctx, &remote.HistorySeekReq{TxId: tx.id, Table: string(name), K: k, Ts: ts})
+func (tx *tx) HistorySeek(name kv.Domain, k []byte, ts uint64) (v []byte, ok bool, err error) {
+	reply, err := tx.db.remoteKV.HistorySeek(tx.ctx, &remote.HistorySeekReq{TxId: tx.id, Table: name.String(), K: k, Ts: ts})
 	if err != nil {
 		return nil, false, err
 	}
 	return reply.V, reply.Ok, nil
 }
-func (tx *tx) HistoryRange(name kv.History, fromTs, toTs int, asc order.By, limit int) (it stream.KV, err error) {
+func (tx *tx) HistoryRange(name kv.Domain, fromTs, toTs int, asc order.By, limit int) (it stream.KV, err error) {
 	return stream.PaginateKV(func(pageToken string) (keys, vals [][]byte, nextPageToken string, err error) {
-		reply, err := tx.db.remoteKV.HistoryRange(tx.ctx, &remote.HistoryRangeReq{TxId: tx.id, Table: string(name), FromTs: int64(fromTs), ToTs: int64(toTs), OrderAscend: bool(asc), Limit: int64(limit), PageToken: pageToken})
+		reply, err := tx.db.remoteKV.HistoryRange(tx.ctx, &remote.HistoryRangeReq{TxId: tx.id, Table: name.String(), FromTs: int64(fromTs), ToTs: int64(toTs), OrderAscend: bool(asc), Limit: int64(limit), PageToken: pageToken})
 		if err != nil {
 			return nil, nil, "", err
 		}

erigon-lib/kv/remotedbserver/remotedbserver.go

@@ -561,7 +561,11 @@ func (s *KvServer) HistorySeek(_ context.Context, req *remote.HistorySeekReq) (r
 		if !ok {
 			return errors.New("server DB doesn't implement kv.Temporal interface")
 		}
-		reply.V, reply.Ok, err = ttx.HistorySeek(kv.History(req.Table), req.K, req.Ts)
+		domain, err := kv.String2Domain(req.Table)
+		if err != nil {
+			return err
+		}
+		reply.V, reply.Ok, err = ttx.HistorySeek(domain, req.K, req.Ts)
 		if err != nil {
 			return err
 		}
@@ -629,7 +633,11 @@ func (s *KvServer) HistoryRange(_ context.Context, req *remote.HistoryRangeReq)
 		if !ok {
 			return fmt.Errorf("server DB doesn't implement kv.Temporal interface")
 		}
-		it, err := ttx.HistoryRange(kv.History(req.Table), fromTs, int(req.ToTs), order.By(req.OrderAscend), limit)
+		domain, err := kv.String2Domain(req.Table)
+		if err != nil {
+			return err
+		}
+		it, err := ttx.HistoryRange(domain, fromTs, int(req.ToTs), order.By(req.OrderAscend), limit)
 		if err != nil {
 			return err
 		}

erigon-lib/kv/tables.go

@@ -126,7 +126,7 @@ AccountsHistory and StorageHistory - indices designed to serve next 2 type of re
 1. what is smallest block number >= X where account A changed
 2. get last shard of A - to append there new block numbers
 
-Task 1. is part of "get historical state" operation (see `core/state:GetAsOf`):
+Task 1. is part of "get historical state" operation (see `core/state:DomainGetAsOf`):
 If `db.seekInFiles(A+bigEndian(X))` returns non-last shard -
 
 	then get block number from shard value Y := RoaringBitmap(shard_value).GetGte(X)
@@ -863,14 +863,6 @@ const (
 	DomainLen        Domain = 5
 )
 
-const (
-	AccountsHistory   History = "AccountsHistory"
-	StorageHistory    History = "StorageHistory"
-	CodeHistory       History = "CodeHistory"
-	CommitmentHistory History = "CommitmentHistory"
-	ReceiptHistory    History = "ReceiptHistory"
-)
-
 const (
 	AccountsHistoryIdx   InvertedIdx = "AccountsHistoryIdx"
 	StorageHistoryIdx    InvertedIdx = "StorageHistoryIdx"

erigon-lib/kv/temporal/kv_temporal.go

@@ -218,10 +218,10 @@ func (tx *Tx) DomainGetAsOf(name kv.Domain, key, key2 []byte, ts uint64) (v []by
 	if key2 != nil {
 		key = append(common.Copy(key), key2...)
 	}
-	return tx.filesTx.DomainGetAsOf(tx.MdbxTx, name, key, ts)
+	return tx.filesTx.GetAsOf(tx.MdbxTx, name, key, ts)
 }
 
-func (tx *Tx) HistorySeek(name kv.History, key []byte, ts uint64) (v []byte, ok bool, err error) {
+func (tx *Tx) HistorySeek(name kv.Domain, key []byte, ts uint64) (v []byte, ok bool, err error) {
 	return tx.filesTx.HistorySeek(name, key, ts, tx.MdbxTx)
 }
 
@@ -234,7 +234,7 @@ func (tx *Tx) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs int, asc or
 	return timestamps, nil
 }
 
-func (tx *Tx) HistoryRange(name kv.History, fromTs, toTs int, asc order.By, limit int) (stream.KV, error) {
+func (tx *Tx) HistoryRange(name kv.Domain, fromTs, toTs int, asc order.By, limit int) (stream.KV, error) {
 	it, err := tx.filesTx.HistoryRange(name, fromTs, toTs, asc, limit, tx.MdbxTx)
 	if err != nil {
 		return nil, err

erigon-lib/state/aggregator.go

@@ -1732,48 +1732,24 @@ func (ac *AggregatorRoTx) IndexRange(name kv.InvertedIdx, k []byte, fromTs, toTs
 
 // -- range end
 
-func (ac *AggregatorRoTx) HistorySeek(name kv.History, key []byte, ts uint64, tx kv.Tx) (v []byte, ok bool, err error) {
-	switch name {
-	case kv.AccountsHistory:
-		v, ok, err = ac.d[kv.AccountsDomain].ht.HistorySeek(key, ts, tx)
+func (ac *AggregatorRoTx) HistorySeek(domain kv.Domain, key []byte, ts uint64, tx kv.Tx) (v []byte, ok bool, err error) {
+	switch domain {
+	case kv.AccountsDomain:
+		v, ok, err = ac.d[domain].ht.HistorySeek(key, ts, tx)
 		if err != nil {
 			return nil, false, err
 		}
 		if !ok || len(v) == 0 {
 			return v, ok, nil
 		}
 		return v, true, nil
-	case kv.StorageHistory:
-		return ac.d[kv.StorageDomain].ht.HistorySeek(key, ts, tx)
-	case kv.CodeHistory:
-		return ac.d[kv.CodeDomain].ht.HistorySeek(key, ts, tx)
-	case kv.CommitmentHistory:
-		return ac.d[kv.CommitmentDomain].ht.HistorySeek(key, ts, tx)
-	case kv.ReceiptHistory:
-		return ac.d[kv.ReceiptDomain].ht.HistorySeek(key, ts, tx)
-	//case kv.GasUsedHistory:
-	//	return ac.d[kv.GasUsedDomain].ht.HistorySeek(key, ts, tx)
 	default:
-		panic(fmt.Sprintf("unexpected: %s", name))
+		return ac.d[domain].ht.HistorySeek(key, ts, tx)
 	}
 }
 
-func (ac *AggregatorRoTx) HistoryRange(name kv.History, fromTs, toTs int, asc order.By, limit int, tx kv.Tx) (it stream.KV, err error) {
-	//TODO: aggTx to store array of histories
-	var domainName kv.Domain
-
-	switch name {
-	case kv.AccountsHistory:
-		domainName = kv.AccountsDomain
-	case kv.StorageHistory:
-		domainName = kv.StorageDomain
-	case kv.CodeHistory:
-		domainName = kv.CodeDomain
-	default:
-		return nil, fmt.Errorf("unexpected history name: %s", name)
-	}
-
-	hr, err := ac.d[domainName].ht.HistoryRange(fromTs, toTs, asc, limit, tx)
+func (ac *AggregatorRoTx) HistoryRange(domain kv.Domain, fromTs, toTs int, asc order.By, limit int, tx kv.Tx) (it stream.KV, err error) {
+	hr, err := ac.d[domain].ht.HistoryRange(fromTs, toTs, asc, limit, tx)
 	if err != nil {
 		return nil, err
 	}
@@ -1844,11 +1820,11 @@ func (ac *AggregatorRoTx) DomainRange(ctx context.Context, tx kv.Tx, domain kv.D
 func (ac *AggregatorRoTx) DomainRangeLatest(tx kv.Tx, domain kv.Domain, from, to []byte, limit int) (stream.KV, error) {
 	return ac.d[domain].DomainRangeLatest(tx, from, to, limit)
 }
-func (ac *AggregatorRoTx) DomainGetAsOfFile(name kv.Domain, key []byte, ts uint64) (v []byte, ok bool, err error) {
+func (ac *AggregatorRoTx) GetAsOfFile(name kv.Domain, key []byte, ts uint64) (v []byte, ok bool, err error) {
 	return ac.d[name].GetAsOfFile(key, ts)
 }
 
-func (ac *AggregatorRoTx) DomainGetAsOf(tx kv.Tx, name kv.Domain, key []byte, ts uint64) (v []byte, ok bool, err error) {
+func (ac *AggregatorRoTx) GetAsOf(tx kv.Tx, name kv.Domain, key []byte, ts uint64) (v []byte, ok bool, err error) {
 	return ac.d[name].GetAsOf(key, ts, tx)
 }
 func (ac *AggregatorRoTx) GetLatest(domain kv.Domain, k, k2 []byte, tx kv.Tx) (v []byte, step uint64, ok bool, err error) {