Skip to content

Commit

Permalink
erigondb: domain/history/index configurations (#12848)
Browse files Browse the repository at this point in the history
  • Loading branch information
@awskii
awskii authored Nov 23, 2024
1 parent 51d4de6 commit 834eaab
Show file tree
Hide file tree
Showing 11 changed files with 357 additions and 273 deletions.
111 changes: 81 additions & 30 deletions erigon-lib/state/aggregator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,60 +155,104 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6
}

cfg := domainCfg{
name: kv.AccountsDomain, valuesTable: kv.TblAccountVals,
restrictSubsetFileDeletions: a.commitmentValuesTransform,

integrity: integrityCheck,
compression: seg.CompressNone,

hist: histCfg{
iiCfg: iiCfg{salt: salt, dirs: dirs, db: db},
withLocalityIndex: false, withExistenceIndex: false, compression: seg.CompressNone, historyLargeValues: false,
valuesTable: kv.TblAccountHistoryVals,
compression: seg.CompressNone,

withLocalityIndex: false, historyLargeValues: false,

iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg,
aggregationStep: aggregationStep, keysTable: kv.TblAccountHistoryKeys, valuesTable: kv.TblAccountIdx},
},
restrictSubsetFileDeletions: a.commitmentValuesTransform,
}
if a.d[kv.AccountsDomain], err = NewDomain(cfg, aggregationStep, kv.AccountsDomain, kv.TblAccountVals, kv.TblAccountHistoryKeys, kv.TblAccountHistoryVals, kv.TblAccountIdx, integrityCheck, logger); err != nil {
if a.d[kv.AccountsDomain], err = NewDomain(cfg, logger); err != nil {
return nil, err
}
cfg = domainCfg{
name: kv.StorageDomain, valuesTable: kv.TblStorageVals,
restrictSubsetFileDeletions: a.commitmentValuesTransform,

integrity: integrityCheck,
compression: seg.CompressKeys,

hist: histCfg{
iiCfg: iiCfg{salt: salt, dirs: dirs, db: db},
withLocalityIndex: false, withExistenceIndex: false, compression: seg.CompressNone, historyLargeValues: false,
valuesTable: kv.TblStorageHistoryVals,
compression: seg.CompressNone,

withLocalityIndex: false, historyLargeValues: false,

iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg,
aggregationStep: aggregationStep, keysTable: kv.TblStorageHistoryKeys, valuesTable: kv.TblStorageIdx},
},
restrictSubsetFileDeletions: a.commitmentValuesTransform,
compress: seg.CompressKeys,
}
if a.d[kv.StorageDomain], err = NewDomain(cfg, aggregationStep, kv.StorageDomain, kv.TblStorageVals, kv.TblStorageHistoryKeys, kv.TblStorageHistoryVals, kv.TblStorageIdx, integrityCheck, logger); err != nil {
if a.d[kv.StorageDomain], err = NewDomain(cfg, logger); err != nil {
return nil, err
}
cfg = domainCfg{
name: kv.CodeDomain, valuesTable: kv.TblCodeVals,
restrictSubsetFileDeletions: a.commitmentValuesTransform,

integrity: integrityCheck,
compression: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet
largeValues: true,

hist: histCfg{
iiCfg: iiCfg{salt: salt, dirs: dirs, db: db},
withLocalityIndex: false, withExistenceIndex: false, historyLargeValues: true,
valuesTable: kv.TblCodeHistoryVals,
compression: seg.CompressKeys | seg.CompressVals,

withLocalityIndex: false, historyLargeValues: true,

iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg,
aggregationStep: aggregationStep, keysTable: kv.TblCodeHistoryKeys, valuesTable: kv.TblCodeIdx},
},
largeVals: true,
compress: seg.CompressVals, // compress Code with keys doesn't show any profit. compress of values show 4x ratio on eth-mainnet and 2.5x ratio on bor-mainnet
}
if a.d[kv.CodeDomain], err = NewDomain(cfg, aggregationStep, kv.CodeDomain, kv.TblCodeVals, kv.TblCodeHistoryKeys, kv.TblCodeHistoryVals, kv.TblCodeIdx, integrityCheck, logger); err != nil {
if a.d[kv.CodeDomain], err = NewDomain(cfg, logger); err != nil {
return nil, err
}
cfg = domainCfg{
name: kv.CommitmentDomain, valuesTable: kv.TblCommitmentVals,
restrictSubsetFileDeletions: a.commitmentValuesTransform,

replaceKeysInValues: a.commitmentValuesTransform,
integrity: integrityCheck,
compression: seg.CompressKeys,

hist: histCfg{
iiCfg: iiCfg{salt: salt, dirs: dirs, db: db},
withLocalityIndex: false, withExistenceIndex: false, compression: seg.CompressNone, historyLargeValues: false,
valuesTable: kv.TblCommitmentHistoryVals,
compression: seg.CompressNone,

snapshotsDisabled: true,
withLocalityIndex: false, historyLargeValues: false,

iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg,
aggregationStep: aggregationStep, keysTable: kv.TblCommitmentHistoryKeys, valuesTable: kv.TblCommitmentIdx},
},
replaceKeysInValues: a.commitmentValuesTransform,
restrictSubsetFileDeletions: a.commitmentValuesTransform,
compress: seg.CompressKeys,
}
if a.d[kv.CommitmentDomain], err = NewDomain(cfg, aggregationStep, kv.CommitmentDomain, kv.TblCommitmentVals, kv.TblCommitmentHistoryKeys, kv.TblCommitmentHistoryVals, kv.TblCommitmentIdx, integrityCheck, logger); err != nil {
if a.d[kv.CommitmentDomain], err = NewDomain(cfg, logger); err != nil {
return nil, err
}
cfg = domainCfg{
name: kv.ReceiptDomain, valuesTable: kv.TblReceiptVals,
compression: seg.CompressNone, //seg.CompressKeys | seg.CompressVals,
integrity: integrityCheck,

hist: histCfg{
iiCfg: iiCfg{salt: salt, dirs: dirs, db: db},
withLocalityIndex: false, withExistenceIndex: false,
compression: seg.CompressNone, historyLargeValues: false,
valuesTable: kv.TblReceiptHistoryVals,
compression: seg.CompressNone,

withLocalityIndex: false, historyLargeValues: false,

iiCfg: iiCfg{salt: salt, dirs: dirs, db: db, withExistence: false, compressorCfg: seg.DefaultCfg,
aggregationStep: aggregationStep, keysTable: kv.TblReceiptHistoryKeys, valuesTable: kv.TblReceiptIdx},
},
compress: seg.CompressNone, //seg.CompressKeys | seg.CompressVals,
}
if a.d[kv.ReceiptDomain], err = NewDomain(cfg, aggregationStep, kv.ReceiptDomain, kv.TblReceiptVals, kv.TblReceiptHistoryKeys, kv.TblReceiptHistoryVals, kv.TblReceiptIdx, integrityCheck, logger); err != nil {
if a.d[kv.ReceiptDomain], err = NewDomain(cfg, logger); err != nil {
return nil, err
}
if err := a.registerII(kv.LogAddrIdxPos, salt, dirs, db, aggregationStep, kv.FileLogAddressIdx, kv.TblLogAddressKeys, kv.TblLogAddressIdx, logger); err != nil {
Expand Down Expand Up @@ -274,9 +318,16 @@ func getStateIndicesSalt(baseDir string) (salt *uint32, err error) {
}

func (a *Aggregator) registerII(idx kv.InvertedIdxPos, salt *uint32, dirs datadir.Dirs, db kv.RoDB, aggregationStep uint64, filenameBase, indexKeysTable, indexTable string, logger log.Logger) error {
idxCfg := iiCfg{salt: salt, dirs: dirs, db: db}
idxCfg := iiCfg{
salt: salt, dirs: dirs, db: db,
aggregationStep: aggregationStep,
filenameBase: filenameBase,
keysTable: indexKeysTable,
valuesTable: indexTable,
compression: seg.CompressNone,
}
var err error
a.iis[idx], err = NewInvertedIndex(idxCfg, aggregationStep, filenameBase, indexKeysTable, indexTable, nil, logger)
a.iis[idx], err = NewInvertedIndex(idxCfg, logger)
if err != nil {
return err
}
Expand Down Expand Up @@ -362,7 +413,7 @@ func (a *Aggregator) SetCompressWorkers(i int) {
d.compressCfg.Workers = i
}
for _, ii := range a.iis {
ii.compressCfg.Workers = i
ii.compressorCfg.Workers = i
}
}

Expand Down Expand Up @@ -686,7 +737,7 @@ func (a *Aggregator) buildFiles(ctx context.Context, step uint64) error {
return err
}

switch ii.indexKeysTable {
switch ii.keysTable {
case kv.TblLogTopicsKeys:
static.ivfs[kv.LogTopicIdxPos] = sf
case kv.TblLogAddressKeys:
Expand All @@ -696,7 +747,7 @@ func (a *Aggregator) buildFiles(ctx context.Context, step uint64) error {
case kv.TblTracesToKeys:
static.ivfs[kv.TracesToIdxPos] = sf
default:
panic("unknown index " + ii.indexKeysTable)
panic("unknown index " + ii.keysTable)
}
return nil
})
Expand Down
Loading

0 comments on commit 834eaab

Please sign in to comment.