From 234947a3a530abdf98c818cb1a4dcfd622d9d7d5 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Thu, 5 Dec 2024 18:41:04 +0700 Subject: [PATCH] speedup files open (#12995) `KnownCfg` call is very expensive --- erigon-lib/chain/snapcfg/util.go | 11 ++--------- erigon-lib/downloader/util.go | 4 +++- turbo/snapshotsync/caplin_state_snapshots.go | 4 +++- turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 6 ++++-- turbo/snapshotsync/snapshots.go | 5 ++--- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/erigon-lib/chain/snapcfg/util.go b/erigon-lib/chain/snapcfg/util.go index 67d6010a4fe..24161e6ef1e 100644 --- a/erigon-lib/chain/snapcfg/util.go +++ b/erigon-lib/chain/snapcfg/util.go @@ -465,23 +465,16 @@ func Seedable(networkName string, info snaptype.FileInfo) bool { return KnownCfg(networkName).Seedable(info) } -func IsFrozen(networkName string, info snaptype.FileInfo) bool { - if networkName == "" { - return false - } - return KnownCfg(networkName).IsFrozen(info) -} - func MergeLimitFromCfg(cfg *Cfg, snapType snaptype.Enum, fromBlock uint64) uint64 { return cfg.MergeLimit(snapType, fromBlock) } func MaxSeedableSegment(chain string, dir string) uint64 { var _max uint64 - + segConfig := KnownCfg(chain) if list, err := snaptype.Segments(dir); err == nil { for _, info := range list { - if Seedable(chain, info) && info.Type.Enum() == snaptype.MinCoreEnum && info.To > _max { + if segConfig.Seedable(info) && info.Type.Enum() == snaptype.MinCoreEnum && info.To > _max { _max = info.To } } diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go index bc06a0573c4..2de24028e6f 100644 --- a/erigon-lib/downloader/util.go +++ b/erigon-lib/downloader/util.go @@ -78,6 +78,8 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool) return nil, err } + segConfig := snapcfg.KnownCfg(chainName) + res := make([]string, 0, len(files)) for _, fPath := range files { _, name := filepath.Split(fPath) @@ -97,7 +99,7 @@ func seedableSegmentFiles(dir string, chainName string, skipSeedableCheck bool) if !skipSeedableCheck && (!ok || isStateFile) { continue } - if !skipSeedableCheck && !snapcfg.Seedable(chainName, ff) { + if !skipSeedableCheck && !segConfig.Seedable(ff) { continue } res = append(res, name) diff --git a/turbo/snapshotsync/caplin_state_snapshots.go b/turbo/snapshotsync/caplin_state_snapshots.go index 8cd6f6c1350..c5eb8a5802f 100644 --- a/turbo/snapshotsync/caplin_state_snapshots.go +++ b/turbo/snapshotsync/caplin_state_snapshots.go @@ -269,6 +269,8 @@ func (s *CaplinStateSnapshots) OpenList(fileNames []string, optimistic bool) err s.dirtySegmentsLock.Lock() defer s.dirtySegmentsLock.Unlock() + snConfig := snapcfg.KnownCfg(s.cfg.ChainName) + s.closeWhatNotInList(fileNames) var segmentsMax uint64 var segmentsMaxSet bool @@ -303,7 +305,7 @@ Loop: // segType: f.Type, Unsupported version: f.Version, Range: Range{f.From, f.To}, - frozen: snapcfg.IsFrozen(s.cfg.ChainName, f), + frozen: snConfig.IsFrozen(f), filePath: filePath, } } diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go index 053824b0b5d..7f00f76ab63 100644 --- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go @@ -159,6 +159,8 @@ func (s *CaplinSnapshots) OpenList(fileNames []string, optimistic bool) error { s.dirtyLock.Lock() defer s.dirtyLock.Unlock() + snConfig := snapcfg.KnownCfg(s.cfg.ChainName) + s.closeWhatNotInList(fileNames) var segmentsMax uint64 var segmentsMaxSet bool @@ -191,7 +193,7 @@ Loop: snaptype.BeaconBlocks, f.Version, f.From, f.To, - snapcfg.IsFrozen(s.cfg.ChainName, f)) + snConfig.IsFrozen(f)) } if err := sn.Open(s.dir); err != nil { if errors.Is(err, os.ErrNotExist) { @@ -247,7 +249,7 @@ Loop: snaptype.BlobSidecars, f.Version, f.From, f.To, - snapcfg.IsFrozen(s.cfg.ChainName, f)) + snConfig.IsFrozen(f)) } if err := sn.Open(s.dir); err != nil { if errors.Is(err, os.ErrNotExist) { diff --git a/turbo/snapshotsync/snapshots.go b/turbo/snapshotsync/snapshots.go index cc55fb8346a..b7b92cf39da 100644 --- a/turbo/snapshotsync/snapshots.go +++ b/turbo/snapshotsync/snapshots.go @@ -1019,8 +1019,7 @@ func (s *RoSnapshots) openSegments(fileNames []string, open bool, optimistic boo var segmentsMax uint64 var segmentsMaxSet bool - //fmt.Println("RS", s) - //defer fmt.Println("Done RS", s) + snConfig := snapcfg.KnownCfg(s.cfg.ChainName) for _, fName := range fileNames { f, isState, ok := snaptype.ParseFileName(s.dir, fName) @@ -1054,7 +1053,7 @@ func (s *RoSnapshots) openSegments(fileNames []string, open bool, optimistic boo }) if !exists { - sn = &DirtySegment{segType: f.Type, version: f.Version, Range: Range{f.From, f.To}, frozen: snapcfg.IsFrozen(s.cfg.ChainName, f)} + sn = &DirtySegment{segType: f.Type, version: f.Version, Range: Range{f.From, f.To}, frozen: snConfig.IsFrozen(f)} } if open {