Skip to content

Commit

Permalink
speedup files open: by parallel index open (#13005)
Browse files Browse the repository at this point in the history
Co-authored-by: awskii <[email protected]>
  • Loading branch information
AskAlexSharov and awskii authored Dec 5, 2024
1 parent 644c9a9 commit 853ad98
Showing 1 changed file with 14 additions and 3 deletions.
17 changes: 14 additions & 3 deletions turbo/snapshotsync/snapshots.go
Original file line number Diff line number Diff line change
Expand Up @@ -1019,6 +1019,11 @@ func (s *RoSnapshots) openSegments(fileNames []string, open bool, optimistic boo
var segmentsMax uint64
var segmentsMaxSet bool

wg := &errgroup.Group{}
wg.SetLimit(64)
//fmt.Println("RS", s)
//defer fmt.Println("Done RS", s)

snConfig := snapcfg.KnownCfg(s.cfg.ChainName)

for _, fName := range fileNames {
Expand Down Expand Up @@ -1080,9 +1085,12 @@ func (s *RoSnapshots) openSegments(fileNames []string, open bool, optimistic boo
}

if open {
if err := sn.OpenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
wg.Go(func() error {
if err := sn.OpenIdxIfNeed(s.dir, optimistic); err != nil {
return err
}
return nil
})
}

if f.To > 0 {
Expand All @@ -1095,6 +1103,9 @@ func (s *RoSnapshots) openSegments(fileNames []string, open bool, optimistic boo
if segmentsMaxSet {
s.segmentsMax.Store(segmentsMax)
}
if err := wg.Wait(); err != nil {
return err
}

return nil
}
Expand Down

0 comments on commit 853ad98

Please sign in to comment.