Skip to content

Commit

Permalink
seg: don't reopen files (#12362)
Browse files Browse the repository at this point in the history
only reason why we did it - is because we opened them first time when
downloading is not done yet. but after
#12332 - it can't happen
anymore
  • Loading branch information
AskAlexSharov authored Oct 19, 2024
1 parent 4c47c59 commit 2f9a3a3
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 19 deletions.
57 changes: 42 additions & 15 deletions turbo/snapshotsync/freezeblocks/block_snapshots.go
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,21 @@ func (s *DirtySegment) Version() snaptype.Version {
return s.version
}

func (s *DirtySegment) Indexed() bool {
if s.Decompressor == nil {
return false
}
if len(s.indexes) != len(s.Type().Indexes()) {
return false
}
for _, idx := range s.indexes {
if idx == nil {
return false
}
}
return true
}

func (s *DirtySegment) Index(index ...snaptype.Index) *recsplit.Index {
if len(index) == 0 {
index = []snaptype.Index{{}}
Expand Down Expand Up @@ -276,10 +291,9 @@ func (s *DirtySegment) isSubSetOf(j *DirtySegment) bool {
}

func (s *DirtySegment) reopenSeg(dir string) (err error) {
if s.refcount.Load() > 0 {
return
if s.Decompressor != nil {
return nil
}
s.closeSeg()
s.Decompressor, err = seg.NewDecompressor(filepath.Join(dir, s.FileName()))
if err != nil {
return fmt.Errorf("%w, fileName: %s", err, s.FileName())
Expand Down Expand Up @@ -350,23 +364,24 @@ func (s *DirtySegment) reopenIdxIfNeed(dir string, optimistic bool) (err error)
}

func (s *DirtySegment) reopenIdx(dir string) (err error) {
if s.refcount.Load() > 0 {
return nil
}

s.closeIdx()
if s.Decompressor == nil {
return nil
}
for len(s.indexes) < len(s.Type().Indexes()) {
s.indexes = append(s.indexes, nil)
}

for _, fileName := range s.Type().IdxFileNames(s.version, s.from, s.to) {
index, err := recsplit.OpenIndex(filepath.Join(dir, fileName))
for i, fileName := range s.Type().IdxFileNames(s.version, s.from, s.to) {
if s.indexes[i] != nil {
continue
}

index, err := recsplit.OpenIndex(filepath.Join(dir, fileName))
if err != nil {
return fmt.Errorf("%w, fileName: %s", err, fileName)
}

s.indexes = append(s.indexes, index)
s.indexes[i] = index
}

return nil
Expand Down Expand Up @@ -609,12 +624,11 @@ func (s *RoSnapshots) recalcVisibleFiles() {
if seg.canDelete.Load() {
continue
}
if seg.Decompressor == nil {
if !seg.Indexed() {
continue
}
if seg.indexes == nil {
break
}

//protect from overlaps overlaps
for len(newVisibleSegments) > 0 && newVisibleSegments[len(newVisibleSegments)-1].src.isSubSetOf(seg) {
newVisibleSegments[len(newVisibleSegments)-1].src = nil
newVisibleSegments = newVisibleSegments[:len(newVisibleSegments)-1]
Expand All @@ -629,6 +643,18 @@ func (s *RoSnapshots) recalcVisibleFiles() {
return true
})

// protect from gaps
if len(newVisibleSegments) > 0 {
prevEnd := newVisibleSegments[0].from
for i, seg := range newVisibleSegments {
if seg.from != prevEnd {
newVisibleSegments = newVisibleSegments[:i] //remove tail if see gap
break
}
prevEnd = seg.to
}
}

value.VisibleSegments = newVisibleSegments
var to uint64
if len(newVisibleSegments) > 0 {
Expand All @@ -638,6 +664,7 @@ func (s *RoSnapshots) recalcVisibleFiles() {
return true
})

// all types must have same hight
minMaxVisibleBlock := slices.Min(maxVisibleBlocks)
s.segments.Scan(func(segtype snaptype.Enum, value *segments) bool {
if minMaxVisibleBlock == 0 {
Expand Down
6 changes: 2 additions & 4 deletions turbo/snapshotsync/freezeblocks/block_snapshots_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,17 @@ import (
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"

"github.com/erigontech/erigon-lib/log/v3"
"github.com/erigontech/erigon/turbo/testlog"

"github.com/erigontech/erigon-lib/chain/networkname"
"github.com/erigontech/erigon-lib/chain/snapcfg"
"github.com/erigontech/erigon-lib/downloader/snaptype"
"github.com/erigontech/erigon-lib/log/v3"
"github.com/erigontech/erigon-lib/recsplit"
"github.com/erigontech/erigon-lib/seg"

"github.com/erigontech/erigon/common/math"
coresnaptype "github.com/erigontech/erigon/core/snaptype"
"github.com/erigontech/erigon/eth/ethconfig"
"github.com/erigontech/erigon/params"
"github.com/erigontech/erigon/turbo/testlog"
)

func createTestSegmentFile(t *testing.T, from, to uint64, name snaptype.Enum, dir string, version snaptype.Version, logger log.Logger) {
Expand Down

0 comments on commit 2f9a3a3

Please sign in to comment.