From 2f9a3a3a46eac107c67f0f535e0a22a012ab071c Mon Sep 17 00:00:00 2001
From: Alex Sharov <AskAlexSharov@gmail.com>
Date: Sat, 19 Oct 2024 09:31:48 +0700
Subject: [PATCH] seg: don't reopen files (#12362)

only reason why we did it - is because we opened them first time when
downloading is not done yet. but after
https://github.com/erigontech/erigon/pull/12332 - it can't happen
anymore
---
 .../freezeblocks/block_snapshots.go           | 57 ++++++++++++++-----
 .../freezeblocks/block_snapshots_test.go      |  6 +-
 2 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index bc82e899684..a1d205a7596 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -249,6 +249,21 @@ func (s *DirtySegment) Version() snaptype.Version {
 	return s.version
 }
 
+func (s *DirtySegment) Indexed() bool {
+	if s.Decompressor == nil {
+		return false
+	}
+	if len(s.indexes) != len(s.Type().Indexes()) {
+		return false
+	}
+	for _, idx := range s.indexes {
+		if idx == nil {
+			return false
+		}
+	}
+	return true
+}
+
 func (s *DirtySegment) Index(index ...snaptype.Index) *recsplit.Index {
 	if len(index) == 0 {
 		index = []snaptype.Index{{}}
@@ -276,10 +291,9 @@ func (s *DirtySegment) isSubSetOf(j *DirtySegment) bool {
 }
 
 func (s *DirtySegment) reopenSeg(dir string) (err error) {
-	if s.refcount.Load() > 0 {
-		return
+	if s.Decompressor != nil {
+		return nil
 	}
-	s.closeSeg()
 	s.Decompressor, err = seg.NewDecompressor(filepath.Join(dir, s.FileName()))
 	if err != nil {
 		return fmt.Errorf("%w, fileName: %s", err, s.FileName())
@@ -350,23 +364,24 @@ func (s *DirtySegment) reopenIdxIfNeed(dir string, optimistic bool) (err error)
 }
 
 func (s *DirtySegment) reopenIdx(dir string) (err error) {
-	if s.refcount.Load() > 0 {
-		return nil
-	}
-
-	s.closeIdx()
 	if s.Decompressor == nil {
 		return nil
 	}
+	for len(s.indexes) < len(s.Type().Indexes()) {
+		s.indexes = append(s.indexes, nil)
+	}
 
-	for _, fileName := range s.Type().IdxFileNames(s.version, s.from, s.to) {
-		index, err := recsplit.OpenIndex(filepath.Join(dir, fileName))
+	for i, fileName := range s.Type().IdxFileNames(s.version, s.from, s.to) {
+		if s.indexes[i] != nil {
+			continue
+		}
 
+		index, err := recsplit.OpenIndex(filepath.Join(dir, fileName))
 		if err != nil {
 			return fmt.Errorf("%w, fileName: %s", err, fileName)
 		}
 
-		s.indexes = append(s.indexes, index)
+		s.indexes[i] = index
 	}
 
 	return nil
@@ -609,12 +624,11 @@ func (s *RoSnapshots) recalcVisibleFiles() {
 				if seg.canDelete.Load() {
 					continue
 				}
-				if seg.Decompressor == nil {
+				if !seg.Indexed() {
 					continue
 				}
-				if seg.indexes == nil {
-					break
-				}
+
+				//protect from overlaps overlaps
 				for len(newVisibleSegments) > 0 && newVisibleSegments[len(newVisibleSegments)-1].src.isSubSetOf(seg) {
 					newVisibleSegments[len(newVisibleSegments)-1].src = nil
 					newVisibleSegments = newVisibleSegments[:len(newVisibleSegments)-1]
@@ -629,6 +643,18 @@ func (s *RoSnapshots) recalcVisibleFiles() {
 			return true
 		})
 
+		// protect from gaps
+		if len(newVisibleSegments) > 0 {
+			prevEnd := newVisibleSegments[0].from
+			for i, seg := range newVisibleSegments {
+				if seg.from != prevEnd {
+					newVisibleSegments = newVisibleSegments[:i] //remove tail if see gap
+					break
+				}
+				prevEnd = seg.to
+			}
+		}
+
 		value.VisibleSegments = newVisibleSegments
 		var to uint64
 		if len(newVisibleSegments) > 0 {
@@ -638,6 +664,7 @@ func (s *RoSnapshots) recalcVisibleFiles() {
 		return true
 	})
 
+	// all types must have same hight
 	minMaxVisibleBlock := slices.Min(maxVisibleBlocks)
 	s.segments.Scan(func(segtype snaptype.Enum, value *segments) bool {
 		if minMaxVisibleBlock == 0 {
diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go
index 5350ed8032e..1faa7e5bb6c 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go
@@ -26,19 +26,17 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/slices"
 
-	"github.com/erigontech/erigon-lib/log/v3"
-	"github.com/erigontech/erigon/turbo/testlog"
-
 	"github.com/erigontech/erigon-lib/chain/networkname"
 	"github.com/erigontech/erigon-lib/chain/snapcfg"
 	"github.com/erigontech/erigon-lib/downloader/snaptype"
+	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon-lib/recsplit"
 	"github.com/erigontech/erigon-lib/seg"
-
 	"github.com/erigontech/erigon/common/math"
 	coresnaptype "github.com/erigontech/erigon/core/snaptype"
 	"github.com/erigontech/erigon/eth/ethconfig"
 	"github.com/erigontech/erigon/params"
+	"github.com/erigontech/erigon/turbo/testlog"
 )
 
 func createTestSegmentFile(t *testing.T, from, to uint64, name snaptype.Enum, dir string, version snaptype.Version, logger log.Logger) {