add commitment files

erigontech · Oct 8, 2024 · 56fe30b · 56fe30b
1 parent f777c20
commit 56fe30b
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 10 deletions.
diff --git a/erigon-lib/common/datadir/dirs.go b/erigon-lib/common/datadir/dirs.go
@@ -38,6 +38,7 @@ type Dirs struct {
 	Chaindata       string
 	Tmp             string
 	Snap            string
+	SnapCommitment  string
 	SnapIdx         string
 	SnapHistory     string
 	SnapDomain      string
@@ -68,6 +69,7 @@ func New(datadir string) Dirs {
 		Chaindata:       filepath.Join(datadir, "chaindata"),
 		Tmp:             filepath.Join(datadir, "temp"),
 		Snap:            filepath.Join(datadir, "snapshots"),
+		SnapCommitment:  filepath.Join(datadir, "snapshots", "commitment"),
 		SnapIdx:         filepath.Join(datadir, "snapshots", "idx"),
 		SnapHistory:     filepath.Join(datadir, "snapshots", "history"),
 		SnapDomain:      filepath.Join(datadir, "snapshots", "domain"),
@@ -82,7 +84,7 @@ func New(datadir string) Dirs {
 	}
 
 	dir.MustExist(dirs.Chaindata, dirs.Tmp,
-		dirs.SnapIdx, dirs.SnapHistory, dirs.SnapDomain, dirs.SnapAccessors,
+		dirs.SnapCommitment, dirs.SnapIdx, dirs.SnapHistory, dirs.SnapDomain, dirs.SnapAccessors,
 		dirs.Downloader, dirs.TxPool, dirs.Nodes, dirs.CaplinBlobs, dirs.CaplinIndexing, dirs.CaplinLatest, dirs.CaplinGenesis)
 	return dirs
 }

diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
@@ -2974,6 +2974,25 @@ func (d *Downloader) torrentCompleted(tName string, tHash metainfo.Hash) {
 		path: tName,
 		hash: hash,
 	}
+
+	// create commitment file for .seg/.idx files
+	if !strings.HasSuffix(tName, ".seg") || !strings.HasSuffix(tName, ".idx") {
+		return
+	}
+	tName = strings.ReplaceAll(tName, ".seg", ".txt")
+	tName = strings.ReplaceAll(tName, ".idx", ".txt")
+	commitmentFile := filepath.Join(d.cfg.Dirs.SnapCommitment, tName)
+	cf, err := os.Create(commitmentFile)
+	if err != nil {
+		d.logger.Error("[snapshots] failed to create commitment file", "file", tName)
+	} else {
+		defer cf.Close()
+	}
+	_, err = cf.WriteAt(tHash[:], 0)
+	if err != nil {
+		d.logger.Error("[snapshots] failed to write to commitment file", "file", tName)
+	}
+
 }
 
 // Notify GrpcServer subscribers about completed torrent

diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -750,6 +750,34 @@ func (s *RoSnapshots) InitSegments(fileNames []string) error {
 	return nil
 }
 
+func (s *RoSnapshots) integrityCheck(fName string) bool {
+	return integrityCheck(s.dir, fName)
+}
+
+func integrityCheck(dir, fName string) bool {
+	file := filepath.Join(dir, fName)
+	f, err := os.Stat(file)
+	if os.IsNotExist(err) {
+		return false
+	}
+	torrentFile := filepath.Join(dir, fName) + ".torrent"
+	if _, err := os.Stat(torrentFile); os.IsNotExist(err) {
+		// torrent file not exists means that file is created locally, in this case file must be complete
+		return true
+	}
+	fName = strings.ReplaceAll(fName, ".seg", ".txt")
+	fName = strings.ReplaceAll(fName, ".idx", ".txt")
+	commitmentFile := filepath.Join(dir, "commitment", fName)
+	cf, err := os.Stat(commitmentFile)
+	if os.IsNotExist(err) {
+		return false
+	}
+	if cf.ModTime().After(f.ModTime()) {
+		return true
+	}
+	return false
+}
+
 func (s *RoSnapshots) rebuildSegments(fileNames []string, open bool, optimistic bool) error {
 	var segmentsMax uint64
 	var segmentsMaxSet bool
@@ -792,6 +820,9 @@ func (s *RoSnapshots) rebuildSegments(fileNames []string, open bool, optimistic
 		}
 
 		if open && sn.Decompressor == nil {
+			if !s.integrityCheck(fName) {
+				continue
+			}
 			if err := sn.reopenSeg(s.dir); err != nil {
 				if errors.Is(err, os.ErrNotExist) {
 					if optimistic {
@@ -814,9 +845,18 @@ func (s *RoSnapshots) rebuildSegments(fileNames []string, open bool, optimistic
 			segtype.DirtySegments.Set(sn)
 		}
 
-		if open && sn.indexes == nil {
-			if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
-				return err
+		if open && len(sn.indexes) != len(sn.Type().Indexes()) {
+			reopen := true
+			for _, idxFileName := range sn.Type().IdxFileNames(sn.version, sn.from, sn.to) {
+				if !s.integrityCheck(idxFileName) {
+					reopen = false
+					break
+				}
+			}
+			if reopen {
+				if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
+					return err
+				}
 			}
 		}
 

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -185,6 +185,10 @@ func (s *CaplinSnapshots) Close() {
 	s.closeWhatNotInList(nil)
 }
 
+func (s *CaplinSnapshots) integrityCheck(fName string) bool {
+	return integrityCheck(s.dir, fName)
+}
+
 // ReopenList stops on optimistic=false, continue opening files on optimistic=true
 func (s *CaplinSnapshots) ReopenList(fileNames []string, optimistic bool) error {
 	defer s.recalcVisibleFiles()
@@ -229,6 +233,9 @@ Loop:
 				}
 			}
 			if sn.Decompressor == nil {
+				if !s.integrityCheck(fName) {
+					continue
+				}
 				if err := sn.reopenSeg(s.dir); err != nil {
 					if errors.Is(err, os.ErrNotExist) {
 						if optimistic {
@@ -251,9 +258,18 @@ Loop:
 				// then make segment available even if index open may fail
 				s.BeaconBlocks.DirtySegments.Set(sn)
 			}
-			if sn.indexes == nil {
-				if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
-					return err
+			if len(sn.indexes) != len(sn.Type().Indexes()) {
+				reopen := true
+				for _, idxFileName := range sn.Type().IdxFileNames(sn.version, sn.from, sn.to) {
+					if !s.integrityCheck(idxFileName) {
+						reopen = false
+						break
+					}
+				}
+				if reopen {
+					if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
+						return err
+					}
 				}
 			}
 			// Only bob sidecars count for progression
@@ -290,6 +306,9 @@ Loop:
 				}
 			}
 			if sn.Decompressor == nil {
+				if !s.integrityCheck(fName) {
+					continue
+				}
 				if err := sn.reopenSeg(s.dir); err != nil {
 					if errors.Is(err, os.ErrNotExist) {
 						if optimistic {
@@ -312,9 +331,18 @@ Loop:
 				// then make segment available even if index open may fail
 				s.BlobSidecars.DirtySegments.Set(sn)
 			}
-			if sn.indexes == nil {
-				if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
-					return err
+			if len(sn.indexes) != len(sn.Type().Indexes()) {
+				reopen := true
+				for _, idxFileName := range sn.Type().IdxFileNames(sn.version, sn.from, sn.to) {
+					if !s.integrityCheck(idxFileName) {
+						reopen = false
+						break
+					}
+				}
+				if reopen {
+					if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
+						return err
+					}
 				}
 			}
 		}