Caplin: Add switch to enable/disable static files generation (#12131)

cl/antiquary/antiquary.go

@@ -44,27 +44,28 @@ const safetyMargin = 10_000 // We retire snapshots 10k blocks after the finalize
 
 // Antiquary is where the snapshots go, aka old history, it is what keep track of the oldest records.
 type Antiquary struct {
-	mainDB                kv.RwDB                  // this is the main DB
-	blobStorage           blob_storage.BlobStorage // this is the blob storage
-	dirs                  datadir.Dirs
-	downloader            proto_downloader.DownloaderClient
-	logger                log.Logger
-	sn                    *freezeblocks.CaplinSnapshots
-	snReader              freezeblocks.BeaconSnapshotReader
-	snBuildSema           *semaphore.Weighted // semaphore for building only one type (blocks, caplin, v3) at a time
-	ctx                   context.Context
-	backfilled            *atomic.Bool
-	blobBackfilled        *atomic.Bool
-	cfg                   *clparams.BeaconChainConfig
-	states, blocks, blobs bool
-	validatorsTable       *state_accessors.StaticValidatorTable
-	genesisState          *state.CachingBeaconState
+	mainDB                         kv.RwDB                  // this is the main DB
+	blobStorage                    blob_storage.BlobStorage // this is the blob storage
+	dirs                           datadir.Dirs
+	downloader                     proto_downloader.DownloaderClient
+	logger                         log.Logger
+	sn                             *freezeblocks.CaplinSnapshots
+	snReader                       freezeblocks.BeaconSnapshotReader
+	snBuildSema                    *semaphore.Weighted // semaphore for building only one type (blocks, caplin, v3) at a time
+	ctx                            context.Context
+	backfilled                     *atomic.Bool
+	blobBackfilled                 *atomic.Bool
+	cfg                            *clparams.BeaconChainConfig
+	states, blocks, blobs, snapgen bool
+
+	validatorsTable *state_accessors.StaticValidatorTable
+	genesisState    *state.CachingBeaconState
 	// set to nil
 	currentState *state.CachingBeaconState
 	balances32   []byte
 }
 
-func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, logger log.Logger, states, blocks, blobs bool, snBuildSema *semaphore.Weighted) *Antiquary {
+func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, logger log.Logger, states, blocks, blobs, snapgen bool, snBuildSema *semaphore.Weighted) *Antiquary {
 	backfilled := &atomic.Bool{}
 	blobBackfilled := &atomic.Bool{}
 	backfilled.Store(false)
@@ -87,6 +88,7 @@ func NewAntiquary(ctx context.Context, blobStorage blob_storage.BlobStorage, gen
 		genesisState:    genesisState,
 		blocks:          blocks,
 		blobs:           blobs,
+		snapgen:         snapgen,
 	}
 }
 
@@ -278,6 +280,9 @@ const caplinSnapshotBuildSemaWeight int64 = 1
 
 // Antiquate will antiquate a specific block range (aka. retire snapshots), this should be ran in the background.
 func (a *Antiquary) antiquate(from, to uint64) error {
+	if !a.snapgen {
+		return nil
+	}
 	if a.snBuildSema != nil {
 		if !a.snBuildSema.TryAcquire(caplinSnapshotBuildSemaWeight) {
 			return nil
@@ -358,6 +363,15 @@ func (a *Antiquary) loopBlobs(ctx context.Context) {
 }
 
 func (a *Antiquary) antiquateBlobs() error {
+	if !a.snapgen {
+		return nil
+	}
+	if a.snBuildSema != nil {
+		if !a.snBuildSema.TryAcquire(caplinSnapshotBuildSemaWeight) {
+			return nil
+		}
+		defer a.snBuildSema.TryAcquire(caplinSnapshotBuildSemaWeight)
+	}
 	roTx, err := a.mainDB.BeginRo(a.ctx)
 	if err != nil {
 		return err

cl/antiquary/state_antiquary_test.go

@@ -41,7 +41,7 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, true, nil)
+	a := NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, true, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 }
 

cl/beacon/handler/utils_test.go

@@ -76,7 +76,7 @@ func setupTestingHandler(t *testing.T, v clparams.StateVersion, logger log.Logge
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &bcfg, datadir.New("/tmp"), nil, db, nil, reader, logger, true, true, false, nil)
+	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &bcfg, datadir.New("/tmp"), nil, db, nil, reader, logger, true, true, false, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 	// historical states reader below
 	statesReader := historical_states_reader.NewHistoricalStatesReader(&bcfg, reader, vt, preState)

cl/clparams/config.go

@@ -40,11 +40,12 @@ import (
 var LatestStateFileName = "latest.ssz_snappy"
 
 type CaplinConfig struct {
-	Backfilling         bool
-	BlobBackfilling     bool
-	BlobPruningDisabled bool
-	Archive             bool
-	NetworkId           NetworkType
+	Backfilling               bool
+	BlobBackfilling           bool
+	BlobPruningDisabled       bool
+	Archive                   bool
+	SnapshotGenerationEnabled bool
+	NetworkId                 NetworkType
 	// DisableCheckpointSync is optional and is used to disable checkpoint sync used by default in the node
 	DisabledCheckpointSync bool
 	// CaplinMeVRelayUrl is optional and is used to connect to the external builder service.

cl/persistence/state/historical_states_reader/historical_states_reader_test.go

@@ -41,7 +41,7 @@ func runTest(t *testing.T, blocks []*cltypes.SignedBeaconBlock, preState, postSt
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, true, nil)
+	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, true, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 	// Now lets test it against the reader
 	tx, err := db.BeginRw(ctx)

cl/sentinel/sentinel_requests_test.go

@@ -55,7 +55,7 @@ func loadChain(t *testing.T) (db kv.RwDB, blocks []*cltypes.SignedBeaconBlock, f
 
 	ctx := context.Background()
 	vt := state_accessors.NewStaticValidatorTable()
-	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, false, nil)
+	a := antiquary.NewAntiquary(ctx, nil, preState, vt, &clparams.MainnetBeaconConfig, datadir.New("/tmp"), nil, db, nil, reader, log.New(), true, true, false, false, nil)
 	require.NoError(t, a.IncrementBeaconState(ctx, blocks[len(blocks)-1].Block.Slot+33))
 	return
 }

cmd/capcli/cli.go

@@ -176,7 +176,7 @@ func (c *Chain) Run(ctx *Context) error {
 	}
 
 	downloader := network.NewBackwardBeaconDownloader(ctx, beacon, nil, nil, db)
-	cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, false, false, false, nil), csn, db, nil, beaconConfig, true, false, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, nil, blobStorage, log.Root())
+	cfg := stages.StageHistoryReconstruction(downloader, antiquary.NewAntiquary(ctx, nil, nil, nil, nil, dirs, nil, nil, nil, nil, nil, false, false, false, false, nil), csn, db, nil, beaconConfig, true, false, true, bRoot, bs.Slot(), "/tmp", 300*time.Millisecond, nil, nil, blobStorage, log.Root())
 	return stages.SpawnStageHistoryDownload(cfg, ctx, log.Root())
 }
 

cmd/caplin/caplin1/run.go

@@ -376,7 +376,7 @@ func RunCaplinService(ctx context.Context, engine execution_client.ExecutionEngi
 		}
 	}
 
-	antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, csn, rcsn, logger, states, backfilling, blobBackfilling, snBuildSema)
+	antiq := antiquary.NewAntiquary(ctx, blobStorage, genesisState, vTables, beaconConfig, dirs, snDownloader, indexDB, csn, rcsn, logger, states, backfilling, blobBackfilling, config.SnapshotGenerationEnabled, snBuildSema)
 	// Create the antiquary
 	go func() {
 		if err := antiq.Loop(); err != nil {

cmd/utils/flags.go

@@ -1016,6 +1016,11 @@ var (
 		Usage: "enables archival node in caplin",
 		Value: false,
 	}
+	CaplinEnableSnapshotGeneration = cli.BoolFlag{
+		Name:  "caplin.snapgen",
+		Usage: "enables snapshot generation in caplin",
+		Value: false,
+	}
 	BeaconApiAllowCredentialsFlag = cli.BoolFlag{
 		Name:  "beacon.api.cors.allow-credentials",
 		Usage: "set the cors' allow credentials",
@@ -1713,6 +1718,7 @@ func setBeaconAPI(ctx *cli.Context, cfg *ethconfig.Config) error {
 func setCaplin(ctx *cli.Context, cfg *ethconfig.Config) {
 	// Caplin's block's backfilling is enabled if any of the following flags are set
 	cfg.CaplinConfig.Backfilling = ctx.Bool(CaplinBackfillingFlag.Name) || ctx.Bool(CaplinArchiveFlag.Name) || ctx.Bool(CaplinBlobBackfillingFlag.Name)
+	cfg.CaplinConfig.SnapshotGenerationEnabled = ctx.Bool(CaplinEnableSnapshotGeneration.Name)
 	// More granularity here.
 	cfg.CaplinConfig.BlobBackfilling = ctx.Bool(CaplinBlobBackfillingFlag.Name)
 	cfg.CaplinConfig.BlobPruningDisabled = ctx.Bool(CaplinDisableBlobPruningFlag.Name)

turbo/cli/default_flags.go

@@ -216,6 +216,7 @@ var DefaultFlags = []cli.Flag{
 	&utils.CaplinDisableBlobPruningFlag,
 	&utils.CaplinDisableCheckpointSyncFlag,
 	&utils.CaplinArchiveFlag,
+	&utils.CaplinEnableSnapshotGeneration,
 	&utils.CaplinMevRelayUrl,
 	&utils.CaplinValidatorMonitorFlag,
 	&utils.CaplinCustomConfigFlag,