Lower memory consumption for testnet and integration tests.

Fantom-foundation · Dec 2, 2024 · a484991 · a484991
1 parent 0b774c3
commit a484991
Show file tree

Hide file tree

Showing 14 changed files with 71 additions and 23 deletions.
diff --git a/cmd/sonicd/app/launcher.go b/cmd/sonicd/app/launcher.go
@@ -67,6 +67,7 @@ func initFlags() {
 		flags.CacheFlag,
 		flags.LiveDbCacheFlag,
 		flags.ArchiveCacheFlag,
+		flags.StateDbCacheCapacityFlag,
 	}
 	networkingFlags = []cli.Flag{
 		flags.BootnodesFlag,

diff --git a/cmd/sonicd/app/run_test.go b/cmd/sonicd/app/run_test.go
@@ -23,7 +23,12 @@ func tmpdir(t *testing.T) string {
 }
 
 func initFakenetDatadir(dataDir string, validatorsNum idx.Validator) {
-	genesisStore := makefakegenesis.FakeGenesisStore(validatorsNum, futils.ToFtm(1000000000), futils.ToFtm(5000000))
+	genesisStore := makefakegenesis.FakeGenesisStore(
+		validatorsNum,
+		futils.ToFtm(1000000000),
+		futils.ToFtm(5000000),
+		1024, // 1024 bytes is both small and safe value used by tests
+	)
 	defer genesisStore.Close()
 
 	if err := genesis.ImportGenesisStore(genesis.ImportParams{

diff --git a/cmd/sonictool/app/genesis.go b/cmd/sonictool/app/genesis.go
@@ -100,7 +100,9 @@ func jsonGenesisImport(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to load JSON genesis: %w", err)
 	}
-	genesisStore, err := makefakegenesis.ApplyGenesisJson(genesisJson)
+
+	stateDbCacheCapacity := ctx.Int(flags.StateDbCacheCapacityFlag.Name)
+	genesisStore, err := makefakegenesis.ApplyGenesisJson(genesisJson, stateDbCacheCapacity)
 	if err != nil {
 		return fmt.Errorf("failed to prepare JSON genesis: %w", err)
 	}
@@ -139,7 +141,12 @@ func fakeGenesisImport(ctx *cli.Context) error {
 		return err
 	}
 
-	genesisStore := makefakegenesis.FakeGenesisStore(idx.Validator(validatorsNumber), futils.ToFtm(1000000000), futils.ToFtm(5000000))
+	genesisStore := makefakegenesis.FakeGenesisStore(
+		idx.Validator(validatorsNumber),
+		futils.ToFtm(1000000000),
+		futils.ToFtm(5000000),
+		ctx.Int(flags.StateDbCacheCapacityFlag.Name),
+	)
 	defer genesisStore.Close()
 	return genesis.ImportGenesisStore(genesis.ImportParams{
 		GenesisStore:  genesisStore,

diff --git a/cmd/sonictool/app/main.go b/cmd/sonictool/app/main.go
@@ -62,6 +62,9 @@ Initialize the database using data from the experimental genesis file.
 					ArgsUsage: "<validators>",
 					Action:    fakeGenesisImport,
 					Flags: []cli.Flag{
+						flags.LiveDbCacheFlag,
+						flags.ArchiveCacheFlag,
+						flags.StateDbCacheCapacityFlag,
 						ModeFlag,
 					},
 					Description: `

diff --git a/config/config.go b/config/config.go
@@ -341,6 +341,10 @@ func MakeAllConfigsFromFile(ctx *cli.Context, configFile string) (*Config, error
 		cfg.Lachesis.SuppressFramePanic = true
 	}
 
+	if ctx.IsSet(flags.StateDbCacheCapacityFlag.Name) {
+		cfg.OperaStore.EVM.Cache.StateDbCapacity = ctx.Int(flags.StateDbCacheCapacityFlag.Name)
+	}
+
 	return &cfg, nil
 }
 

diff --git a/config/flags/flags.go b/config/flags/flags.go
@@ -365,4 +365,10 @@ var (
 			"Setting this value to <=2000 will result in pre-confugired cache capacity of 2KB", CacheFlag.Name),
 		Value: 0,
 	}
+	StateDbCacheCapacityFlag = cli.IntFlag{
+		Name: "statedb.cache",
+		Usage: "Size of StateDb instances cache in bytes. Leaving this blank (which is generally recommended), or setting" +
+			"this to <1 will automatically set the cache capacity a hard-coded constant.",
+		Value: 0,
+	}
 )
diff --git a/gossip/common_test.go b/gossip/common_test.go
@@ -143,7 +143,15 @@ func newTestEnv(firstEpoch idx.Epoch, validatorsNum idx.Validator, tb testing.TB
 	rules.Blocks.MaxEmptyBlockSkipPeriod = 0
 	rules.Emitter.Interval = 0
 
-	genStore := makefakegenesis.FakeGenesisStoreWithRulesAndStart(validatorsNum, utils.ToFtm(genesisBalance), utils.ToFtm(genesisStake), rules, firstEpoch, 2)
+	genStore := makefakegenesis.FakeGenesisStoreWithRulesAndStart(
+		validatorsNum,
+		utils.ToFtm(genesisBalance),
+		utils.ToFtm(genesisStake),
+		rules,
+		firstEpoch,
+		2,
+		1024,
+	)
 	genesis := genStore.Genesis()
 
 	store, err := NewMemStore(tb)

diff --git a/gossip/evmstore/config.go b/gossip/evmstore/config.go
@@ -19,6 +19,8 @@ type (
 		EvmBlocksNum int
 		// Cache size for EvmBlock (size in bytes).
 		EvmBlocksSize uint
+		// Cache size for StateDb instances (0 for DB-selected default)
+		StateDbCapacity int
 	}
 	// StoreConfig is a config for store db.
 	StoreConfig struct {
@@ -38,11 +40,11 @@ type (
 func DefaultStoreConfig(scale cachescale.Func) StoreConfig {
 	return StoreConfig{
 		Cache: StoreCacheConfig{
-			ReceiptsSize:      scale.U(4 * opt.MiB),
-			ReceiptsBlocks:    scale.I(4000),
-			TxPositions:       scale.I(20000),
-			EvmBlocksNum:      scale.I(5000),
-			EvmBlocksSize:     scale.U(6 * opt.MiB),
+			ReceiptsSize:   scale.U(4 * opt.MiB),
+			ReceiptsBlocks: scale.I(4000),
+			TxPositions:    scale.I(20000),
+			EvmBlocksNum:   scale.I(5000),
+			EvmBlocksSize:  scale.U(6 * opt.MiB),
 		},
 		StateDb: carmen.Parameters{
 			Variant:      "go-file",

diff --git a/gossip/evmstore/store.go b/gossip/evmstore/store.go
@@ -78,7 +78,7 @@ func (s *Store) Open() error {
 	if err != nil {
 		return fmt.Errorf("failed to create carmen state; %s", err)
 	}
-	s.liveStateDb = carmen.CreateStateDBUsing(s.carmenState)
+	s.liveStateDb = carmen.CreateCustomStateDBUsing(s.carmenState, s.cfg.Cache.StateDbCapacity)
 	return nil
 }
 

diff --git a/gossip/handler_fuzz.go b/gossip/handler_fuzz.go
@@ -68,7 +68,12 @@ func makeFuzzedHandler() (h *handler, err error) {
 		genesisStake   = 2 * 4e6
 	)
 
-	genStore := makefakegenesis.FakeGenesisStore(genesisStakers, utils.ToFtm(genesisBalance), utils.ToFtm(genesisStake))
+	genStore := makefakegenesis.FakeGenesisStore(
+		genesisStakers,
+		utils.ToFtm(genesisBalance),
+		utils.ToFtm(genesisStake),
+		0, // 0 sets default value
+	)
 	genesis := genStore.Genesis()
 
 	config := DefaultConfig(cachescale.Identity)

diff --git a/integration/makefakegenesis/genesis.go b/integration/makefakegenesis/genesis.go
@@ -42,16 +42,23 @@ func FakeKey(n idx.ValidatorID) *ecdsa.PrivateKey {
 	return evmcore.FakeKey(uint32(n))
 }
 
-func FakeGenesisStore(num idx.Validator, balance, stake *big.Int) *genesisstore.Store {
-	return FakeGenesisStoreWithRules(num, balance, stake, opera.FakeNetRules())
+func FakeGenesisStore(num idx.Validator, balance, stake *big.Int, stateDbCacheCapacity int) *genesisstore.Store {
+	return FakeGenesisStoreWithRules(num, balance, stake, opera.FakeNetRules(), stateDbCacheCapacity)
 }
 
-func FakeGenesisStoreWithRules(num idx.Validator, balance, stake *big.Int, rules opera.Rules) *genesisstore.Store {
-	return FakeGenesisStoreWithRulesAndStart(num, balance, stake, rules, 2, 1)
+func FakeGenesisStoreWithRules(num idx.Validator, balance, stake *big.Int, rules opera.Rules, stateDbCacheCapacity int) *genesisstore.Store {
+	return FakeGenesisStoreWithRulesAndStart(num, balance, stake, rules, 2, 1, stateDbCacheCapacity)
 }
 
-func FakeGenesisStoreWithRulesAndStart(num idx.Validator, balance, stake *big.Int, rules opera.Rules, epoch idx.Epoch, block idx.Block) *genesisstore.Store {
-	builder := makegenesis.NewGenesisBuilder()
+func FakeGenesisStoreWithRulesAndStart(
+	num idx.Validator,
+	balance, stake *big.Int,
+	rules opera.Rules,
+	epoch idx.Epoch,
+	block idx.Block,
+	stateDbCacheCapacity int,
+) *genesisstore.Store {
+	builder := makegenesis.NewGenesisBuilder(stateDbCacheCapacity)
 
 	validators := GetFakeValidators(num)
 

diff --git a/integration/makefakegenesis/json.go b/integration/makefakegenesis/json.go
@@ -61,12 +61,12 @@ func LoadGenesisJson(filename string) (*GenesisJson, error) {
 	return &decoded, nil
 }
 
-func ApplyGenesisJson(json *GenesisJson) (*genesisstore.Store, error) {
+func ApplyGenesisJson(json *GenesisJson, stateDbCacheCapacity int) (*genesisstore.Store, error) {
 	if json.BlockZeroTime.IsZero() {
 		return nil, fmt.Errorf("block zero time must be set")
 	}
 
-	builder := makegenesis.NewGenesisBuilder()
+	builder := makegenesis.NewGenesisBuilder(stateDbCacheCapacity)
 
 	fmt.Printf("Building genesis file - rules: %+v\n", json.Rules)
 

diff --git a/integration/makegenesis/genesis.go b/integration/makegenesis/genesis.go
@@ -113,7 +113,7 @@ func (b *GenesisBuilder) CurrentHash() hash.Hash {
 	return er.Hash()
 }
 
-func NewGenesisBuilder() *GenesisBuilder {
+func NewGenesisBuilder(stateDbCacheCapacity int) *GenesisBuilder {
 	carmenDir, err := os.MkdirTemp("", "opera-tmp-genesis")
 	if err != nil {
 		panic(fmt.Errorf("failed to create temporary dir for GenesisBuilder: %v", err))
@@ -128,7 +128,7 @@ func NewGenesisBuilder() *GenesisBuilder {
 	if err != nil {
 		panic(fmt.Errorf("failed to create carmen state; %s", err))
 	}
-	carmenStateDb := carmen.CreateStateDBUsing(carmenState)
+	carmenStateDb := carmen.CreateCustomStateDBUsing(carmenState, stateDbCacheCapacity)
 	tmpStateDB := evmstore.CreateCarmenStateDb(carmenStateDb)
 	return &GenesisBuilder{
 		tmpStateDB:    tmpStateDB,

diff --git a/tests/integration_test_net.go b/tests/integration_test_net.go
@@ -200,15 +200,14 @@ func startIntegrationTestNet(directory string, args []string) (*IntegrationTestN
 		"--datadir", result.stateDir(),
 		"--statedb.livecache", "1",
 		"--statedb.archivecache", "1",
+		"--statedb.cache", "1024",
 	}, args...)
 	if err := sonictool.Run(); err != nil {
 		os.Args = originalArgs
 		return nil, fmt.Errorf("failed to initialize the test network: %w", err)
 	}
 	os.Args = originalArgs
 
-	os.Args = originalArgs
-
 	if err := result.start(); err != nil {
 		return nil, fmt.Errorf("failed to start the test network: %w", err)
 	}
@@ -273,6 +272,7 @@ func (n *IntegrationTestNet) start() error {
 			// database memory usage options
 			"--statedb.livecache", "1",
 			"--statedb.archivecache", "1",
+			"--statedb.cache", "1024",
 		}
 
 		err := sonicd.Run()