From 948f08b6086dab51d079e8cf61b2d32f65d15465 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Thu, 16 Nov 2023 10:07:29 -0500 Subject: [PATCH] config: move crash and stateproof DB defaults to hot dir (#5817) --- config/config_test.go | 124 +++++++++++++++++++++++++++++++++++++--- config/localTemplate.go | 57 +++++++++++++++--- node/follower_node.go | 2 +- node/node.go | 2 +- node/node_test.go | 6 +- 5 files changed, 170 insertions(+), 21 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 87b4cc4d43..ef58bffb9d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -675,6 +675,12 @@ func TestEnsureAbsDir(t *testing.T) { require.Equal(t, testDirectory+"/myGenesisID", t2Abs) } +type tLogger struct{ t *testing.T } + +func (l tLogger) Infof(fmts string, args ...interface{}) { + l.t.Logf(fmts, args...) +} + // TestEnsureAndResolveGenesisDirs confirms that paths provided in the config are resolved to absolute paths and are created if relevant func TestEnsureAndResolveGenesisDirs(t *testing.T) { partitiontest.PartitionTest(t) @@ -689,7 +695,7 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) { cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof") cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint") - paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID") + paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t}) require.NoError(t, err) // confirm that the paths are absolute, and contain the genesisID @@ -711,7 +717,7 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) { cfg := GetDefaultLocal() testDirectory := t.TempDir() - paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID") + paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t}) require.NoError(t, err) // confirm that if only the root is specified, it is used for all directories require.Equal(t, testDirectory+"/myGenesisID", paths.TrackerGenesisDir) @@ -731,21 +737,125 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) { cold := filepath.Join(testDirectory, "cold") cfg.HotDataDir = hot cfg.ColdDataDir = cold - paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID") + paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t}) require.NoError(t, err) // confirm that if hot/cold are specified, hot/cold are used for appropriate directories require.Equal(t, hot+"/myGenesisID", paths.TrackerGenesisDir) require.DirExists(t, paths.TrackerGenesisDir) require.Equal(t, cold+"/myGenesisID", paths.BlockGenesisDir) require.DirExists(t, paths.BlockGenesisDir) - require.Equal(t, cold+"/myGenesisID", paths.CrashGenesisDir) + require.Equal(t, hot+"/myGenesisID", paths.CrashGenesisDir) require.DirExists(t, paths.CrashGenesisDir) - require.Equal(t, cold+"/myGenesisID", paths.StateproofGenesisDir) + require.Equal(t, hot+"/myGenesisID", paths.StateproofGenesisDir) require.DirExists(t, paths.StateproofGenesisDir) require.Equal(t, cold+"/myGenesisID", paths.CatchpointGenesisDir) require.DirExists(t, paths.CatchpointGenesisDir) } +func TestEnsureAndResolveGenesisDirs_migrate(t *testing.T) { + partitiontest.PartitionTest(t) + + cfg := GetDefaultLocal() + testDirectory := t.TempDir() + cfg.HotDataDir = filepath.Join(testDirectory, "hot") + cfg.ColdDataDir = filepath.Join(testDirectory, "cold") + coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID") + hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID") + err := os.MkdirAll(coldDir, 0755) + require.NoError(t, err) + // put a crash.sqlite file in the ColdDataDir + err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644) + require.NoError(t, err) + // put a stateproof.sqlite file in the ColdDataDir + err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644) + require.NoError(t, err) + // Resolve + paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t}) + require.NoError(t, err) + // Confirm that crash.sqlite was moved to HotDataDir + require.DirExists(t, paths.CrashGenesisDir) + require.Equal(t, hotDir, paths.CrashGenesisDir) + require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite")) + require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite-shm")) + require.FileExists(t, filepath.Join(hotDir, "crash.sqlite")) + require.FileExists(t, filepath.Join(hotDir, "crash.sqlite-shm")) + // Confirm that stateproof.sqlite was moved to HotDataDir + require.DirExists(t, paths.StateproofGenesisDir) + require.Equal(t, hotDir, paths.StateproofGenesisDir) + require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite")) + require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal")) + require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite")) + require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal")) +} + +func TestEnsureAndResolveGenesisDirs_migrateCrashFail(t *testing.T) { + partitiontest.PartitionTest(t) + + cfg := GetDefaultLocal() + testDirectory := t.TempDir() + cfg.HotDataDir = filepath.Join(testDirectory, "hot") + cfg.ColdDataDir = filepath.Join(testDirectory, "cold") + coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID") + hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID") + err := os.MkdirAll(coldDir, 0755) + require.NoError(t, err) + err = os.MkdirAll(hotDir, 0755) + require.NoError(t, err) + // put a crash.sqlite file in the ColdDataDir + err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644) + require.NoError(t, err) + // also put a crash.sqlite file in the HotDataDir + err = os.WriteFile(filepath.Join(hotDir, "crash.sqlite"), []byte("test"), 0644) + require.NoError(t, err) + // Resolve + paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t}) + require.Error(t, err) + require.Empty(t, paths) + // Confirm that crash.sqlite was not moved to HotDataDir + require.FileExists(t, filepath.Join(coldDir, "crash.sqlite")) + require.FileExists(t, filepath.Join(coldDir, "crash.sqlite-shm")) + require.FileExists(t, filepath.Join(hotDir, "crash.sqlite")) + require.NoFileExists(t, filepath.Join(hotDir, "crash.sqlite-shm")) +} + +func TestEnsureAndResolveGenesisDirs_migrateSPFail(t *testing.T) { + partitiontest.PartitionTest(t) + + cfg := GetDefaultLocal() + testDirectory := t.TempDir() + cfg.HotDataDir = filepath.Join(testDirectory, "hot") + cfg.ColdDataDir = filepath.Join(testDirectory, "cold") + coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID") + hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID") + err := os.MkdirAll(coldDir, 0755) + require.NoError(t, err) + err = os.MkdirAll(hotDir, 0755) + require.NoError(t, err) + // put a stateproof.sqlite file in the ColdDataDir + err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644) + require.NoError(t, err) + // also put a stateproof.sqlite-wal file in the HotDataDir + err = os.WriteFile(filepath.Join(hotDir, "stateproof.sqlite-wal"), []byte("test"), 0644) + require.NoError(t, err) + // Resolve + paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t}) + require.Error(t, err) + require.Empty(t, paths) + // Confirm that stateproof.sqlite was not moved to HotDataDir + require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite")) + require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal")) + require.NoFileExists(t, filepath.Join(hotDir, "stateproof.sqlite")) + require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal")) +} + // TestEnsureAndResolveGenesisDirsError confirms that if a path can't be created, an error is returned func TestEnsureAndResolveGenesisDirsError(t *testing.T) { partitiontest.PartitionTest(t) @@ -761,7 +871,7 @@ func TestEnsureAndResolveGenesisDirsError(t *testing.T) { cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint") // first try an error with an empty root dir - paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID") + paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID", tLogger{t: t}) require.Empty(t, paths) require.Error(t, err) require.Contains(t, err.Error(), "rootDir is required") @@ -769,7 +879,7 @@ func TestEnsureAndResolveGenesisDirsError(t *testing.T) { require.NoError(t, os.Chmod(testDirectory, 0200)) // now try an error with a root dir that can't be written to - paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID") + paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t}) require.Empty(t, paths) require.Error(t, err) require.Contains(t, err.Error(), "permission denied") diff --git a/config/localTemplate.go b/config/localTemplate.go index 6748801209..07a9bf5eb0 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -111,13 +111,13 @@ type Local struct { // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. // If not specified, the node will use the ColdDataDir. CatchpointDir string `version[31]:""` - // StateproofDir is an optional directory to store stateproof data. + // StateproofDir is an optional directory to persist state about observed and issued state proof messages. // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. - // If not specified, the node will use the ColdDataDir. + // If not specified, the node will use the HotDataDir. StateproofDir string `version[31]:""` - // CrashDBDir is an optional directory to store the crash database. + // CrashDBDir is an optional directory to persist agreement's consensus participation state. // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. - // If not specified, the node will use the ColdDataDir. + // If not specified, the node will use the HotDataDir CrashDBDir string `version[31]:""` // LogFileDir is an optional directory to store the log, node.log @@ -785,9 +785,13 @@ func (cfg *Local) ResolveLogPaths(rootDir string) (liveLog, archive string) { return liveLog, archive } +type logger interface { + Infof(format string, args ...interface{}) +} + // EnsureAndResolveGenesisDirs will resolve the supplied config paths to absolute paths, and will create the genesis directories of each // returns a ResolvedGenesisDirs struct with the resolved paths for use during runtime -func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (ResolvedGenesisDirs, error) { +func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string, logger logger) (ResolvedGenesisDirs, error) { var resolved ResolvedGenesisDirs var err error if rootDir != "" { @@ -843,27 +847,62 @@ func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (Resolv } else { resolved.CatchpointGenesisDir = resolved.ColdGenesisDir } - // if StateproofDir is not set, use ColdDataDir + // if StateproofDir is not set, use HotDataDir if cfg.StateproofDir != "" { resolved.StateproofGenesisDir, err = ensureAbsGenesisDir(cfg.StateproofDir, genesisID) if err != nil { return ResolvedGenesisDirs{}, err } } else { - resolved.StateproofGenesisDir = resolved.ColdGenesisDir + resolved.StateproofGenesisDir = resolved.HotGenesisDir + // if separate HotDataDir and ColdDataDir was configured, but StateproofDir was not configured + if resolved.ColdGenesisDir != resolved.HotGenesisDir { + // move existing stateproof DB files from ColdDataDir to HotDataDir + moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, StateProofFileName, StateProofFileName+"-shm", StateProofFileName+"-wal") + if moveErr != nil { + return ResolvedGenesisDirs{}, fmt.Errorf("error moving stateproof DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr) + } + } } - // if CrashDBDir is not set, use ColdDataDir + // if CrashDBDir is not set, use HotDataDir if cfg.CrashDBDir != "" { resolved.CrashGenesisDir, err = ensureAbsGenesisDir(cfg.CrashDBDir, genesisID) if err != nil { return ResolvedGenesisDirs{}, err } } else { - resolved.CrashGenesisDir = resolved.ColdGenesisDir + resolved.CrashGenesisDir = resolved.HotGenesisDir + // if separate HotDataDir and ColdDataDir was configured, but CrashDBDir was not configured + if resolved.ColdGenesisDir != resolved.HotGenesisDir { + // move existing crash DB files from ColdDataDir to HotDataDir + moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, CrashFilename, CrashFilename+"-shm", CrashFilename+"-wal") + if moveErr != nil { + return ResolvedGenesisDirs{}, fmt.Errorf("error moving crash DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr) + } + } } return resolved, nil } +func moveDirIfExists(logger logger, srcdir, dstdir string, files ...string) error { + // first, check if any files already exist in dstdir, and quit if so + for _, file := range files { + if _, err := os.Stat(filepath.Join(dstdir, file)); err == nil { + return fmt.Errorf("destination file %s already exists, not overwriting", filepath.Join(dstdir, file)) + } + } + // then, check if any files exist in srcdir, and move them to dstdir + for _, file := range files { + if _, err := os.Stat(filepath.Join(srcdir, file)); err == nil { + if err := os.Rename(filepath.Join(srcdir, file), filepath.Join(dstdir, file)); err != nil { + return fmt.Errorf("failed to move file %s from %s to %s: %v", file, srcdir, dstdir, err) + } + logger.Infof("Moved DB file %s from ColdDataDir %s to HotDataDir %s", file, srcdir, dstdir) + } + } + return nil +} + // AdjustConnectionLimits updates RestConnectionsSoftLimit, RestConnectionsHardLimit, IncomingConnectionsLimit // if requiredFDs greater than maxFDs func (cfg *Local) AdjustConnectionLimits(requiredFDs, maxFDs uint64) bool { diff --git a/node/follower_node.go b/node/follower_node.go index 66790b1291..c61c379577 100644 --- a/node/follower_node.go +++ b/node/follower_node.go @@ -83,7 +83,7 @@ func MakeFollower(log logging.Logger, rootDir string, cfg config.Local, phoneboo node.genesisHash = genesis.Hash() node.devMode = genesis.DevMode var err error - node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID()) + node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID(), log) if err != nil { return nil, err } diff --git a/node/node.go b/node/node.go index 4c18ad1d51..e1f79907c8 100644 --- a/node/node.go +++ b/node/node.go @@ -183,7 +183,7 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd node.devMode = genesis.DevMode node.config = cfg var err error - node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID()) + node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID(), log) if err != nil { return nil, err } diff --git a/node/node_test.go b/node/node_test.go index c905fa78da..55cbae9368 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -598,7 +598,7 @@ func TestConfiguredDataDirs(t *testing.T) { require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "ledger.tracker.sqlite")) // confirm the stateproof db in the genesis dir of hot data dir - require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "stateproof.sqlite")) + require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "stateproof.sqlite")) // confirm cold data dir exists and contains a genesis dir require.DirExists(t, filepath.Join(testDirCold, genesis.ID())) @@ -609,8 +609,8 @@ func TestConfiguredDataDirs(t *testing.T) { // confirm the partregistry is in the genesis dir of cold data dir require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "partregistry.sqlite")) - // confirm the partregistry is in the genesis dir of cold data dir - require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "crash.sqlite")) + // confirm the agreement crash DB is in the genesis dir of hot data dir + require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "crash.sqlite")) } // TestConfiguredResourcePaths tests to see that when TrackerDbFilePath, BlockDbFilePath, StateproofDir, and CrashFilePath are set, underlying resources are created in the correct locations