config: move crash and stateproof DB defaults to hot dir (algorand#5817)

config/config_test.go

@@ -675,6 +675,12 @@ func TestEnsureAbsDir(t *testing.T) {
 	require.Equal(t, testDirectory+"/myGenesisID", t2Abs)
 }
 
+type tLogger struct{ t *testing.T }
+
+func (l tLogger) Infof(fmts string, args ...interface{}) {
+	l.t.Logf(fmts, args...)
+}
+
 // TestEnsureAndResolveGenesisDirs confirms that paths provided in the config are resolved to absolute paths and are created if relevant
 func TestEnsureAndResolveGenesisDirs(t *testing.T) {
 	partitiontest.PartitionTest(t)
@@ -689,7 +695,7 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) {
 	cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof")
 	cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
 
-	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.NoError(t, err)
 
 	// confirm that the paths are absolute, and contain the genesisID
@@ -711,7 +717,7 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) {
 
 	cfg := GetDefaultLocal()
 	testDirectory := t.TempDir()
-	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.NoError(t, err)
 	// confirm that if only the root is specified, it is used for all directories
 	require.Equal(t, testDirectory+"/myGenesisID", paths.TrackerGenesisDir)
@@ -731,21 +737,125 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) {
 	cold := filepath.Join(testDirectory, "cold")
 	cfg.HotDataDir = hot
 	cfg.ColdDataDir = cold
-	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.NoError(t, err)
 	// confirm that if hot/cold are specified, hot/cold are used for appropriate directories
 	require.Equal(t, hot+"/myGenesisID", paths.TrackerGenesisDir)
 	require.DirExists(t, paths.TrackerGenesisDir)
 	require.Equal(t, cold+"/myGenesisID", paths.BlockGenesisDir)
 	require.DirExists(t, paths.BlockGenesisDir)
-	require.Equal(t, cold+"/myGenesisID", paths.CrashGenesisDir)
+	require.Equal(t, hot+"/myGenesisID", paths.CrashGenesisDir)
 	require.DirExists(t, paths.CrashGenesisDir)
-	require.Equal(t, cold+"/myGenesisID", paths.StateproofGenesisDir)
+	require.Equal(t, hot+"/myGenesisID", paths.StateproofGenesisDir)
 	require.DirExists(t, paths.StateproofGenesisDir)
 	require.Equal(t, cold+"/myGenesisID", paths.CatchpointGenesisDir)
 	require.DirExists(t, paths.CatchpointGenesisDir)
 }
 
+func TestEnsureAndResolveGenesisDirs_migrate(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+	testDirectory := t.TempDir()
+	cfg.HotDataDir = filepath.Join(testDirectory, "hot")
+	cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
+	coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
+	hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
+	err := os.MkdirAll(coldDir, 0755)
+	require.NoError(t, err)
+	// put a crash.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// put a stateproof.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// Resolve
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
+	require.NoError(t, err)
+	// Confirm that crash.sqlite was moved to HotDataDir
+	require.DirExists(t, paths.CrashGenesisDir)
+	require.Equal(t, hotDir, paths.CrashGenesisDir)
+	require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite"))
+	require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite-shm"))
+	require.FileExists(t, filepath.Join(hotDir, "crash.sqlite"))
+	require.FileExists(t, filepath.Join(hotDir, "crash.sqlite-shm"))
+	// Confirm that stateproof.sqlite was moved to HotDataDir
+	require.DirExists(t, paths.StateproofGenesisDir)
+	require.Equal(t, hotDir, paths.StateproofGenesisDir)
+	require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite"))
+	require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal"))
+	require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite"))
+	require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal"))
+}
+
+func TestEnsureAndResolveGenesisDirs_migrateCrashFail(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+	testDirectory := t.TempDir()
+	cfg.HotDataDir = filepath.Join(testDirectory, "hot")
+	cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
+	coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
+	hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
+	err := os.MkdirAll(coldDir, 0755)
+	require.NoError(t, err)
+	err = os.MkdirAll(hotDir, 0755)
+	require.NoError(t, err)
+	// put a crash.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// also put a crash.sqlite file in the HotDataDir
+	err = os.WriteFile(filepath.Join(hotDir, "crash.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// Resolve
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
+	require.Error(t, err)
+	require.Empty(t, paths)
+	// Confirm that crash.sqlite was not moved to HotDataDir
+	require.FileExists(t, filepath.Join(coldDir, "crash.sqlite"))
+	require.FileExists(t, filepath.Join(coldDir, "crash.sqlite-shm"))
+	require.FileExists(t, filepath.Join(hotDir, "crash.sqlite"))
+	require.NoFileExists(t, filepath.Join(hotDir, "crash.sqlite-shm"))
+}
+
+func TestEnsureAndResolveGenesisDirs_migrateSPFail(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+	testDirectory := t.TempDir()
+	cfg.HotDataDir = filepath.Join(testDirectory, "hot")
+	cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
+	coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
+	hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
+	err := os.MkdirAll(coldDir, 0755)
+	require.NoError(t, err)
+	err = os.MkdirAll(hotDir, 0755)
+	require.NoError(t, err)
+	// put a stateproof.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// also put a stateproof.sqlite-wal file in the HotDataDir
+	err = os.WriteFile(filepath.Join(hotDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// Resolve
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
+	require.Error(t, err)
+	require.Empty(t, paths)
+	// Confirm that stateproof.sqlite was not moved to HotDataDir
+	require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite"))
+	require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal"))
+	require.NoFileExists(t, filepath.Join(hotDir, "stateproof.sqlite"))
+	require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal"))
+}
+
 // TestEnsureAndResolveGenesisDirsError confirms that if a path can't be created, an error is returned
 func TestEnsureAndResolveGenesisDirsError(t *testing.T) {
 	partitiontest.PartitionTest(t)
@@ -761,15 +871,15 @@ func TestEnsureAndResolveGenesisDirsError(t *testing.T) {
 	cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
 
 	// first try an error with an empty root dir
-	paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID")
+	paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID", tLogger{t: t})
 	require.Empty(t, paths)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "rootDir is required")
 
 	require.NoError(t, os.Chmod(testDirectory, 0200))
 
 	// now try an error with a root dir that can't be written to
-	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.Empty(t, paths)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "permission denied")

config/localTemplate.go

@@ -111,13 +111,13 @@ type Local struct {
 	// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
 	// If not specified, the node will use the ColdDataDir.
 	CatchpointDir string `version[31]:""`
-	// StateproofDir is an optional directory to store stateproof data.
+	// StateproofDir is an optional directory to persist state about observed and issued state proof messages.
 	// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
-	// If not specified, the node will use the ColdDataDir.
+	// If not specified, the node will use the HotDataDir.
 	StateproofDir string `version[31]:""`
-	// CrashDBDir is an optional directory to store the crash database.
+	// CrashDBDir is an optional directory to persist agreement's consensus participation state.
 	// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
-	// If not specified, the node will use the ColdDataDir.
+	// If not specified, the node will use the HotDataDir
 	CrashDBDir string `version[31]:""`
 
 	// LogFileDir is an optional directory to store the log, node.log
@@ -785,9 +785,13 @@ func (cfg *Local) ResolveLogPaths(rootDir string) (liveLog, archive string) {
 	return liveLog, archive
 }
 
+type logger interface {
+	Infof(format string, args ...interface{})
+}
+
 // EnsureAndResolveGenesisDirs will resolve the supplied config paths to absolute paths, and will create the genesis directories of each
 // returns a ResolvedGenesisDirs struct with the resolved paths for use during runtime
-func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (ResolvedGenesisDirs, error) {
+func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string, logger logger) (ResolvedGenesisDirs, error) {
 	var resolved ResolvedGenesisDirs
 	var err error
 	if rootDir != "" {
@@ -843,27 +847,62 @@ func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (Resolv
 	} else {
 		resolved.CatchpointGenesisDir = resolved.ColdGenesisDir
 	}
-	// if StateproofDir is not set, use ColdDataDir
+	// if StateproofDir is not set, use HotDataDir
 	if cfg.StateproofDir != "" {
 		resolved.StateproofGenesisDir, err = ensureAbsGenesisDir(cfg.StateproofDir, genesisID)
 		if err != nil {
 			return ResolvedGenesisDirs{}, err
 		}
 	} else {
-		resolved.StateproofGenesisDir = resolved.ColdGenesisDir
+		resolved.StateproofGenesisDir = resolved.HotGenesisDir
+		// if separate HotDataDir and ColdDataDir was configured, but StateproofDir was not configured
+		if resolved.ColdGenesisDir != resolved.HotGenesisDir {
+			// move existing stateproof DB files from ColdDataDir to HotDataDir
+			moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, StateProofFileName, StateProofFileName+"-shm", StateProofFileName+"-wal")
+			if moveErr != nil {
+				return ResolvedGenesisDirs{}, fmt.Errorf("error moving stateproof DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr)
+			}
+		}
 	}
-	// if CrashDBDir is not set, use ColdDataDir
+	// if CrashDBDir is not set, use HotDataDir
 	if cfg.CrashDBDir != "" {
 		resolved.CrashGenesisDir, err = ensureAbsGenesisDir(cfg.CrashDBDir, genesisID)
 		if err != nil {
 			return ResolvedGenesisDirs{}, err
 		}
 	} else {
-		resolved.CrashGenesisDir = resolved.ColdGenesisDir
+		resolved.CrashGenesisDir = resolved.HotGenesisDir
+		// if separate HotDataDir and ColdDataDir was configured, but CrashDBDir was not configured
+		if resolved.ColdGenesisDir != resolved.HotGenesisDir {
+			// move existing crash DB files from ColdDataDir to HotDataDir
+			moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, CrashFilename, CrashFilename+"-shm", CrashFilename+"-wal")
+			if moveErr != nil {
+				return ResolvedGenesisDirs{}, fmt.Errorf("error moving crash DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr)
+			}
+		}
 	}
 	return resolved, nil
 }
 
+func moveDirIfExists(logger logger, srcdir, dstdir string, files ...string) error {
+	// first, check if any files already exist in dstdir, and quit if so
+	for _, file := range files {
+		if _, err := os.Stat(filepath.Join(dstdir, file)); err == nil {
+			return fmt.Errorf("destination file %s already exists, not overwriting", filepath.Join(dstdir, file))
+		}
+	}
+	// then, check if any files exist in srcdir, and move them to dstdir
+	for _, file := range files {
+		if _, err := os.Stat(filepath.Join(srcdir, file)); err == nil {
+			if err := os.Rename(filepath.Join(srcdir, file), filepath.Join(dstdir, file)); err != nil {
+				return fmt.Errorf("failed to move file %s from %s to %s: %v", file, srcdir, dstdir, err)
+			}
+			logger.Infof("Moved DB file %s from ColdDataDir %s to HotDataDir %s", file, srcdir, dstdir)
+		}
+	}
+	return nil
+}
+
 // AdjustConnectionLimits updates RestConnectionsSoftLimit, RestConnectionsHardLimit, IncomingConnectionsLimit
 // if requiredFDs greater than maxFDs
 func (cfg *Local) AdjustConnectionLimits(requiredFDs, maxFDs uint64) bool {

node/follower_node.go

@@ -83,7 +83,7 @@ func MakeFollower(log logging.Logger, rootDir string, cfg config.Local, phoneboo
 	node.genesisHash = genesis.Hash()
 	node.devMode = genesis.DevMode
 	var err error
-	node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID())
+	node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID(), log)
 	if err != nil {
 		return nil, err
 	}

node/node.go

@@ -183,7 +183,7 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd
 	node.devMode = genesis.DevMode
 	node.config = cfg
 	var err error
-	node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID())
+	node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID(), log)
 	if err != nil {
 		return nil, err
 	}

node/node_test.go

@@ -598,7 +598,7 @@ func TestConfiguredDataDirs(t *testing.T) {
 	require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "ledger.tracker.sqlite"))
 
 	// confirm the stateproof db in the genesis dir of hot data dir
-	require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "stateproof.sqlite"))
+	require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "stateproof.sqlite"))
 
 	// confirm cold data dir exists and contains a genesis dir
 	require.DirExists(t, filepath.Join(testDirCold, genesis.ID()))
@@ -609,8 +609,8 @@ func TestConfiguredDataDirs(t *testing.T) {
 	// confirm the partregistry is in the genesis dir of cold data dir
 	require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "partregistry.sqlite"))
 
-	// confirm the partregistry is in the genesis dir of cold data dir
-	require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "crash.sqlite"))
+	// confirm the agreement crash DB is in the genesis dir of hot data dir
+	require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "crash.sqlite"))
 }
 
 // TestConfiguredResourcePaths tests to see that when TrackerDbFilePath, BlockDbFilePath, StateproofDir, and CrashFilePath are set, underlying resources are created in the correct locations