Add a test which checks for proper finish of Bodies/Receipts on mutliple tries

NethermindNode.Core/Helpers/CommandExecutor.cs

@@ -11,7 +11,6 @@ public static void RemoveDirectory(string absolutePath, NLog.Logger logger)
         if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
         {
             processInfo = new ProcessStartInfo("rm", $"-r {absolutePath}");
-            logger.Debug("Removing path: " + absolutePath);
         }
         else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
         {
@@ -21,6 +20,7 @@ public static void RemoveDirectory(string absolutePath, NLog.Logger logger)
         {
             throw new ArgumentOutOfRangeException();
         }
+        logger.Debug("Removing path: " + absolutePath);
 
         processInfo.CreateNoWindow = true;
         processInfo.UseShellExecute = false;
@@ -35,6 +35,37 @@ public static void RemoveDirectory(string absolutePath, NLog.Logger logger)
         }
     }
 
+    public static void BackupDirectory(string absolutePath, string backupPath, NLog.Logger logger)
+    {
+        ProcessStartInfo processInfo;
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            processInfo = new ProcessStartInfo("cp", $"-r {absolutePath} {backupPath}");
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            processInfo = new ProcessStartInfo("xcopy", $"{ConvertFromWslPathToWindowsPath(absolutePath)} {ConvertFromWslPathToWindowsPath(backupPath)} /E /H /K /O /X");
+        }
+        else
+        {
+            throw new ArgumentOutOfRangeException();
+        }
+        logger.Debug("Creating a backup of: " + absolutePath + " in a path: " + backupPath);
+
+        processInfo.CreateNoWindow = true;
+        processInfo.UseShellExecute = false;
+
+        using (var process = new Process())
+        {
+            process.StartInfo = processInfo;
+            process.Start();
+            // long as it may be huge backup
+            process.WaitForExit(3000000);
+
+            process.Close();
+        }
+    }
+
     public static string ConvertFromWslPathToWindowsPath(string wslPath)
     {
         string windowsPath = "";

NethermindNode.Core/Helpers/DockerCommands.cs

@@ -50,6 +50,11 @@ public static string GetDockerDetails(string containerName, string dataToFetch,
         return result;
     }
 
+    public static void RecreateDockerCompose(string containerName, string dockerComposePath, Logger logger)
+    {
+        DockerCommandExecute("compose -f " + dockerComposePath + " create  --force-recreate " + containerName, logger);
+    }
+
     public static string GetExecutionDataPath(Logger logger)
     {
         return GetDockerDetails(ConfigurationHelper.Instance["execution-container-name"], "{{ range .Mounts }}{{ if eq .Destination \\\"/nethermind/data\\\" }}{{ .Source }}{{ end }}{{ end }}", logger).Trim();

NethermindNode.Core/Helpers/DockerComposeHelper.cs

@@ -0,0 +1,55 @@
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+
+namespace NethermindNode.Core.Helpers
+{
+    public static class DockerComposeHelper
+    {
+        public static Dictionary<string, object> ReadDockerCompose(string filePath)
+        {
+            var deserializer = new DeserializerBuilder()
+                .WithNamingConvention(UnderscoredNamingConvention.Instance)
+                .Build();
+
+            using var reader = new StreamReader(filePath);
+            var yaml = reader.ReadToEnd();
+            var result = deserializer.Deserialize<Dictionary<string, object>>(yaml);
+
+            return result;
+        }
+
+        public static void WriteDockerCompose(Dictionary<string, object> compose, string filePath)
+        {
+            var serializer = new SerializerBuilder()
+                .WithNamingConvention(UnderscoredNamingConvention.Instance)
+                .Build();
+
+            var yaml = serializer.Serialize(compose);
+            using var writer = new StreamWriter(filePath);
+            Console.WriteLine("Writing at path: " + filePath);
+            writer.Write(yaml);
+        }
+
+        public static void RemoveCommandFlag(Dictionary<string, object> dockerCompose, string serviceName, string flagToRemove)
+        {
+            var services = (Dictionary<object, object>)dockerCompose["services"];
+            if (services.ContainsKey(serviceName))
+            {
+                var service = (Dictionary<object, object>)services[serviceName];
+                var command = (List<object>)service["command"];
+                command.Remove(flagToRemove);
+            }
+        }
+
+        public static void AddCommandFlag(Dictionary<string, object> dockerCompose, string serviceName, string flagToAdd)
+        {
+            var services = (Dictionary<object, object>)dockerCompose["services"];
+            if (services.ContainsKey(serviceName))
+            {
+                var service = (Dictionary<object, object>)services[serviceName];
+                var command = (List<object>)service["command"];
+                command.Add(flagToAdd);
+            }
+        }
+    }
+}

NethermindNode.Core/NethermindNode.Core.csproj

@@ -16,6 +16,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
     <PackageReference Include="NLog" Version="5.1.3" />
     <PackageReference Include="Notion.Net" Version="4.1.0" />
+    <PackageReference Include="YamlDotNet" Version="15.1.2" />
 	<None Include="..\config.json" Link="config.json" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 

NethermindNode.Tests/NethermindNode.Tests.csproj

@@ -35,7 +35,10 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="NunitXml.TestLogger" Version="3.0.131" />
+    <PackageReference Include="Polly" Version="8.3.1" />
+    <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
     <PackageReference Include="System.Configuration.ConfigurationManager" Version="7.0.0" />
+    <PackageReference Include="YamlDotNet" Version="15.1.2" />
 	<None Include="..\config.json" Link="config.json" CopyToOutputDirectory="Always" />
   </ItemGroup>
 

NethermindNode.Tests/Tests/Pruning/JsonRpcPruning.cs

@@ -44,7 +44,7 @@ public void ShouldPruneDbUsingAdminRpc()
             var parameters = $"";
             var result = HttpExecutor.ExecuteNethermindJsonRpcCommand("admin_prune", parameters, TestItems.RpcAddress, Logger).Result.Item1;
 
-            Assert.IsTrue(result.Contains("Starting"), $"Result should contains \"Starting\" but it doesn't. Result content: {result}");
+            Assert.IsTrue(result.ToLowerInvariant().Contains("starting"), $"Result should contains \"starting\" but it doesn't. Result content: {result}");
 
             // Wait for maximum 60 seconds for pruning to be properly started
             Stopwatch stopwatch = new Stopwatch();

NethermindNode.Tests/Tests/SyncingNode/BodiesAndReceiptsFinalization.cs

@@ -0,0 +1,108 @@
+using NethermindNode.Core.Helpers;
+using Polly;
+using Polly.Contrib.WaitAndRetry;
+
+namespace NethermindNode.Tests.SyncingNode
+{
+    [TestFixture]
+    public class BodiesAndReceiptsTests : BaseTest
+    {
+
+        private static readonly NLog.Logger Logger = NLog.LogManager.GetLogger(TestContext.CurrentContext.Test.Name);
+
+        [TestCase(10)]
+        [Category("BodiesAndReceipts")]
+        [Description(
+            """
+                !!!!! Test to be used when NonValidator mode is true. !!!!!
+
+                1. Wait for node to be synced (eth_syncing returns false)
+                2. Stop a execution node
+                3. Create a backup of database
+                4. Restart node but without NonValidator node flags
+                5. Wait until end of sync (eth_syncing returns false)
+                6. Check if logs "Fast blocks bodies task completed." and "Fast blocks receipts task completed." are there - those two should always been present meaning that tasks are properly finished.
+                7. Stop a node, restore backup, redo steps 5 and 6
+
+                WHY?
+                If there is no such logs, there is high chance we "lost" something and we should investigate what is missing 
+            """
+            )]
+        public void ShouldResyncBodiesAndReceiptsAfterNonValidator(int repeatCount)
+        {
+            Logger.Info("***Starting test: ShouldResyncBodiesAndReceiptsAfterNonValidator***");
+
+            // 1
+            NodeInfo.WaitForNodeToBeReady(Logger);
+
+            var delay = Backoff.ConstantBackoff(TimeSpan.FromSeconds(3), retryCount: 100);
+
+            var retryPolicy = Policy
+            .HandleResult<string>(s => !s.Contains("SnapSync") && !s.Contains("StateNodes"))
+            .WaitAndRetry(delay);
+
+            string result = retryPolicy.Execute(() => NodeInfo.GetCurrentStage(Logger));
+
+            NodeInfo.WaitForNodeToBeSynced(Logger);
+
+            // 2
+            DockerCommands.StopDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger);
+
+            // 3
+            var execPath = DockerCommands.GetExecutionDataPath(Logger);
+            CommandExecutor.BackupDirectory(execPath + "/nethermind_db", execPath + "/nethermind_db_backup" , Logger);
+
+            // 4
+            string[] flagsToRemove =
+            {
+                "--Sync.NonValidatorNode=true",
+                "--Sync.DownloadBodiesInFastSync=false",
+                "--Sync.DownloadReceiptsInFastSync=false"
+            };
+
+            Logger.Info("Reading docker compose file at path: " + execPath + "/../docker-compose.yml");
+            var dockerCompose = DockerComposeHelper.ReadDockerCompose(execPath + "/../docker-compose.yml");
+            foreach (var flag in flagsToRemove)
+            {
+                Console.WriteLine("Removing: " + flag);
+                DockerComposeHelper.RemoveCommandFlag(dockerCompose, "execution", flag);
+            }
+            DockerComposeHelper.WriteDockerCompose(dockerCompose, execPath + "/../docker-compose.yml");
+            DockerCommands.RecreateDockerCompose("execution", execPath + "/../docker-compose.yml", Logger);
+            DockerCommands.StartDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger);
+
+            // 5-6-7
+            for (int i = 0; i < repeatCount; i++)
+            {
+                NodeInfo.WaitForNodeToBeReady(Logger);
+                NodeInfo.WaitForNodeToBeSynced(Logger);
+                var bodiesLine = DockerCommands.GetDockerLogs(ConfigurationHelper.Instance["execution-container-name"], "Fast blocks bodies task completed.");
+                var receiptsLine = DockerCommands.GetDockerLogs(ConfigurationHelper.Instance["execution-container-name"], "Fast blocks receipts task completed.");
+
+                Assert.IsTrue(bodiesLine.Count() > 0, "Bodies log line missing - verify with getBlockByNumber.");
+                Assert.IsTrue(receiptsLine.Count() > 0, "Receipts log line missing - verify with getReceipt");
+
+                DockerCommands.StopDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger);
+
+                CommandExecutor.RemoveDirectory(execPath + "/nethermind_db", Logger);
+                CommandExecutor.BackupDirectory(execPath + "/nethermind_db_backup", execPath + "/nethermind_db", Logger);
+
+                // For logs cleanup purpose only
+                DockerCommands.RecreateDockerCompose("execution", execPath + "/../docker-compose.yml", Logger);
+
+                DockerCommands.StartDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger);
+            }
+
+            // Restore to previous state
+            foreach (var flag in flagsToRemove)
+            {
+                Console.WriteLine("Adding: " + flag);
+                DockerComposeHelper.AddCommandFlag(dockerCompose, "execution", flag);
+            }
+            DockerComposeHelper.WriteDockerCompose(dockerCompose, execPath + "/../docker-compose.yml");
+            DockerCommands.RecreateDockerCompose("execution", execPath + "/../docker-compose.yml", Logger);
+            DockerCommands.StopDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger);
+            DockerCommands.StartDockerContainer(ConfigurationHelper.Instance["execution-container-name"], Logger);
+        }
+    }
+}