Faster sync tests

src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs

@@ -161,4 +161,7 @@ public interface ISyncConfig : IConfig
 
     [ConfigItem(Description = "_Technical._ Min distance of state sync from best suggested header.", DefaultValue = "32", HiddenFromDocs = true)]
     int StateMinDistanceFromHead { get; set; }
+
+    [ConfigItem(Description = "_Technical._ Run explicit GC after state sync finished.", DefaultValue = "true", HiddenFromDocs = true)]
+    bool GCOnStateSyncFinished { get; set; }
 }

src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs

@@ -72,6 +72,7 @@ public string? PivotHash
         public bool TrieHealing { get; set; } = true;
         public int StateMaxDistanceFromHead { get; set; } = 128;
         public int StateMinDistanceFromHead { get; set; } = 32;
+        public bool GCOnStateSyncFinished { get; set; } = true;
 
         public override string ToString()
         {

src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using Nethermind.Blockchain;
+using Nethermind.Core.Test.Builders;
+using Nethermind.Db;
+using NonBlocking;
+
+namespace Nethermind.Core.Test;
+
+public class CachedBlockTreeBuilder
+{
+    private static ConcurrentDictionary<string, CachedDb> _cachedDbs = new();
+    private record CachedDb(
+        MemDb blocksDb,
+        MemDb metadataDb,
+        MemDb headersDb,
+        MemDb blockNumbersDb,
+        MemDb blockInfo
+    );
+
+    public static IBlockTree BuildCached(string key, Func<BlockTreeBuilder> blockTreeBuilderFactory)
+    {
+        if (_cachedDbs.TryGetValue(key, out CachedDb? db))
+        {
+            return Build.A.BlockTree()
+                .WithBlocksDb(MemDb.CopyFrom(db.blocksDb))
+                .WithMetadataDb(MemDb.CopyFrom(db.metadataDb))
+                .WithHeadersDb(MemDb.CopyFrom(db.headersDb))
+                .WithBlocksNumberDb(MemDb.CopyFrom(db.blockNumbersDb))
+                .WithBlockInfoDb(MemDb.CopyFrom(db.blockInfo))
+                .TestObject;
+        }
+        else
+        {
+            BlockTreeBuilder builder = blockTreeBuilderFactory();
+            CachedDb cachedValue = new CachedDb(
+                MemDb.CopyFrom(builder.BlocksDb),
+                MemDb.CopyFrom(builder.MetadataDb),
+                MemDb.CopyFrom(builder.HeadersDb),
+                MemDb.CopyFrom(builder.BlockNumbersDb),
+                MemDb.CopyFrom(builder.BlockInfoDb)
+            );
+            _cachedDbs.TryAdd(key, cachedValue);
+
+            return builder.TestObject;
+        }
+    }
+
+    public static IBlockTree OfLength(int length)
+    {
+        return BuildCached($"{nameof(CachedBlockTreeBuilder)}-{length}", () => Build.A.BlockTree().OfChainLength(length));
+    }
+}

src/Nethermind/Nethermind.Db/MemDb.cs

@@ -27,6 +27,17 @@ public MemDb(string name)
             Name = name;
         }
 
+        public static MemDb CopyFrom(IDb anotherDb)
+        {
+            MemDb newDb = new MemDb();
+            foreach (KeyValuePair<byte[], byte[]> kv in anotherDb.GetAll())
+            {
+                newDb[kv.Key] = kv.Value;
+            }
+
+            return newDb;
+        }
+
         public MemDb() : this(0, 0)
         {
         }

src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs

@@ -276,7 +276,7 @@ public async Task Does_not_deadlock_on_replace_peer()
         peerAllocationStrategy
             .Allocate(Arg.Any<PeerInfo?>(), Arg.Any<IEnumerable<PeerInfo>>(), Arg.Any<INodeStatsManager>(), Arg.Any<IBlockTree>())
             .Returns(new PeerInfo(syncPeer1));
-        SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks);
+        SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks, null);
         peerAllocation.AllocateBestPeer(new List<PeerInfo>(), Substitute.For<INodeStatsManager>(), ctx.BlockTree);
         ctx.PeerPool
             .Allocate(Arg.Any<IPeerAllocationStrategy>(), Arg.Any<AllocationContexts>(), Arg.Any<int>())
@@ -332,7 +332,7 @@ public void No_old_bodies_and_receipts()
             BlockTreeScenario = blockTrees,
         };
 
-        ctx.Feed = new FastSyncFeed(new SyncConfig
+        ctx.Feed = new FastSyncFeed(new TestSyncConfig
         {
             NonValidatorNode = true,
             DownloadBodiesInFastSync = false,
@@ -349,7 +349,7 @@ public void No_old_bodies_and_receipts()
         peerAllocationStrategy
             .Allocate(Arg.Any<PeerInfo?>(), Arg.Any<IEnumerable<PeerInfo>>(), Arg.Any<INodeStatsManager>(), Arg.Any<IBlockTree>())
             .Returns(peerInfo);
-        SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks);
+        SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks, null);
         peerAllocation.AllocateBestPeer(new List<PeerInfo>(), Substitute.For<INodeStatsManager>(), ctx.BlockTree);
 
         ctx.PeerPool
@@ -479,13 +479,13 @@ public MergeConfig MergeConfig
 
         public PoSSwitcher PosSwitcher => _posSwitcher ??= new(
             MergeConfig,
-            new SyncConfig(),
+            new TestSyncConfig(),
             MetadataDb,
             BlockTree,
             SpecProvider,
             new ChainSpec(),
             LimboLogs.Instance);
-        public BeaconPivot BeaconPivot => _beaconPivot ??= new(new SyncConfig(), MetadataDb, BlockTree, _posSwitcher!, LimboLogs.Instance);
+        public BeaconPivot BeaconPivot => _beaconPivot ??= new(new TestSyncConfig(), MetadataDb, BlockTree, _posSwitcher!, LimboLogs.Instance);
 
         protected override IBetterPeerStrategy BetterPeerStrategy => _betterPeerStrategy ??=
             new MergeBetterPeerStrategy(new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), PosSwitcher, BeaconPivot, LimboLogs.Instance);
@@ -498,7 +498,7 @@ public IChainLevelHelper ChainLevelHelper
                 _chainLevelHelper ??= new ChainLevelHelper(
                     BlockTree,
                     BeaconPivot,
-                    new SyncConfig(),
+                    new TestSyncConfig(),
                     LimboLogs.Instance);
             set => _chainLevelHelper = value;
         }

src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs

@@ -38,6 +38,7 @@
 using NUnit.Framework;
 using BlockTree = Nethermind.Blockchain.BlockTree;
 using System.Diagnostics.CodeAnalysis;
+using Nethermind.Core.Test;
 
 namespace Nethermind.Synchronization.Test;
 
@@ -108,7 +109,7 @@ public async Task Ancestor_lookup_simple()
     {
         Context ctx = new()
         {
-            BlockTree = Build.A.BlockTree().OfChainLength(1024).TestObject,
+            BlockTree = CachedBlockTreeBuilder.OfLength(1024),
         };
         BlockDownloader downloader = ctx.BlockDownloader;
 
@@ -139,7 +140,7 @@ public async Task Ancestor_lookup_headers()
     {
         Context ctx = new()
         {
-            BlockTree = Build.A.BlockTree().OfChainLength(1024).TestObject,
+            BlockTree = CachedBlockTreeBuilder.OfLength(1024),
         };
         BlockDownloader downloader = ctx.BlockDownloader;
 
@@ -168,7 +169,7 @@ public void Ancestor_failure()
     {
         Context ctx = new()
         {
-            BlockTree = Build.A.BlockTree().OfChainLength(2048 + 1).TestObject,
+            BlockTree = CachedBlockTreeBuilder.OfLength(2048 + 1),
         };
         BlockDownloader downloader = ctx.BlockDownloader;
 
@@ -186,7 +187,7 @@ public void Ancestor_failure_blocks()
     {
         Context ctx = new()
         {
-            BlockTree = Build.A.BlockTree().OfChainLength(2048 + 1).TestObject,
+            BlockTree = CachedBlockTreeBuilder.OfLength(2048 + 1),
         };
         BlockDownloader downloader = ctx.BlockDownloader;
 
@@ -1024,11 +1025,9 @@ public IBlockValidator BlockValidator
 
         private SyncDispatcher<BlocksRequest>? _dispatcher;
         public SyncDispatcher<BlocksRequest> Dispatcher => _dispatcher ??= new SyncDispatcher<BlocksRequest>(
-            new SyncConfig()
+            new TestSyncConfig()
             {
                 MaxProcessingThreads = 0,
-                SyncDispatcherEmptyRequestDelayMs = 1,
-                SyncDispatcherAllocateTimeoutMs = 1
             },
             Feed!,
             BlockDownloader,
@@ -1062,7 +1061,7 @@ private class SyncPeerMock : ISyncPeer
         private readonly ReceiptsMessageSerializer _receiptsSerializer = new(MainnetSpecProvider.Instance);
         private readonly Response _flags;
 
-        public BlockTree BlockTree { get; private set; } = null!;
+        public IBlockTree BlockTree { get; private set; } = null!;
         private IReceiptStorage _receiptStorage = new InMemoryReceiptStorage();
 
         public string Name => "Mock";

src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs

@@ -54,7 +54,7 @@ public void Setup()
 
         _pivotBlock = _syncingFromBlockTree.FindBlock(99, BlockTreeLookupOptions.None)!;
 
-        _syncConfig = new SyncConfig()
+        _syncConfig = new TestSyncConfig()
         {
             FastSync = true,
             PivotHash = _pivotBlock.Hash!.ToString(),

src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs

@@ -14,6 +14,7 @@
 using Nethermind.Core.Collections;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
+using Nethermind.Core.Test;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Logging;
 using Nethermind.Stats.Model;
@@ -40,7 +41,7 @@ public Task Will_fail_if_launched_without_fast_blocks_enabled()
             HeadersSyncFeed _ = new HeadersSyncFeed(
                 blockTree: blockTree,
                 syncPeerPool: Substitute.For<ISyncPeerPool>(),
-                syncConfig: new SyncConfig(),
+                syncConfig: new TestSyncConfig(),
                 syncReport: Substitute.For<ISyncReport>(),
                 logManager: LimboLogs.Instance);
         });
@@ -56,7 +57,7 @@ public async Task Can_prepare_3_requests_in_a_row()
         HeadersSyncFeed feed = new(
             blockTree: blockTree,
             syncPeerPool: Substitute.For<ISyncPeerPool>(),
-            syncConfig: new SyncConfig
+            syncConfig: new TestSyncConfig
             {
                 FastSync = true,
                 PivotNumber = "1000",
@@ -89,7 +90,7 @@ public async Task When_next_header_hash_update_is_delayed_do_not_drop_peer()
         ResettableHeaderSyncFeed feed = new(
             blockTree: blockTree,
             syncPeerPool: syncPeerPool,
-            syncConfig: new SyncConfig
+            syncConfig: new TestSyncConfig
             {
                 FastSync = true,
                 PivotNumber = "1000",
@@ -149,7 +150,7 @@ public async Task Can_prepare_several_request_and_ignore_request_from_previous_s
         using ResettableHeaderSyncFeed feed = new(
             blockTree: blockTree,
             syncPeerPool: Substitute.For<ISyncPeerPool>(),
-            syncConfig: new SyncConfig
+            syncConfig: new TestSyncConfig
             {
                 FastSync = true,
                 PivotNumber = "500",
@@ -196,7 +197,7 @@ public async Task Will_dispatch_when_only_partially_processed_dependency()
         using HeadersSyncFeed feed = new(
             blockTree: blockTree,
             syncPeerPool: Substitute.For<ISyncPeerPool>(),
-            syncConfig: new SyncConfig
+            syncConfig: new TestSyncConfig
             {
                 FastSync = true,
                 PivotNumber = pivot.Number.ToString(),
@@ -267,7 +268,7 @@ public async Task Can_reset_and_not_hang_when_a_batch_is_processing()
         ResettableHeaderSyncFeed feed = new(
             blockTree: blockTree,
             syncPeerPool: Substitute.For<ISyncPeerPool>(),
-            syncConfig: new SyncConfig
+            syncConfig: new TestSyncConfig
             {
                 FastSync = true,
                 PivotNumber = "500",
@@ -321,7 +322,7 @@ public async Task Can_keep_returning_nulls_after_all_batches_were_prepared()
         HeadersSyncFeed feed = new(
             blockTree: blockTree,
             syncPeerPool: Substitute.For<ISyncPeerPool>(),
-            syncConfig: new SyncConfig
+            syncConfig: new TestSyncConfig
             {
                 FastSync = true,
                 PivotNumber = "1000",
@@ -349,7 +350,7 @@ public async Task Finishes_when_all_downloaded()
         report.HeadersInQueue.Returns(new MeasuredProgress());
         MeasuredProgress measuredProgress = new();
         report.FastBlocksHeaders.Returns(measuredProgress);
-        HeadersSyncFeed feed = new(blockTree, Substitute.For<ISyncPeerPool>(), new SyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance);
+        HeadersSyncFeed feed = new(blockTree, Substitute.For<ISyncPeerPool>(), new TestSyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance);
         await feed.PrepareRequest();
         blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).TestObject);
         using HeadersSyncBatch? result = await feed.PrepareRequest();
@@ -372,7 +373,7 @@ public async Task Can_resume_downloading_from_parent_of_lowest_inserted_header()
         report.HeadersInQueue.Returns(new MeasuredProgress());
         report.FastBlocksHeaders.Returns(new MeasuredProgress());
 
-        HeadersSyncFeed feed = new(blockTree, Substitute.For<ISyncPeerPool>(), new SyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance);
+        HeadersSyncFeed feed = new(blockTree, Substitute.For<ISyncPeerPool>(), new TestSyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance);
         feed.InitializeFeed();
         using HeadersSyncBatch? result = await feed.PrepareRequest();
 
@@ -406,8 +407,8 @@ public async Task Can_resume_downloading_from_parent_of_lowest_inserted_header()
     [TestCase(0, 192, 1, false, true)]
     public async Task Can_insert_all_good_headers_from_dependent_batch_with_missing_or_null_headers(int nullIndex, int count, int increment, bool shouldReport, bool useNulls)
     {
-        var peerChain = Build.A.BlockTree().OfChainLength(1000).TestObject;
-        var syncConfig = new SyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" };
+        var peerChain = CachedBlockTreeBuilder.OfLength(1000);
+        var syncConfig = new TestSyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" };
 
         IBlockTree localBlockTree = Build.A.BlockTree(peerChain.FindBlock(0, BlockTreeLookupOptions.None)!, null).WithSyncConfig(syncConfig).TestObject;
         const int lowestInserted = 999;
@@ -472,7 +473,7 @@ public async Task Will_never_lose_batch_on_invalid_batch()
         HeadersSyncFeed feed = new(
             blockTree,
             Substitute.For<ISyncPeerPool>(),
-            new SyncConfig
+            new TestSyncConfig
             {
                 FastSync = true,
                 PivotNumber = "1000",
@@ -538,7 +539,7 @@ public async Task Will_never_lose_batch_on_invalid_batch()
     public void IsFinished_returns_false_when_headers_not_downloaded()
     {
         IBlockTree blockTree = Substitute.For<IBlockTree>();
-        SyncConfig syncConfig = new()
+        TestSyncConfig syncConfig = new()
         {
             FastSync = true,
             DownloadBodiesInFastSync = true,

src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs

@@ -100,7 +100,7 @@ public void Setup()
         _blockTree = Substitute.For<IBlockTree>();
         _metadataDb = new TestMemDb();
 
-        _syncConfig = new SyncConfig { FastSync = true };
+        _syncConfig = new TestSyncConfig { FastSync = true };
         _syncConfig.PivotNumber = _pivotNumber.ToString();
         _syncConfig.PivotHash = Keccak.Zero.ToString();
 
@@ -140,7 +140,7 @@ private ReceiptsSyncFeed CreateFeed()
     [Test]
     public void Should_throw_when_fast_block_not_enabled()
     {
-        _syncConfig = new SyncConfig { FastSync = false };
+        _syncConfig = new TestSyncConfig { FastSync = false };
         Assert.Throws<InvalidOperationException>(
             () => _feed = new ReceiptsSyncFeed(
                 _specProvider,
@@ -408,7 +408,7 @@ public async Task Can_sync_final_batch()
     public void Is_fast_block_receipts_finished_returns_false_when_receipts_not_downloaded()
     {
         _blockTree = Substitute.For<IBlockTree>();
-        _syncConfig = new SyncConfig()
+        _syncConfig = new TestSyncConfig()
         {
             FastSync = true,
             DownloadBodiesInFastSync = true,
@@ -430,7 +430,7 @@ public void Is_fast_block_receipts_finished_returns_false_when_receipts_not_down
     public void Is_fast_block_bodies_finished_returns_true_when_bodies_not_downloaded_and_we_do_not_want_to_download_bodies()
     {
         _blockTree = Substitute.For<IBlockTree>();
-        _syncConfig = new SyncConfig()
+        _syncConfig = new TestSyncConfig()
         {
             FastSync = true,
             DownloadBodiesInFastSync = false,

src/Nethermind/Nethermind.Synchronization.Test/FastSync/SnapProtocolTests/StateSyncDispatcherTests.cs

@@ -18,6 +18,7 @@
 using Nethermind.Core.Crypto;
 using System.Net;
 using FluentAssertions;
+using Nethermind.Core.Test;
 using Nethermind.Trie;
 
 namespace Nethermind.Synchronization.Test.FastSync.SnapProtocolTests;
@@ -43,7 +44,7 @@ public void Setup()
     {
         _logManager = LimboLogs.Instance;
 
-        BlockTree blockTree = Build.A.BlockTree().OfChainLength((int)BlockTree.BestSuggestedHeader!.Number).TestObject;
+        IBlockTree blockTree = CachedBlockTreeBuilder.OfLength((int)BlockTree.BestSuggestedHeader!.Number);
         ITimerFactory timerFactory = Substitute.For<ITimerFactory>();
         _pool = new SyncPeerPool(blockTree, new NodeStatsManager(timerFactory, LimboLogs.Instance), new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), LimboLogs.Instance, 25);
         _pool.Start();