From 9067bb6109ca2ab39a25072b26ab4dece05d0c0c Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 28 Nov 2024 20:56:39 +0800 Subject: [PATCH 1/3] Faster sync tests --- .../Synchronization/ISyncConfig.cs | 3 + .../Synchronization/SyncConfig.cs | 1 + .../CachedBlockTreeBuilder.cs | 55 +++++++++++++++++++ src/Nethermind/Nethermind.Db/MemDb.cs | 11 ++++ .../BlockDownloaderTests.Merge.cs | 12 ++-- .../BlockDownloaderTests.cs | 15 +++-- .../FastBlocks/BodiesSyncFeedTests.cs | 2 +- .../FastBlocks/FastHeadersSyncTests.cs | 27 ++++----- .../FastBlocks/ReceiptsSyncFeedTests.cs | 8 +-- .../StateSyncDispatcherTests.cs | 3 +- .../FastSync/StateSyncFeedTestsBase.cs | 9 ++- .../OldStyleFullSynchronizerTests.cs | 7 +-- .../MultiSyncModeSelectorFastSyncTests.cs | 4 +- .../MultiSyncModeSelectorTests.Scenario.cs | 2 +- .../ParallelSync/SyncDispatcherTests.cs | 10 +--- .../ReceiptSyncFeedTests.cs | 2 +- .../SnapSync/ProgressTrackerTests.cs | 4 +- .../RecreateStateFromAccountRangesTests.cs | 16 +++--- .../RecreateStateFromStorageRangesTests.cs | 10 ++-- .../SnapSync/SnapProviderTests.cs | 8 +-- .../SnapSync/SnapServerTest.cs | 6 +- .../SnapSync/StateSyncPivotTest.cs | 2 +- .../SyncServerTests.cs | 28 +++++----- .../SynchronizerModuleTests.cs | 2 +- .../TestSyncConfig.cs | 17 ++++++ .../ParallelSync/SyncFeed.cs | 1 - .../Peers/SyncPeerAllocation.cs | 9 +-- .../Peers/SyncPeerPool.cs | 2 +- .../Synchronizer.cs | 14 +++++ 29 files changed, 190 insertions(+), 100 deletions(-) create mode 100644 src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs create mode 100644 src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs index f2dc73eb650..4716b0560b9 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs @@ -161,4 +161,7 @@ public interface ISyncConfig : IConfig [ConfigItem(Description = "_Technical._ Min distance of state sync from best suggested header.", DefaultValue = "32", HiddenFromDocs = true)] int StateMinDistanceFromHead { get; set; } + + [ConfigItem(Description = "_Technical._ Run explicit GC after state sync finished.", DefaultValue = "true", HiddenFromDocs = true)] + bool GCOnStateSyncFinished { get; set; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs index 8e3cc5c125b..3e99020f751 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs @@ -72,6 +72,7 @@ public string? PivotHash public bool TrieHealing { get; set; } = true; public int StateMaxDistanceFromHead { get; set; } = 128; public int StateMinDistanceFromHead { get; set; } = 32; + public bool GCOnStateSyncFinished { get; set; } = true; public override string ToString() { diff --git a/src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs b/src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs new file mode 100644 index 00000000000..edee22a6939 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Blockchain; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using NonBlocking; + +namespace Nethermind.Core.Test; + +public class CachedBlockTreeBuilder +{ + private static ConcurrentDictionary _cachedDbs = new(); + private record CachedDb( + MemDb blocksDb, + MemDb metadataDb, + MemDb headersDb, + MemDb blockNumbersDb, + MemDb blockInfo + ); + + public static IBlockTree BuildCached(string key, Func blockTreeBuilderFactory) + { + if (_cachedDbs.TryGetValue(key, out CachedDb? db)) + { + return Build.A.BlockTree() + .WithBlocksDb(MemDb.CopyFrom(db.blocksDb)) + .WithMetadataDb(MemDb.CopyFrom(db.metadataDb)) + .WithHeadersDb(MemDb.CopyFrom(db.headersDb)) + .WithBlocksNumberDb(MemDb.CopyFrom(db.blockNumbersDb)) + .WithBlockInfoDb(MemDb.CopyFrom(db.blockInfo)) + .TestObject; + } + else + { + BlockTreeBuilder builder = blockTreeBuilderFactory(); + CachedDb cachedValue = new CachedDb( + MemDb.CopyFrom(builder.BlocksDb), + MemDb.CopyFrom(builder.MetadataDb), + MemDb.CopyFrom(builder.HeadersDb), + MemDb.CopyFrom(builder.BlockNumbersDb), + MemDb.CopyFrom(builder.BlockInfoDb) + ); + _cachedDbs.TryAdd(key, cachedValue); + + return builder.TestObject; + } + } + + public static IBlockTree OfLength(int length) + { + return BuildCached($"{nameof(CachedBlockTreeBuilder)}-{length}", () => Build.A.BlockTree().OfChainLength(length)); + } +} diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index aaad49b326e..43f759b8320 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -27,6 +27,17 @@ public MemDb(string name) Name = name; } + public static MemDb CopyFrom(IDb anotherDb) + { + MemDb newDb = new MemDb(); + foreach (KeyValuePair kv in anotherDb.GetAll()) + { + newDb[kv.Key] = kv.Value; + } + + return newDb; + } + public MemDb() : this(0, 0) { } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs index 4638459df26..886cf38112c 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs @@ -276,7 +276,7 @@ public async Task Does_not_deadlock_on_replace_peer() peerAllocationStrategy .Allocate(Arg.Any(), Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new PeerInfo(syncPeer1)); - SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks); + SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks, null); peerAllocation.AllocateBestPeer(new List(), Substitute.For(), ctx.BlockTree); ctx.PeerPool .Allocate(Arg.Any(), Arg.Any(), Arg.Any()) @@ -332,7 +332,7 @@ public void No_old_bodies_and_receipts() BlockTreeScenario = blockTrees, }; - ctx.Feed = new FastSyncFeed(new SyncConfig + ctx.Feed = new FastSyncFeed(new TestSyncConfig { NonValidatorNode = true, DownloadBodiesInFastSync = false, @@ -349,7 +349,7 @@ public void No_old_bodies_and_receipts() peerAllocationStrategy .Allocate(Arg.Any(), Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(peerInfo); - SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks); + SyncPeerAllocation peerAllocation = new(peerAllocationStrategy, AllocationContexts.Blocks, null); peerAllocation.AllocateBestPeer(new List(), Substitute.For(), ctx.BlockTree); ctx.PeerPool @@ -479,13 +479,13 @@ public MergeConfig MergeConfig public PoSSwitcher PosSwitcher => _posSwitcher ??= new( MergeConfig, - new SyncConfig(), + new TestSyncConfig(), MetadataDb, BlockTree, SpecProvider, new ChainSpec(), LimboLogs.Instance); - public BeaconPivot BeaconPivot => _beaconPivot ??= new(new SyncConfig(), MetadataDb, BlockTree, _posSwitcher!, LimboLogs.Instance); + public BeaconPivot BeaconPivot => _beaconPivot ??= new(new TestSyncConfig(), MetadataDb, BlockTree, _posSwitcher!, LimboLogs.Instance); protected override IBetterPeerStrategy BetterPeerStrategy => _betterPeerStrategy ??= new MergeBetterPeerStrategy(new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), PosSwitcher, BeaconPivot, LimboLogs.Instance); @@ -498,7 +498,7 @@ public IChainLevelHelper ChainLevelHelper _chainLevelHelper ??= new ChainLevelHelper( BlockTree, BeaconPivot, - new SyncConfig(), + new TestSyncConfig(), LimboLogs.Instance); set => _chainLevelHelper = value; } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs index 1f3929efe6c..dcd985cd73b 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs @@ -38,6 +38,7 @@ using NUnit.Framework; using BlockTree = Nethermind.Blockchain.BlockTree; using System.Diagnostics.CodeAnalysis; +using Nethermind.Core.Test; namespace Nethermind.Synchronization.Test; @@ -108,7 +109,7 @@ public async Task Ancestor_lookup_simple() { Context ctx = new() { - BlockTree = Build.A.BlockTree().OfChainLength(1024).TestObject, + BlockTree = CachedBlockTreeBuilder.OfLength(1024), }; BlockDownloader downloader = ctx.BlockDownloader; @@ -139,7 +140,7 @@ public async Task Ancestor_lookup_headers() { Context ctx = new() { - BlockTree = Build.A.BlockTree().OfChainLength(1024).TestObject, + BlockTree = CachedBlockTreeBuilder.OfLength(1024), }; BlockDownloader downloader = ctx.BlockDownloader; @@ -168,7 +169,7 @@ public void Ancestor_failure() { Context ctx = new() { - BlockTree = Build.A.BlockTree().OfChainLength(2048 + 1).TestObject, + BlockTree = CachedBlockTreeBuilder.OfLength(2048 + 1), }; BlockDownloader downloader = ctx.BlockDownloader; @@ -186,7 +187,7 @@ public void Ancestor_failure_blocks() { Context ctx = new() { - BlockTree = Build.A.BlockTree().OfChainLength(2048 + 1).TestObject, + BlockTree = CachedBlockTreeBuilder.OfLength(2048 + 1), }; BlockDownloader downloader = ctx.BlockDownloader; @@ -1024,11 +1025,9 @@ public IBlockValidator BlockValidator private SyncDispatcher? _dispatcher; public SyncDispatcher Dispatcher => _dispatcher ??= new SyncDispatcher( - new SyncConfig() + new TestSyncConfig() { MaxProcessingThreads = 0, - SyncDispatcherEmptyRequestDelayMs = 1, - SyncDispatcherAllocateTimeoutMs = 1 }, Feed!, BlockDownloader, @@ -1062,7 +1061,7 @@ private class SyncPeerMock : ISyncPeer private readonly ReceiptsMessageSerializer _receiptsSerializer = new(MainnetSpecProvider.Instance); private readonly Response _flags; - public BlockTree BlockTree { get; private set; } = null!; + public IBlockTree BlockTree { get; private set; } = null!; private IReceiptStorage _receiptStorage = new InMemoryReceiptStorage(); public string Name => "Mock"; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs index 5fbc4f1d977..2eaaecf493b 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs @@ -54,7 +54,7 @@ public void Setup() _pivotBlock = _syncingFromBlockTree.FindBlock(99, BlockTreeLookupOptions.None)!; - _syncConfig = new SyncConfig() + _syncConfig = new TestSyncConfig() { FastSync = true, PivotHash = _pivotBlock.Hash!.ToString(), diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs index 7f9d9e93c3b..2f7277c0af9 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs @@ -14,6 +14,7 @@ using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; +using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; using Nethermind.Logging; using Nethermind.Stats.Model; @@ -40,7 +41,7 @@ public Task Will_fail_if_launched_without_fast_blocks_enabled() HeadersSyncFeed _ = new HeadersSyncFeed( blockTree: blockTree, syncPeerPool: Substitute.For(), - syncConfig: new SyncConfig(), + syncConfig: new TestSyncConfig(), syncReport: Substitute.For(), logManager: LimboLogs.Instance); }); @@ -56,7 +57,7 @@ public async Task Can_prepare_3_requests_in_a_row() HeadersSyncFeed feed = new( blockTree: blockTree, syncPeerPool: Substitute.For(), - syncConfig: new SyncConfig + syncConfig: new TestSyncConfig { FastSync = true, PivotNumber = "1000", @@ -89,7 +90,7 @@ public async Task When_next_header_hash_update_is_delayed_do_not_drop_peer() ResettableHeaderSyncFeed feed = new( blockTree: blockTree, syncPeerPool: syncPeerPool, - syncConfig: new SyncConfig + syncConfig: new TestSyncConfig { FastSync = true, PivotNumber = "1000", @@ -149,7 +150,7 @@ public async Task Can_prepare_several_request_and_ignore_request_from_previous_s using ResettableHeaderSyncFeed feed = new( blockTree: blockTree, syncPeerPool: Substitute.For(), - syncConfig: new SyncConfig + syncConfig: new TestSyncConfig { FastSync = true, PivotNumber = "500", @@ -196,7 +197,7 @@ public async Task Will_dispatch_when_only_partially_processed_dependency() using HeadersSyncFeed feed = new( blockTree: blockTree, syncPeerPool: Substitute.For(), - syncConfig: new SyncConfig + syncConfig: new TestSyncConfig { FastSync = true, PivotNumber = pivot.Number.ToString(), @@ -267,7 +268,7 @@ public async Task Can_reset_and_not_hang_when_a_batch_is_processing() ResettableHeaderSyncFeed feed = new( blockTree: blockTree, syncPeerPool: Substitute.For(), - syncConfig: new SyncConfig + syncConfig: new TestSyncConfig { FastSync = true, PivotNumber = "500", @@ -321,7 +322,7 @@ public async Task Can_keep_returning_nulls_after_all_batches_were_prepared() HeadersSyncFeed feed = new( blockTree: blockTree, syncPeerPool: Substitute.For(), - syncConfig: new SyncConfig + syncConfig: new TestSyncConfig { FastSync = true, PivotNumber = "1000", @@ -349,7 +350,7 @@ public async Task Finishes_when_all_downloaded() report.HeadersInQueue.Returns(new MeasuredProgress()); MeasuredProgress measuredProgress = new(); report.FastBlocksHeaders.Returns(measuredProgress); - HeadersSyncFeed feed = new(blockTree, Substitute.For(), new SyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance); + HeadersSyncFeed feed = new(blockTree, Substitute.For(), new TestSyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance); await feed.PrepareRequest(); blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).TestObject); using HeadersSyncBatch? result = await feed.PrepareRequest(); @@ -372,7 +373,7 @@ public async Task Can_resume_downloading_from_parent_of_lowest_inserted_header() report.HeadersInQueue.Returns(new MeasuredProgress()); report.FastBlocksHeaders.Returns(new MeasuredProgress()); - HeadersSyncFeed feed = new(blockTree, Substitute.For(), new SyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance); + HeadersSyncFeed feed = new(blockTree, Substitute.For(), new TestSyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }, report, LimboLogs.Instance); feed.InitializeFeed(); using HeadersSyncBatch? result = await feed.PrepareRequest(); @@ -406,8 +407,8 @@ public async Task Can_resume_downloading_from_parent_of_lowest_inserted_header() [TestCase(0, 192, 1, false, true)] public async Task Can_insert_all_good_headers_from_dependent_batch_with_missing_or_null_headers(int nullIndex, int count, int increment, bool shouldReport, bool useNulls) { - var peerChain = Build.A.BlockTree().OfChainLength(1000).TestObject; - var syncConfig = new SyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }; + var peerChain = CachedBlockTreeBuilder.OfLength(1000); + var syncConfig = new TestSyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }; IBlockTree localBlockTree = Build.A.BlockTree(peerChain.FindBlock(0, BlockTreeLookupOptions.None)!, null).WithSyncConfig(syncConfig).TestObject; const int lowestInserted = 999; @@ -472,7 +473,7 @@ public async Task Will_never_lose_batch_on_invalid_batch() HeadersSyncFeed feed = new( blockTree, Substitute.For(), - new SyncConfig + new TestSyncConfig { FastSync = true, PivotNumber = "1000", @@ -538,7 +539,7 @@ public async Task Will_never_lose_batch_on_invalid_batch() public void IsFinished_returns_false_when_headers_not_downloaded() { IBlockTree blockTree = Substitute.For(); - SyncConfig syncConfig = new() + TestSyncConfig syncConfig = new() { FastSync = true, DownloadBodiesInFastSync = true, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs index b8d065984e9..3ba80233122 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs @@ -100,7 +100,7 @@ public void Setup() _blockTree = Substitute.For(); _metadataDb = new TestMemDb(); - _syncConfig = new SyncConfig { FastSync = true }; + _syncConfig = new TestSyncConfig { FastSync = true }; _syncConfig.PivotNumber = _pivotNumber.ToString(); _syncConfig.PivotHash = Keccak.Zero.ToString(); @@ -140,7 +140,7 @@ private ReceiptsSyncFeed CreateFeed() [Test] public void Should_throw_when_fast_block_not_enabled() { - _syncConfig = new SyncConfig { FastSync = false }; + _syncConfig = new TestSyncConfig { FastSync = false }; Assert.Throws( () => _feed = new ReceiptsSyncFeed( _specProvider, @@ -408,7 +408,7 @@ public async Task Can_sync_final_batch() public void Is_fast_block_receipts_finished_returns_false_when_receipts_not_downloaded() { _blockTree = Substitute.For(); - _syncConfig = new SyncConfig() + _syncConfig = new TestSyncConfig() { FastSync = true, DownloadBodiesInFastSync = true, @@ -430,7 +430,7 @@ public void Is_fast_block_receipts_finished_returns_false_when_receipts_not_down public void Is_fast_block_bodies_finished_returns_true_when_bodies_not_downloaded_and_we_do_not_want_to_download_bodies() { _blockTree = Substitute.For(); - _syncConfig = new SyncConfig() + _syncConfig = new TestSyncConfig() { FastSync = true, DownloadBodiesInFastSync = false, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/SnapProtocolTests/StateSyncDispatcherTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/SnapProtocolTests/StateSyncDispatcherTests.cs index 80a7a517763..00260452ef6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/SnapProtocolTests/StateSyncDispatcherTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/SnapProtocolTests/StateSyncDispatcherTests.cs @@ -18,6 +18,7 @@ using Nethermind.Core.Crypto; using System.Net; using FluentAssertions; +using Nethermind.Core.Test; using Nethermind.Trie; namespace Nethermind.Synchronization.Test.FastSync.SnapProtocolTests; @@ -43,7 +44,7 @@ public void Setup() { _logManager = LimboLogs.Instance; - BlockTree blockTree = Build.A.BlockTree().OfChainLength((int)BlockTree.BestSuggestedHeader!.Number).TestObject; + IBlockTree blockTree = CachedBlockTreeBuilder.OfLength((int)BlockTree.BestSuggestedHeader!.Number); ITimerFactory timerFactory = Substitute.For(); _pool = new SyncPeerPool(blockTree, new NodeStatsManager(timerFactory, LimboLogs.Instance), new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), LimboLogs.Instance, 25); _pool.Start(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTestsBase.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTestsBase.cs index 59ae28883b5..43a9b7d42f5 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTestsBase.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTestsBase.cs @@ -33,8 +33,6 @@ using NSubstitute; using NUnit.Framework; -using BlockTree = Nethermind.Blockchain.BlockTree; - namespace Nethermind.Synchronization.Test.FastSync { public class StateSyncFeedTestsBase @@ -113,9 +111,8 @@ protected IContainer PrepareDownloader(DbContext dbContext, Action protected ContainerBuilder BuildTestContainerBuilder(DbContext dbContext, int syncDispatcherAllocateTimeoutMs = 10) { ContainerBuilder containerBuilder = new ContainerBuilder() - .AddModule(new TestSynchronizerModule(new SyncConfig() + .AddModule(new TestSynchronizerModule(new TestSyncConfig() { - SyncDispatcherEmptyRequestDelayMs = 1, SyncDispatcherAllocateTimeoutMs = syncDispatcherAllocateTimeoutMs, // there is a test for requested nodes which get affected if allocate timeout FastSync = true })) @@ -125,7 +122,9 @@ protected ContainerBuilder BuildTestContainerBuilder(DbContext dbContext, int sy .AddSingleton(dbContext.LocalNodeStorage) // Use factory function to make it lazy in case test need to replace IBlockTree - .AddSingleton((ctx) => Build.A.BlockTree().WithStateRoot(dbContext.RemoteStateTree.RootHash).OfChainLength((int)BlockTree.BestSuggestedHeader!.Number).TestObject) + .AddSingleton((ctx) => CachedBlockTreeBuilder.BuildCached( + $"{nameof(StateSyncFeedTestsBase)}{dbContext.RemoteStateTree.RootHash}{BlockTree.BestSuggestedHeader!.Number}", + () => Build.A.BlockTree().WithStateRoot(dbContext.RemoteStateTree.RootHash).OfChainLength((int)BlockTree.BestSuggestedHeader!.Number))) .Add(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index 150ae4dde4f..9b4d5721819 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -55,12 +55,7 @@ public async Task Setup() ITimerFactory timerFactory = Substitute.For(); NodeStatsManager stats = new(timerFactory, LimboLogs.Instance); - SyncConfig syncConfig = new() - { - MultiSyncModeSelectorLoopTimerMs = 1, - SyncDispatcherEmptyRequestDelayMs = 1, - SyncDispatcherAllocateTimeoutMs = 1 - }; + SyncConfig syncConfig = new TestSyncConfig(); NodeStorage nodeStorage = new NodeStorage(_stateDb); TrieStore trieStore = new(nodeStorage, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorFastSyncTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorFastSyncTests.cs index 8e677c9e2db..bda40539303 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorFastSyncTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorFastSyncTests.cs @@ -621,7 +621,7 @@ public void Switch_correctly_from_full_sync_to_state_nodes_catch_up() syncPeerPool.InitializedPeers.Returns(peerInfos); syncPeerPool.AllPeers.Returns(peerInfos); - ISyncConfig syncConfig = new SyncConfig() { FastSyncCatchUpHeightDelta = 2 }; + ISyncConfig syncConfig = new TestSyncConfig() { FastSyncCatchUpHeightDelta = 2 }; syncConfig.FastSync = true; TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); @@ -671,7 +671,7 @@ public void Changed_event_no_longer_gets_blocked_when_invoking_delegates() syncPeerPool.InitializedPeers.Returns(peerInfos); syncPeerPool.AllPeers.Returns(peerInfos); - ISyncConfig syncConfig = new SyncConfig + ISyncConfig syncConfig = new TestSyncConfig { FastSyncCatchUpHeightDelta = 2, FastSync = true, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs index 87ab10fe7bd..8e94ff818b5 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs @@ -150,7 +150,7 @@ public class ScenarioBuilder public ISyncProgressResolver SyncProgressResolver { get; set; } = null!; - public ISyncConfig SyncConfig { get; } = new SyncConfig(); + public ISyncConfig SyncConfig { get; } = new TestSyncConfig(); public IBeaconSyncStrategy BeaconSyncStrategy { get; set; } = No.BeaconSync; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/SyncDispatcherTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/SyncDispatcherTests.cs index 52af5db10bc..a842b0329f6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/SyncDispatcherTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/SyncDispatcherTests.cs @@ -255,12 +255,7 @@ public async Task Simple_test_sync() TestSyncFeed syncFeed = new(); TestDownloader downloader = new TestDownloader(); SyncDispatcher dispatcher = new( - new SyncConfig() - { - MultiSyncModeSelectorLoopTimerMs = 1, - SyncDispatcherEmptyRequestDelayMs = 1, - SyncDispatcherAllocateTimeoutMs = 1 - }, + new TestSyncConfig(), syncFeed, downloader, new TestSyncPeerPool(), @@ -287,10 +282,9 @@ public async Task Test_release_before_processing_complete(bool isMultiSync, int TestDownloader downloader = new TestDownloader(); SyncDispatcher dispatcher = new( - new SyncConfig() + new TestSyncConfig() { MaxProcessingThreads = processingThread, - SyncDispatcherEmptyRequestDelayMs = 1, }, syncFeed, downloader, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs index eef137e816f..e709fa75f92 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs @@ -56,7 +56,7 @@ public void Setup() _pivotBlock = _syncingFromBlockTree.FindBlock(99, BlockTreeLookupOptions.None)!; - _syncConfig = new SyncConfig() + _syncConfig = new TestSyncConfig() { FastSync = true, PivotHash = _pivotBlock.Hash!.ToString(), diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs index 216f970cf18..bbb039b8402 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs @@ -146,7 +146,7 @@ public void Will_mark_progress_and_flush_when_finished() .WithStateRoot(Keccak.EmptyTreeHash) .TestObject).TestObject; TestMemDb memDb = new(); - SyncConfig syncConfig = new SyncConfig() { SnapSyncAccountRangePartitionCount = 1 }; + SyncConfig syncConfig = new TestSyncConfig() { SnapSyncAccountRangePartitionCount = 1 }; using ProgressTracker progressTracker = new(memDb, syncConfig, new StateSyncPivot(blockTree, syncConfig, LimboLogs.Instance), LimboLogs.Instance); progressTracker.IsFinished(out SnapSyncBatch? request); @@ -164,7 +164,7 @@ public void Will_mark_progress_and_flush_when_finished() private ProgressTracker CreateProgressTracker(int accountRangePartition = 1) { BlockTree blockTree = Build.A.BlockTree().WithBlocks(Build.A.Block.WithStateRoot(Keccak.EmptyTreeHash).TestObject).TestObject; - SyncConfig syncConfig = new SyncConfig() { SnapSyncAccountRangePartitionCount = accountRangePartition }; + SyncConfig syncConfig = new TestSyncConfig() { SnapSyncAccountRangePartitionCount = accountRangePartition }; return new(new MemDb(), syncConfig, new StateSyncPivot(blockTree, syncConfig, LimboLogs.Instance), LimboLogs.Instance); } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs index 5cadd953f6c..5e74e84aca6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromAccountRangesTests.cs @@ -113,7 +113,7 @@ public void RecreateAccountStateFromOneRangeWithNonExistenceProof() byte[][] firstProof = CreateProofForPath(Keccak.Zero.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); @@ -132,7 +132,7 @@ public void RecreateAccountStateFromOneRangeWithExistenceProof() byte[][] firstProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[0].Path.Bytes); byte[][] lastProof = CreateProofForPath(TestItem.Tree.AccountsWithPaths[5].Path.Bytes); - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); @@ -148,7 +148,7 @@ public void RecreateAccountStateFromOneRangeWithoutProof() { Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); @@ -165,7 +165,7 @@ public void RecreateAccountStateFromMultipleRange() Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" // output state - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); @@ -201,7 +201,7 @@ public void RecreateAccountStateFromMultipleRange_InReverseOrder() Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" // output state - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); @@ -234,7 +234,7 @@ public void RecreateAccountStateFromMultipleRange_OutOfOrder() Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" // output state - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); @@ -267,7 +267,7 @@ public void RecreateAccountStateFromMultipleOverlappingRange() Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" // output state - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); @@ -390,7 +390,7 @@ public void MissingAccountFromRange() Hash256 rootHash = _inputTree.RootHash; // "0x8c81279168edc449089449bc0f2136fc72c9645642845755633cf259cd97988b" // output state - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); IDb db = container.ResolveKeyed(DbNames.State); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs index e5f6f90e3fe..043951df9d2 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/RecreateStateFromStorageRangesTests.cs @@ -51,7 +51,7 @@ public void RecreateStorageStateFromOneRangeWithNonExistenceProof() _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); var proof = accountProofCollector.BuildResult(); - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); var result = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); @@ -68,7 +68,7 @@ public void RecreateAccountStateFromOneRangeWithExistenceProof() _inputStateTree!.Accept(accountProofCollector, _inputStateTree.RootHash); var proof = accountProofCollector.BuildResult(); - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); var result = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray()); @@ -81,7 +81,7 @@ public void RecreateStorageStateFromOneRangeWithoutProof() { Hash256 rootHash = _inputStorageTree!.RootHash; // "..." - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); var result = snapProvider.AddStorageRange(1, _pathWithAccount, rootHash, TestItem.Tree.SlotsWithPaths[0].Path, TestItem.Tree.SlotsWithPaths); @@ -95,7 +95,7 @@ public void RecreateAccountStateFromMultipleRange() Hash256 rootHash = _inputStorageTree!.RootHash; // "..." // output state - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); @@ -127,7 +127,7 @@ public void MissingAccountFromRange() Hash256 rootHash = _inputStorageTree!.RootHash; // "..." // output state - using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new SyncConfig())).Build(); + using IContainer container = new ContainerBuilder().AddModule(new TestSynchronizerModule(new TestSyncConfig())).Build(); SnapProvider snapProvider = container.Resolve(); AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path }); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs index 883875647ff..38d5fe2e231 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs @@ -37,7 +37,7 @@ public class SnapProviderTests public void AddAccountRange_AccountListIsEmpty_ThrowArgumentException() { using IContainer container = new ContainerBuilder() - .AddModule(new TestSynchronizerModule(new SyncConfig())) + .AddModule(new TestSynchronizerModule(new TestSyncConfig())) .Build(); SnapProvider snapProvider = container.Resolve(); @@ -55,7 +55,7 @@ public void AddAccountRange_AccountListIsEmpty_ThrowArgumentException() public void AddAccountRange_ResponseHasEmptyListOfAccountsAndOneProof_ReturnsExpiredRootHash() { using IContainer container = new ContainerBuilder() - .AddModule(new TestSynchronizerModule(new SyncConfig())) + .AddModule(new TestSynchronizerModule(new TestSyncConfig())) .Build(); SnapProvider snapProvider = container.Resolve(); @@ -85,7 +85,7 @@ public void AddAccountRange_SetStartRange_ToAfterLastPath() (SnapServer ss, Hash256 root) = BuildSnapServerFromEntries(entries); using IContainer container = new ContainerBuilder() - .AddModule(new TestSynchronizerModule(new SyncConfig() + .AddModule(new TestSynchronizerModule(new TestSyncConfig() { SnapSyncAccountRangePartitionCount = 1 })) @@ -127,7 +127,7 @@ public void AddAccountRange_ShouldNotStoreStorageAfterLimit() (SnapServer ss, Hash256 root) = BuildSnapServerFromEntries(entries); using IContainer container = new ContainerBuilder() - .AddModule(new TestSynchronizerModule(new SyncConfig() + .AddModule(new TestSynchronizerModule(new TestSyncConfig() { SnapSyncAccountRangePartitionCount = 2 })) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs index edbdfb04fc0..951fe44fc9d 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs @@ -41,7 +41,7 @@ private Context CreateContext(ILastNStateRootTracker? stateRootTracker = null) SnapServer server = new(store.AsReadOnly(), codeDbServer, stateRootTracker ?? CreateConstantStateRootTracker(true), LimboLogs.Instance); MemDb clientStateDb = new(); - using ProgressTracker progressTracker = new(clientStateDb, new SyncConfig(), new StateSyncPivot(null!, new SyncConfig(), LimboLogs.Instance), LimboLogs.Instance); + using ProgressTracker progressTracker = new(clientStateDb, new TestSyncConfig(), new StateSyncPivot(null!, new TestSyncConfig(), LimboLogs.Instance), LimboLogs.Instance); INodeStorage nodeStorage = new NodeStorage(clientStateDb); @@ -263,7 +263,7 @@ public void TestGetStorageRange() dbProviderClient.RegisterDb(DbNames.State, new MemDb()); dbProviderClient.RegisterDb(DbNames.Code, new MemDb()); - using ProgressTracker progressTracker = new(dbProviderClient.StateDb, new SyncConfig(), new StateSyncPivot(null!, new SyncConfig(), LimboLogs.Instance), LimboLogs.Instance); + using ProgressTracker progressTracker = new(dbProviderClient.StateDb, new TestSyncConfig(), new StateSyncPivot(null!, new TestSyncConfig(), LimboLogs.Instance), LimboLogs.Instance); SnapProvider snapProvider = new(progressTracker, dbProviderClient.CodeDb, new NodeStorage(dbProviderClient.StateDb), LimboLogs.Instance); (IOwnedReadOnlyList> storageSlots, IOwnedReadOnlyList proofs) = @@ -299,7 +299,7 @@ public void TestGetStorageRangeMulti() dbProviderClient.RegisterDb(DbNames.State, new MemDb()); dbProviderClient.RegisterDb(DbNames.Code, new MemDb()); - using ProgressTracker progressTracker = new(dbProviderClient.StateDb, new SyncConfig(), new StateSyncPivot(null!, new SyncConfig(), LimboLogs.Instance), LimboLogs.Instance); + using ProgressTracker progressTracker = new(dbProviderClient.StateDb, new TestSyncConfig(), new StateSyncPivot(null!, new TestSyncConfig(), LimboLogs.Instance), LimboLogs.Instance); SnapProvider snapProvider = new(progressTracker, dbProviderClient.CodeDb, new NodeStorage(dbProviderClient.StateDb), LimboLogs.Instance); Hash256 startRange = Keccak.Zero; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/StateSyncPivotTest.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/StateSyncPivotTest.cs index ce160cb1274..b455a35b1d1 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/StateSyncPivotTest.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/StateSyncPivotTest.cs @@ -30,7 +30,7 @@ int newBestHeader .Returns((ci) => Build.A.BlockHeader.WithNumber((long)ci[0]).TestObject); Synchronization.FastSync.StateSyncPivot stateSyncPivot = new Synchronization.FastSync.StateSyncPivot(blockTree, - new SyncConfig() + new TestSyncConfig() { StateMinDistanceFromHead = minDistance, StateMaxDistanceFromHead = maxDistance, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs index 8ca63c93f95..c17070946b6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs @@ -99,7 +99,7 @@ public void Can_accept_new_valid_blocks(bool sealOk, bool validationOk, bool acc sealValidator, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -141,7 +141,7 @@ public void Can_accept_blocks_that_are_fine() Always.Valid, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -178,7 +178,7 @@ public void Should_accept_or_not_blocks_depends_on_sync_mode(SyncMode syncMode, Always.Valid, ctx.PeerPool, staticSelector, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -211,7 +211,7 @@ public void Terminal_block_with_lower_td_should_not_change_best_suggested_but_sh { TerminalTotalDifficulty = $"{testSpecProvider.TerminalTotalDifficulty}" }, - new SyncConfig(), + new TestSyncConfig(), new MemDb(), localBlockTree, testSpecProvider, @@ -246,7 +246,7 @@ public void Terminal_block_with_lower_td_should_not_change_best_suggested_but_sh Always.Valid, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, testSpecProvider, LimboLogs.Instance); @@ -420,7 +420,7 @@ private Context CreateMergeContext(int blockTreeChainLength, UInt256 ttd) { TerminalTotalDifficulty = $"{ttd}" }, - new SyncConfig(), + new TestSyncConfig(), new MemDb(), localBlockTree, testSpecProvider, @@ -469,7 +469,7 @@ private Context CreateMergeContext(int blockTreeChainLength, UInt256 ttd) sealEngine, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, testSpecProvider, LimboLogs.Instance); @@ -509,7 +509,7 @@ public void Will_not_reject_block_with_bad_total_diff_but_will_reset_diff_to_nul Always.Valid, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -541,7 +541,7 @@ public void Rejects_new_old_blocks() sealValidator, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -568,7 +568,7 @@ public async Task Broadcast_NewBlock_on_arrival() Always.Valid, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -604,7 +604,7 @@ public async Task Skip_known_block() Always.Valid, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -645,7 +645,7 @@ public async Task Broadcast_NewBlock_on_arrival_to_sqrt_of_peers([Values(1, 2, 3 Always.Valid, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -683,7 +683,7 @@ public void GetNodeData_returns_cached_trie_nodes() sealValidator, ctx.PeerPool, StaticSelector.Full, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -723,7 +723,7 @@ public Context() Always.Valid, PeerPool, selector, - new SyncConfig(), + new TestSyncConfig(), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs index cb4d9e62d99..10e0481d5fe 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs @@ -29,7 +29,7 @@ public IContainer CreateTestContainer() IBlockProcessingQueue blockQueue = Substitute.For(); return new ContainerBuilder() - .AddModule(new SynchronizerModule(new SyncConfig() + .AddModule(new SynchronizerModule(new TestSyncConfig() { FastSync = true, VerifyTrieOnStateSyncFinished = true diff --git a/src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs b/src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs new file mode 100644 index 00000000000..49c02ff6d2a --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain.Synchronization; + +namespace Nethermind.Synchronization.Test; + +public class TestSyncConfig : SyncConfig +{ + public TestSyncConfig() + { + GCOnStateSyncFinished = false; + MultiSyncModeSelectorLoopTimerMs = 1; + SyncDispatcherEmptyRequestDelayMs = 1; + SyncDispatcherAllocateTimeoutMs = 1; + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs index 5bbee478f58..d645646e527 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncFeed.cs @@ -44,7 +44,6 @@ private void ChangeState(SyncFeedState newState) public virtual void Finish() { ChangeState(SyncFeedState.Finished); - GC.Collect(2, GCCollectionMode.Aggressive, true, true); } public Task FeedTask => _taskCompletionSource?.Task ?? Task.CompletedTask; public abstract void SyncModeSelectorOnChanged(SyncMode current); diff --git a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs index cfb7c3380f6..0e2c80e9990 100644 --- a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs +++ b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs @@ -13,12 +13,12 @@ namespace Nethermind.Synchronization.Peers { public class SyncPeerAllocation { - public static SyncPeerAllocation FailedAllocation = new(NullStrategy.Instance, AllocationContexts.None); + public static SyncPeerAllocation FailedAllocation = new(NullStrategy.Instance, AllocationContexts.None, null); /// /// this should be used whenever we change IsAllocated property on PeerInfo- /// - private static readonly Lock _allocationLock = new(); + private readonly Lock? _allocationLock; private readonly IPeerAllocationStrategy _peerAllocationStrategy; public AllocationContexts Contexts { get; } @@ -29,14 +29,15 @@ public class SyncPeerAllocation public bool HasPeer => Current is not null; public SyncPeerAllocation(PeerInfo peerInfo, AllocationContexts contexts) - : this(new StaticStrategy(peerInfo), contexts) + : this(new StaticStrategy(peerInfo), contexts, null) { } - public SyncPeerAllocation(IPeerAllocationStrategy peerAllocationStrategy, AllocationContexts contexts) + public SyncPeerAllocation(IPeerAllocationStrategy peerAllocationStrategy, AllocationContexts contexts, Lock? allocationLock) { _peerAllocationStrategy = peerAllocationStrategy; Contexts = contexts; + _allocationLock = allocationLock ?? new Lock(); } public void AllocateBestPeer( diff --git a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerPool.cs b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerPool.cs index 263617effbe..9d85a746ca3 100644 --- a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerPool.cs +++ b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerPool.cs @@ -328,7 +328,7 @@ public async Task Allocate( using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _refreshLoopCancellation.Token); - SyncPeerAllocation allocation = new(peerAllocationStrategy, allocationContexts); + SyncPeerAllocation allocation = new(peerAllocationStrategy, allocationContexts, _isAllocatedChecks); while (true) { if (TryAllocateOnce(peerAllocationStrategy, allocationContexts, allocation)) diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 4aa227df376..410697ade01 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -90,10 +90,24 @@ public virtual void Start() SyncModeSelector.Changed += syncReport.SyncModeSelectorOnChanged; + if (syncConfig.GCOnStateSyncFinished) + { + SyncModeSelector.Changed += GCOnStateSyncFinished; + } + // Make unit test faster. SyncModeSelector.Update(); } + private void GCOnStateSyncFinished(object? sender, SyncModeChangedEventArgs e) + { + // State nodes finished + if ((e.Previous & SyncMode.StateNodes) != 0 && (e.Current & SyncMode.StateNodes) == 0) + { + GC.Collect(2, GCCollectionMode.Aggressive, true, true); + } + } + private void StartFullSyncComponents() { fullSyncComponent.BlockDownloader.SyncEvent += DownloaderOnSyncEvent; From 6356105b2bfef9a2f4096bee1b54cd3388deb47b Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 29 Nov 2024 08:10:52 +0800 Subject: [PATCH 2/3] Update src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs Co-authored-by: Lukasz Rozmej --- .../Nethermind.Synchronization/Peers/SyncPeerAllocation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs index 0e2c80e9990..e605f00643b 100644 --- a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs +++ b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeerAllocation.cs @@ -33,7 +33,7 @@ public SyncPeerAllocation(PeerInfo peerInfo, AllocationContexts contexts) { } - public SyncPeerAllocation(IPeerAllocationStrategy peerAllocationStrategy, AllocationContexts contexts, Lock? allocationLock) + public SyncPeerAllocation(IPeerAllocationStrategy peerAllocationStrategy, AllocationContexts contexts, Lock? allocationLock = null) { _peerAllocationStrategy = peerAllocationStrategy; Contexts = contexts; From 926ac8853f4e4751bb1eb5bd1c29e0a9cc63433f Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Mon, 2 Dec 2024 09:50:45 +0800 Subject: [PATCH 3/3] Addressing comment --- .../Nethermind.Blockchain/Synchronization/ISyncConfig.cs | 2 +- .../Nethermind.Blockchain/Synchronization/SyncConfig.cs | 2 +- .../Nethermind.Core.Test/CachedBlockTreeBuilder.cs | 2 +- .../Nethermind.Synchronization.Test/TestSyncConfig.cs | 2 +- .../ParallelSync/SyncModeChangedEventArgs.cs | 5 +++++ .../Nethermind.Synchronization/Synchronizer.cs | 9 ++++----- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs index 4716b0560b9..4694d5ead47 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs @@ -163,5 +163,5 @@ public interface ISyncConfig : IConfig int StateMinDistanceFromHead { get; set; } [ConfigItem(Description = "_Technical._ Run explicit GC after state sync finished.", DefaultValue = "true", HiddenFromDocs = true)] - bool GCOnStateSyncFinished { get; set; } + bool GCOnFeedFinished { get; set; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs index 3e99020f751..6ce0c295c5e 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs @@ -72,7 +72,7 @@ public string? PivotHash public bool TrieHealing { get; set; } = true; public int StateMaxDistanceFromHead { get; set; } = 128; public int StateMinDistanceFromHead { get; set; } = 32; - public bool GCOnStateSyncFinished { get; set; } = true; + public bool GCOnFeedFinished { get; set; } = true; public override string ToString() { diff --git a/src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs b/src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs index edee22a6939..143f360b202 100644 --- a/src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs @@ -42,7 +42,7 @@ public static IBlockTree BuildCached(string key, Func blockTre MemDb.CopyFrom(builder.BlockNumbersDb), MemDb.CopyFrom(builder.BlockInfoDb) ); - _cachedDbs.TryAdd(key, cachedValue); + _cachedDbs.GetOrAdd(key, cachedValue); return builder.TestObject; } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs b/src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs index 49c02ff6d2a..27141b0d4a1 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/TestSyncConfig.cs @@ -9,7 +9,7 @@ public class TestSyncConfig : SyncConfig { public TestSyncConfig() { - GCOnStateSyncFinished = false; + GCOnFeedFinished = false; MultiSyncModeSelectorLoopTimerMs = 1; SyncDispatcherEmptyRequestDelayMs = 1; SyncDispatcherAllocateTimeoutMs = 1; diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncModeChangedEventArgs.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncModeChangedEventArgs.cs index 916a8d2bf2d..f531004a4e8 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncModeChangedEventArgs.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncModeChangedEventArgs.cs @@ -15,5 +15,10 @@ public SyncModeChangedEventArgs(SyncMode previous, SyncMode current) public SyncMode Previous { get; } public SyncMode Current { get; } + + public bool WasModeFinished(SyncMode mode) + { + return (Previous & mode) != 0 && (Current & mode) == 0; + } } } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 410697ade01..1870d8df483 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -90,19 +90,18 @@ public virtual void Start() SyncModeSelector.Changed += syncReport.SyncModeSelectorOnChanged; - if (syncConfig.GCOnStateSyncFinished) + if (syncConfig.GCOnFeedFinished) { - SyncModeSelector.Changed += GCOnStateSyncFinished; + SyncModeSelector.Changed += GCOnFeedFinished; } // Make unit test faster. SyncModeSelector.Update(); } - private void GCOnStateSyncFinished(object? sender, SyncModeChangedEventArgs e) + private void GCOnFeedFinished(object? sender, SyncModeChangedEventArgs e) { - // State nodes finished - if ((e.Previous & SyncMode.StateNodes) != 0 && (e.Current & SyncMode.StateNodes) == 0) + if (e.WasModeFinished(SyncMode.StateNodes) || e.WasModeFinished(SyncMode.FastReceipts) || e.WasModeFinished(SyncMode.FastBlocks)) { GC.Collect(2, GCCollectionMode.Aggressive, true, true); }