Apply-load benchmark updates:

- Use the new function that doesn't access storage, but does emit events - Generate synthetic bucket list with configurable number of entries/ledgers - Tighten the resources estimation - Wait for merges in-between benchmark runs
stellar · Dec 10, 2024 · 9666c91 · 9666c91
1 parent 0241e79
commit 9666c91
Show file tree

Hide file tree

Showing 20 changed files with 848 additions and 357 deletions.
diff --git a/Builds/VisualStudio/stellar-core.vcxproj b/Builds/VisualStudio/stellar-core.vcxproj
@@ -481,20 +481,26 @@ exit /b 0
     <ClCompile Include="..\..\lib\spdlog.cpp" />
     <ClCompile Include="..\..\lib\tracy\public\TracyClient.cpp" />
     <ClCompile Include="..\..\lib\util\siphash.cpp" />
-    <ClCompile Include="..\..\src\bucket\Bucket.cpp" />
     <ClCompile Include="..\..\src\bucket\BucketApplicator.cpp" />
+    <ClCompile Include="..\..\src\bucket\BucketBase.cpp" />
     <ClCompile Include="..\..\src\bucket\BucketIndexImpl.cpp" />
     <ClCompile Include="..\..\src\bucket\BucketInputIterator.cpp" />
-    <ClCompile Include="..\..\src\bucket\BucketList.cpp" />
-    <ClCompile Include="..\..\src\bucket\BucketListSnapshot.cpp" />
-    <ClCompile Include="..\..\src\bucket\BucketManagerImpl.cpp" />
+    <ClCompile Include="..\..\src\bucket\BucketListBase.cpp" />
+    <ClCompile Include="..\..\src\bucket\BucketListSnapshotBase.cpp" />
+    <ClCompile Include="..\..\src\bucket\BucketManager.cpp" />
     <ClCompile Include="..\..\src\bucket\BucketMergeMap.cpp" />
     <ClCompile Include="..\..\src\bucket\BucketOutputIterator.cpp" />
     <ClCompile Include="..\..\src\bucket\BucketSnapshot.cpp" />
     <ClCompile Include="..\..\src\bucket\BucketSnapshotManager.cpp" />
+    <ClCompile Include="..\..\src\bucket\BucketUtils.cpp" />
     <ClCompile Include="..\..\src\bucket\FutureBucket.cpp" />
+    <ClCompile Include="..\..\src\bucket\HotArchiveBucket.cpp" />
+    <ClCompile Include="..\..\src\bucket\HotArchiveBucketList.cpp" />
+    <ClCompile Include="..\..\src\bucket\LiveBucket.cpp" />
+    <ClCompile Include="..\..\src\bucket\LiveBucketList.cpp" />
     <ClCompile Include="..\..\src\bucket\MergeKey.cpp" />
     <ClCompile Include="..\..\src\bucket\PublishQueueBuckets.cpp" />
+    <ClCompile Include="..\..\src\bucket\SearchableBucketList.cpp" />
     <ClCompile Include="..\..\src\bucket\test\BucketIndexTests.cpp" />
     <ClCompile Include="..\..\src\bucket\test\BucketListTests.cpp" />
     <ClCompile Include="..\..\src\bucket\test\BucketManagerTests.cpp" />
@@ -942,23 +948,28 @@ exit /b 0
     <ClInclude Include="..\..\lib\fmt\include\fmt\format.h" />
     <ClInclude Include="..\..\lib\util\siphash.h" />
     <ClInclude Include="..\..\lib\util\stdrandom.h" />
-    <ClInclude Include="..\..\src\bucket\Bucket.h" />
     <ClInclude Include="..\..\src\bucket\BucketApplicator.h" />
+    <ClInclude Include="..\..\src\bucket\BucketBase.h" />
     <ClInclude Include="..\..\src\bucket\BucketIndex.h" />
     <ClInclude Include="..\..\src\bucket\BucketIndexImpl.h" />
     <ClInclude Include="..\..\src\bucket\BucketInputIterator.h" />
-    <ClInclude Include="..\..\src\bucket\BucketList.h" />
-    <ClInclude Include="..\..\src\bucket\BucketListSnapshot.h" />
+    <ClInclude Include="..\..\src\bucket\BucketListBase.h" />
+    <ClInclude Include="..\..\src\bucket\BucketListSnapshotBase.h" />
     <ClInclude Include="..\..\src\bucket\BucketManager.h" />
-    <ClInclude Include="..\..\src\bucket\BucketManagerImpl.h" />
     <ClInclude Include="..\..\src\bucket\BucketMergeMap.h" />
     <ClInclude Include="..\..\src\bucket\BucketOutputIterator.h" />
     <ClInclude Include="..\..\src\bucket\BucketSnapshot.h" />
     <ClInclude Include="..\..\src\bucket\BucketSnapshotManager.h" />
+    <ClInclude Include="..\..\src\bucket\BucketUtils.h" />
     <ClInclude Include="..\..\src\bucket\FutureBucket.h" />
+    <ClInclude Include="..\..\src\bucket\HotArchiveBucket.h" />
+    <ClInclude Include="..\..\src\bucket\HotArchiveBucketList.h" />
     <ClInclude Include="..\..\src\bucket\LedgerCmp.h" />
+    <ClInclude Include="..\..\src\bucket\LiveBucket.h" />
+    <ClInclude Include="..\..\src\bucket\LiveBucketList.h" />
     <ClInclude Include="..\..\src\bucket\MergeKey.h" />
     <ClInclude Include="..\..\src\bucket\PublishQueueBuckets.h" />
+    <ClInclude Include="..\..\src\bucket\SearchableBucketList.h" />
     <ClInclude Include="..\..\src\bucket\test\BucketTestUtils.h" />
     <ClInclude Include="..\..\src\catchup\ApplyBucketsWork.h" />
     <ClInclude Include="..\..\src\catchup\ApplyBufferedLedgersWork.h" />

diff --git a/Builds/VisualStudio/stellar-core.vcxproj.filters b/Builds/VisualStudio/stellar-core.vcxproj.filters
@@ -576,9 +576,6 @@
     <ClCompile Include="..\..\src\bucket\test\BucketTestUtils.cpp">
       <Filter>bucket\tests</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\bucket\Bucket.cpp">
-      <Filter>bucket</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\src\bucket\BucketApplicator.cpp">
       <Filter>bucket</Filter>
     </ClCompile>
@@ -588,12 +585,6 @@
     <ClCompile Include="..\..\src\bucket\BucketInputIterator.cpp">
       <Filter>bucket</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\bucket\BucketList.cpp">
-      <Filter>bucket</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\src\bucket\BucketManagerImpl.cpp">
-      <Filter>bucket</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\src\bucket\BucketMergeMap.cpp">
       <Filter>bucket</Filter>
     </ClCompile>
@@ -1323,9 +1314,6 @@
     <ClCompile Include="..\..\src\main\SettingsUpgradeUtils.cpp">
       <Filter>main</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\bucket\BucketListSnapshot.cpp">
-      <Filter>bucket</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\src\bucket\BucketSnapshot.cpp">
       <Filter>bucket</Filter>
     </ClCompile>
@@ -1383,6 +1371,36 @@
     <ClCompile Include="..\..\src\history\CheckpointBuilder.cpp">
       <Filter>history</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\bucket\BucketBase.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\BucketListBase.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\BucketListSnapshotBase.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\HotArchiveBucket.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\HotArchiveBucketList.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\LiveBucket.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\LiveBucketList.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\SearchableBucketList.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\BucketManager.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\bucket\BucketUtils.cpp">
+      <Filter>bucket</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\lib\util\cpptoml.h">
@@ -1796,9 +1814,6 @@
     <ClInclude Include="..\..\src\bucket\test\BucketTestUtils.h">
       <Filter>bucket\tests</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\bucket\Bucket.h">
-      <Filter>bucket</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\src\bucket\BucketApplicator.h">
       <Filter>bucket</Filter>
     </ClInclude>
@@ -1811,15 +1826,9 @@
     <ClInclude Include="..\..\src\bucket\BucketInputIterator.h">
       <Filter>bucket</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\bucket\BucketList.h">
-      <Filter>bucket</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\src\bucket\BucketManager.h">
       <Filter>bucket</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\bucket\BucketManagerImpl.h">
-      <Filter>bucket</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\src\bucket\BucketMergeMap.h">
       <Filter>bucket</Filter>
     </ClInclude>
@@ -2357,9 +2366,6 @@
     <ClInclude Include="..\..\src\main\SettingsUpgradeUtils.h">
       <Filter>main</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\bucket\BucketListSnapshot.h">
-      <Filter>bucket</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\src\bucket\BucketSnapshot.h">
       <Filter>bucket</Filter>
     </ClInclude>
@@ -2416,6 +2422,33 @@
     <ClInclude Include="..\..\src\history\CheckpointBuilder.h">
       <Filter>history</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\bucket\BucketBase.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\BucketListBase.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\BucketListSnapshotBase.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\HotArchiveBucket.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\HotArchiveBucketList.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\LiveBucket.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\LiveBucketList.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\SearchableBucketList.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\bucket\BucketUtils.h">
+      <Filter>bucket</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\AUTHORS" />

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/bucket/BucketIndexImpl.cpp b/src/bucket/BucketIndexImpl.cpp
@@ -272,7 +272,7 @@ BucketIndexImpl<BucketIndex::RangeIndex>::saveToDisk(
 
     {
         OutputFileStream out(ctx, !bm.getConfig().DISABLE_XDR_FSYNC);
-        out.open(tmpFilename);
+        out.open(tmpFilename.string());
         cereal::BufferedAsioOutputArchive ar(out);
         ar(mData);
     }

diff --git a/src/history/HistoryManagerImpl.cpp b/src/history/HistoryManagerImpl.cpp
@@ -560,13 +560,13 @@ HistoryManagerImpl::getPublishQueueStates()
 {
     ZoneScoped;
     std::vector<HistoryArchiveState> states;
-    forEveryQueuedCheckpoint(publishQueuePath(mApp.getConfig()).string(),
-                             [&](uint32_t seq, std::string const& f) {
-                                 HistoryArchiveState has;
-                                 auto fullPath =
-                                     publishQueuePath(mApp.getConfig()) / f;
-                                 states.push_back(loadCheckpointHAS(fullPath));
-                             });
+    forEveryQueuedCheckpoint(
+        publishQueuePath(mApp.getConfig()).string(),
+        [&](uint32_t seq, std::string const& f) {
+            HistoryArchiveState has;
+            auto fullPath = publishQueuePath(mApp.getConfig()) / f;
+            states.push_back(loadCheckpointHAS(fullPath.string()));
+        });
     return states;
 }
 

diff --git a/src/ledger/LedgerManager.h b/src/ledger/LedgerManager.h
@@ -135,6 +135,7 @@ class LedgerManager
     virtual SorobanNetworkConfig& getMutableSorobanNetworkConfig() = 0;
     virtual std::vector<TransactionMetaFrame> const&
     getLastClosedLedgerTxMeta() = 0;
+    virtual void storeCurrentLedgerForTest(LedgerHeader const& header) = 0;
 #endif
 
     // Return the (changing) number of seconds since the LCL closed.

diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp
@@ -580,6 +580,12 @@ LedgerManagerImpl::getLastClosedLedgerTxMeta()
 {
     return mLastLedgerTxMeta;
 }
+
+void
+LedgerManagerImpl::storeCurrentLedgerForTest(LedgerHeader const& header)
+{
+    storeCurrentLedger(header, true, true);
+}
 #endif
 
 SorobanMetrics&

diff --git a/src/ledger/LedgerManagerImpl.h b/src/ledger/LedgerManagerImpl.h
@@ -166,6 +166,7 @@ class LedgerManagerImpl : public LedgerManager
     std::vector<TransactionMetaFrame> const&
     getLastClosedLedgerTxMeta() override;
     TransactionResultSet mLatestTxResultSet{};
+    void storeCurrentLedgerForTest(LedgerHeader const& header) override;
 #endif
 
     uint64_t secondsSinceLastLedgerClose() const override;

diff --git a/src/ledger/SorobanMetrics.cpp b/src/ledger/SorobanMetrics.cpp
@@ -132,6 +132,8 @@ SorobanMetrics::SorobanMetrics(medida::MetricsRegistry& metrics)
           {"soroban", "host-fn-op", "ledger-cpu-insns-ratio"}))
     , mLedgerHostFnCpuInsnsRatioExclVm(metrics.NewHistogram(
           {"soroban", "host-fn-op", "ledger-cpu-insns-ratio-excl-vm"}))
+    , mHostFnOpDeclaredInsnsUsageRatio(metrics.NewHistogram(
+          {"soroban", "host-fn-op", "declared-cpu-insns-usage-ratio"}))
 {
 }
 

diff --git a/src/ledger/SorobanMetrics.h b/src/ledger/SorobanMetrics.h
@@ -71,6 +71,7 @@ class SorobanMetrics
     medida::Timer& mHostFnOpInvokeTimeNsecsExclVm;
     medida::Histogram& mHostFnOpInvokeTimeFsecsCpuInsnRatio;
     medida::Histogram& mHostFnOpInvokeTimeFsecsCpuInsnRatioExclVm;
+    medida::Histogram& mHostFnOpDeclaredInsnsUsageRatio;
     medida::Meter& mHostFnOpMaxRwKeyByte;
     medida::Meter& mHostFnOpMaxRwDataByte;
     medida::Meter& mHostFnOpMaxRwCodeByte;