From ebe3884493a46e1761f0036971404e405f8c7d3a Mon Sep 17 00:00:00 2001
From: Stefan Marr <git@stefan-marr.de>
Date: Wed, 14 Aug 2024 00:17:27 +0100
Subject: [PATCH 1/2] Reimplement All as benchmark
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It’s run with the normal harness.
Though, it needs a long classpath:

 -cp Smalltalk:Examples/Benchmarks/TestSuite:Examples/Benchmarks/Richards:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody:Examples/Benchmarks/Json:Examples/Benchmarks/GraphSearch:Examples/Benchmarks/LanguageFeatures Examples/Benchmarks/BenchmarkHarness.som All 3 10

Signed-off-by: Stefan Marr <git@stefan-marr.de>
---
 Examples/Benchmarks/All.som | 98 +++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 52 deletions(-)

diff --git a/Examples/Benchmarks/All.som b/Examples/Benchmarks/All.som
index 50199541..0e49abde 100644
--- a/Examples/Benchmarks/All.som
+++ b/Examples/Benchmarks/All.som
@@ -20,63 +20,57 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 "
 
-All = BenchmarkHarness (
-    | summedAverage |
+All = Benchmark (
+    oneTimeSetup = (
+        self all do: [:pair |
+            system load: (pair at: 1)
+        ]
+    )
 
     all = (
-        ^ Fibonacci, Dispatch, Bounce, Loop, Permute, Queens, List, Recurse,
-          Storage, Sieve, BubbleSort, QuickSort, Sum, Towers, TreeSort,
-          IntegerLoop, FieldLoop
-    )
-    
-    run: params = (
-        params length < 2
-            ifTrue:  [ self exec: 100 ]
-            ifFalse: [ self exec: (params at: 2) asInteger ]
-    )
-    
-    printUsage = (
-      './som.sh -cp Smalltalk Examples/Benchmarks/All.som [number-of-iterations]' println.
-      '' println.
-      '  number-of-iterations - the number of time each benchmark is executed, default: 1' println.
-    )
-    
-    initialize = (
-        super initialize.
-        summedAverage := 0.
+        ^ #(#Richards 1),
+          #(#DeltaBlue 100),
+          #(#NBody 1000),
+          #(#Json 1),
+          #(#GraphSearch 7),
+          #(#PageRank 75),
+          #(#Fannkuch 7),
+          #(#Fibonacci 10),
+          #(#Dispatch 10),
+          #(#Bounce 10),
+          #(#Loop 100),
+          #(#Permute 10),
+          #(#Queens 10),
+          #(#List 2),
+          #(#Recurse 12),
+          #(#Storage 8),
+          #(#Sieve 20),
+          #(#BubbleSort 15),
+          #(#QuickSort 15),
+          #(#Sum 40),
+          #(#Towers 2),
+          #(#TreeSort 7),
+          #(#IntegerLoop 7),
+          #(#FieldLoop 1),
+          #(#WhileLoop 30),
+          #(#Mandelbrot 50),
+          #(#Test 1)
     )
-    
-    exec: iterations = (
-        'Start execution of all benchmarks. Iterations: ' print.
-        iterations println.
-        
-        self all do: [:cls |
-            self initialize.
-            self benchmarkClass: cls.
-            self printAll: false.
-            self maxRuntime: 3. "seconds"
-            self numIterations: iterations.
-            self warmUp: 10.
 
-            self runBenchmark.
+    benchmark = (
+        self all do: [:pair |
+            | benchName size benchClass bench |
+            benchName := pair at: 1.
+            size := pair at: 2.
+
+            benchClass := system load: benchName.
+            bench := benchClass new.
+            bench oneTimeSetup.
+            (bench innerBenchmarkLoop: size) ifFalse: [
+              self error: benchName + ' failed with incorrect result' ].
         ].
-        self printTotal.
+        ^ true
     )
-    
-    reportBenchmark: bench result: total = (
-        '' println.
-        'Benchmark: ' print.
-        bench name println.
 
-        ('   Iterations: ' + numIterations + ' (elapsed time ' + (total // 1000) round
-            + ' ms)') println.
-        ('   AVERAGE: ' + ((total // numIterations) // 1000) round + ' ms') println.
-        
-        summedAverage := summedAverage + (total // numIterations).
-    )
-    
-    printTotal = (
-        ('Summed Average Runtime: ' + (summedAverage // 1000) round asString + ' ms') println.
-    )
-    
+    verifyResult: result = ( ^ result )
 )

From d1ed7da084d5420eaf08bd5098ac310156b31466 Mon Sep 17 00:00:00 2001
From: Stefan Marr <git@stefan-marr.de>
Date: Wed, 21 Aug 2024 00:53:14 +0100
Subject: [PATCH 2/2] Test All Benchmark on GitHub Actions

Signed-off-by: Stefan Marr <git@stefan-marr.de>
---
 .github/workflows/ci.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b8fbd89f..4903abef 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -178,3 +178,10 @@ jobs:
           ${{ matrix.som }} -cp ../Smalltalk:../TestSuite ../Examples/Benchmarks/TestSuite/TestTestSuite.som
           ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som --gc TestGC100 1 1
           ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som Test100 1 1
+
+      - name: Test All Benchmark
+        # ignore VMs that give slighly different results for tests
+        if: ${{ matrix.repo != 'som-rs.git' && matrix.som != 'spec' && matrix.repo != 'JsSOM.git' && matrix.repo != 'PySOM.git' }}
+        run: |
+          cd som-vm
+          ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite:../Examples/Benchmarks/Richards:../Examples/Benchmarks/DeltaBlue:../Examples/Benchmarks/NBody:../Examples/Benchmarks/Json:../Examples/Benchmarks/GraphSearch:../Examples/Benchmarks/LanguageFeatures ../Examples/Benchmarks/BenchmarkHarness.som All 1 1