diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8fbd89f..4903abef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,3 +178,10 @@ jobs: ${{ matrix.som }} -cp ../Smalltalk:../TestSuite ../Examples/Benchmarks/TestSuite/TestTestSuite.som ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som --gc TestGC100 1 1 ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som Test100 1 1 + + - name: Test All Benchmark + # ignore VMs that give slighly different results for tests + if: ${{ matrix.repo != 'som-rs.git' && matrix.som != 'spec' && matrix.repo != 'JsSOM.git' && matrix.repo != 'PySOM.git' }} + run: | + cd som-vm + ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite:../Examples/Benchmarks/Richards:../Examples/Benchmarks/DeltaBlue:../Examples/Benchmarks/NBody:../Examples/Benchmarks/Json:../Examples/Benchmarks/GraphSearch:../Examples/Benchmarks/LanguageFeatures ../Examples/Benchmarks/BenchmarkHarness.som All 1 1 diff --git a/Examples/Benchmarks/All.som b/Examples/Benchmarks/All.som index 50199541..0e49abde 100644 --- a/Examples/Benchmarks/All.som +++ b/Examples/Benchmarks/All.som @@ -20,63 +20,57 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. " -All = BenchmarkHarness ( - | summedAverage | +All = Benchmark ( + oneTimeSetup = ( + self all do: [:pair | + system load: (pair at: 1) + ] + ) all = ( - ^ Fibonacci, Dispatch, Bounce, Loop, Permute, Queens, List, Recurse, - Storage, Sieve, BubbleSort, QuickSort, Sum, Towers, TreeSort, - IntegerLoop, FieldLoop - ) - - run: params = ( - params length < 2 - ifTrue: [ self exec: 100 ] - ifFalse: [ self exec: (params at: 2) asInteger ] - ) - - printUsage = ( - './som.sh -cp Smalltalk Examples/Benchmarks/All.som [number-of-iterations]' println. - '' println. - ' number-of-iterations - the number of time each benchmark is executed, default: 1' println. - ) - - initialize = ( - super initialize. - summedAverage := 0. + ^ #(#Richards 1), + #(#DeltaBlue 100), + #(#NBody 1000), + #(#Json 1), + #(#GraphSearch 7), + #(#PageRank 75), + #(#Fannkuch 7), + #(#Fibonacci 10), + #(#Dispatch 10), + #(#Bounce 10), + #(#Loop 100), + #(#Permute 10), + #(#Queens 10), + #(#List 2), + #(#Recurse 12), + #(#Storage 8), + #(#Sieve 20), + #(#BubbleSort 15), + #(#QuickSort 15), + #(#Sum 40), + #(#Towers 2), + #(#TreeSort 7), + #(#IntegerLoop 7), + #(#FieldLoop 1), + #(#WhileLoop 30), + #(#Mandelbrot 50), + #(#Test 1) ) - - exec: iterations = ( - 'Start execution of all benchmarks. Iterations: ' print. - iterations println. - - self all do: [:cls | - self initialize. - self benchmarkClass: cls. - self printAll: false. - self maxRuntime: 3. "seconds" - self numIterations: iterations. - self warmUp: 10. - self runBenchmark. + benchmark = ( + self all do: [:pair | + | benchName size benchClass bench | + benchName := pair at: 1. + size := pair at: 2. + + benchClass := system load: benchName. + bench := benchClass new. + bench oneTimeSetup. + (bench innerBenchmarkLoop: size) ifFalse: [ + self error: benchName + ' failed with incorrect result' ]. ]. - self printTotal. + ^ true ) - - reportBenchmark: bench result: total = ( - '' println. - 'Benchmark: ' print. - bench name println. - (' Iterations: ' + numIterations + ' (elapsed time ' + (total // 1000) round - + ' ms)') println. - (' AVERAGE: ' + ((total // numIterations) // 1000) round + ' ms') println. - - summedAverage := summedAverage + (total // numIterations). - ) - - printTotal = ( - ('Summed Average Runtime: ' + (summedAverage // 1000) round asString + ' ms') println. - ) - + verifyResult: result = ( ^ result ) )