Test unit test benchmark (#118)

.github/workflows/ci.yml

@@ -10,26 +10,26 @@ jobs:
       fail-fast: false # we want all jobs to run, because they may fail independently
       matrix:
         include:
-          - name: SOM++
-            repo: SOMpp.git
-            apt: libcppunit-dev
-            build: "cmake . && make SOM++"
-            som:   "./som.sh"
-            not-up-to-date: true
-
-          - name: CSOM
-            repo: CSOM.git
-            build: |
-              export COMPILER=gcc
-              export ARCH=64bit
-              make
-            som:   "./som.sh"
-            not-up-to-date: true
+          # - name: SOM++
+          #   repo: SOMpp.git
+          #   apt: libcppunit-dev
+          #   build: "cmake . && make SOM++"
+          #   som: "./som.sh"
+          #   not-up-to-date: true
+
+          # - name: CSOM
+          #   repo: CSOM.git
+          #   build: |
+          #     export COMPILER=gcc
+          #     export ARCH=64bit
+          #     make
+          #   som: "./som.sh"
+          #   not-up-to-date: true
 
           - name: JsSOM
             repo: JsSOM.git
             build: ""
-            som:   "./som.sh"
+            som: "./som.sh"
             not-up-to-date: true
 
           - name: PySOM
@@ -39,7 +39,7 @@ jobs:
 
           - name: SOM (Java)
             build: make
-            som:  ./som.sh
+            som: ./som.sh
             repo: som-java.git
             not-up-to-date: false
 
@@ -59,7 +59,7 @@ jobs:
             not-up-to-date: false
 
           - name: Specification Tests
-            som:  spec
+            som: spec
             not-up-to-date: false
 
           - name: SOM-RS
@@ -69,17 +69,17 @@ jobs:
             som-tests: " -c ../Smalltalk ../TestSuite -- TestHarness"
             not-up-to-date: false
 
-          - name: ykSOM
-            repo: yksom.git
-            build: "cargo build"
-            som:   "cargo run -- "
-            som-tests: "--cp ../Smalltalk ../TestSuite/TestHarness.som"
-            not-up-to-date: true
+          # - name: ykSOM
+          #   repo: yksom.git
+          #   build: "cargo build"
+          #   som: "cargo run -- "
+          #   som-tests: "--cp ../Smalltalk ../TestSuite/TestHarness.som"
+          #   not-up-to-date: true
 
     name: ${{ matrix.name }}
     steps:
       - name: Checkout SOM Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Checkout SOM VM Repository
         if: matrix.som != 'spec'
@@ -168,12 +168,13 @@ jobs:
         run: |
           cd som-vm
           ${{ matrix.som }} -cp ../Smalltalk:../TestSuite:../SomSom/src/compiler:../SomSom/src/interpreter:../SomSom/src/primitives:../SomSom/src/vm:../SomSom/src/vmobjects  ../SomSom/tests/SomSomTests.som
-      
+
       - name: Test Unit Test Benchmark
         if: ${{ matrix.repo == 'som-java.git' }}
         run: |
           cd Examples/Benchmarks/TestSuite
           ./duplicate-tests.sh
           cd ../../../som-vm
+          ${{ matrix.som }} -cp ../Smalltalk:../TestSuite ../Examples/Benchmarks/TestSuite/TestTestSuite.som
           ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som --gc TestGC100 1 1
           ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som Test100 1 1

Examples/Benchmarks/TestSuite/TestCommon.som

@@ -33,7 +33,7 @@ TestCommon = Benchmark (
     numExecs = (
       self error: 'Implement in subclass and return the number of times a test is executed'
     )
-    
+
     oneTimeSetup = (
       "Load all Tests. We don't really want to benchmark the parser."
       self tests
@@ -77,17 +77,17 @@ TestCommon = Benchmark (
       runner run.
       runner hasFailures ifTrue: [system exit: 1]
     )
-    
+
     benchmark = (
         failOnUnsupportedOptionals := false.
         ^ self runAllSuites
     )
-    
+
     verifyResult: result = (
       "result do: [:e | e println ]."
       ^ (result at: 1) = (190 * self numExecs) and: [
           (result at: 2) = (0 * self numExecs) and: [
-            (result at: 3) = (189 * self numExecs) and: [
+            (result at: 3) = (190 * self numExecs) and: [
               (result at: 4) = (992 * self numExecs)
         ] ] ]
     )

Examples/Benchmarks/TestSuite/TestTestSuite.som

@@ -0,0 +1,30 @@
+TestTestSuite = TestHarness (
+  tests = ( "Now ordered by alphabetical order to improve maintainability"
+        ^ Array1Test,
+          Block1Test,
+          Boolean1Test,
+          ClassLoading1Test,
+          ClassStructure1Test,
+          Closure1Test,
+          Coercion1Test,
+          CompilerReturn1Test,
+          Dictionary1Test,
+          DoesNotUnderstand1Test,
+          Double1Test,
+          Empty1Test,
+          Global1Test,
+          Hash1Test,
+          Integer1Test,
+          Preliminary1Test,
+          Reflection1Test,
+          SelfBlock1Test,
+          Set1Test,
+          SpecialSelectors1Test,
+          String1Test,
+          Super1Test,
+          Symbol1Test,
+          System1Test,
+          Vector1Test
+    )
+)
+

Examples/Benchmarks/TestSuite/VectorTest.som

@@ -54,7 +54,7 @@ VectorTest = TestCase (
       self assert: 1 equals: v first ].
 
     1 to: 10 do: [:i |
-      self assert: 1 equals: v first.
+      self assert: i equals: v first.
       v removeFirst ]
   )
 
@@ -148,7 +148,7 @@ VectorTest = TestCase (
     self assert: 1 equals: (arr at: 1).
     self assert: 2 equals: (arr at: 2).
   )
-  
+
   testAsSet = (
     | v set |
     v := Vector new.
@@ -157,38 +157,38 @@ VectorTest = TestCase (
     v append: 3.
     v append: 4.
     self assert: 4 equals: v size.
-    
+
     set := v asSet.
     self assert: 4 equals: set size.
-    
+
     v append: 1.
     v append: 1.
     v append: 1.
-    
+
     self assert: 4 + 3 equals: v size.
-    
+
     set := v asSet.
     self assert: 4 equals: set size.
   )
-  
+
   testIsEmpty = (
     | v |
     v := Vector new.
     self assert: v isEmpty.
-    
+
     v append: 1.
     self deny: v isEmpty.
-    
+
     v removeFirst.
     self assert: v isEmpty.
-    
+
     v append: #ee.
     self deny: v isEmpty.
-    
+
     v removeFirst.
-    self assert: v isEmpty.    
+    self assert: v isEmpty.
   )
-  
+
   testRemoveObj = (
     | v |
     v := Vector new.
@@ -200,31 +200,31 @@ VectorTest = TestCase (
     v append: #f.
     v append: #g.
     v append: #h.
-    
+
     self assert: 8 equals: v size.
-    
+
     self deny: (v remove: #aa).
     self assert: (v remove: #e).
     self assert: 7 equals: v size.
   )
-  
+
   testAppendComma = (
     | v |
     v := Vector new.
     v, #a.
     v, #b.
-    
+
     self assert: 2 equals: v size.
     self assert: (v contains: #a).
     self assert: (v contains: #b).
   )
-  
+
   testDoIndexes = (
     | i v |
     v := Vector new.
     v doIndexes: [:j |
       self assert: false ].
-    
+
     v appendAll: #(1 2 3 4 5).
     i := 1.
     v doIndexes: [:j |
@@ -233,7 +233,7 @@ VectorTest = TestCase (
     ].
     self assert: 6 equals: i.
   )
-  
+
   testDo = (
     | i v |
     v := Vector new.