mate.conf

# Config file for ReBench
# Config format is YAML (see http://yaml.org/ for detailed spec)

# this run definition will be choosen if no parameters are given to rebench.py
standard_experiment: all
standard_data_file: 'mateV2.data'

runs:
    number_of_data_points: 100
    
are_we_fast_yet:
    ## this is ignored by rebench
    iteration_numbers:
        fast_vm_macro:      &FAST_VM_MACRO      1500
        slow_vm_macro:      &SLOW_VM_MACRO      800
        fast_vm_micro:      &FAST_VM_MICRO      800
        slow_vm_micro:      &SLOW_VM_MICRO      400
        inlining: &INLINING_VALUES
#            - 1500
            - 2500
#            - 3500
#            - 4000
#            - 6000
#            - 8000

# settings and requirements for statistic evaluation
# Note, this is only used for explorative runs, for the final experiments, we
# rely soley on a fixed configuration of data points.
#statistics:
#    confidence_level: 0.95
 
# settings for quick runs, useful for fast feedback during experiments
quick_runs:
    number_of_data_points: 3
    max_time: 60   # time in seconds

# definition of benchmark suites
benchmark_suites:
    classic-macro-steady:
        gauge_adapter: RebenchLog
        command: &MACRO_CMD " -cp .:LanguageFeatures:Json:Richards:DeltaBlue:NBody:GraphSearch:CD:Havlak:../../Smalltalk:../../Smalltalk/Mate:../../Smalltalk/Mate/MOP BenchmarkHarness %(benchmark)s %(variable)s 0"
        variable_values: *SLOW_VM_MACRO ## the number iterations measured
        max_runtime: 60000
        location: Examples/Benchmarks
        benchmarks: &MACRO_STEADY
            - Richards:
                extra_args: "20"
            - DeltaBlue:
                extra_args: "1200"
            - Mandelbrot:
                extra_args: "500"
            - NBody:
                extra_args: "50000"
            - Json:
                extra_args: "50"
            - GraphSearch:
                extra_args: "30"
            - PageRank:
                extra_args: "1000"
            - CD:
                extra_args: "250"
            - Havlak:
                extra_args: "1500"                
                
#        variable_values: &INLINING_VALUES
#            - 1500
#            - 2500
#            - 3500
#            - 4000
#            - 6000
#            - 8000
            
    classic-micro-steady:
        gauge_adapter: RebenchLog
        command: *MACRO_CMD 
        max_runtime: 60000
        location: Examples/Benchmarks
        variable_values: *SLOW_VM_MICRO
        benchmarks: &MICRO_STEADY
            - Bounce:
                extra_args: "200"
            - BubbleSort:
                extra_args: "300"
            - Dispatch:
                extra_args: "2000"
            - Fannkuch:
                extra_args: "8"
            - Fibonacci:
                extra_args: "300"
            - FieldLoop:
                extra_args: "300"
            - IntegerLoop:
                extra_args: "8000"
            - List:
                extra_args: "200"
            - Loop:
                extra_args: "1000"
            - Permute:
                extra_args: "300"
            - Queens:
                extra_args: "300"
            - QuickSort:
                extra_args: "300"
            - Recurse:
                extra_args: "300"
            - Sieve:
                extra_args: "500"
            - Storage:
                extra_args: "200"
            - Sum:
                extra_args: "1000"
            - Towers:
                extra_args: "200"
            - TreeSort:
                extra_args: "100"
            - WhileLoop:
                extra_args: "3000"
#        variable_values: *INLINING_VALUES

    classic-steady-java-micro:
        gauge_adapter: RebenchLog
        command: " -cp classic-benchmarks.jar som.%(benchmark)s %(variable)s"
        max_runtime: 60000
        location: ../Classic-Benchmarks
        variable_values: *SLOW_VM_MICRO
        benchmarks: *MICRO_STEADY
    
    classic-steady-java-macro:
        gauge_adapter: RebenchLog
        command: " -cp classic-benchmarks.jar som.%(benchmark)s %(variable)s"
        max_runtime: 60000
        location: ../Classic-Benchmarks
        variable_values: *SLOW_VM_MICRO
        benchmarks: *MICRO_STEADY    

    #Configuration of the SOM Mate microbenchmarks
    mop-operations-steady:
        gauge_adapter: RebenchLog
        command: ":Examples/Benchmarks/Mate/IndividualOperations Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks: &INDIVIDUAL
            - FieldRead:
                extra_args: "750 0 2000000"
            - FieldWrite:
                extra_args: "750 0 2000000"
            - MessageSend:
                extra_args: "750 0 2000000"
            - SeveralObjectsFieldRead:
                extra_args: "750 0 100000"                
            - SeveralObjectsFieldRead2:
                extra_args: "750 0 100000"                
            - VMReflectiveFieldRead:
                extra_args: "750 0 2000000"
            - VMReflectiveFieldWrite:
                extra_args: "750 0 2000000"
            - VMReflectiveLayoutFieldWrite:
                extra_args: "750 0 2000000"
            - VMReflectiveLayoutFieldRead:
                extra_args: "750 0 2000000"
            - VMReflectiveMethodActivation:
                extra_args: "750 0 2000000"
            - VMReflectiveMessageSend:
                extra_args: "750 0 2000000"
            - VMReflectiveReturn:
                extra_args: "750 0 2000000"    
            - AllOperations:
                extra_args: "750 0 2000000"
            - VMReflectiveAllOperations:
                extra_args: "750 0 2000000"                
            - VMReflectiveSeveralObjectsFieldRead:
                extra_args: "750 0 100000"       
            - VMReflectiveSeveralObjectsFieldRead2:
                extra_args: "750 0 100000"       
            - VMReflectiveSeveralObjectsFieldReadOneMO:
                extra_args: "750 0 100000"                                    
            - VMReflectiveSeveralObjectsFieldReadOneMO2:
                extra_args: "750 0 100000"                                    
        variable_values: *INLINING_VALUES
        
    #Configuration of the Delegation Proxies for read only
    readonly:
        gauge_adapter: RebenchLog
        command: &READONLY_CMD ":Examples/Benchmarks/Mate:Examples/Benchmarks/Mate/DelegationProxies Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
            - SumKeys:
                extra_args: "1200 0 80"
            - DelegationProxiesSumKeys:
                extra_args: "1200 0 80"
        variable_values: *INLINING_VALUES                
    
    readonly-interp:
        gauge_adapter: RebenchLog
        command: *READONLY_CMD
        max_runtime: 60000
        benchmarks:
            - SumKeys:
                extra_args: "200 0 5"
            - DelegationProxiesSumKeys:
                extra_args: "200 0 5"
        variable_values: *INLINING_VALUES                

    #Configuration of the Mate Handles for readonly
    readonly-handles:
        gauge_adapter: RebenchLog
        command: &READONLYHANDLE_CMD ":Examples/Benchmarks/Mate:Examples/Benchmarks/Mate/DelegationProxies Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
            - ReadonlySumKeys:
                extra_args: "1200 0 80"
        variable_values: *INLINING_VALUES                

    readonly-handles-interp:
        gauge_adapter: RebenchLog
        command: *READONLYHANDLE_CMD
        max_runtime: 60000
        benchmarks:
            - ReadonlySumKeys:
                extra_args: "200 0 5"
        variable_values: *INLINING_VALUES                

    tracing-mate:
        gauge_adapter: RebenchLog
        command: ":Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody:Examples/Benchmarks/DeltaBlueInstrumented:Examples/Benchmarks/Mate:Examples/Benchmarks/Mate/Tracing:Examples/Benchmarks/Json Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
            - DeltaBlueTrace:
                extra_args: "1500 0 1200"
            - NBodyTrace:
                extra_args: "1500 0 50000"
            - JsonTrace:
                extra_args: "150 0 1"
            - QuickSortTrace:
                extra_args: "800 0 300"
            - JsonBig:
               extra_args: "150 0 1"
        variable_values: *INLINING_VALUES                

    tracing-mateNBody:
        gauge_adapter: RebenchLog
        command: ":Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody:Examples/Benchmarks/DeltaBlueInstrumented:Examples/Benchmarks/Mate:Examples/Benchmarks/Mate/Tracing:Examples/Benchmarks/Json Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
            - NBodyTrace:
                extra_args: "1500 0 50000"
        variable_values: 7000

    #Configuration of the Tracing Benchmarks
    tracing-standard:
        gauge_adapter: RebenchLog
        command: ":Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/DeltaBlueInstrumented:Examples/Benchmarks/Mate/Tracing:Examples/Benchmarks/Json Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
#            - DeltaBlueInstrumented:
#                extra_args: "1500 0 1200"
            - JsonBig:
               extra_args: "150 0 1"
        variable_values: *INLINING_VALUES

    #In moviesColumnar the inner iterations represent the problem size
    movies:
        gauge_adapter: RebenchLog
        command: &CMD_MOVIES ":Examples/Benchmarks/Mate:Examples/Benchmarks/Mate/Columnar BenchmarkHarness %(benchmark)s"
        max_runtime: 60000
        benchmarks:
            - MoviesAggregate:
                extra_args: "100 0 %(variable)s" 
            - MoviesFilter:
                extra_args: "100 0 %(variable)s"
            - MoviesMap:
                extra_args: "100 0 %(variable)s"                                
        variable_values: &VAR_MOVIES 
            - 10000
            - 50000
            - 100000
            - 500000
            - 1000000

    movies-columnar:
        gauge_adapter: RebenchLog
        command: *CMD_MOVIES
        max_runtime: 60000
        benchmarks:
            - MoviesColumnarAggregate:
                extra_args: "100 0 %(variable)s"
            - MoviesColumnarFilter:
                extra_args: "100 0 %(variable)s"
            - MoviesColumnarMap:
                extra_args: "100 0 %(variable)s"
        variable_values: *VAR_MOVIES
        
    layoutVsObjectBasedEnvironment:
        gauge_adapter: RebenchLog
        command: " -cp .:../../Smalltalk:../../Smalltalk/Mate:../../Smalltalk/Mate/MOP:Mate/LayoutInObjVsEnv:Mate/LayoutInObjVsEnv/AuxiliaryClasses BenchmarkHarness %(benchmark)s %(variable)s 0"
        max_runtime: 60000
        location: Examples/Benchmarks
        benchmarks:
            - PolymorphicSite:
                extra_args: "100000"
        variable_values: *SLOW_VM_MICRO ## the number iterations measured 
         
    
# VMs have a name and are specified by a path and the binary to be executed
virtual_machines:
    TruffleSOM-interpreter:
        path: .
        binary: som
        args: "-G -cp Smalltalk:Smalltalk/FileSystem/Core:Smalltalk/FileSystem/Disk:Smalltalk/FileSystem/Streams:Smalltalk/Collections/Streams:Examples/Benchmarks"
    TruffleSOM-graal:
        path: . 
        binary: fast
        args: ""
    TruffleSOM-graal-enterprise:
        path: . 
        binary: oficial
        args: "-A"
    TruffleMate-interpreter:
        path: . 
        binary: som
        args: "--mate -activateMate -G -cp Smalltalk:Smalltalk/Mate:Smalltalk/Mate/MOP:Smalltalk/FileSystem/Core:Smalltalk/FileSystem/Disk:Smalltalk/FileSystem/Streams:Smalltalk/Collections/Streams:Examples/Benchmarks"
    TruffleMate-graal:
        path: . 
        binary: fast 
        args: "--mate -activateMate"
    TruffleMate-graal-enterprise:
        path: . 
        binary: oficial
        args: "-A --mate -activateMate"
    TruffleMate-graal-deactivated:
        path: . 
        binary: som
        args: "--mate -it %(variable)s -cp Smalltalk:Smalltalk/Mate:Smalltalk/Mate/MOP:Smalltalk/FileSystem/Core:Smalltalk/FileSystem/Disk:Smalltalk/FileSystem/Streams:Smalltalk/Collections/Streams:Examples/Benchmarks"

    Java:
        path: .
        binary: java8.sh
#        args: "-Xms2048m -server "
                
# define the benchmarks to be executed for a re-executable benchmark run
experiments:

    Inherent:
        data_file: inherent.data
        benchmark:
            - classic-micro-steady
            - classic-macro-steady
        executions:
            - TruffleSOM-graal
            - TruffleSOM-graal-enterprise
            - TruffleMate-graal
            - TruffleMate-graal-enterprise
#            - TruffleMate-graal-enterprise
#            - TruffleMate-graal-deactivated-moreInlining

    InherentNaive:
        data_file: inherentNaive.data
        benchmark:
            - classic-micro-steady
            - classic-macro-steady
        executions:
            - TruffleMate-graal
            - TruffleMate-graal-enterprise

    Inherent-Interpreter:
        benchmark:
            - classic-micro-steady-interp
            - classic-macro-steady-interp
            - readonly-interp
            - readonly-handles-interp
        executions:
            - TruffleSOM-interpreter
            - TruffleMate-interpreter
            
    IndividualActivations:
        data_file: individualActivations.data
        benchmark:
            - mop-operations-steady
        executions:
            - TruffleMate-graal
    
    Readonly:
        data_file: readonly.data
        executions:
            - TruffleMate-graal:
                benchmark: 
                    - readonly-handles
                    - readonly
#            - TruffleSOM-graal:
#                benchmark: 
#                    readonly
        
    Tracing:
        data_file: tracing.data
        executions:
            - TruffleSOM-graal:
                benchmark: tracing-standard
            - TruffleMate-graal:            
                benchmark:  
                    - tracing-mate
                    - tracing-mateNBody

    Movies:
        data_file: columnar.data
        executions:
            - TruffleSOM-graal-enterprise:
                benchmark: 
                    - movies
            - TruffleMate-graal-enterprise:            
                benchmark: 
                    - movies
                    - movies-columnar                    
                                

    EnvInObjVsEnvInLayout:
        data_file: objVsLayout.data
        description: The performance comparison of MATE with metaobjects in the layout or in the object itself
        executions:
            - TruffleMate-graal-enterprise:
                benchmark: 
                    - layoutVsObjectBasedEnvironment    
    
    Baseline:
        data_file: baseline.data
        description: The comparison of Truffle SOM/MATE in peak performance with Java
        executions:
            - TruffleSOM-graal-enterprise:
                benchmark: 
                    - classic-macro-steady
                    - classic-micro-steady
            - TruffleMate-graal-enterprise:
                benchmark: 
                    - classic-macro-steady
                    - classic-micro-steady    
#            - Java:
#                benchmark: 
#                    - classic-steady-java-macro
#                    - classic-steady-java-micro