diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 4b20391225..0000000000 --- a/.drone.yml +++ /dev/null @@ -1,128 +0,0 @@ -kind: pipeline -name: 5.0.0+stable+serial - -platform: - os: linux - arch: amd64 - -steps: -- name: 5.0.0+stable - image: ocaml/opam:ubuntu-20.04-ocaml-4.12 - commands: - - sudo apt-get update && sudo apt-get -y install wget pkg-config libgmp-dev m4 libdw-dev jq python3-pip autoconf jo - - pip3 install intervaltree - - sudo chown -R opam . - - eval $(opam env) - - opam update - - opam install dune.2.9.0 - - eval $(opam env) - - export ITER=1 - - export OPAM_DISABLE_SANDBOXING=true - - TAG='"run_in_ci"' make run_config_filtered.json - - USE_SYS_DUNE_HACK=1 RUN_CONFIG_JSON=run_config_filtered.json make ocaml-versions/5.0.0+stable.bench - - ls _results - - cat _results/* - ---- -kind: pipeline -name: 5.0.0+stable+parallel - -platform: - os: linux - arch: amd64 - -steps: -- name: 5.0.0+stable+parallel - image: ocaml/opam:ubuntu-20.04-ocaml-4.12 - commands: - - sudo apt-get update && sudo apt-get -y install wget pkg-config libgmp-dev m4 libdw-dev jq python3-pip autoconf jo - - pip3 install intervaltree - - sudo chown -R opam . - - eval $(opam env) - - opam update - - opam install dune.2.9.0 - - eval $(opam env) - - export ITER=1 - - export OPAM_DISABLE_SANDBOXING=true - - TAG='"run_in_ci"' make multicore_parallel_run_config_filtered.json - - TAG='"macro_bench"' make multicore_parallel_run_config_filtered_filtered.json - - make multicore_parallel_run_config_filtered_filtered_2domains.json - - USE_SYS_DUNE_HACK=1 BUILD_BENCH_TARGET=multibench_parallel RUN_CONFIG_JSON=multicore_parallel_run_config_filtered_filtered_2domains.json make ocaml-versions/5.0.0+stable.bench - - ls _results - - cat _results/* - ---- -kind: pipeline -name: 5.0.0+trunk+serial - -platform: - os: linux - arch: amd64 - -steps: -- name: 5.0.0+trunk+serial - image: ocaml/opam:ubuntu-20.04-ocaml-4.12 - commands: - - sudo apt-get update && sudo apt-get -y install wget pkg-config libgmp-dev m4 libdw-dev jq python3-pip autoconf jo - - pip3 install intervaltree - - sudo chown -R opam . - - eval $(opam env) - - opam update - - opam install dune.2.9.0 - - eval $(opam env) - - export ITER=1 - - export OPAM_DISABLE_SANDBOXING=true - - TAG='"run_in_ci"' make run_config_filtered.json - - USE_SYS_DUNE_HACK=1 RUN_CONFIG_JSON=run_config_filtered.json make ocaml-versions/5.0.0+trunk.bench - - ls _results - - cat _results/* - ---- -kind: pipeline -name: 5.0.0+trunk+parallel - -platform: - os: linux - arch: amd64 - -steps: -- name: 5.0.0+trunk+parallel - image: ocaml/opam:ubuntu-20.04-ocaml-4.12 - commands: - - sudo apt-get update && sudo apt-get -y install wget pkg-config libgmp-dev m4 libdw-dev jq python3-pip autoconf jo - - pip3 install intervaltree - - sudo chown -R opam . - - eval $(opam env) - - opam update - - opam install dune.2.9.0 - - eval $(opam env) - - export ITER=1 - - export OPAM_DISABLE_SANDBOXING=true - - TAG='"run_in_ci"' make multicore_parallel_run_config_filtered.json - - TAG='"macro_bench"' make multicore_parallel_run_config_filtered_filtered.json - - make multicore_parallel_run_config_filtered_filtered_2domains.json - - USE_SYS_DUNE_HACK=1 BUILD_BENCH_TARGET=multibench_parallel RUN_CONFIG_JSON=multicore_parallel_run_config_filtered_filtered_2domains.json make ocaml-versions/5.0.0+trunk.bench - - ls _results - - cat _results/* - ---- -kind: pipeline -name: test_notebooks - -platform: - os: linux - arch: amd64 - -steps: -- name: test_notebooks - image: ocaml/opam:ubuntu-20.04-ocaml-4.12 - commands: - - sudo apt-get update && sudo apt-get -y install wget pkg-config libgmp-dev m4 libdw-dev jq python3-pip jo - - export PATH=$PATH:/home/opam/.local/bin - - pip3 install jupyter nbconvert seaborn pandas - - cd notebooks/sequential - - sudo chmod 777 . - - jupyter nbconvert --to html --execute sequential.ipynb - - cd ../parallel - - sudo chmod 777 . - - jupyter nbconvert --to html --execute parallel.ipynb diff --git a/README.md b/README.md index 7ba11ca2fe..d2c4754a8d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ dependencies. Sandmark uses opam, with a static local repository, to build external libraries and applications. It then builds any sandmark OCaml benchmarks and any data dependencies. Following this it runs the -benchmarks as defined in the `run_config.json` +benchmarks as defined in the `sequential.json` These stages are implemented in: @@ -57,7 +57,7 @@ These stages are implemented in: - Runplan: the list of benchmarks which will run along with the measurement wrapper (e.g. orun or perf) is specified in - `run_config.json`. This config file is used to generate dune files + `sequential.json`. This config file is used to generate dune files which will run the benchmarks. - Build: dune is used to build all the sandmark OCaml benchmarks that @@ -65,7 +65,7 @@ These stages are implemented in: - Execute: dune is used to execute all the benchmarks sepcified in the runplan using the benchmark wrapper defined in - `run_config.json` and specified via the `RUN_BENCH_TARGET` variable + `sequential.json` and specified via the `RUN_BENCH_TARGET` variable passed to the makefile. ## Configuration of the compiler build @@ -141,7 +141,7 @@ sudo setcap cap_sys_nice=ep /usr/bin/chrt ### Configuring the benchmark runs A config file can be specified with the environment variable `RUN_CONFIG_JSON`, -and the default value is `run_config.json`. This file lists the executable to +and the default value is `sequential.json`. This file lists the executable to run and the wrapper which will be used to collect data (e.g. orun or perf). You can edit this file to change benchmark parameters or wrappers. @@ -177,13 +177,13 @@ current tags are: The benchmarking machine `turing` is an Intel Xeon Gold 5120 CPU with 64GB of RAM housed at IITM. -The `run_config.json` file may be filtered based on the tag. For example, +The `sequential.json` file may be filtered based on the tag. For example, ```bash -$ TAG='"macro_bench"' make run_config_filtered.json +$ TAG='"macro_bench"' make sequential_filtered.json ``` -filters the `run_config.json` file to only contain the benchmarks tagged as +filters the `sequential.json` file to only contain the benchmarks tagged as `macro_bench`. ### Running benchmarks @@ -261,17 +261,17 @@ You can add new benchmarks as follows: - **Add commands to run your applications:** Add an entry for your benchmark run to the appropriate config file; - `run_config.json` for sequential benchmarks and - `multicore_parallel_run_config.json` for parallel benchmarks. + `sequential.json` for sequential benchmarks and + `parallel_turing.json` for parallel benchmarks. ### Config files The `*_config.json` files used to build benchmarks - - **run_config.json** : Runs sequential benchmarks with stock OCaml variants in CI and sandmark-nightly on the IITM machine(turing) - - **multicore_parallel_run_config.json** : Runs parallel benchmarks with multicore OCaml in CI and sandmark-nightly on the IITM machine(turing) - - **multicore_parallel_navajo_run_config.json** : Runs parallel benchmarks with multicore OCaml in sandmark-nightly on Navajo (AMD EPYC 7551 32-Core Processor) machine - - **micro_multicore.json** : To locally run multicore specific micro benchmarks + - **sequential.json** : Runs sequential benchmarks with stock OCaml variants in CI and sandmark-nightly on the IITM machine(turing) + - **parallel_turing.json** : Runs parallel benchmarks with multicore OCaml in CI and sandmark-nightly on the IITM machine(turing) + - **parallel_navajo.json** : Runs parallel benchmarks with multicore OCaml in sandmark-nightly on Navajo (AMD EPYC 7551 32-Core Processor) machine + - **micro.json** : To locally run multicore specific micro benchmarks ### Benchmarks status diff --git a/micro_multicore.json b/micro.json similarity index 100% rename from micro_multicore.json rename to micro.json diff --git a/ocaml-versions/custom.json b/ocaml-versions/custom.json index bc65c42bac..6e899d3778 100644 --- a/ocaml-versions/custom.json +++ b/ocaml-versions/custom.json @@ -2,36 +2,36 @@ { "url": "https://github.com/ocaml/ocaml/archive/trunk.tar.gz", "tag": "macro_bench", - "config_json": "run_config_filtered.json", + "config_json": "sequential_filtered.json", "name": "5.0.0+trunk+sequential", "expiry": "2100-01-01" }, { "url": "https://github.com/ocaml/ocaml/archive/b73cbbea4bc40ffd26a459d594a39b99cec4273d.zip", "tag": "macro_bench", - "config_json": "run_config_filtered.json", + "config_json": "sequential_filtered.json", "name": "5.0.0+stable+sequential", "expiry": "2100-01-01" }, { "url": "https://github.com/ocaml/ocaml/archive/trunk.tar.gz", "tag": "macro_bench", - "config_json": "multicore_parallel_run_config_filtered.json", + "config_json": "parallel_turing_filtered.json", "name": "5.0.0+trunk+parallel", "expiry": "2100-01-01" }, { "url": "https://github.com/ocaml/ocaml/archive/b73cbbea4bc40ffd26a459d594a39b99cec4273d.zip", "tag": "macro_bench", - "config_json": "multicore_parallel_run_config_filtered.json", + "config_json": "parallel_turing_filtered.json", "name": "5.0.0+stable+parallel", "expiry": "2100-01-01" }, { "url": "https://github.com/sadiqj/ocaml/archive/refs/heads/eventring-pr.zip", "tag": "run_in_ci", - "config_json": "run_config_filtered.json", + "config_json": "sequential_filtered.json", "name": "5.0.0+trunk+sadiqj+pr10964", "expiry": "2022-02-25" } -] +] \ No newline at end of file diff --git a/ocaml-versions/custom_navajo.json b/ocaml-versions/custom_navajo.json index 591749cdb2..862c96e212 100644 --- a/ocaml-versions/custom_navajo.json +++ b/ocaml-versions/custom_navajo.json @@ -2,28 +2,28 @@ { "url": "https://github.com/ocaml/ocaml/archive/trunk.tar.gz", "tag": "macro_bench", - "config_json": "run_config_filtered.json", + "config_json": "sequential_filtered.json", "name": "5.0.0+trunk+sequential", "expiry": "2100-01-01" }, { "url": "https://github.com/ocaml/ocaml/archive/trunk.tar.gz", "tag": "macro_bench", - "config_json": "multicore_parallel_navajo_run_config_filtered.json", + "config_json": "parallel_navajo_filtered.json", "name": "5.0.0+trunk+parallel", "expiry": "2100-01-01" }, { "url": "https://github.com/gadmm/ocaml/archive/refs/heads/multicore_async_actions.zip", "tag": "macro_bench", - "config_json": "run_config_filtered.json", + "config_json": "sequential_filtered.json", "name": "5.0.0+trunk+sequential+gadmm+pr11057", "expiry": "2022-03-20" }, { "url": "https://github.com/gadmm/ocaml/archive/refs/heads/multicore_async_actions.zip", "tag": "macro_bench", - "config_json": "multicore_parallel_navajo_run_config_filtered.json", + "config_json": "parallel_navajo_filtered.json", "name": "5.0.0+trunk+parallel+gadmm+pr11057", "expiry": "2022-03-20" } diff --git a/ocaml-versions/custom_turing.json b/ocaml-versions/custom_turing.json index 861e3925fc..e4f1a18982 100644 --- a/ocaml-versions/custom_turing.json +++ b/ocaml-versions/custom_turing.json @@ -2,28 +2,28 @@ { "url": "https://github.com/ocaml/ocaml/archive/trunk.tar.gz", "tag": "macro_bench", - "config_json": "run_config_filtered.json", + "config_json": "sequential_filtered.json", "name": "5.0.0+trunk+sequential", "expiry": "2100-01-01" }, { "url": "https://github.com/ocaml/ocaml/archive/trunk.tar.gz", "tag": "macro_bench", - "config_json": "multicore_parallel_run_config_filtered.json", + "config_json": "parallel_turing_filtered.json", "name": "5.0.0+trunk+parallel", "expiry": "2100-01-01" }, { "url": "https://github.com/gadmm/ocaml/archive/refs/heads/multicore_async_actions.zip", "tag": "macro_bench", - "config_json": "run_config_filtered.json", + "config_json": "sequential_filtered.json", "name": "5.0.0+trunk+sequential+gadmm+pr11057", "expiry": "2022-03-20" }, { "url": "https://github.com/gadmm/ocaml/archive/refs/heads/multicore_async_actions.zip", "tag": "macro_bench", - "config_json": "multicore_parallel_run_config_filtered.json", + "config_json": "parallel_turing_filtered.json", "name": "5.0.0+trunk+parallel+gadmm+pr11057", "expiry": "2022-03-20" } diff --git a/multicore_parallel_navajo_run_config.json b/parallel_navajo.json similarity index 100% rename from multicore_parallel_navajo_run_config.json rename to parallel_navajo.json diff --git a/multicore_parallel_run_config.json b/parallel_turing.json similarity index 100% rename from multicore_parallel_run_config.json rename to parallel_turing.json diff --git a/multicore_parallel_run_config_filtered_filtered_2domains.json b/parallel_turing_filtered_filtered_2domains.json similarity index 100% rename from multicore_parallel_run_config_filtered_filtered_2domains.json rename to parallel_turing_filtered_filtered_2domains.json diff --git a/run_all_custom.sh b/run_all_custom.sh index 8ea202edeb..ef2b364e8b 100644 --- a/run_all_custom.sh +++ b/run_all_custom.sh @@ -68,14 +68,14 @@ while [ $i -lt ${COUNT} ]; do CONFIG_TAG=`jq -r '.['$i'].tag // "macro_bench"' "${CUSTOM_FILE}"` if [ $j -eq 0 ]; then - CONFIG_RUN_JSON="run_config_filtered.json" + CONFIG_RUN_JSON="sequential_filtered.json" CONFIG_NAME="${CONFIG_NAME}+sequential" else CONFIG_NAME="${CONFIG_NAME}+parallel" if [ "${HOSTNAME}" == "navajo" ]; then - CONFIG_RUN_JSON="multicore_parallel_navajo_run_config_filtered.json" + CONFIG_RUN_JSON="parallel_navajo_filtered.json" else - CONFIG_RUN_JSON="multicore_parallel_run_config_filtered.json" + CONFIG_RUN_JSON="parallel_turing_filtered.json" fi fi diff --git a/run_all_parallel.sh b/run_all_parallel.sh index d8d558b21e..7f78deecef 100644 --- a/run_all_parallel.sh +++ b/run_all_parallel.sh @@ -6,10 +6,10 @@ # sudo setcap cap_sys_nice=ep /usr/bin/chrt # -TAG='"macro_bench"' make multicore_parallel_run_config_filtered.json +TAG='"macro_bench"' make parallel_turing_filtered.json USE_SYS_DUNE_HACK=1 \ RUN_BENCH_TARGET=run_orunchrt \ BUILD_BENCH_TARGET=multibench_parallel \ - RUN_CONFIG_JSON=multicore_parallel_run_config_filtered.json \ + RUN_CONFIG_JSON=parallel_turing_filtered.json \ make ocaml-versions/5.0.0+stable.bench diff --git a/run_all_serial.sh b/run_all_serial.sh index add843933b..8a4c8bf3a8 100644 --- a/run_all_serial.sh +++ b/run_all_serial.sh @@ -1,5 +1,5 @@ #!/bin/bash -TAG='"macro_bench"' make run_config_filtered.json +TAG='"macro_bench"' make sequential_filtered.json -USE_SYS_DUNE_HACK=1 RUN_CONFIG_JSON=run_config_filtered.json make ocaml-versions/5.0.0+stable.bench +USE_SYS_DUNE_HACK=1 RUN_CONFIG_JSON=sequential_filtered.json make ocaml-versions/5.0.0+stable.bench diff --git a/run_config.json b/sequential.json similarity index 100% rename from run_config.json rename to sequential.json