Issue 274: Add continuous benchmark structure (#298)

* add benchmark workflow

* make sure to install benchmark deps

* add benchmark folder

* add infracstructure to run benchmarks

* add a proposed structure for future benchmarks

* fix action indexing

* tweak action for detection of local EpiAware

* tweak action for detection of local EpiAware

* add benchmark run

* move benchmarks to the top level and wrap everything in a utility package

* add in subdirectory builds

* Update benchmark.yaml

* Update benchmark.yaml

* Update benchmark.yaml

* Update benchmark.yaml

* Update benchmark.yaml

* Update benchmark.yaml

* Update benchmark.yaml

* Update Rtwithoutrenewal.jl

* Update benchmark.yaml

* Update benchmark.yaml

* Update benchmark.yaml

* Update Project.toml

* Update benchmark.yaml

* Update benchmark.yaml

* Update benchmark.yaml

.github/workflows/benchmark.yaml

@@ -0,0 +1,36 @@
+name: Run benchmarks
+
+on:
+  pull_request:
+    paths:
+      - 'EpiAware/**'
+  merge_group:
+    paths:
+      - 'EpiAware/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/cache@v1
+      - name: Install dependencies
+        run: |
+          julia --project=benchmark -e 'using Pkg;
+          Pkg.rm("EpiAware");
+          Pkg.rm("EpiAwarePipeline");
+          Pkg.resolve();
+          Pkg.instantiate();
+          Pkg.develop(path = "./EpiAware");
+          Pkg.develop(path = "./pipeline")'
+      - name: Run benchmarks
+        run: julia --project=benchmark -e 'using BenchmarkCI; BenchmarkCI.judge(; baseline = "origin/main")'
+      - name: Post results
+        run: julia --project=benchmark -e 'using BenchmarkCI; BenchmarkCI.postjudge()'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -385,3 +385,7 @@ docs/site/
 
 #Ignore generated Pluto notebooks
 EpiAware/docs/src/examples/*.md
+
+# benchmark ignore
+/.benchmarkci
+/benchmark/*.json

EpiAware/.gitignore

@@ -42,3 +42,7 @@ docs/build
 *.zip
 
 .DS_Store
+
+# benchmark ignore
+/.benchmarkci
+/benchmark/*.json

Project.toml

@@ -0,0 +1,8 @@
+name = "Rtwithoutrenewal"
+uuid = "5f18122a-2e46-11ef-2c26-2d57cbc8fe48"
+authors = ["Samuel Abbott <[email protected]>", "Samuel Brand <[email protected]>", "Zachary Susswein <[email protected]>"]
+version = "0.1.0-DEV"
+
+[deps]
+EpiAware = "b2eeebe4-5992-4301-9193-7ebc9f62c855"
+EpiAwarePipeline = "66314692-3678-4cbb-b3ac-6fcb54505317"

benchmark/Project.toml

@@ -0,0 +1,7 @@
+[deps]
+BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+EpiAware = "b2eeebe4-5992-4301-9193-7ebc9f62c855"
+EpiAwarePipeline = "66314692-3678-4cbb-b3ac-6fcb54505317"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"

benchmark/bench/EpiAwareUtils/EpiAwareUtils.jl

@@ -0,0 +1,10 @@
+module BenchEpiAwareUtils
+
+using BenchmarkTools, EpiAware
+
+suite = BenchmarkGroup()
+
+include("censored_pmf.jl")
+
+end
+BenchEpiAwareUtils.suite

benchmark/bench/EpiAwareUtils/censored_pmf.jl

@@ -0,0 +1,5 @@
+let
+    using Distributions
+    dist = Exponential(1.0)
+    suite["censored_pmf"] = @benchmarkable(censored_pmf($dist, Δd = 1.0, D = 3.0))
+end

benchmark/benchmarks.jl

@@ -0,0 +1,11 @@
+using BenchmarkTools
+SUITE = BenchmarkGroup()
+for folder in readdir(joinpath(@__DIR__, "bench"))
+    if isdir(joinpath(@__DIR__, "bench", folder))
+        suite_name = basename(folder)
+        suite_file = joinpath(@__DIR__, "bench", folder, "$suite_name.jl")
+        if isfile(suite_file)
+            SUITE[suite_name] = include(suite_file)
+        end
+    end
+end

benchmark/runbenchmarks.jl

@@ -0,0 +1,11 @@
+using PkgBenchmark
+benchmarkpkg(
+    dirname(@__DIR__),
+    BenchmarkConfig(
+        env = Dict(
+        "JULIA_NUM_THREADS" => "1",
+        "OMP_NUM_THREADS" => "1"
+    ),
+    ),
+    resultfile = joinpath(@__DIR__, "result.json")
+)

src/Rtwithoutrenewal.jl

@@ -0,0 +1,6 @@
+module Rtwithoutrenewal
+
+using EpiAware
+using EpiAwarePipeline
+
+end