Automate benchmarks (#22)

* Test buildjet for ARM benchmarks

* Add throughput plot generation

---------

Co-authored-by: Olivier Giniaux <[email protected]>

.github/workflows/bench.yml

@@ -0,0 +1,59 @@
+name: Benchmark
+
+on:
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  benchmark-x86:
+    name: Benchmark X86
+    runs-on: buildjet-2vcpu-ubuntu-2204
+
+    permissions:
+      contents: write
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Benchmark
+      run: cargo bench --bench throughput
+
+    - name: Commit & Push Plots
+      uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: Update Benchmark Plots
+        file_pattern: '*.svg'
+        commit_user_name: Benchmark Bot
+        commit_user_email: [email protected]
+        commit_author: Author <[email protected]>
+
+  benchmark-arm:
+    name: Benchmark ARM
+    needs: benchmark-x86
+    runs-on: buildjet-2vcpu-ubuntu-2204-arm
+
+    permissions:
+      contents: write
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+
+    - name: Benchmark
+      run: cargo bench --bench throughput
+
+    - name: Commit & Push Plots
+      uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: Update Benchmark Plots
+        commit_options: '--amend --no-edit'
+        push_options: '--force'
+        skip_fetch: true
+        file_pattern: '*.svg'
+        commit_user_name: Benchmark Bot
+        commit_user_email: [email protected]
+        commit_author: Author <[email protected]>
+

.github/workflows/rust.yml → .github/workflows/build_test.yml

@@ -8,20 +8,16 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  RUSTFLAGS: "-C target-cpu=native"
 
 jobs:
-  build:
+  build_test:
 
+    name: Build & Test
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
-    - name: Switch to nightly
-      run: rustup default nightly
     - name: Build
       run: cargo build --release
-    - name: Run tests
-      run: cargo test --release
-    - name: Benchmark
-      run: cargo bench --bench throughput
+    - name: Test
+      run: cargo test --release

Cargo.toml

@@ -26,8 +26,10 @@ rand = "0.8"
 
 [dev-dependencies]
 rstest = "0.18.2"
-criterion = { version = "0.5.1" }
 lazy_static = { version = "1.3" }
+# Benchmarks
+criterion = { version = "0.5.1" }
+plotters = "0.3.5"
 # Other hash algorithms, for comparison.
 ahash = "0.8.3"
 t1ha = "0.1.0"

README.md

@@ -51,6 +51,8 @@ To run the benchmarks: `cargo bench --bench throughput`.
 
 ### Intel Ice Lake (x86 64-bit) (GCP n2-standard-2)
 
+https://github.com/ogxd/gxhash/blob/af980cb313f3d16efc6e63956eb9ca4ddd70ee30/src/lib.rs#L4C1-L8C1
+
 | Method      |    4 |    16 |    64 |   256 |  1024 |   4096 |  16384 |
 |-------------|-----:|------:|------:|------:|------:|-------:|-------:|
 | gxhash-avx2 | 4189 | 16734 | 46142 | 72679 | 96109 | 102202 | 100845 |

benches/throughput/main.rs

@@ -2,18 +2,18 @@ mod result_processor;
 
 use result_processor::*;
 
+use std::hash::Hasher;
 use std::hint::black_box;
 use std::time::{Instant, Duration};
 use std::alloc::{alloc, dealloc, Layout};
 use std::slice;
-use std::hash::Hasher;
 
 use rand::Rng;
 
 use gxhash::*;
 
 const ITERATIONS: u32 = 1000;
-const MAX_RUN_DURATION: Duration = Duration::from_millis(1000);
+const MAX_RUN_DURATION: Duration = Duration::from_millis(100);
 const FORCE_NO_INLINING: bool = false;
 
 fn main() {
@@ -30,11 +30,17 @@ fn main() {
     let mut processor = ResultProcessor::default();
 
     // GxHash
-    let algo_name = if cfg!(feature = "avx2") { "gxhash-avx2" } else { "gxhash" };
-    benchmark(&mut processor, slice, algo_name, |data: &[u8], seed: i64| -> u64 {
+    benchmark(&mut processor, slice, "gxhash", |data: &[u8], seed: i64| -> u64 {
         gxhash64(data, seed)
     });
 
+    // GxHash-AVX2
+    if cfg!(feature = "avx2") {
+        benchmark(&mut processor, slice, "gxhash-avx2", |data: &[u8], seed: i64| -> u64 {
+            gxhash64(data, seed)
+        });
+    }
+
     // XxHash (twox-hash)
     benchmark(&mut processor, slice, "xxhash", |data: &[u8], seed: u64| -> u64 {
         twox_hash::xxh3::hash64_with_seed(data, seed)
@@ -76,6 +82,8 @@ fn main() {
         fnv_hasher.finish()
     });
 
+    processor.finish();
+
     // Free benchmark data
     unsafe { dealloc(ptr, layout) };
 }
@@ -122,6 +130,8 @@ fn time<F>(iterations: u32, delegate: &F) -> Duration
     where F: Fn() -> u64
 {
     let now = Instant::now();
+    // Bench the same way to what is done in criterion.rs
+    // https://github.com/bheisler/criterion.rs/blob/e1a8c9ab2104fbf2d15f700d0038b2675054a2c8/src/bencher.rs#L87
     for _ in 0..iterations {  
         if FORCE_NO_INLINING {
             black_box(execute_noinlining(delegate));
@@ -132,6 +142,8 @@ fn time<F>(iterations: u32, delegate: &F) -> Duration
     now.elapsed()
 }
 
+// Some algorithm are more likely to be inlined than others.
+// This puts then all at the same level. But is it fair?
 #[inline(never)]
 fn execute_noinlining<F>(delegate: &F) -> u64
     where F: Fn() -> u64

benches/throughput/result_processor.rs

@@ -1,3 +1,6 @@
+use gxhash::GxHashMap;
+use plotters::prelude::*;
+
 #[cfg(feature = "bench-csv")]
 #[derive(Default)]
 pub struct ResultProcessor {
@@ -68,21 +71,105 @@ impl ResultProcessor {
     }
 }
 
-#[cfg(all(not(feature = "bench-csv"),not(feature = "bench-md")))]
+// #[cfg(all(not(feature = "bench-csv"), not(feature = "bench-md")))]
+// #[derive(Default)]
+// pub struct ResultProcessor;
+
+// #[cfg(all(not(feature = "bench-csv"), not(feature = "bench-md")))]
+// impl ResultProcessor {
+//     pub fn on_start(&mut self, name: &str) {
+//         println!("{}", name);
+//     }
+
+//     pub fn on_result(&mut self, input_size: usize, throughput: f64) {
+//         println!("  | {} > {:.2}", input_size, throughput);
+//     }
+
+//     pub fn on_end(&mut self) {
+//         println!();
+//     }
+// }
+
+//#[cfg(feature = "bench-plot")]
 #[derive(Default)]
-pub struct ResultProcessor;
+pub struct ResultProcessor {
+    series: Vec<(String, Vec<(usize, f64)>)>
+}
 
-#[cfg(all(not(feature = "bench-csv"),not(feature = "bench-md")))]
+//#[cfg(feature = "bench-plot")]
 impl ResultProcessor {
+
     pub fn on_start(&mut self, name: &str) {
-        println!("{}", name);
+        println!("Started '{}'...", name);
+        self.series.push((name.to_string(), Vec::new()));
     }
 
     pub fn on_result(&mut self, input_size: usize, throughput: f64) {
-        println!("  | {} > {:.2}", input_size, throughput);
+        let len = self.series.len();
+        let serie = self.series.get_mut(len - 1).unwrap();
+
+        serie.1.push((input_size, throughput));
     }
 
     pub fn on_end(&mut self) {
-        println!();
+
+    }
+
+    pub fn finish(&self) {
+        let arch = std::env::consts::ARCH;
+        let file_name = format!("benches/throughput/{}.svg", arch);
+
+        let canvas = SVGBackend::new(file_name.as_str(), (600, 400)).into_drawing_area();
+        canvas.fill(&WHITE).unwrap();
+
+        let x_min = self.series.iter().next().unwrap().1.iter().map(|(x, _)| *x as u32).min().unwrap();
+        let x_max = self.series.iter().next().unwrap().1.iter().map(|(x, _)| *x as u32).max().unwrap();
+
+        let y_min = 0u32;
+        let y_max = self.series.iter().flat_map(|inner_map| inner_map.1.iter()).map(|(_, y)| (1.05 * *y) as u32).max().unwrap();
+
+        let mut chart = ChartBuilder::on(&canvas)
+            .caption(format!("Throughput ({})", arch), ("sans-serif", (5).percent_height()))
+            .set_label_area_size(LabelAreaPosition::Left, (14).percent())
+            .set_label_area_size(LabelAreaPosition::Bottom, (10).percent())
+            .margin((1).percent())
+            .build_cartesian_2d(
+                (x_min..x_max)
+                    .log_scale()
+                    .with_key_points(self.series.iter().next().unwrap().1.iter().map(|(x, _)| *x as u32).collect::<Vec<u32>>()),
+                    (y_min..y_max)
+                    //.log_scale(),
+            ).unwrap();
+
+        chart
+            .configure_mesh()
+            .max_light_lines(1)
+            .x_desc("Input Size (bytes)")
+            .y_desc("Throughput (MiB/s)")
+            .draw().unwrap();
+
+        let mut color_idx = 0;
+        for (name, values) in self.series.iter() {
+            let color = Palette99::pick(color_idx);
+            color_idx += 1;
+            let data: Vec<_> = values.iter().map(|(x, y)| (*x as u32, *y as u32)).collect();
+            chart
+                .draw_series(LineSeries::new(data,
+                    color.stroke_width(2),
+                )).unwrap()
+                .label(name)
+                .legend(move |(x, y)| Rectangle::new([(x, y - 5), (x + 10, y + 5)], color.filled()));
+        }
+
+        chart
+            .configure_series_labels()
+            .border_style(BLACK)
+            .background_style(RGBColor(255, 255, 255))
+            .draw().unwrap();
+
+        // To avoid the IO failure being ignored silently, we manually call the present function
+        canvas.present().expect("Unable to write result to file, please make sure 'plotters-doc-data' dir exists under current dir");
+
+        println!("Finished");
     }
 }