From a96401d4b7f6984bbd0a378e5f6ac0625a322f7a Mon Sep 17 00:00:00 2001 From: Author Date: Tue, 21 Nov 2023 23:14:02 +0100 Subject: [PATCH] Update Benchmark Plots --- .github/workflows/bench.yml | 59 ++++++ .../workflows/{rust.yml => build_test.yml} | 12 +- Cargo.toml | 4 +- README.md | 2 + benches/throughput/aarch64.svg | 183 ++++++++++++++++++ benches/throughput/main.rs | 20 +- benches/throughput/result_processor.rs | 99 +++++++++- 7 files changed, 360 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/bench.yml rename .github/workflows/{rust.yml => build_test.yml} (53%) create mode 100644 benches/throughput/aarch64.svg diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..7bbc483 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,59 @@ +name: Benchmark + +on: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + benchmark-x86: + name: Benchmark X86 + runs-on: buildjet-2vcpu-ubuntu-2204 + + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + + - name: Benchmark + run: cargo bench --bench throughput + + - name: Commit & Push Plots + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Update Benchmark Plots + file_pattern: '*.svg' + commit_user_name: Benchmark Bot + commit_user_email: benchmark-bot@noreply.com + commit_author: Author + + benchmark-arm: + name: Benchmark ARM + needs: benchmark-x86 + runs-on: buildjet-2vcpu-ubuntu-2204-arm + + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Benchmark + run: cargo bench --bench throughput + + - name: Commit & Push Plots + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Update Benchmark Plots + commit_options: '--amend --no-edit' + push_options: '--force' + skip_fetch: true + file_pattern: '*.svg' + commit_user_name: Benchmark Bot + commit_user_email: benchmark-bot@noreply.com + commit_author: Author + \ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/build_test.yml similarity index 53% rename from .github/workflows/rust.yml rename to .github/workflows/build_test.yml index a21302c..a178298 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/build_test.yml @@ -8,20 +8,16 @@ on: env: CARGO_TERM_COLOR: always - RUSTFLAGS: "-C target-cpu=native" jobs: - build: + build_test: + name: Build & Test runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Switch to nightly - run: rustup default nightly - name: Build run: cargo build --release - - name: Run tests - run: cargo test --release - - name: Benchmark - run: cargo bench --bench throughput \ No newline at end of file + - name: Test + run: cargo test --release \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 5d606d3..1e2c912 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,8 +26,10 @@ rand = "0.8" [dev-dependencies] rstest = "0.18.2" -criterion = { version = "0.5.1" } lazy_static = { version = "1.3" } +# Benchmarks +criterion = { version = "0.5.1" } +plotters = "0.3.5" # Other hash algorithms, for comparison. ahash = "0.8.3" t1ha = "0.1.0" diff --git a/README.md b/README.md index 924ba6c..1ecf265 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ To run the benchmarks: `cargo bench --bench throughput`. ### Intel Ice Lake (x86 64-bit) (GCP n2-standard-2) +https://github.com/ogxd/gxhash/blob/af980cb313f3d16efc6e63956eb9ca4ddd70ee30/src/lib.rs#L4C1-L8C1 + | Method | 4 | 16 | 64 | 256 | 1024 | 4096 | 16384 | |-------------|-----:|------:|------:|------:|------:|-------:|-------:| | gxhash-avx2 | 4189 | 16734 | 46142 | 72679 | 96109 | 102202 | 100845 | diff --git a/benches/throughput/aarch64.svg b/benches/throughput/aarch64.svg new file mode 100644 index 0000000..e5619c2 --- /dev/null +++ b/benches/throughput/aarch64.svg @@ -0,0 +1,183 @@ + + + +Throughput (aarch64) + + + + + + + + + +Throughput (MiB/s) + + +Input Size (bytes) + + + + + + + + + + + + + + + + + + + + + + + + + + + +0 + + + +5000 + + + +10000 + + + +15000 + + + +20000 + + + +25000 + + + +30000 + + + + +4 + + + +8 + + + +16 + + + +32 + + + +64 + + + +128 + + + +256 + + + +512 + + + +1024 + + + +2048 + + + +4096 + + + +8192 + + + +16384 + + + +32768 + + + +65536 + + + +131072 + + + +262144 + + + + + + + + + + + + + +gxhash + + +xxhash + + +ahash + + +t1ha0 + + +seahash + + +metrohash + + +highwayhash + + +fnv-1a + + + + + + + + + + diff --git a/benches/throughput/main.rs b/benches/throughput/main.rs index 2ed5819..1848f0f 100644 --- a/benches/throughput/main.rs +++ b/benches/throughput/main.rs @@ -2,18 +2,18 @@ mod result_processor; use result_processor::*; +use std::hash::Hasher; use std::hint::black_box; use std::time::{Instant, Duration}; use std::alloc::{alloc, dealloc, Layout}; use std::slice; -use std::hash::Hasher; use rand::Rng; use gxhash::*; const ITERATIONS: u32 = 1000; -const MAX_RUN_DURATION: Duration = Duration::from_millis(1000); +const MAX_RUN_DURATION: Duration = Duration::from_millis(100); const FORCE_NO_INLINING: bool = false; fn main() { @@ -30,11 +30,17 @@ fn main() { let mut processor = ResultProcessor::default(); // GxHash - let algo_name = if cfg!(feature = "avx2") { "gxhash-avx2" } else { "gxhash" }; - benchmark(&mut processor, slice, algo_name, |data: &[u8], seed: i64| -> u64 { + benchmark(&mut processor, slice, "gxhash", |data: &[u8], seed: i64| -> u64 { gxhash64(data, seed) }); + // GxHash-AVX2 + if cfg!(feature = "avx2") { + benchmark(&mut processor, slice, "gxhash-avx2", |data: &[u8], seed: i64| -> u64 { + gxhash64(data, seed) + }); + } + // XxHash (twox-hash) benchmark(&mut processor, slice, "xxhash", |data: &[u8], seed: u64| -> u64 { twox_hash::xxh3::hash64_with_seed(data, seed) @@ -76,6 +82,8 @@ fn main() { fnv_hasher.finish() }); + processor.finish(); + // Free benchmark data unsafe { dealloc(ptr, layout) }; } @@ -122,6 +130,8 @@ fn time(iterations: u32, delegate: &F) -> Duration where F: Fn() -> u64 { let now = Instant::now(); + // Bench the same way to what is done in criterion.rs + // https://github.com/bheisler/criterion.rs/blob/e1a8c9ab2104fbf2d15f700d0038b2675054a2c8/src/bencher.rs#L87 for _ in 0..iterations { if FORCE_NO_INLINING { black_box(execute_noinlining(delegate)); @@ -132,6 +142,8 @@ fn time(iterations: u32, delegate: &F) -> Duration now.elapsed() } +// Some algorithm are more likely to be inlined than others. +// This puts then all at the same level. But is it fair? #[inline(never)] fn execute_noinlining(delegate: &F) -> u64 where F: Fn() -> u64 diff --git a/benches/throughput/result_processor.rs b/benches/throughput/result_processor.rs index 88a4faf..375641e 100644 --- a/benches/throughput/result_processor.rs +++ b/benches/throughput/result_processor.rs @@ -1,3 +1,6 @@ +use gxhash::GxHashMap; +use plotters::prelude::*; + #[cfg(feature = "bench-csv")] #[derive(Default)] pub struct ResultProcessor { @@ -68,21 +71,105 @@ impl ResultProcessor { } } -#[cfg(all(not(feature = "bench-csv"),not(feature = "bench-md")))] +// #[cfg(all(not(feature = "bench-csv"), not(feature = "bench-md")))] +// #[derive(Default)] +// pub struct ResultProcessor; + +// #[cfg(all(not(feature = "bench-csv"), not(feature = "bench-md")))] +// impl ResultProcessor { +// pub fn on_start(&mut self, name: &str) { +// println!("{}", name); +// } + +// pub fn on_result(&mut self, input_size: usize, throughput: f64) { +// println!(" | {} > {:.2}", input_size, throughput); +// } + +// pub fn on_end(&mut self) { +// println!(); +// } +// } + +//#[cfg(feature = "bench-plot")] #[derive(Default)] -pub struct ResultProcessor; +pub struct ResultProcessor { + series: Vec<(String, Vec<(usize, f64)>)> +} -#[cfg(all(not(feature = "bench-csv"),not(feature = "bench-md")))] +//#[cfg(feature = "bench-plot")] impl ResultProcessor { + pub fn on_start(&mut self, name: &str) { - println!("{}", name); + println!("Started '{}'...", name); + self.series.push((name.to_string(), Vec::new())); } pub fn on_result(&mut self, input_size: usize, throughput: f64) { - println!(" | {} > {:.2}", input_size, throughput); + let len = self.series.len(); + let serie = self.series.get_mut(len - 1).unwrap(); + + serie.1.push((input_size, throughput)); } pub fn on_end(&mut self) { - println!(); + + } + + pub fn finish(&self) { + let arch = std::env::consts::ARCH; + let file_name = format!("benches/throughput/{}.svg", arch); + + let canvas = SVGBackend::new(file_name.as_str(), (600, 400)).into_drawing_area(); + canvas.fill(&WHITE).unwrap(); + + let x_min = self.series.iter().next().unwrap().1.iter().map(|(x, _)| *x as u32).min().unwrap(); + let x_max = self.series.iter().next().unwrap().1.iter().map(|(x, _)| *x as u32).max().unwrap(); + + let y_min = 0u32; + let y_max = self.series.iter().flat_map(|inner_map| inner_map.1.iter()).map(|(_, y)| (1.05 * *y) as u32).max().unwrap(); + + let mut chart = ChartBuilder::on(&canvas) + .caption(format!("Throughput ({})", arch), ("sans-serif", (5).percent_height())) + .set_label_area_size(LabelAreaPosition::Left, (14).percent()) + .set_label_area_size(LabelAreaPosition::Bottom, (10).percent()) + .margin((1).percent()) + .build_cartesian_2d( + (x_min..x_max) + .log_scale() + .with_key_points(self.series.iter().next().unwrap().1.iter().map(|(x, _)| *x as u32).collect::>()), + (y_min..y_max) + //.log_scale(), + ).unwrap(); + + chart + .configure_mesh() + .max_light_lines(1) + .x_desc("Input Size (bytes)") + .y_desc("Throughput (MiB/s)") + .draw().unwrap(); + + let mut color_idx = 0; + for (name, values) in self.series.iter() { + let color = Palette99::pick(color_idx); + color_idx += 1; + let data: Vec<_> = values.iter().map(|(x, y)| (*x as u32, *y as u32)).collect(); + chart + .draw_series(LineSeries::new(data, + color.stroke_width(2), + )).unwrap() + .label(name) + .legend(move |(x, y)| Rectangle::new([(x, y - 5), (x + 10, y + 5)], color.filled())); + } + + chart + .configure_series_labels() + .border_style(BLACK) + .background_style(RGBColor(255, 255, 255)) + .draw().unwrap(); + + // To avoid the IO failure being ignored silently, we manually call the present function + canvas.present().expect("Unable to write result to file, please make sure 'plotters-doc-data' dir exists under current dir"); + + println!("Finished"); } } \ No newline at end of file