From 197dbf088eaaac3826c8615be5e78146b5cbeb65 Mon Sep 17 00:00:00 2001 From: Matt Stam Date: Thu, 29 Feb 2024 19:42:13 -0800 Subject: [PATCH] ci: runs benchmarks on different architectures and run multiple times (#340) --- .github/workflows/eval.yml | 11 +++- eval.sh | 5 +- eval/src/main.rs | 123 +++++++++++++++++++------------------ 3 files changed, 74 insertions(+), 65 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 0cef5acfdf..7477a2c6f3 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -7,7 +7,14 @@ on: jobs: benchmark: name: Benchmark - runs-on: self-hosted + runs-on: ${{ matrix.runner_label }} + strategy: + matrix: + include: + - runner_label: self-hosted-x86 + arch: x86 + - runner_label: self-hosted-arm64 + arch: arm64 if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')" env: CARGO_NET_GIT_FETCH_WITH_CLI: "true" @@ -40,5 +47,5 @@ jobs: - name: Upload Benchmark as Artifact uses: actions/upload-artifact@v2 with: - name: benchmark-results + name: benchmark-results-${{ matrix.arch }} path: benchmark.csv diff --git a/eval.sh b/eval.sh index 79d4d19c54..7701c3b998 100755 --- a/eval.sh +++ b/eval.sh @@ -4,6 +4,7 @@ set -e declare -a programs=("fibonacci" "ssz-withdrawals" "tendermint") declare -a hash_functions=("poseidon" "blake3" "keccak256") declare -a shard_sizes=("262144" "524288" "1048576" "2097152" "4194304") +declare -i runs=5 root_directory=$(pwd) @@ -33,9 +34,9 @@ for program in "${programs[@]}"; do cd "${root_directory}/eval" for hash_fn in "${hash_functions[@]}"; do for shard_size in "${shard_sizes[@]}"; do - echo "Running $program with hash function $hash_fn and shard size $shard_size" + echo "Running $program with hash function $hash_fn and shard size $shard_size, $runs times" if ! CARGO_NET_GIT_FETCH_WITH_CLI=true RUSTFLAGS='-C target-cpu=native' cargo run -p sp1-eval --release -- \ - --program $program --hashfn $hash_fn --shard-size $shard_size --benchmark-path "$benchmark_path" --elf-path "$elf_path"; then + --program $program --hashfn $hash_fn --shard-size $shard_size --benchmark-path "$benchmark_path" --elf-path "$elf_path" --runs $runs; then echo "Error running evaluation for $program with hash function $hash_fn and shard size $shard_size" fi done diff --git a/eval/src/main.rs b/eval/src/main.rs index 37984c98f0..8e50e10fcb 100644 --- a/eval/src/main.rs +++ b/eval/src/main.rs @@ -75,126 +75,127 @@ struct EvalArgs { #[arg(long)] pub elf_path: String, + + #[arg(long, default_value_t = 1)] + pub runs: usize, } fn main() { let args = EvalArgs::parse(); - let elf_path = args.elf_path; - println!("ELF path: {}", elf_path); + // Load the program. + let elf_path = &args.elf_path; + let program = Program::from_elf(elf_path); + let cycles = get_cycles(program.clone()); - let benchmark_path = args.benchmark_path; - println!("Benchmark path: {}", benchmark_path); + // Initialize total duration counters. + let mut total_execution_duration = 0f64; + let mut total_prove_duration = 0f64; + let mut total_verify_duration = 0f64; + + // Perform runs. + for _ in 0..args.runs { + let elf = fs::read(elf_path).expect("Failed to read ELF file"); + let (execution_duration, prove_duration, verify_duration) = + run_evaluation(&args.hashfn, &program, &elf); + + // Accumulate durations. + total_execution_duration += execution_duration; + total_prove_duration += prove_duration; + total_verify_duration += verify_duration; + } - // Read the program from the file system. - let program = Program::from_elf(&elf_path); - let elf = fs::read(&elf_path).unwrap(); + // Calculate average durations. + let avg_execution_duration = total_execution_duration / args.runs as f64; + let avg_prove_duration = total_prove_duration / args.runs as f64; + let avg_verify_duration = total_verify_duration / args.runs as f64; - // Compute some statistics. - let cycles = get_cycles(program.clone()); - println!("sp1 cycles: {}", cycles); + let report = PerformanceReport { + program: args.program, + hashfn: args.hashfn.to_string(), + shard_size: args.shard_size, + cycles, + speed: cycles as f64 / avg_prove_duration, + execution_duration: avg_execution_duration, + prove_duration: avg_prove_duration, + verify_duration: avg_verify_duration, + }; - // Setup the prover. - std::env::set_var("SHARD_SIZE", format!("{}", args.shard_size)); - if args.shard_size & (args.shard_size - 1) != 0 { - panic!("shard_size must be a power of 2"); + // Write the report. + if let Err(e) = write_report(report, &args.benchmark_path) { + eprintln!("Failed to write report: {}", e); } +} - let (execution_duration, prove_duration, verify_duration) = match args.hashfn { +fn run_evaluation(hashfn: &HashFnId, program: &Program, elf: &[u8]) -> (f64, f64, f64) { + match hashfn { HashFnId::Blake3 => { - // Execute the runtime. - let mut runtime = Runtime::new(program); + let mut runtime = Runtime::new(program.clone()); let execution_start = Instant::now(); runtime.run(); - let execution_duration = execution_start.elapsed(); + let execution_duration = execution_start.elapsed().as_secs_f64(); - // Generate the proof. let config = BabyBearBlake3::new(); let prove_start = Instant::now(); - let proof = prove_core(config, runtime); - let prove_duration = prove_start.elapsed(); + let proof = prove_core(config.clone(), runtime); + let prove_duration = prove_start.elapsed().as_secs_f64(); let proof = SP1ProofWithIO { stdin: SP1Stdin::new(), stdout: SP1Stdout::new(), proof, }; - // Verify the proof. - let config = BabyBearBlake3::new(); let verify_start = Instant::now(); - SP1Verifier::verify_with_config(&elf, &proof, config).unwrap(); - let verify_duration = verify_start.elapsed(); + SP1Verifier::verify_with_config(elf, &proof, config).unwrap(); + let verify_duration = verify_start.elapsed().as_secs_f64(); (execution_duration, prove_duration, verify_duration) } HashFnId::Poseidon => { - // Execute the runtime. - let mut runtime = Runtime::new(program); + let mut runtime = Runtime::new(program.clone()); let execution_start = Instant::now(); runtime.run(); - let execution_duration = execution_start.elapsed(); + let execution_duration = execution_start.elapsed().as_secs_f64(); - // Generate the proof. let config = BabyBearPoseidon2::new(); let prove_start = Instant::now(); - let proof = prove_core(config, runtime); - let prove_duration = prove_start.elapsed(); + let proof = prove_core(config.clone(), runtime); + let prove_duration = prove_start.elapsed().as_secs_f64(); let proof = SP1ProofWithIO { stdin: SP1Stdin::new(), stdout: SP1Stdout::new(), proof, }; - // Verify the proof. - let config = BabyBearPoseidon2::new(); let verify_start = Instant::now(); - SP1Verifier::verify_with_config(&elf, &proof, config).unwrap(); - let verify_duration = verify_start.elapsed(); + SP1Verifier::verify_with_config(elf, &proof, config).unwrap(); + let verify_duration = verify_start.elapsed().as_secs_f64(); (execution_duration, prove_duration, verify_duration) } HashFnId::Keccak256 => { - // Execute the runtime. - let mut runtime = Runtime::new(program); + let mut runtime = Runtime::new(program.clone()); let execution_start = Instant::now(); runtime.run(); - let execution_duration = execution_start.elapsed(); + let execution_duration = execution_start.elapsed().as_secs_f64(); - // Generate the proof. let config = BabyBearKeccak::new(); let prove_start = Instant::now(); - let proof = prove_core(config, runtime); - let prove_duration = prove_start.elapsed(); + let proof = prove_core(config.clone(), runtime); + let prove_duration = prove_start.elapsed().as_secs_f64(); let proof = SP1ProofWithIO { stdin: SP1Stdin::new(), stdout: SP1Stdout::new(), proof, }; - // Verify the proof. - let config = BabyBearKeccak::new(); let verify_start = Instant::now(); - SP1Verifier::verify_with_config(&elf, &proof, config).unwrap(); - let verify_duration = verify_start.elapsed(); + SP1Verifier::verify_with_config(elf, &proof, config).unwrap(); + let verify_duration = verify_start.elapsed().as_secs_f64(); (execution_duration, prove_duration, verify_duration) } - _ => panic!("unsupported hash function"), - }; - - let report = PerformanceReport { - program: args.program, - hashfn: args.hashfn.to_string(), - shard_size: args.shard_size, - cycles, - speed: (cycles as f64) / prove_duration.as_secs_f64(), - execution_duration: execution_duration.as_secs_f64(), - prove_duration: prove_duration.as_secs_f64(), - verify_duration: verify_duration.as_secs_f64(), - }; - - if let Err(e) = write_report(report, &benchmark_path) { - eprintln!("Failed to write report: {}", e); + _ => panic!("Unsupported hash function"), } }