ci: runs benchmarks on different architectures and run multiple times (…

…#340)

.github/workflows/eval.yml

@@ -7,7 +7,14 @@ on:
 jobs:
   benchmark:
     name: Benchmark
-    runs-on: self-hosted
+    runs-on: ${{ matrix.runner_label }}
+    strategy:
+      matrix:
+        include:
+          - runner_label: self-hosted-x86
+            arch: x86
+          - runner_label: self-hosted-arm64
+            arch: arm64
     if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')"
     env:
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
@@ -40,5 +47,5 @@ jobs:
       - name: Upload Benchmark as Artifact
         uses: actions/upload-artifact@v2
         with:
-          name: benchmark-results
+          name: benchmark-results-${{ matrix.arch }}
           path: benchmark.csv

eval.sh

@@ -4,6 +4,7 @@ set -e
 declare -a programs=("fibonacci" "ssz-withdrawals" "tendermint")
 declare -a hash_functions=("poseidon" "blake3" "keccak256")
 declare -a shard_sizes=("262144" "524288" "1048576" "2097152" "4194304")
+declare -i runs=5
 
 root_directory=$(pwd)
 
@@ -33,9 +34,9 @@ for program in "${programs[@]}"; do
     cd "${root_directory}/eval"
     for hash_fn in "${hash_functions[@]}"; do
         for shard_size in "${shard_sizes[@]}"; do
-            echo "Running $program with hash function $hash_fn and shard size $shard_size"
+            echo "Running $program with hash function $hash_fn and shard size $shard_size, $runs times"
             if ! CARGO_NET_GIT_FETCH_WITH_CLI=true RUSTFLAGS='-C target-cpu=native' cargo run -p sp1-eval --release -- \
-                --program $program --hashfn $hash_fn --shard-size $shard_size --benchmark-path "$benchmark_path" --elf-path "$elf_path"; then
+                --program $program --hashfn $hash_fn --shard-size $shard_size --benchmark-path "$benchmark_path" --elf-path "$elf_path" --runs $runs; then
                 echo "Error running evaluation for $program with hash function $hash_fn and shard size $shard_size"
             fi
         done

eval/src/main.rs

@@ -75,126 +75,127 @@ struct EvalArgs {
 
     #[arg(long)]
     pub elf_path: String,
+
+    #[arg(long, default_value_t = 1)]
+    pub runs: usize,
 }
 
 fn main() {
     let args = EvalArgs::parse();
 
-    let elf_path = args.elf_path;
-    println!("ELF path: {}", elf_path);
+    // Load the program.
+    let elf_path = &args.elf_path;
+    let program = Program::from_elf(elf_path);
+    let cycles = get_cycles(program.clone());
 
-    let benchmark_path = args.benchmark_path;
-    println!("Benchmark path: {}", benchmark_path);
+    // Initialize total duration counters.
+    let mut total_execution_duration = 0f64;
+    let mut total_prove_duration = 0f64;
+    let mut total_verify_duration = 0f64;
+
+    // Perform runs.
+    for _ in 0..args.runs {
+        let elf = fs::read(elf_path).expect("Failed to read ELF file");
+        let (execution_duration, prove_duration, verify_duration) =
+            run_evaluation(&args.hashfn, &program, &elf);
+
+        // Accumulate durations.
+        total_execution_duration += execution_duration;
+        total_prove_duration += prove_duration;
+        total_verify_duration += verify_duration;
+    }
 
-    // Read the program from the file system.
-    let program = Program::from_elf(&elf_path);
-    let elf = fs::read(&elf_path).unwrap();
+    // Calculate average durations.
+    let avg_execution_duration = total_execution_duration / args.runs as f64;
+    let avg_prove_duration = total_prove_duration / args.runs as f64;
+    let avg_verify_duration = total_verify_duration / args.runs as f64;
 
-    // Compute some statistics.
-    let cycles = get_cycles(program.clone());
-    println!("sp1 cycles: {}", cycles);
+    let report = PerformanceReport {
+        program: args.program,
+        hashfn: args.hashfn.to_string(),
+        shard_size: args.shard_size,
+        cycles,
+        speed: cycles as f64 / avg_prove_duration,
+        execution_duration: avg_execution_duration,
+        prove_duration: avg_prove_duration,
+        verify_duration: avg_verify_duration,
+    };
 
-    // Setup the prover.
-    std::env::set_var("SHARD_SIZE", format!("{}", args.shard_size));
-    if args.shard_size & (args.shard_size - 1) != 0 {
-        panic!("shard_size must be a power of 2");
+    // Write the report.
+    if let Err(e) = write_report(report, &args.benchmark_path) {
+        eprintln!("Failed to write report: {}", e);
     }
+}
 
-    let (execution_duration, prove_duration, verify_duration) = match args.hashfn {
+fn run_evaluation(hashfn: &HashFnId, program: &Program, elf: &[u8]) -> (f64, f64, f64) {
+    match hashfn {
         HashFnId::Blake3 => {
-            // Execute the runtime.
-            let mut runtime = Runtime::new(program);
+            let mut runtime = Runtime::new(program.clone());
             let execution_start = Instant::now();
             runtime.run();
-            let execution_duration = execution_start.elapsed();
+            let execution_duration = execution_start.elapsed().as_secs_f64();
 
-            // Generate the proof.
             let config = BabyBearBlake3::new();
             let prove_start = Instant::now();
-            let proof = prove_core(config, runtime);
-            let prove_duration = prove_start.elapsed();
+            let proof = prove_core(config.clone(), runtime);
+            let prove_duration = prove_start.elapsed().as_secs_f64();
             let proof = SP1ProofWithIO {
                 stdin: SP1Stdin::new(),
                 stdout: SP1Stdout::new(),
                 proof,
             };
 
-            // Verify the proof.
-            let config = BabyBearBlake3::new();
             let verify_start = Instant::now();
-            SP1Verifier::verify_with_config(&elf, &proof, config).unwrap();
-            let verify_duration = verify_start.elapsed();
+            SP1Verifier::verify_with_config(elf, &proof, config).unwrap();
+            let verify_duration = verify_start.elapsed().as_secs_f64();
 
             (execution_duration, prove_duration, verify_duration)
         }
         HashFnId::Poseidon => {
-            // Execute the runtime.
-            let mut runtime = Runtime::new(program);
+            let mut runtime = Runtime::new(program.clone());
             let execution_start = Instant::now();
             runtime.run();
-            let execution_duration = execution_start.elapsed();
+            let execution_duration = execution_start.elapsed().as_secs_f64();
 
-            // Generate the proof.
             let config = BabyBearPoseidon2::new();
             let prove_start = Instant::now();
-            let proof = prove_core(config, runtime);
-            let prove_duration = prove_start.elapsed();
+            let proof = prove_core(config.clone(), runtime);
+            let prove_duration = prove_start.elapsed().as_secs_f64();
             let proof = SP1ProofWithIO {
                 stdin: SP1Stdin::new(),
                 stdout: SP1Stdout::new(),
                 proof,
             };
 
-            // Verify the proof.
-            let config = BabyBearPoseidon2::new();
             let verify_start = Instant::now();
-            SP1Verifier::verify_with_config(&elf, &proof, config).unwrap();
-            let verify_duration = verify_start.elapsed();
+            SP1Verifier::verify_with_config(elf, &proof, config).unwrap();
+            let verify_duration = verify_start.elapsed().as_secs_f64();
 
             (execution_duration, prove_duration, verify_duration)
         }
         HashFnId::Keccak256 => {
-            // Execute the runtime.
-            let mut runtime = Runtime::new(program);
+            let mut runtime = Runtime::new(program.clone());
             let execution_start = Instant::now();
             runtime.run();
-            let execution_duration = execution_start.elapsed();
+            let execution_duration = execution_start.elapsed().as_secs_f64();
 
-            // Generate the proof.
             let config = BabyBearKeccak::new();
             let prove_start = Instant::now();
-            let proof = prove_core(config, runtime);
-            let prove_duration = prove_start.elapsed();
+            let proof = prove_core(config.clone(), runtime);
+            let prove_duration = prove_start.elapsed().as_secs_f64();
             let proof = SP1ProofWithIO {
                 stdin: SP1Stdin::new(),
                 stdout: SP1Stdout::new(),
                 proof,
             };
 
-            // Verify the proof.
-            let config = BabyBearKeccak::new();
             let verify_start = Instant::now();
-            SP1Verifier::verify_with_config(&elf, &proof, config).unwrap();
-            let verify_duration = verify_start.elapsed();
+            SP1Verifier::verify_with_config(elf, &proof, config).unwrap();
+            let verify_duration = verify_start.elapsed().as_secs_f64();
 
             (execution_duration, prove_duration, verify_duration)
         }
-        _ => panic!("unsupported hash function"),
-    };
-
-    let report = PerformanceReport {
-        program: args.program,
-        hashfn: args.hashfn.to_string(),
-        shard_size: args.shard_size,
-        cycles,
-        speed: (cycles as f64) / prove_duration.as_secs_f64(),
-        execution_duration: execution_duration.as_secs_f64(),
-        prove_duration: prove_duration.as_secs_f64(),
-        verify_duration: verify_duration.as_secs_f64(),
-    };
-
-    if let Err(e) = write_report(report, &benchmark_path) {
-        eprintln!("Failed to write report: {}", e);
+        _ => panic!("Unsupported hash function"),
     }
 }