From c989dd01d09f5ca975920dbf37e2e2ae507d877d Mon Sep 17 00:00:00 2001 From: John Guibas Date: Mon, 7 Oct 2024 21:28:30 -0700 Subject: [PATCH] feat: g16/plonk for network (#1617) --- .github/workflows/suite.yml | 204 ++++++++++++++++++------------------ crates/perf/README.md | 25 +++++ crates/perf/src/main.rs | 10 +- crates/perf/workflow.sh | 74 +++++++++++++ 4 files changed, 206 insertions(+), 107 deletions(-) create mode 100644 crates/perf/README.md create mode 100755 crates/perf/workflow.sh diff --git a/.github/workflows/suite.yml b/.github/workflows/suite.yml index 72928bdff2..8faa6d7dc2 100644 --- a/.github/workflows/suite.yml +++ b/.github/workflows/suite.yml @@ -2,14 +2,16 @@ name: Testing Suite on: workflow_dispatch: - pull_request: - branches: - - "**" - paths: - - "crates/**" - - "examples/**" - - "Cargo.toml" - - ".github/workflows/**" + inputs: + cpu_workloads: + description: "list of cpu workloads to run" + required: true + cuda_workloads: + description: "list of cuda workloads to run" + required: true + network_workloads: + description: "list of network workloads to run" + required: true merge_group: concurrency: @@ -17,123 +19,117 @@ concurrency: cancel-in-progress: true jobs: - test-x86: - name: ${{ matrix.workload.s3_folder }} (${{ matrix.workload.mode }}) - runs-on: ${{ matrix.workload.mode == 'cuda' && fromJSON('["runs-on", "family=g6.16xlarge", "ami=ami-0a63dc9cb9e934ba3", "spot=false", "run-id=${{ github.run_id }}"]') || fromJSON('["runs-on", "runner=64cpu-linux-x64", "spot=false", "run-id=${{ github.run_id }}"]') }} - if: ${{ contains(github.event.head_commit.message, '(run-suite)') }} + test-cpu: strategy: matrix: - workload: - # Fibonacci 17k - # - { s3_folder: fibonacci-17k, mode: cpu } - - { s3_folder: fibonacci-17k, mode: cuda } - # - { s3_folder: fibonacci-17k, mode: network } - # SSZ Withdrawals - # - { s3_folder: ssz-withdrawals, mode: cpu } - - { s3_folder: ssz-withdrawals, mode: cuda } - # - { s3_folder: ssz-withdrawals, mode: network } - # Tendermint - # - { s3_folder: tendermint, mode: cpu } - - { s3_folder: tendermint, mode: cuda } - # - { s3_folder: tendermint, mode: network } - # RSP 20526624 - # - { s3_folder: rsp-20526624, mode: cpu } - - { s3_folder: rsp-20526624, mode: cuda } - # - { s3_folder: rsp-20526624, mode: network } - # RSA - # - { s3_folder: rsa, mode: cpu } - - { s3_folder: rsa, mode: cuda } - # - { s3_folder: rsa, mode: network } - # Regex - # - { s3_folder: regex, mode: cpu } - - { s3_folder: regex, mode: cuda } - # - { s3_folder: regex, mode: network } - # Chess - # - { s3_folder: chess, mode: cpu } - - { s3_folder: chess, mode: cuda } - # - { s3_folder: chess, mode: network } - # Tendermint - # - { s3_folder: tendermint, mode: cpu } - - { s3_folder: tendermint, mode: cuda } - # - { s3_folder: tendermint, mode: network } - # RSP 20526624 - # - { s3_folder: rsp-20526624, mode: cpu } - - { s3_folder: rsp-20526624, mode: cuda } - # - { s3_folder: rsp-20526624, mode: network } - # RSA - # - { s3_folder: rsa, mode: cpu } - - { s3_folder: rsa, mode: cuda } - # - { s3_folder: rsa, mode: network } - # Regex - # - { s3_folder: regex, mode: cpu } - - { s3_folder: regex, mode: cuda } - # - { s3_folder: regex, mode: network } - # JSON - # - { s3_folder: json, mode: cpu } - - { s3_folder: json, mode: cuda } - # - { s3_folder: json, mode: network } - # Chess - # - { s3_folder: chess, mode: cpu } - - { s3_folder: chess, mode: cuda } - # - { s3_folder: chess, mode: network } - # Blobstream 01j6z63fgafrc8jeh0k12gbtvw - # - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cpu } - - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cuda } - # # - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: network } - # Blobstream 01j6z95bdme9svevmfyc974bja - # - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cpu } - - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cuda } - # # - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: network } - # Blobstream 01j6z9ak0ke9srsppgywgke6fj - # - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cpu } - - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cuda } - # - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: network } - # Vector 01j6xsv35re96tkgyda115320t - # - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cpu } - - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cuda } - # - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: network } - # Vector 01j6xzy366ff5tbkzcrs8pma02 - # - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cpu } - - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cuda } - # - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: network } - # Vector 01j6y06de0fdaafemr8b1t69z3 - # - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cpu } - - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cuda } - # - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: network } - # Raiko A7 10 - # - { s3_folder: raiko-a7-10, mode: cpu } - - { s3_folder: raiko-a7-10, mode: cuda } - # - { s3_folder: raiko-a7-10, mode: network } - # OP Succinct - # - { s3_folder: op-succinct-op-sepolia-range-17685896-17685897, mode: network } - # - { s3_folder: op-succinct-op-sepolia-range-17985900-17985905, mode: network } - # - { s3_folder: op-succinct-op-sepolia-range-18129400-18129401, mode: network } + workload: ${{ fromJSON(github.event.inputs.cpu_workloads) }} + name: ${{ matrix.workload }} (cpu) + runs-on: ["runs-on", "runner=64cpu-linux-x64", "spot=false", "run-id=${{ github.run_id }}"] steps: - name: Checkout sources uses: actions/checkout@v4 + - name: Setup CI uses: ./.github/actions/setup + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_S3 }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3 }} aws-region: us-west-2 + - name: Copy files from S3 run: | mkdir -p workdir - aws s3 cp s3://sp1-testing-suite/${{ matrix.workload.s3_folder }}/program.bin workdir/program.bin - aws s3 cp s3://sp1-testing-suite/${{ matrix.workload.s3_folder }}/stdin.bin workdir/stdin.bin + aws s3 cp s3://sp1-testing-suite/${{ matrix.workload }}/program.bin workdir/program.bin + aws s3 cp s3://sp1-testing-suite/${{ matrix.workload }}/stdin.bin workdir/stdin.bin + - name: Run sp1-perf uses: actions-rs/cargo@v1 with: command: run toolchain: 1.81.0 - args: --release -p sp1-perf -- --program workdir/program.bin --stdin workdir/stdin.bin --mode ${{ matrix.workload.mode }} + args: --release -p sp1-perf -- --program workdir/program.bin --stdin workdir/stdin.bin --mode cpu env: - RUST_LOG: debug + RUST_LOG: info RUSTFLAGS: -Copt-level=3 -Ctarget-cpu=native RUST_BACKTRACE: 1 - SP1_PROVER: ${{ matrix.workload.mode == 'cpu' && 'local' || matrix.workload.mode == 'cuda' && 'cuda' || matrix.workload.mode == 'network' && 'network' }} - SP1_PRIVATE_KEY: ${{ secrets.SP1_PRIVATE_KEY }} - PROVER_NETWORK_RPC: https://43xadiwbrdkiquizwybgpzdhcu0pgvux.lambda-url.us-east-2.on.aws/ + + test-cuda: + strategy: + matrix: + workload: ${{ fromJSON(github.event.inputs.cuda_workloads) }} + name: ${{ matrix.workload }} (gpu) + runs-on: ["runs-on", "family=g6.4xlarge", "ami=ami-0a63dc9cb9e934ba3", "spot=false", "run-id=${{ github.run_id }}"] + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Setup CI + uses: ./.github/actions/setup + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_S3 }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3 }} + aws-region: us-west-2 + + - name: Copy files from S3 + run: | + mkdir -p workdir + aws s3 cp s3://sp1-testing-suite/${{ matrix.workload }}/program.bin workdir/program.bin + aws s3 cp s3://sp1-testing-suite/${{ matrix.workload }}/stdin.bin workdir/stdin.bin + + - name: Run sp1-perf + uses: actions-rs/cargo@v1 + with: + command: run + toolchain: 1.81.0 + args: --release -p sp1-perf -- --program workdir/program.bin --stdin workdir/stdin.bin --mode cuda + env: + RUST_LOG: info + RUSTFLAGS: -Copt-level=3 -Ctarget-cpu=native + RUST_BACKTRACE: 1 + SP1_PROVER: cuda + + test-network: + strategy: + matrix: + workload: ${{ fromJSON(github.event.inputs.network_workloads) }} + name: ${{ matrix.workload }} (network) + runs-on: ["runs-on", "runner=4cpu-linux-x64", "spot=false", "run-id=${{ github.run_id }}"] + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Setup CI + uses: ./.github/actions/setup + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_S3 }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3 }} + aws-region: us-west-2 + + - name: Copy files from S3 + run: | + mkdir -p workdir + aws s3 cp s3://sp1-testing-suite/${{ matrix.workload }}/program.bin workdir/program.bin + aws s3 cp s3://sp1-testing-suite/${{ matrix.workload }}/stdin.bin workdir/stdin.bin + + - name: Run sp1-perf + uses: actions-rs/cargo@v1 + with: + command: run + toolchain: 1.81.0 + args: --release -p sp1-perf -- --program workdir/program.bin --stdin workdir/stdin.bin --mode network + env: + RUST_LOG: info + RUSTFLAGS: -Copt-level=3 -Ctarget-cpu=native + RUST_BACKTRACE: 1 + SP1_PROVER: network + SP1_PRIVATE_KEY: ${{ secrets.SP1_PRIVATE_KEY }} + PROVER_NETWORK_RPC: https://rpc-staging.succinct.xyz \ No newline at end of file diff --git a/crates/perf/README.md b/crates/perf/README.md new file mode 100644 index 0000000000..337921b6c8 --- /dev/null +++ b/crates/perf/README.md @@ -0,0 +1,25 @@ +# SP1 Testing Suite + +## Prerequisites + +- [GitHub CLI](https://cli.github.com/) + +## Run the testing suite + +Set the workloads you want to run in the `workflow.sh` file. The workloads are keys in the +`sp1-testing-suite` s3 bucket. + +``` +CPU_WORKLOADS=("fibonacci-17k" "ssz-withdrawals") +CUDA_WORKLOADS=() +NETWORK_WORKLOADS=() +``` + +Run the workflow. +``` +./workflow.sh +``` + +## View the results + +Visit the [actions](https://github.com/succinctlabs/sp1/actions) tab on GitHub to view the results. \ No newline at end of file diff --git a/crates/perf/src/main.rs b/crates/perf/src/main.rs index 3db79b045c..631b1ea49c 100644 --- a/crates/perf/src/main.rs +++ b/crates/perf/src/main.rs @@ -155,10 +155,14 @@ fn main() { let prover = ProverClient::network(); let (_, _) = time_operation(|| prover.execute(&elf, stdin.clone())); - let (core_proof, _) = - time_operation(|| prover.prove(&pk, stdin).compressed().run().unwrap()); + let (proof, _) = + time_operation(|| prover.prove(&pk, stdin.clone()).groth16().run().unwrap()); - let (_, _) = time_operation(|| prover.verify(&core_proof, &vk)); + let (_, _) = time_operation(|| prover.verify(&proof, &vk)); + + let (proof, _) = time_operation(|| prover.prove(&pk, stdin).plonk().run().unwrap()); + + let (_, _) = time_operation(|| prover.verify(&proof, &vk)); } }; } diff --git a/crates/perf/workflow.sh b/crates/perf/workflow.sh new file mode 100755 index 0000000000..b9dd82e8f6 --- /dev/null +++ b/crates/perf/workflow.sh @@ -0,0 +1,74 @@ +#! /bin/bash + +# Get the current git branch. +GIT_REF=$(git rev-parse --abbrev-ref HEAD) + +# Define the list of CPU workloads. +CPU_WORKLOADS=( + "fibonacci-17k" + "ssz-withdrawals" + "tendermint" + "rsp-20526624" + "rsa" + "regex" + "chess" + "json" + "blobstream-01j6z63fgafrc8jeh0k12gbtvw" + "blobstream-01j6z95bdme9svevmfyc974bja" + "blobstream-01j6z9ak0ke9srsppgywgke6fj" + "vector-01j6xsv35re96tkgyda115320t" + "vector-01j6xzy366ff5tbkzcrs8pma02" + "vector-01j6y06de0fdaafemr8b1t69z3" + "raiko-a7-10" +) + +# Define the list of CUDA workloads. +CUDA_WORKLOADS=( + "fibonacci-17k" + "ssz-withdrawals" + "tendermint" + "rsp-20526624" + "rsa" + "regex" + "chess" + "json" + "blobstream-01j6z63fgafrc8jeh0k12gbtvw" + "blobstream-01j6z95bdme9svevmfyc974bja" + "blobstream-01j6z9ak0ke9srsppgywgke6fj" + "vector-01j6xsv35re96tkgyda115320t" + "vector-01j6xzy366ff5tbkzcrs8pma02" + "vector-01j6y06de0fdaafemr8b1t69z3" + "raiko-a7-10" +) + +# Define the list of network workloads. +NETWORK_WORKLOADS=( + "fibonacci-17k" + "ssz-withdrawals" + "tendermint" + "rsp-20526624" + "rsa" + "regex" + "chess" + "json" + "blobstream-01j6z63fgafrc8jeh0k12gbtvw" + "blobstream-01j6z95bdme9svevmfyc974bja" + "blobstream-01j6z9ak0ke9srsppgywgke6fj" + "vector-01j6xsv35re96tkgyda115320t" + "vector-01j6xzy366ff5tbkzcrs8pma02" + "vector-01j6y06de0fdaafemr8b1t69z3" + "raiko-a7-10" + "op-succinct-op-sepolia-range-17685896-17685897" + "op-succinct-op-sepolia-range-17985900-17985905" + "op-succinct-op-sepolia-range-18129400-18129401" +) + +# Create a JSON object with the list of workloads. +WORKLOADS=$(jq -n \ + --arg cpu "$(printf '%s\n' "${CPU_WORKLOADS[@]}" | jq -R . | jq -s 'map(select(length > 0))')" \ + --arg cuda "$(printf '%s\n' "${CUDA_WORKLOADS[@]}" | jq -R . | jq -s 'map(select(length > 0))')" \ + --arg network "$(printf '%s\n' "${NETWORK_WORKLOADS[@]}" | jq -R . | jq -s 'map(select(length > 0))')" \ + '{cpu_workloads: $cpu, cuda_workloads: $cuda, network_workloads: $network}') + +# Run the workflow with the list of workloads. +echo $WORKLOADS | gh workflow run suite.yml --ref $GIT_REF --json \ No newline at end of file