From a0b519255b2e162d7dbae11818efa71f9088ae4e Mon Sep 17 00:00:00 2001 From: Federica Date: Wed, 4 Oct 2023 12:00:01 -0300 Subject: [PATCH 1/6] Add benchmarks --- Makefile | 21 +++++++++++++++++-- cairo_programs/benchmarks/fibonacci_10K.cairo | 14 +++++++++++++ cairo_programs/benchmarks/fibonacci_50k.cairo | 14 +++++++++++++ scripts/run_benchmarks.sh | 16 ++++++++++++++ 4 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 cairo_programs/benchmarks/fibonacci_10K.cairo create mode 100644 cairo_programs/benchmarks/fibonacci_50k.cairo create mode 100644 scripts/run_benchmarks.sh diff --git a/Makefile b/Makefile index 9954a52f..e98df69d 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,13 @@ $(CAIRO_VM_CLI): # NOTE: This is super flaky, DO NOT move this section below the non proof mode one # or things will go wrong. +BENCH_DIR=cairo_programs/benchmarks +BENCH_FILES:=$(wildcard $(BENCH_DIR)/*.cairo) +COMPILED_BENCHES:=$(patsubst $(BENCH_DIR)/%.cairo, $(BENCH_DIR)/%.json, $(BENCH_FILES)) + +$(BENCH_DIR)/%.json: $(BENCH_DIR)/%.cairo + cairo-compile --cairo_path="$(BENCH_DIR)" $< --output $@ --proof_mode + TEST_PROOF_DIR=cairo_programs/proof_programs TEST_PROOF_FILES:=$(wildcard $(TEST_PROOF_DIR)/*.cairo) COMPILED_PROOF_TESTS:=$(patsubst $(TEST_PROOF_DIR)/%.cairo, $(TEST_PROOF_DIR)/%.json, $(TEST_PROOF_FILES)) @@ -83,7 +90,7 @@ build: @cp pkg/lambdaworks/lib/lambdaworks/target/release/liblambdaworks.a pkg/lambdaworks/lib @cd pkg/starknet_crypto/lib/starknet_crypto && cargo build --release @cp pkg/starknet_crypto/lib/starknet_crypto/target/release/libstarknet_crypto.a pkg/starknet_crypto/lib - @go build ./... + @go build -ldflags "-w -s" ./... fmt: gofmt -w pkg @@ -95,6 +102,10 @@ clean: rm -f $(TEST_DIR)/*.json rm -f $(TEST_DIR)/*.memory rm -f $(TEST_DIR)/*.trace + rm -f $(TEST_PROOF_DIR)/*.json + rm -f $(TEST_PROOF_DIR)/*.memory + rm -f $(TEST_PROOF_DIR)/*.trace + rm -f $(BENCH_DIR)/*.json cd pkg/lambdaworks/lib/lambdaworks && cargo clean rm -f pkg/lambdaworks/lib/liblambdaworks.a cd pkg/starknet_crypto/lib/starknet_crypto && cargo clean @@ -104,9 +115,12 @@ clean: clean_files: rm -f $(TEST_DIR)/*.json - rm -f $(TEST_DIR)/proof_programs/*.json rm -f $(TEST_DIR)/*.memory rm -f $(TEST_DIR)/*.trace + rm -f $(TEST_PROOF_DIR)/*.json + rm -f $(TEST_PROOF_DIR)/*.memory + rm -f $(TEST_PROOF_DIR)/*.trace + rm -f $(BENCH_DIR)/*.json demo_fibonacci: clean_files build_cairo_vm_cli build @echo "Compiling fibonacci program..." @@ -172,3 +186,6 @@ compare_proof_memory: build_cairo_vm_cli $(CAIRO_RS_PROOF_MEM) $(CAIRO_GO_PROOF_ clean_trace_and_memory_files: rm -f $(TEST_DIR)/*.rs.* && rm -f $(TEST_DIR)/*.go.* && rm -f $(TEST_PROOF_DIR)/*.rs.* && rm -f $(TEST_PROOF_DIR)/*.go.* + +compare_benchmarks: $(COMPILED_BENCHES) build + sh scripts/run_benchmarks.sh diff --git a/cairo_programs/benchmarks/fibonacci_10K.cairo b/cairo_programs/benchmarks/fibonacci_10K.cairo new file mode 100644 index 00000000..12ab5771 --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_10K.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 10000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +} diff --git a/cairo_programs/benchmarks/fibonacci_50k.cairo b/cairo_programs/benchmarks/fibonacci_50k.cairo new file mode 100644 index 00000000..f5d32b85 --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_50k.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 50000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +} \ No newline at end of file diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh new file mode 100644 index 00000000..d0704113 --- /dev/null +++ b/scripts/run_benchmarks.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh +tests_path="cairo_programs/benchmarks" + +set -e + +for file in $(ls $tests_path | grep .cairo | sed -E 's/\.cairo//'); do + echo "Running $file benchmark" + + export PATH="$(pyenv root)/shims:$PATH" + + hyperfine \ + -n "Cairo VM (CPython)" "PYENV_VERSION=3.9.15 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \ + -n "Cairo VM (PyPy)" "PYENV_VERSION=pypy3.9-7.3.9 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \ + -n "cairo-vm (Rust)" "cairo-vm/target/release/cairo-vm-cli $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" \ + -n "cairo-vm.go (Go)" "go run cmd/cli/main.go $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" +done From 2ef0a0ff45c58f66fb55562b57686ad691ec27f2 Mon Sep 17 00:00:00 2001 From: Federica Date: Wed, 4 Oct 2023 12:02:56 -0300 Subject: [PATCH 2/6] Fix file --- cairo_programs/benchmarks/fibonacci_50k.cairo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cairo_programs/benchmarks/fibonacci_50k.cairo b/cairo_programs/benchmarks/fibonacci_50k.cairo index f5d32b85..e8358f0f 100644 --- a/cairo_programs/benchmarks/fibonacci_50k.cairo +++ b/cairo_programs/benchmarks/fibonacci_50k.cairo @@ -11,4 +11,4 @@ func fib(first_element, second_element, n) -> (res: felt) { tempvar y = first_element + second_element; return fib(second_element, y, n - 1); -} \ No newline at end of file +} From 0e2901a4ed0a7619d44e062de935eeef97ef8a1c Mon Sep 17 00:00:00 2001 From: Federica Date: Wed, 4 Oct 2023 12:04:57 -0300 Subject: [PATCH 3/6] Add warmup --- scripts/run_benchmarks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh index d0704113..e79b3a5a 100644 --- a/scripts/run_benchmarks.sh +++ b/scripts/run_benchmarks.sh @@ -8,7 +8,7 @@ for file in $(ls $tests_path | grep .cairo | sed -E 's/\.cairo//'); do export PATH="$(pyenv root)/shims:$PATH" - hyperfine \ + hyperfine -w 5 \ -n "Cairo VM (CPython)" "PYENV_VERSION=3.9.15 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \ -n "Cairo VM (PyPy)" "PYENV_VERSION=pypy3.9-7.3.9 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \ -n "cairo-vm (Rust)" "cairo-vm/target/release/cairo-vm-cli $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" \ From a05319dd0820124cbf81eebcae6b1cb9659158a2 Mon Sep 17 00:00:00 2001 From: Federica Date: Wed, 4 Oct 2023 12:42:53 -0300 Subject: [PATCH 4/6] Add light version with only rust and go --- Makefile | 3 +++ scripts/run_benchmarks_light.sh | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 scripts/run_benchmarks_light.sh diff --git a/Makefile b/Makefile index e98df69d..6765dedd 100644 --- a/Makefile +++ b/Makefile @@ -189,3 +189,6 @@ clean_trace_and_memory_files: compare_benchmarks: $(COMPILED_BENCHES) build sh scripts/run_benchmarks.sh + +compare_benchmarks_light: $(COMPILED_BENCHES) build + sh scripts/run_benchmarks_light.sh diff --git a/scripts/run_benchmarks_light.sh b/scripts/run_benchmarks_light.sh new file mode 100644 index 00000000..a37b3d0d --- /dev/null +++ b/scripts/run_benchmarks_light.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh +tests_path="cairo_programs/benchmarks" + +set -e + +for file in $(ls $tests_path | grep .cairo | sed -E 's/\.cairo//'); do + echo "Running $file benchmark" + + export PATH="$(pyenv root)/shims:$PATH" + + hyperfine -w 5 \ + -n "cairo-vm (Rust)" "cairo-vm/target/release/cairo-vm-cli $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" \ + -n "cairo-vm.go (Go)" "go run cmd/cli/main.go $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" +done \ No newline at end of file From 2d34a3453c7cf112650717bd3f2b8fd478aefb61 Mon Sep 17 00:00:00 2001 From: Federica Date: Wed, 4 Oct 2023 12:43:15 -0300 Subject: [PATCH 5/6] Add bench --- cairo_programs/benchmarks/fibonacci_100k.cairo | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 cairo_programs/benchmarks/fibonacci_100k.cairo diff --git a/cairo_programs/benchmarks/fibonacci_100k.cairo b/cairo_programs/benchmarks/fibonacci_100k.cairo new file mode 100644 index 00000000..5fa6116c --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_100k.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 100000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +} \ No newline at end of file From 82ddc91e1f89787ac2d85fb6df9b36624c01a65f Mon Sep 17 00:00:00 2001 From: Federica Date: Wed, 4 Oct 2023 13:01:10 -0300 Subject: [PATCH 6/6] Add benchmark program --- cairo_programs/benchmarks/fibonacci_100k.cairo | 2 +- cairo_programs/benchmarks/fibonacci_500k.cairo | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 cairo_programs/benchmarks/fibonacci_500k.cairo diff --git a/cairo_programs/benchmarks/fibonacci_100k.cairo b/cairo_programs/benchmarks/fibonacci_100k.cairo index 5fa6116c..2ffca7d9 100644 --- a/cairo_programs/benchmarks/fibonacci_100k.cairo +++ b/cairo_programs/benchmarks/fibonacci_100k.cairo @@ -11,4 +11,4 @@ func fib(first_element, second_element, n) -> (res: felt) { tempvar y = first_element + second_element; return fib(second_element, y, n - 1); -} \ No newline at end of file +} diff --git a/cairo_programs/benchmarks/fibonacci_500k.cairo b/cairo_programs/benchmarks/fibonacci_500k.cairo new file mode 100644 index 00000000..10a9cdc5 --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_500k.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 500000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +}