diff --git a/Makefile b/Makefile index 9954a52f..6765dedd 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,13 @@ $(CAIRO_VM_CLI): # NOTE: This is super flaky, DO NOT move this section below the non proof mode one # or things will go wrong. +BENCH_DIR=cairo_programs/benchmarks +BENCH_FILES:=$(wildcard $(BENCH_DIR)/*.cairo) +COMPILED_BENCHES:=$(patsubst $(BENCH_DIR)/%.cairo, $(BENCH_DIR)/%.json, $(BENCH_FILES)) + +$(BENCH_DIR)/%.json: $(BENCH_DIR)/%.cairo + cairo-compile --cairo_path="$(BENCH_DIR)" $< --output $@ --proof_mode + TEST_PROOF_DIR=cairo_programs/proof_programs TEST_PROOF_FILES:=$(wildcard $(TEST_PROOF_DIR)/*.cairo) COMPILED_PROOF_TESTS:=$(patsubst $(TEST_PROOF_DIR)/%.cairo, $(TEST_PROOF_DIR)/%.json, $(TEST_PROOF_FILES)) @@ -83,7 +90,7 @@ build: @cp pkg/lambdaworks/lib/lambdaworks/target/release/liblambdaworks.a pkg/lambdaworks/lib @cd pkg/starknet_crypto/lib/starknet_crypto && cargo build --release @cp pkg/starknet_crypto/lib/starknet_crypto/target/release/libstarknet_crypto.a pkg/starknet_crypto/lib - @go build ./... + @go build -ldflags "-w -s" ./... fmt: gofmt -w pkg @@ -95,6 +102,10 @@ clean: rm -f $(TEST_DIR)/*.json rm -f $(TEST_DIR)/*.memory rm -f $(TEST_DIR)/*.trace + rm -f $(TEST_PROOF_DIR)/*.json + rm -f $(TEST_PROOF_DIR)/*.memory + rm -f $(TEST_PROOF_DIR)/*.trace + rm -f $(BENCH_DIR)/*.json cd pkg/lambdaworks/lib/lambdaworks && cargo clean rm -f pkg/lambdaworks/lib/liblambdaworks.a cd pkg/starknet_crypto/lib/starknet_crypto && cargo clean @@ -104,9 +115,12 @@ clean: clean_files: rm -f $(TEST_DIR)/*.json - rm -f $(TEST_DIR)/proof_programs/*.json rm -f $(TEST_DIR)/*.memory rm -f $(TEST_DIR)/*.trace + rm -f $(TEST_PROOF_DIR)/*.json + rm -f $(TEST_PROOF_DIR)/*.memory + rm -f $(TEST_PROOF_DIR)/*.trace + rm -f $(BENCH_DIR)/*.json demo_fibonacci: clean_files build_cairo_vm_cli build @echo "Compiling fibonacci program..." @@ -172,3 +186,9 @@ compare_proof_memory: build_cairo_vm_cli $(CAIRO_RS_PROOF_MEM) $(CAIRO_GO_PROOF_ clean_trace_and_memory_files: rm -f $(TEST_DIR)/*.rs.* && rm -f $(TEST_DIR)/*.go.* && rm -f $(TEST_PROOF_DIR)/*.rs.* && rm -f $(TEST_PROOF_DIR)/*.go.* + +compare_benchmarks: $(COMPILED_BENCHES) build + sh scripts/run_benchmarks.sh + +compare_benchmarks_light: $(COMPILED_BENCHES) build + sh scripts/run_benchmarks_light.sh diff --git a/cairo_programs/benchmarks/fibonacci_100k.cairo b/cairo_programs/benchmarks/fibonacci_100k.cairo new file mode 100644 index 00000000..2ffca7d9 --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_100k.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 100000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +} diff --git a/cairo_programs/benchmarks/fibonacci_10K.cairo b/cairo_programs/benchmarks/fibonacci_10K.cairo new file mode 100644 index 00000000..12ab5771 --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_10K.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 10000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +} diff --git a/cairo_programs/benchmarks/fibonacci_500k.cairo b/cairo_programs/benchmarks/fibonacci_500k.cairo new file mode 100644 index 00000000..10a9cdc5 --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_500k.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 500000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +} diff --git a/cairo_programs/benchmarks/fibonacci_50k.cairo b/cairo_programs/benchmarks/fibonacci_50k.cairo new file mode 100644 index 00000000..e8358f0f --- /dev/null +++ b/cairo_programs/benchmarks/fibonacci_50k.cairo @@ -0,0 +1,14 @@ +func main() { + fib(1, 1, 50000); + + ret; +} + +func fib(first_element, second_element, n) -> (res: felt) { + if (n == 0) { + return (second_element,); + } + + tempvar y = first_element + second_element; + return fib(second_element, y, n - 1); +} diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh new file mode 100644 index 00000000..e79b3a5a --- /dev/null +++ b/scripts/run_benchmarks.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh +tests_path="cairo_programs/benchmarks" + +set -e + +for file in $(ls $tests_path | grep .cairo | sed -E 's/\.cairo//'); do + echo "Running $file benchmark" + + export PATH="$(pyenv root)/shims:$PATH" + + hyperfine -w 5 \ + -n "Cairo VM (CPython)" "PYENV_VERSION=3.9.15 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \ + -n "Cairo VM (PyPy)" "PYENV_VERSION=pypy3.9-7.3.9 cairo-run --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak --program $tests_path/$file.json" \ + -n "cairo-vm (Rust)" "cairo-vm/target/release/cairo-vm-cli $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" \ + -n "cairo-vm.go (Go)" "go run cmd/cli/main.go $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" +done diff --git a/scripts/run_benchmarks_light.sh b/scripts/run_benchmarks_light.sh new file mode 100644 index 00000000..a37b3d0d --- /dev/null +++ b/scripts/run_benchmarks_light.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh +tests_path="cairo_programs/benchmarks" + +set -e + +for file in $(ls $tests_path | grep .cairo | sed -E 's/\.cairo//'); do + echo "Running $file benchmark" + + export PATH="$(pyenv root)/shims:$PATH" + + hyperfine -w 5 \ + -n "cairo-vm (Rust)" "cairo-vm/target/release/cairo-vm-cli $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" \ + -n "cairo-vm.go (Go)" "go run cmd/cli/main.go $tests_path/$file.json --proof_mode --memory_file /dev/null --trace_file /dev/null --layout starknet_with_keccak" +done \ No newline at end of file