Skip to content

Create Open Source GUI for Golem in TypeScript #398

Create Open Source GUI for Golem in TypeScript

Create Open Source GUI for Golem in TypeScript #398

Workflow file for this run

name: On Demand Benchmark
on:
issue_comment:
types: [created]
jobs:
run-benchmark:
if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/run-benchmark') }}
permissions: write-all
runs-on: ubuntu-latest-large
steps:
# https://github.com/actions/checkout/issues/331#issuecomment-1567674767
- id: "get-branch"
run: echo ::set-output name=branch::$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')
env:
REPO: ${{ github.repository }}
PR_NO: ${{ github.event.issue.number }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
ref: ${{ steps.get-branch.outputs.branch }}
submodules: recursive
- name: Fetch tag
run: git fetch origin --deepen=1
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
override: true
- uses: Swatinem/rust-cache@v2
with:
shared-key: benchmark-tests
cache-all-crates: true
- name: Run sccache-cache
uses: mozilla-actions/[email protected]
- uses: davidB/rust-cargo-make@v1
- name: Install Protoc
uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Redis
uses: shogo82148/[email protected]
with:
redis-version: latest
auto-start: false
- name: Build Benchmark Report Job
run: |
echo "Building all targets including benchmark-report job"
cargo make build-release
pushd test-components/rust-service
cargo build -p rust-http-service --release
popd
- name: Run Benchmark
run: |
echo "Running all benchmark jobs and saving to files"
- name: Run benchmark and report
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ISSUE_NUMBER: ${{ github.event.issue.number }}
WORKFLOW_ID: ${{ github.run_id }}
COMMENT_BODY: ${{ github.event.comment.body }}
run: |
#! /bin/bash
# Create a dummy benchmark result
mkdir -p ./benchmark-data-current
echo "Starting cold start large benchmark"
./target/release/benchmark_cold_start_large --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_cold_start_large.json
echo "Successfully ran cold start large benchmark"
echo "Starting cold start medium benchmark"
./target/release/benchmark_cold_start_medium --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_cold_start_medium.json
echo "Successfully ran cold start medium benchmark"
echo "Starting cold start small benchmark"
./target/release/benchmark_cold_start_small --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_cold_start_small.json
echo "Successfully ran cold start small benchmark"
echo "Starting cold start large benchmark without component compilation"
./target/release/benchmark_cold_start_large --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child --compilation-service-disabled > benchmark-data-current/benchmark_cold_start_large_no_compilation.json
echo "Successfully ran cold start large benchmark without component compilation"
echo "Starting cold start medium benchmark without component compilation"
./target/release/benchmark_cold_start_medium --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child --compilation-service-disabled > benchmark-data-current/benchmark_cold_start_medium_no_compilation.json
echo "Successfully ran cold start medium benchmark without component compilation"
echo "Starting cold start small benchmark without component compilation"
./target/release/benchmark_cold_start_small --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child --compilation-service-disabled > benchmark-data-current/benchmark_cold_start_small_no_compilation.json
echo "Successfully ran cold start small benchmark without component compilation"
echo "Starting durability overhead benchmark"
./target/release/benchmark_durability_overhead --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_durability_overhead.json
echo "Successfully durability overhead benchmark"
echo "Starting latency large benchmark"
./target/release/benchmark_latency_large --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_latency_large.json
echo "Successfully ran latency large benchmark"
echo "Starting latency medium benchmark"
./target/release/benchmark_latency_medium --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_latency_medium.json
echo "Successfully ran latency medium benchmark"
echo "Starting latency small benchmark"
./target/release/benchmark_latency_small --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_latency_small.json
echo "Successfully ran latency small benchmark"
echo "Starting suspend benchmark"
./target/release/benchmark_suspend_worker --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_suspend_worker.json
echo "Successfully ran suspend benchmark"
echo "Starting RPC benchmark"
./target/release/benchmark_rpc --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 50 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_rpc.json
echo "Successfully ran RPC benchmark"
echo "Starting RPC CPU intensive benchmark"
./target/release/benchmark_rpc_cpu_intensive --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 2 --size 4 --size 8 --size 16 --size 32 --length 10 spawned --mute-child > benchmark-data-current/benchmark_rpc_cpu_intensive.json
echo "Successfully ran RPC CPU intensive benchmark"
echo "Starting RPC large input benchmark"
./target/release/benchmark_rpc_large_input --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 50 --size 100 --length 10 spawned --mute-child > benchmark-data-current/benchmark_rpc_large_input.json
echo "Successfully ran RPC large input benchmark"
echo "Starting throughput benchmark"
./target/release/benchmark_throughput --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 50 --size 100 --size 250 --size 500 --length 10 spawned --mute-child > benchmark-data-current/benchmark_throughput.json
echo "Successfully ran throughput benchmark"
echo "Starting throughput CPU intensive benchmark"
./target/release/benchmark_throughput_cpu_intensive --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 2 --size 4 --size 8 --size 16 --size 32 --length 10 spawned --mute-child > benchmark-data-current/benchmark_throughput_cpu_intensive.json
echo "Successfully ran throughput CPU intensive benchmark"
echo "Starting throughput large input benchmark"
./target/release/benchmark_throughput_large_input --quiet --json --primary-only --iterations 1 --cluster-size 1 --size 1 --size 10 --size 50 --size 100 --length 10 spawned --mute-child > benchmark-data-current/benchmark_throughput_large_input.json
echo "Successfully ran throughput large input benchmark"
echo "Finished Running Benchmark Jobs"
files=$(ls benchmark-data-current)
echo "Checking the following files if it already exists in a previous benchmark run"
for str in ${files[@]}; do
echo $str
done
# Initialize arrays to store present and absent files
present_files=()
absent_files=()
compare_command="./target/release/benchmark_report compare-benchmarks"
get_report_command="./target/release/benchmark_report get-report"
echo "Starting to pull the existing benchmark results if exists"
if git fetch origin main:main && [[ ! "$COMMENT_BODY" =~ refresh ]]; then
# Check each file
for file in ${files[@]}; do
if git checkout main -- "benchmark-data/$file"; then
present_files+=("$file")
compare_command="$compare_command --files $file=$(pwd)/benchmark-data/$file,$(pwd)/benchmark-data-current/$file"
else
absent_files+=("$file")
get_report_command="$get_report_command --files $file=$(pwd)/benchmark-data-current/$file"
fi
done
else
for file in ${files[@]}; do
absent_files+=("$file")
get_report_command="$get_report_command --files $file=$(pwd)/benchmark-data-current/$file"
done
fi
echo $compare_command
echo $get_report_command
# If there are present files, run the comparison
if [ ${#present_files[@]} -ne 0 ]; then
echo "Running comparison with command: $compare_command"
$compare_command > benchmark_comparison.md
COMPARISON_MD=$(<benchmark_comparison.md)
echo $COMPARISON_MD
curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/golemcloud/golem/issues/$ISSUE_NUMBER/comments \
-d "{\"body\":\"$COMPARISON_MD \n RunID: $WORKFLOW_ID \"}"
fi
# Process absent files
if [ ${#absent_files[@]} -ne 0 ]; then
echo "Running report"
$get_report_command > report.md
echo "Successfully ran the report command"
REPORT_MD=$(<report.md)
echo $REPORT_MD
curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/golemcloud/golem/issues/$ISSUE_NUMBER/comments \
-d "{\"body\":\"$REPORT_MD \n RunID: $WORKFLOW_ID \"}"
fi
echo "Done processing files."
- name: "Move benchmark-data-current to benchmark-data"
run: |
# Removing the old benchmark data
rm -rf benchmark-data
rm -rf benchmark_comparison.md
rm -rf report.md
# Moving the current benchmark data to benchmark-data
mv benchmark-data-current benchmark-data
- name: Commit and push benchmark results (not report) back to PR branch
run: |
git branch -a
git config --local user.email "[email protected]"
git config --local user.name "GitHub Action"
git pull
git add benchmark-data
git commit -m "Adding benchmarking data"
if git push; then
echo "Successfully pushed the benchmark data"
else
echo "Benchmark data hasn't been pushed probably due to zero changes"
fi