concrete_compiler_benchmark.yml

name: concrete-compiler benchmark linux-cpu

on:
  workflow_dispatch:
  pull_request:
    paths:
      - .github/workflows/concrete_compiler_benchmark.yml
      - compilers/**
      - backends/**
      - tools/**
  push:
    branches:
      - 'main'
      - 'release/*'

env:
  DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
  SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
  SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
  SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

concurrency:
  group: concrete_compiler_benchmark_${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  setup-instance:
    runs-on: ubuntu-latest
    outputs:
      runner-name: ${{ steps.start-instance.outputs.label }}
    steps:
      - name: Start instance
        id: start-instance
        uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
        with:
          mode: start
          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
          slab-url: ${{ secrets.SLAB_BASE_URL }}
          job-secret: ${{ secrets.JOB_SECRET }}
          backend: aws
          profile: cpu-bench

  build-and-run-benchmarks:
    needs: setup-instance
    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
    steps:
      - name: Checkout concrete
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          submodules: recursive
          fetch-depth: 0
      - name: Set up home
        # "Install rust" step require root user to have a HOME directory which is not set.
        run: |
          echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
      - name: Setup rust toolchain for concrete-cpu
        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
      - name: Build compiler benchmarks
        run: |
          set -e
          git config --global --add safe.directory '*'
          cd compilers/concrete-compiler/compiler
          make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks
      - name: Run compiler benchmarks
        run: |
          set -e
          cd compilers/concrete-compiler/compiler
          make run-cpu-benchmarks
      - name: Upload raw results artifact
        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
        with:
          name: compiler-benchmarks-result
          path: compilers/concrete-compiler/compiler/benchmarks_results.json
      - name: Slack Notification
        if: ${{ failure() && github.ref == 'refs/heads/main' }}
        continue-on-error: true
        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
        env:
          SLACK_COLOR: ${{ job.status }}
          SLACK_MESSAGE: "build-and-run-benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

  parse-and-send-results:
    name: Parse and send results
    needs: [setup-instance, build-and-run-benchmarks]
    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
    steps:
      - name: Download compiler-benchmarks-result
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: compiler-benchmarks-result
      - name: Parse results
        shell: bash
        run: |
          BENCH_DATE=$(date --iso-8601=seconds)
          COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})
          COMMIT_HASH=$(git describe --tags --dirty)
          # TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38)
          python3 ./ci/benchmark_parser.py benchmarks_results.json parsed_benchmark_results.json \
          --database compiler_benchmarks \
          --hardware "hpc7a.96xlarge" \
          --project-version ${COMMIT_HASH} \
          --branch ${{ github.ref_name }} \
          --commit-date ${COMMIT_DATE} \
          --bench-date "${BENCH_DATE}" \
          --throughput
      - name: Upload parsed results artifact
        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
        with:
          name: compiler-benchmarks-parsed-result
          path: parsed_benchmark_results.json
      - name: Checkout Slab repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: zama-ai/slab
          path: slab
          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
      - name: Send data to Slab
        shell: bash
        run: |
          python3 slab/scripts/data_sender.py parsed_benchmark_results.json "${{ secrets.JOB_SECRET }}" --slab-url "${{ secrets.SLAB_URL }}"
      - name: Slack Notification
        if: ${{ failure() && github.ref == 'refs/heads/main' }}
        continue-on-error: true
        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
        env:
          SLACK_COLOR: ${{ job.status }}
          SLACK_MESSAGE: "parse-and-send-results finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

  teardown-instance:
    name: Teardown instance
    needs: [ setup-instance, parse-and-send-results ]
    if: ${{ always() && needs.setup-instance.result != 'skipped' }}
    runs-on: ubuntu-latest
    steps:
      - name: Stop instance
        id: stop-instance
        uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
        with:
          mode: stop
          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
          slab-url: ${{ secrets.SLAB_BASE_URL }}
          job-secret: ${{ secrets.JOB_SECRET }}
          label: ${{ needs.setup-instance.outputs.runner-name }}
      - name: Slack Notification
        if: ${{ failure() }}
        continue-on-error: true
        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
        env:
          SLACK_COLOR: ${{ job.status }}
          SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"