Refactor ci2 #5
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: concrete-compiler test linux-cpu | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - .github/workflows/concrete_compiler_test_cpu.yml | |
| - compilers/** | |
| - backends/** | |
| - tools/** | |
| push: | |
| branches: | |
| - 'main' | |
| - 'release/*' | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| concurrency: | |
| group: concrete_compiler_test_cpu_${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| setup-instance: | |
| name: Setup instance | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: cpu-test | |
| format-and-lint: | |
| name: Format and linting checks | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Format with clang-format (Cpp) | |
| run: | | |
| sudo apt install moreutils | |
| cd compilers/concrete-compiler/compiler | |
| ./scripts/format_cpp.sh | |
| - name: Format with cmake-format (Cmake) | |
| run: | | |
| pip3 install cmakelang | |
| cd compilers/concrete-compiler/compiler | |
| ./scripts/format_cmake.sh | |
| - name: Format with black (Python) | |
| run: | | |
| cd compilers/concrete-compiler/compiler | |
| pip install -r lib/Bindings/Python/requirements_dev.txt | |
| make check-python-format | |
| - name: Lint with pylint (Python) | |
| run: | | |
| cd compilers/concrete-compiler/compiler | |
| # compiler requirements to lint | |
| pip install numpy | |
| make python-lint | |
| - name: Check if sources include the license header | |
| run: .github/workflows/scripts/check_for_license.sh | |
| - name: Slack Notification | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "format-and-lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| build-and-run-test: | |
| name: Build and test compiler in EC2 | |
| needs: [ setup-instance ] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| steps: | |
| - name: Fetch repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Create build dir | |
| run: mkdir build | |
| - name: Build compiler | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| id: build-compiler | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ github.workspace }}/wheels:/wheels | |
| shell: bash | |
| run: | | |
| set -e | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| rm -rf /build/* | |
| make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| - name: Check compiler dialects docs is up to date | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| id: build-compiler-docs | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ github.workspace }}/wheels:/wheels | |
| shell: bash | |
| run: | | |
| set -e | |
| cd /build/tools/concretelang/docs/concretelang/ | |
| sed -i -e 's/\[TOC\]//' *Dialect.md | |
| for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; | |
| - name: Enable complete tests on push to main | |
| if: github.ref == 'refs/heads/main' | |
| run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV | |
| - name: Enable minimal tests otherwise | |
| if: github.ref != 'refs/heads/main' | |
| run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV | |
| - name: Run compiler tests | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| shell: bash | |
| run: | | |
| set -e | |
| rustup toolchain install nightly-2024-09-30 | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| pip install pytest | |
| pip install mypy | |
| dnf install -y libzstd libzstd-devel | |
| sed "s/pytest/python -m pytest/g" -i Makefile | |
| mkdir -p /tmp/concrete_compiler/gpu_tests/ | |
| make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} run-tests | |
| - name: Analyze logs | |
| run: | | |
| cd build/gtest-parallel-logs/passed | |
| ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat | |
| - name: Slack Notification | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| name: Teardown instance | |
| needs: [ setup-instance, build-and-run-test ] | |
| if: ${{ always() && needs.setup-instance.result != 'skipped' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" |