Fixing release check and test docs in concrete-python CI #2920
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: concrete-python tests linux-cpu | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - .github/workflows/concrete_python_tests_linux.yml | |
| - frontends/concrete-python/** | |
| push: | |
| branches: | |
| - 'main' | |
| - 'release/*' | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| concurrency: | |
| group: concrete_python_tests_linux_${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| setup-instance: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: cpu-test | |
| pre-commit-check: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Install platform dependencies | |
| run: | | |
| sudo apt install -y graphviz libgraphviz-dev | |
| - name: Pre-commit Checks | |
| run: | | |
| cd frontends/concrete-python | |
| make venv | |
| source .venv/bin/activate | |
| make pcc | |
| build-python-bindings: | |
| needs: setup-instance | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| steps: | |
| - name: Checkout concrete | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Create concrete build directory | |
| run: mkdir build | |
| - name: Build concrete-compiler python bindings | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| shell: bash | |
| run: | | |
| set -e | |
| rustup toolchain install nightly-2024-09-30 | |
| dnf -y install graphviz graphviz-devel | |
| cd /concrete/frontends/concrete-python | |
| make PYTHON=/opt/python/cp310-cp310/bin/python venv | |
| source .venv/bin/activate | |
| python -m pip install -r /concrete/third_party/llvm-project/mlir/python/requirements.txt | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| for f in $(find /build/tools/concretelang/python_packages/concretelang_core/ -type l); | |
| do | |
| cp --remove-destination $(readlink -e $f) $f | |
| done | |
| cd /concrete/frontends/concrete-python | |
| export COMPILER_BUILD_DIRECTORY="/build" | |
| make whl | |
| - name: Upload concrete-compiler python-bindings | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: concrete-compiler-python-bindings | |
| include-hidden-files: true | |
| retention-days: 3 | |
| path: | | |
| ./build/lib/libConcretelangRuntime.so | |
| ./build/tools/concretelang/python_packages/ | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: concrete-python-wheel | |
| path: frontends/concrete-python/dist/*manylinux*.whl | |
| retention-days: 3 | |
| - name: Slack Notification | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| test-apidoc-up-to-date: | |
| needs: [build-python-bindings] | |
| runs-on: ubuntu-latest | |
| env: | |
| python-version: "3.10" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Setup Python | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 | |
| with: | |
| python-version: ${{ env.python-version }} | |
| - name: Download concrete-compiler python-bindings | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: concrete-python-wheel | |
| path: wheel | |
| - name: Check documentation | |
| run: | | |
| export WHEEL_DIR=${{ github.workspace }}/wheel | |
| export CONCRETE_WHEEL="${WHEEL_DIR}/*.whl" | |
| .github/workflows/scripts/make_apidocs.sh | |
| - name: Upload docs.patch if failure | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 | |
| with: | |
| name: docs-artifacts | |
| path: | | |
| docs.patch | |
| docs/dev/api | |
| retention-days: 3 | |
| - name: Slack Notification | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "test-apidoc-up-to-date finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| test-pytest: | |
| needs: [setup-instance, build-python-bindings] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| steps: | |
| - name: Download concrete-compiler python-bindings | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: concrete-compiler-python-bindings | |
| path: compiler-artifacts | |
| - name: Run pytest | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts | |
| shell: bash | |
| run: | | |
| set -e | |
| export COMPILER_BUILD_DIRECTORY=/compiler-artifacts | |
| cd /concrete/frontends/concrete-python | |
| source .venv/bin/activate | |
| export KEY_CACHE_DIRECTORY=./key-set-cache | |
| mkdir $KEY_CACHE_DIRECTORY | |
| make pytest | |
| - name: Slack Notification | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "test-pytest finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| test-notebooks: | |
| needs: [setup-instance, build-python-bindings] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| steps: | |
| - name: Download concrete-compiler python-bindings | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: concrete-compiler-python-bindings | |
| path: compiler-artifacts | |
| - name: Run pytest | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts | |
| shell: bash | |
| run: | | |
| set -e | |
| export COMPILER_BUILD_DIRECTORY=/compiler-artifacts | |
| cd /concrete/frontends/concrete-python | |
| source .venv/bin/activate | |
| make test-notebooks | |
| - name: Slack Notification | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "test-notebooks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| if: ${{ always() && needs.setup-instance.result != 'skipped' }} | |
| needs: [ setup-instance, test-pytest, test-notebooks ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" |