concrete-python release-gpu #198
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: concrete-python release-gpu | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - 'v[0-9]+.[0-9]+.[0-9]+*' | |
| schedule: | |
| # Nightly Release @ 3AM after each work day | |
| - cron: "0 3 * * 2-6" | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| CUDA_PATH: /usr/local/cuda-11.8 | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| RELEASE_TYPE: ${{ (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) && 'public' || 'nightly' }} | |
| concurrency: | |
| group: concrete_python_release_gpu_${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| setup-instance: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: release | |
| build-linux-x86: | |
| needs: setup-instance | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| steps: | |
| - name: Set up GitHub environment | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| NIGHTLY_VERSION=$(date +"%Y.%m.%d") | |
| NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") | |
| LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) | |
| echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt | |
| git tag nightly-"${NIGHTLY_VERSION}" || true | |
| git push origin nightly-"${NIGHTLY_VERSION}" || true | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| id: build-compiler-bindings | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| shell: bash | |
| run: | | |
| set -e | |
| rm -rf /build/* | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| cd /concrete/frontends/concrete-python | |
| make PYTHON=$PYTHON venv | |
| source .venv/bin/activate | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=OFF Python3_EXECUTABLE=$(which python) \ | |
| CUDA_SUPPORT=ON TIMING_ENABLED=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| cd /concrete/frontends/concrete-python | |
| export COMPILER_BUILD_DIRECTORY="/build" | |
| make whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: frontends/concrete-python/dist/*manylinux*.whl | |
| retention-days: 3 | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "build-linux-x86 (${{matrix.python-version}}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| needs: [ setup-instance, build-linux-x86 ] | |
| if: ${{ always() && needs.setup-instance.result != 'skipped' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| push-wheels: | |
| needs: [build-linux-x86] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| wheel_version: ${{ steps.version.outputs.wheel_version }} | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| path: wheels | |
| merge-multiple: true | |
| - name: Install aws-cli | |
| run: | | |
| aws --version || (curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ | |
| unzip awscliv2.zip && \ | |
| sudo ./aws/install) | |
| - name: Upload wheels to S3 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} | |
| CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }} | |
| run: | | |
| pip install boto3 bigtree | |
| # upload wheels | |
| aws s3 sync ./wheels/ s3://"${S3_BUCKET_NAME}"/gpu/concrete-python | |
| # update indexes and invalidate cloudfront cache | |
| python .github/workflows/scripts/s3_update_html_indexes.py | |
| - name: Output Wheel Version | |
| id: version | |
| run: | | |
| VERSION=$(find ./wheels/ -name '*manylinux*' | head -n1 | cut -d "-" -f2) | |
| echo "VERSION=$VERSION" | |
| echo "wheel_version=$VERSION" >> "$GITHUB_OUTPUT" | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "push-wheels finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| setup-test-instance: | |
| runs-on: ubuntu-latest | |
| needs: [push-wheels] | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: gpu-test | |
| test-linux-x86: | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| fail-fast: false | |
| needs: [setup-test-instance, push-wheels] | |
| runs-on: ${{ needs.setup-test-instance.outputs.runner-name }} | |
| steps: | |
| # HOME is needed by actions-rs/toolchain | |
| - run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Install rust | |
| uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 | |
| with: | |
| toolchain: nightly | |
| default: true | |
| - name: Setup Python | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install concrete-python | |
| run: pip install --pre --extra-index-url https://pypi.zama.ai/gpu/ "concrete-python==${{ needs.push-wheels.outputs.wheel_version }}" | |
| - name: Checkout the repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: repo | |
| - name: Test wheel | |
| run: | | |
| CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python | |
| # Install extra requirements for tests | |
| sudo apt update -y | |
| sudo apt install -y graphviz libgraphviz-dev | |
| pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt | |
| pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt | |
| # Running tests | |
| cd "${CONCRETE_PYTHON}" | |
| make pytest-gpu | |
| teardown-test-instance: | |
| needs: [ setup-test-instance, test-linux-x86 ] | |
| if: ${{ always() && needs.setup-test-instance.result != 'skipped' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@f26b8d611b2e695158fb0a6980834f0612f65ef8 # v1.4.0 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-test-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" |