concrete-python release-gpu #34
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Concrete Python Release (GPU) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: 'Instance ID' | |
| type: string | |
| instance_image_id: | |
| description: 'Instance AMI ID' | |
| type: string | |
| instance_type: | |
| description: 'Instance product type' | |
| type: string | |
| runner_name: | |
| description: 'Action runner name' | |
| type: string | |
| request_id: | |
| description: 'Slab request ID' | |
| type: string | |
| user_inputs: | |
| description: 'either "nightly" or "public" or "private" to specify the release type' | |
| required: true | |
| default: 'nightly' | |
| type: string | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| CUDA_PATH: /usr/local/cuda-11.8 | |
| GCC_VERSION: 11 | |
| RELEASE_TYPE: ${{ inputs.user_inputs }} | |
| jobs: | |
| build-linux-x86: | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| steps: | |
| - name: Log instance configuration | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| echo "User Inputs: ${{ inputs.user_inputs }}" | |
| - name: Set up GitHub environment | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| NIGHTLY_VERSION=$(date +"%Y.%m.%d") | |
| NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") | |
| LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` | |
| echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt | |
| git tag nightly-$NIGHTLY_VERSION || true | |
| git push origin nightly-$NIGHTLY_VERSION || true | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| uses: addnab/docker-run-action@v3 | |
| id: build-compiler-bindings | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| shell: bash | |
| run: | | |
| set -e | |
| rustup toolchain install nightly-2024-01-31 | |
| rm -rf /build/* | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| dnf -y install graphviz graphviz-devel | |
| cd /concrete/frontends/concrete-python | |
| make PYTHON=$PYTHON venv | |
| source .venv/bin/activate | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=OFF Python3_EXECUTABLE=$(which python) \ | |
| CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| cd /concrete/frontends/concrete-python | |
| export COMPILER_BUILD_DIRECTORY="/build" | |
| make whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: frontends/concrete-python/dist/*manylinux*.whl | |
| retention-days: 3 | |
| push: | |
| needs: [build-linux-x86] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| wheel_version: ${{ steps.version.outputs.wheel_version }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: wheels | |
| merge-multiple: true | |
| - name: Install aws-cli if not present | |
| run: | | |
| aws --version || (curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ | |
| unzip awscliv2.zip && \ | |
| sudo ./aws/install) | |
| - name: Upload wheels to S3 | |
| if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} | |
| CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }} | |
| run: | | |
| pip install boto3 bigtree | |
| # upload wheels | |
| aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/gpu/concrete-python | |
| # update indexes and invalidate cloudfront cache | |
| python .github/workflows/scripts/s3_update_html_indexes.py | |
| - name: Output Wheel Version | |
| id: version | |
| run: | | |
| export VERSION=`ls ./wheels/*manylinux* | head -n1 | cut -d "-" -f2` | |
| echo "VERSION=$VERSION" | |
| echo "wheel_version=$VERSION" >> "$GITHUB_OUTPUT" | |
| test-gpu-wheel: | |
| needs: [push] | |
| uses: ./.github/workflows/start_slab.yml | |
| secrets: inherit | |
| with: | |
| command: concrete-python-test-gpu-wheel | |
| user_inputs: "${{ needs.push.outputs.wheel_version }}" |