-
Notifications
You must be signed in to change notification settings - Fork 148
173 lines (147 loc) · 6.05 KB
/
concrete_python_release_gpu.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
name: Concrete Python Release (GPU)
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
user_inputs:
description: 'either "nightly" or "public" or "private" to specify the release type'
required: true
default: 'nightly'
type: string
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
CUDA_PATH: /usr/local/cuda-11.8
GCC_VERSION: 11
RELEASE_TYPE: ${{ inputs.user_inputs }}
jobs:
build-linux-x86:
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
runs-on: ${{ github.event.inputs.runner_name }}
steps:
- name: Log instance configuration
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
echo "User Inputs: ${{ inputs.user_inputs }}"
- name: Set up GitHub environment
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Checkout
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
submodules: recursive
fetch-depth: 0
- name: Set release version (nightly)
if: ${{ env.RELEASE_TYPE == 'nightly' }}
run: |
NIGHTLY_VERSION=$(date +"%Y.%m.%d")
NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d")
LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o`
echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
git tag nightly-$NIGHTLY_VERSION || true
git push origin nightly-$NIGHTLY_VERSION || true
- name: Set release version (public)
if: ${{ env.RELEASE_TYPE == 'public' }}
run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Build wheel
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler-bindings
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
shell: bash
run: |
set -e
rustup toolchain install nightly-2024-01-31
rm -rf /build/*
export PYTHON=${{ format('python{0}', matrix.python-version) }}
echo "Using $PYTHON"
dnf -y install graphviz graphviz-devel
cd /concrete/frontends/concrete-python
make PYTHON=$PYTHON venv
source .venv/bin/activate
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=OFF Python3_EXECUTABLE=$(which python) \
CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
cd /concrete/frontends/concrete-python
export COMPILER_BUILD_DIRECTORY="/build"
make whl
deactivate
- name: Upload wheel
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }}
path: frontends/concrete-python/dist/*manylinux*.whl
retention-days: 3
push:
needs: [build-linux-x86]
runs-on: ubuntu-latest
outputs:
wheel_version: ${{ steps.version.outputs.wheel_version }}
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
with:
path: wheels
merge-multiple: true
- name: Install aws-cli if not present
run: |
aws --version || (curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && \
sudo ./aws/install)
- name: Upload wheels to S3
if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }}
CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }}
run: |
pip install boto3 bigtree
# upload wheels
aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/gpu/concrete-python
# update indexes and invalidate cloudfront cache
python .github/workflows/scripts/s3_update_html_indexes.py
- name: Output Wheel Version
id: version
run: |
export VERSION=`ls ./wheels/*manylinux* | head -n1 | cut -d "-" -f2`
echo "VERSION=$VERSION"
echo "wheel_version=$VERSION" >> "$GITHUB_OUTPUT"
test-gpu-wheel:
needs: [push]
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-test-gpu-wheel
user_inputs: "${{ needs.push.outputs.wheel_version }}"