feat: declare the schema of the Parquet file (#8) #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docker Compose Deployment CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - deploy-* | |
| tags: | |
| - v*.*.* | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| env: | |
| - ${{ startsWith(github.ref, 'refs/tags/v') && 'off-exports-org' || 'off-exports-net' }} | |
| environment: ${{ matrix.env }} | |
| concurrency: ${{ matrix.env }} | |
| steps: | |
| - name: Set various variable for staging deployment | |
| if: matrix.env == 'off-exports-net' | |
| run: | | |
| # configurations | |
| echo "ENVIRONMENT=preprod" >> $GITHUB_ENV | |
| echo "ENABLE_HF_PUSH=0" >> $GITHUB_ENV | |
| echo "ENABLE_S3_PUSH=0" >> $GITHUB_ENV | |
| # deploy target | |
| echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV | |
| echo "SSH_USERNAME=off" >> $GITHUB_ENV | |
| echo "SSH_PROXY_USERNAME=off" >> $GITHUB_ENV | |
| echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV | |
| - name: Set various variable for production deployment | |
| if: matrix.env == 'off-exports-org' | |
| run: | | |
| # configurations | |
| echo "ENVIRONMENT=prod" >> $GITHUB_ENV | |
| echo "ENABLE_HF_PUSH=1" >> $GITHUB_ENV | |
| echo "ENABLE_S3_PUSH=0" >> $GITHUB_ENV | |
| # deploy target | |
| echo "SSH_PROXY_HOST=45.147.209.254" >> $GITHUB_ENV | |
| echo "SSH_USERNAME=off" >> $GITHUB_ENV | |
| echo "SSH_PROXY_USERNAME=off" >> $GITHUB_ENV | |
| echo "SSH_PROTOCOL=tcp" >> $GITHUB_ENV | |
| echo "SSH_HOST=10.3.0.200" >> $GITHUB_ENV | |
| - name: Wait for container build workflow | |
| uses: tomchv/[email protected] | |
| id: wait-build | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| checkName: build | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| intervalSeconds: 10 | |
| timeoutSeconds: 600 # 10m | |
| - name: Do something if build isn't launched | |
| if: steps.wait-build.outputs.conclusion == 'not found' | |
| run: echo job does not exist && true | |
| - name: Do something if build fail | |
| if: steps.wait-build.outputs.conclusion == 'failure' | |
| run: echo fail && false # fail if build fail | |
| - name: Do something if build timeout | |
| if: steps.wait-build.outputs.conclusion == 'timed_out' | |
| run: echo Timeout && false # fail if build time out | |
| - name: Checkout git repository | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{ env.SSH_HOST }} | |
| username: ${{ env.SSH_USERNAME }} | |
| key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| proxy_host: ${{ env.SSH_PROXY_HOST }} | |
| proxy_username: ${{ env.SSH_PROXY_USERNAME }} | |
| proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| script_stop: false | |
| script: | | |
| # Clone Git repository if not already there | |
| [ ! -d '${{ matrix.env }}' ] && git clone --depth 1 https://github.com/${{ github.repository }} ${{ matrix.env }} --no-single-branch 2>&1 | |
| # Go to repository directory | |
| cd ${{ matrix.env }} | |
| # Fetch newest commits (in case it wasn't freshly cloned) | |
| git fetch --depth 1 | |
| # Checkout current commit SHA | |
| git checkout -qf ${{ github.sha }} | |
| - name: Set environment variables | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{ env.SSH_HOST }} | |
| username: ${{ env.SSH_USERNAME }} | |
| key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| proxy_host: ${{ env.SSH_PROXY_HOST }} | |
| proxy_username: ${{ env.SSH_PROXY_USERNAME }} | |
| proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| script_stop: false | |
| script: | | |
| # Go to repository directory | |
| cd ${{ matrix.env }} | |
| # new env | |
| rm .env | |
| touch .env | |
| # Set Docker Compose variables | |
| echo "DOCKER_CLIENT_TIMEOUT=120" >> .env | |
| echo "COMPOSE_HTTP_TIMEOUT=120" >> .env | |
| echo "COMPOSE_PROJECT_NAME=off-exports" >> .env | |
| echo "COMPOSE_PATH_SEPARATOR=;" >> .env | |
| echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env | |
| echo "TAG=sha-${{ github.sha }}" >> .env | |
| echo "RESTART_POLICY=always" >> .env | |
| # App environment variables | |
| echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env | |
| echo "REDIS_HOST=redis" >> .env | |
| echo "NUM_RQ_WORKERS=4" >> .env | |
| echo "ENABLE_HF_PUSH=${{ env.ENABLE_HF_PUSH }}" >> .env | |
| echo "ENABLE_S3_PUSH=${{ env.ENABLE_S3_PUSH }}" >> .env | |
| # Secrets | |
| echo "SENTRY_DSN=${{ secrets.SENTRY_DSN }}" >> .env | |
| echo "HF_TOKEN=${{ secrets.HF_TOKEN }}" >> .env | |
| echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> .env | |
| echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> .env | |
| - name: Create Docker volumes | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{ env.SSH_HOST }} | |
| username: ${{ env.SSH_USERNAME }} | |
| key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| proxy_host: ${{ env.SSH_PROXY_HOST }} | |
| proxy_username: ${{ env.SSH_PROXY_USERNAME }} | |
| proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| script_stop: false | |
| script: | | |
| cd ${{ matrix.env }} | |
| make create_external_volumes | |
| - name: Start services | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{ env.SSH_HOST }} | |
| username: ${{ env.SSH_USERNAME }} | |
| key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| proxy_host: ${{ env.SSH_PROXY_HOST }} | |
| proxy_username: ${{ env.SSH_PROXY_USERNAME }} | |
| proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| script_stop: false | |
| script: | | |
| cd ${{ matrix.env }} | |
| make up | |
| - name: Cleanup obsolete Docker objects | |
| uses: appleboy/ssh-action@master | |
| if: ${{ always() }} | |
| with: | |
| host: ${{ env.SSH_HOST }} | |
| username: ${{ env.SSH_USERNAME }} | |
| key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| proxy_host: ${{ env.SSH_PROXY_HOST }} | |
| proxy_username: ${{ env.SSH_PROXY_USERNAME }} | |
| proxy_key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| script_stop: false | |
| script: | | |
| cd ${{ matrix.env }} | |
| make prune |