Skip to content

rsv: update rsv datasets with more recent data and the B.D.E.1.1 clad… #629

rsv: update rsv datasets with more recent data and the B.D.E.1.1 clad…

rsv: update rsv datasets with more recent data and the B.D.E.1.1 clad… #629

name: build-and-deploy
on:
push:
branches:
- master
- staging
- release
pull_request:
repository_dispatch:
types: build-and-deploy
workflow_dispatch:
workflow_call:
concurrency:
group: cli-${{ github.workflow }}-${{ github.ref_type }}-${{ github.event.pull_request.number || github.ref || github.run_id }}
cancel-in-progress: true
defaults:
run:
shell: bash -euxo pipefail {0}
env:
VERBOSE: 1
jobs:
build-and-deploy-datasets:
# Run only if it's not a fork
if: ${{ !((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) != github.repository))) }}
runs-on: ubuntu-20.04
environment:
name: ${{ github.ref }}
env:
VERBOSE: 0
PYTHONUNBUFFERED: 1
GH_TOKEN: ${{ github.token }}
steps:
- name: config
id: config
run: |
echo 'is_fork=${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) != github.repository)) }}' >> $GITHUB_OUTPUT
echo 'should_deploy=${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/release' }}' >> $GITHUB_OUTPUT
echo 'is_release=${{ github.ref == 'refs/heads/release' }}' >> $GITHUB_OUTPUT
- name: "DEBUG"
run: |
set +x
echo is_fork=${{ steps.config.outputs.is_fork }}
echo should_deploy=${{ steps.config.outputs.should_deploy }}
echo is_release=${{ steps.config.outputs.is_release }}
echo '\n'
echo 'github.ref =' ${{ github.ref }}
echo 'github.event_name =' ${{ github.event_name }}
echo 'github.repository =' ${{ github.repository }}
echo '\n'
echo 'github.event.pull_request.head.repo.fork =' ${{ github.event.pull_request.head.repo.fork }}
echo 'github.event.pull_request.head.repo.full_name =' ${{ github.event.pull_request.head.repo.full_name }}
echo '\n'
echo 'github.event.pull_request.base.repo.fork =' ${{ github.event.pull_request.base.repo.fork }}
echo 'github.event.pull_request.base.repo.full_name =' ${{ github.event.pull_request.base.repo.full_name }}
echo '\n'
echo 'github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name =' ${{ github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name }}
echo '(github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository =' ${{ (github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository }}
echo 'github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository)) =' ${{ github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository)) }}
- name: "Checkout code (fork)"
if: ${{ steps.config.outputs.is_fork == 'true' }}
uses: actions/checkout@v4
with:
fetch-depth: 0 # Number of commits to fetch. 0 indicates all history for all branches and tags.
fetch-tags: true # Whether to fetch tags, even if fetch-depth > 0.
ref: ${{github.event.pull_request.head.ref}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: "Checkout code"
if: ${{ steps.config.outputs.is_fork != 'true' }}
uses: actions/checkout@v4
with:
fetch-depth: 0 # Number of commits to fetch. 0 indicates all history for all branches and tags.
fetch-tags: true # Whether to fetch tags, even if fetch-depth > 0.
ref: ${{ github.head_ref || github.ref }}
- name: "Install system dependencies"
run: |
sudo apt-get install brotli pigz parallel python3 rename --yes -qq >/dev/null
- name: "Install Python dependencies"
run: |
pip3 install -r requirements.txt
- name: "Rebuild datasets (fork)"
if: ${{ steps.config.outputs.is_fork == 'true' }}
run: |
./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --no-pull
- name: "Rebuild, commit and push datasets (non-release branch)"
if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.is_release != 'true' }}
run: |
git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"
./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --push --repo="${GITHUB_REPOSITORY}"
- name: "Rebuild, commit, push and make a release (release branch)"
if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.is_release == 'true' }}
run: |
git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"
./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --release --repo="${GITHUB_REPOSITORY}"
- name: "Install awscli"
if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.should_deploy == 'true' }}
run: |
pushd /tmp >/dev/null
curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip -oqq awscliv2.zip
sudo ./aws/install --update
popd >/dev/null
aws --version
- name: "Deploy dataset server"
if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.should_deploy == 'true' }}
env:
DATA_AWS_S3_BUCKET: ${{ secrets.DATA_AWS_S3_BUCKET }}
DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-2
run: |
./scripts/upload 'data_output/'
- name: "Upload build artifacts: dataset server"
uses: actions/upload-artifact@v4
with:
name: server
path: ./data_output/*
- name: "Upload build artifacts: zip archives"
uses: actions/upload-artifact@v4
with:
name: zips
path: ./data_temp/*