From ced0186d2226c564ed4194454ae39c2b1332c75b Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 12 Apr 2024 13:08:54 -0700 Subject: [PATCH] ingest/phylo GH Actions: add `trial_name` inputs Allow trial builds of the ingest and phylogenetic workflows to be uploaded to trial S3 URLs. The `trial_name` inputs are also added to the `workflow_call` trigger but these are not being used in the ingest-to-phylogenetic workflow yet. A full trial run from ingest to phylogenetic can only work if the phylogenetic workflow accepts inputs from the config, so I'll do this in a separate change. --- .github/workflows/ingest.yaml | 37 ++++++++++++++++++++++++++++- .github/workflows/phylogenetic.yaml | 37 ++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ingest.yaml b/.github/workflows/ingest.yaml index 3e0cc52..c274637 100644 --- a/.github/workflows/ingest.yaml +++ b/.github/workflows/ingest.yaml @@ -16,6 +16,13 @@ on: description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")' required: false type: string + trial_name: + description: | + Trial name for outputs. + If not set, outputs will overwrite files at s3://nextstrain-data/files/workflows/zika/ + If set, outputs will be uploaded to s3://nextstrain-data/files/workflows/zika/trials/. + required: false + type: string workflow_dispatch: inputs: @@ -23,9 +30,35 @@ on: description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")' required: false type: string + trial_name: + description: | + Trial name for outputs. + If not set, outputs will overwrite files at s3://nextstrain-data/files/workflows/zika/ + If set, outputs will be uploaded to s3://nextstrain-data/files/workflows/zika/trials/. + required: false + type: string jobs: + set_config_overrides: + runs-on: ubuntu-latest + steps: + - id: config + name: Set config overrides + env: + TRIAL_NAME: ${{ inputs.trial_name }} + run: | + config="" + if [[ "$TRIAL_NAME" ]]; then + config+="--config" + config+=" s3_dst='s3://nextstrain-data/files/workflows/zika/trials/"$TRIAL_NAME"'" + fi + + echo "config=$config" >> "$GITHUB_OUTPUT" + outputs: + config_overrides: ${{ steps.config.outputs.config }} + ingest: + needs: [set_config_overrides] permissions: id-token: write uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master @@ -37,13 +70,15 @@ jobs: runtime: docker env: | NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }} + CONFIG_OVERRIDES: ${{ needs.set_config_overrides.outputs.config_overrides }} run: | nextstrain build \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ ingest \ upload_all \ - --configfile build-configs/nextstrain-automation/config.yaml + --configfile build-configs/nextstrain-automation/config.yaml \ + $CONFIG_OVERRIDES # Specifying artifact name to differentiate ingest build outputs from # the phylogenetic build outputs artifact-name: ingest-build-output diff --git a/.github/workflows/phylogenetic.yaml b/.github/workflows/phylogenetic.yaml index 391c789..a7dbe23 100644 --- a/.github/workflows/phylogenetic.yaml +++ b/.github/workflows/phylogenetic.yaml @@ -16,6 +16,13 @@ on: description: 'Specific container image to use for phylogenetic workflow (will override the default of "nextstrain build")' required: false type: string + trial_name: + description: | + Trial name for deploying builds. + If not set, builds will overwrite existing builds at s3://nextstrain-data/zika* + If set, builds will be deployed to s3://nextstrain-data/zika_trials_* + required: false + type: string workflow_dispatch: inputs: @@ -23,9 +30,35 @@ on: description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")' required: false type: string + trial_name: + description: | + Trial name for deploying builds. + If not set, builds will overwrite existing builds at s3://nextstrain-data/zika* + If set, builds will be deployed to s3://nextstrain-data/zika_trials_* + required: false + type: string jobs: + set_config_overrides: + runs-on: ubuntu-latest + steps: + - id: config + name: Set config overrides + env: + TRIAL_NAME: ${{ inputs.trial_name }} + run: | + config="" + if [[ "$TRIAL_NAME" ]]; then + config+="--config" + config+=" deploy_url='s3://nextstrain-data/zika_trials_"$TRIAL_NAME"'" + fi + + echo "config=$config" >> "$GITHUB_OUTPUT" + outputs: + config_overrides: ${{ steps.config.outputs.config }} + phylogenetic: + needs: [set_config_overrides] permissions: id-token: write uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master @@ -37,13 +70,15 @@ jobs: runtime: docker env: | NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }} + CONFIG_OVERRIDES: ${{ needs.set_config_overrides.outputs.config_overrides }} run: | nextstrain build \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ phylogenetic \ deploy_all \ - --configfile build-configs/nextstrain-automation/config.yaml + --configfile build-configs/nextstrain-automation/config.yaml \ + $CONFIG_OVERRIDES # Specifying artifact name to differentiate ingest build outputs from # the phylogenetic build outputs artifact-name: phylogenetic-build-output