ingest/phylo GH Actions: add trial_name inputs

Allow trial builds of the ingest and phylogenetic workflows to be uploaded to trial S3 URLs. The `trial_name` inputs are also added to the `workflow_call` trigger but these are not being used in the ingest-to-phylogenetic workflow yet. A full trial run from ingest to phylogenetic can only work if the phylogenetic workflow accepts inputs from the config, so I'll do this in a separate change.
nextstrain · Apr 12, 2024 · ced0186 · ced0186
1 parent 57145f8
commit ced0186
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 2 deletions.
diff --git a/.github/workflows/ingest.yaml b/.github/workflows/ingest.yaml
@@ -16,16 +16,49 @@ on:
         description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
         required: false
         type: string
+      trial_name:
+        description: |
+          Trial name for outputs.
+          If not set, outputs will overwrite files at s3://nextstrain-data/files/workflows/zika/
+          If set, outputs will be uploaded to s3://nextstrain-data/files/workflows/zika/trials/<trial_name>.
+        required: false
+        type: string
 
   workflow_dispatch:
     inputs:
       image:
         description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
         required: false
         type: string
+      trial_name:
+        description: |
+          Trial name for outputs.
+          If not set, outputs will overwrite files at s3://nextstrain-data/files/workflows/zika/
+          If set, outputs will be uploaded to s3://nextstrain-data/files/workflows/zika/trials/<trial_name>.
+        required: false
+        type: string
 
 jobs:
+  set_config_overrides:
+    runs-on: ubuntu-latest
+    steps:
+      - id: config
+        name: Set config overrides
+        env:
+          TRIAL_NAME: ${{ inputs.trial_name }}
+        run: |
+          config=""
+          if [[ "$TRIAL_NAME" ]]; then
+            config+="--config"
+            config+=" s3_dst='s3://nextstrain-data/files/workflows/zika/trials/"$TRIAL_NAME"'"
+          fi
+
+          echo "config=$config" >> "$GITHUB_OUTPUT"
+    outputs:
+      config_overrides: ${{ steps.config.outputs.config }}
+
   ingest:
+    needs: [set_config_overrides]
     permissions:
       id-token: write
     uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
@@ -37,13 +70,15 @@ jobs:
       runtime: docker
       env: |
         NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
+        CONFIG_OVERRIDES: ${{ needs.set_config_overrides.outputs.config_overrides }}
       run: |
         nextstrain build \
           --env AWS_ACCESS_KEY_ID \
           --env AWS_SECRET_ACCESS_KEY \
           ingest \
             upload_all \
-            --configfile build-configs/nextstrain-automation/config.yaml
+            --configfile build-configs/nextstrain-automation/config.yaml \
+            $CONFIG_OVERRIDES
       # Specifying artifact name to differentiate ingest build outputs from
       # the phylogenetic build outputs
       artifact-name: ingest-build-output

diff --git a/.github/workflows/phylogenetic.yaml b/.github/workflows/phylogenetic.yaml
@@ -16,16 +16,49 @@ on:
         description: 'Specific container image to use for phylogenetic workflow (will override the default of "nextstrain build")'
         required: false
         type: string
+      trial_name:
+        description: |
+          Trial name for deploying builds.
+          If not set, builds will overwrite existing builds at s3://nextstrain-data/zika*
+          If set, builds will be deployed to s3://nextstrain-data/zika_trials_<trial_name>*
+        required: false
+        type: string
 
   workflow_dispatch:
     inputs:
       image:
         description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
         required: false
         type: string
+      trial_name:
+        description: |
+          Trial name for deploying builds.
+          If not set, builds will overwrite existing builds at s3://nextstrain-data/zika*
+          If set, builds will be deployed to s3://nextstrain-data/zika_trials_<trial_name>*
+        required: false
+        type: string
 
 jobs:
+  set_config_overrides:
+    runs-on: ubuntu-latest
+    steps:
+      - id: config
+        name: Set config overrides
+        env:
+          TRIAL_NAME: ${{ inputs.trial_name }}
+        run: |
+          config=""
+          if [[ "$TRIAL_NAME" ]]; then
+            config+="--config"
+            config+=" deploy_url='s3://nextstrain-data/zika_trials_"$TRIAL_NAME"'"
+          fi
+
+          echo "config=$config" >> "$GITHUB_OUTPUT"
+    outputs:
+      config_overrides: ${{ steps.config.outputs.config }}
+
   phylogenetic:
+    needs: [set_config_overrides]
     permissions:
       id-token: write
     uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
@@ -37,13 +70,15 @@ jobs:
       runtime: docker
       env: |
         NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
+        CONFIG_OVERRIDES: ${{ needs.set_config_overrides.outputs.config_overrides }}
       run: |
         nextstrain build \
           --env AWS_ACCESS_KEY_ID \
           --env AWS_SECRET_ACCESS_KEY \
           phylogenetic \
             deploy_all \
-            --configfile build-configs/nextstrain-automation/config.yaml
+            --configfile build-configs/nextstrain-automation/config.yaml \
+            $CONFIG_OVERRIDES
       # Specifying artifact name to differentiate ingest build outputs from
       # the phylogenetic build outputs
       artifact-name: phylogenetic-build-output