From 4fc42a5950998653aa4a59501b83d6b319c47535 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Tue, 18 Jun 2024 15:41:31 -0700 Subject: [PATCH 1/3] pathogen-repo-build: Include `AWS_DEFAULT_REGION` in envdir Resolves https://github.com/nextstrain/.github/issues/97 --- .github/workflows/pathogen-repo-build.yaml | 1 + .github/workflows/pathogen-repo-build.yaml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index 2e3a274..f36e375 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -287,6 +287,7 @@ jobs: "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ AWS_ACCESS_KEY_ID \ AWS_SECRET_ACCESS_KEY \ + AWS_DEFAULT_REGION \ ${AWS_SESSION_TOKEN:+AWS_SESSION_TOKEN} # This will overwrite the runtime AWS credential envvars configured above # so if the build is using the aws-batch runtime, the Nextstrain CLI will diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 72427d1..a258adf 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -256,6 +256,7 @@ jobs: "$NEXTSTRAIN_GITHUB_DIR"/bin/write-envdir "$NEXTSTRAIN_RUNTIME_ENVDIR" \ AWS_ACCESS_KEY_ID \ AWS_SECRET_ACCESS_KEY \ + AWS_DEFAULT_REGION \ ${AWS_SESSION_TOKEN:+AWS_SESSION_TOKEN} # This will overwrite the runtime AWS credential envvars configured above From d8640df6b0a0a2573c2c7ffa0630b92bd25a1e52 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Tue, 18 Jun 2024 16:07:06 -0700 Subject: [PATCH 2/3] pathogen-repo-build: Add output `aws-batch-job-id` If the workflow is run as a reusable workflow with the `aws-batch` runtime, the workflow will output the AWS Batch job id. This is useful for re-attaching to a complete AWS Batch job to download the results and use them in subsequent jobs. Resolves https://github.com/nextstrain/.github/issues/79 --- .github/workflows/pathogen-repo-build.yaml | 10 ++++++++++ .github/workflows/pathogen-repo-build.yaml.in | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index f36e375..7af76f3 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -115,6 +115,16 @@ on: This is not supported for builds on AWS Batch because the workflow detaches from the build. Please use the `nextstrain build` command locally to reattach to AWS Batch builds to download outputs. type: string required: false + outputs: + aws-batch-job-id: + description: >- + AWS Batch job id of the build if it was run with the `aws-batch` runtime. This can be used to re-attach to the AWS Batch job with: + + + nextstrain build --aws-batch --attach + + This can be useful to download results from the build and use them in subsequent jobs. + value: ${{ jobs.run-build.outputs.AWS_BATCH_JOB_ID }} workflow_dispatch: inputs: runtime: diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index a258adf..aed6b69 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -162,6 +162,19 @@ on: type: string required: false + outputs: + aws-batch-job-id: + description: >- + AWS Batch job id of the build if it was run with the `aws-batch` runtime. + This can be used to re-attach to the AWS Batch job with: + + nextstrain build --aws-batch --attach + + This can be useful to download results from the build and use them + in subsequent jobs. + + value: ${{ jobs.run-build.outputs.AWS_BATCH_JOB_ID }} + workflow_dispatch: inputs: <<: *inputs From e5d5d4fe054e7dc3414d325b2407c6bf5e678434 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 21 Jun 2024 12:24:26 -0700 Subject: [PATCH 3/3] CI: Verify pathogen-repo-build's aws-batch-job-id output --- .github/workflows/ci.yaml | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c1d1bb2..a0777ab 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,7 +49,7 @@ jobs: id-token: write strategy: matrix: - runtime: [docker, conda, aws-batch] + runtime: [docker, conda] uses: ./.github/workflows/pathogen-repo-build.yaml with: repo: nextstrain/zika-tutorial @@ -60,6 +60,37 @@ jobs: BOGUS_ENVVAR: BOGUS_ENVVAR secrets: inherit + # This job has to be separate from the matrix above because we want + # to check the aws-batch-job-id output from the workflow but matrix outputs + # currently will overwrite each other. + # See + test-pathogen-repo-build-aws-batch: + permissions: + id-token: write + uses: ./.github/workflows/pathogen-repo-build.yaml + with: + repo: nextstrain/zika-tutorial + runtime: aws-batch + run: nextstrain build --env BOGUS_ENVVAR . + artifact-name: build-output-aws-batch + env: | + BOGUS_ENVVAR: BOGUS_ENVVAR + secrets: inherit + + test-pathogen-repo-build-aws-batch-id-output: + needs: [test-pathogen-repo-build-aws-batch] + permissions: + id-token: write + uses: ./.github/workflows/pathogen-repo-build.yaml + with: + repo: nextstrain/zika-tutorial + runtime: aws-batch + run: nextstrain build --attach "$AWS_BATCH_JOB_ID" . + artifact-name: build-output-aws-batch-reattached + env: | + AWS_BATCH_JOB_ID: ${{ needs.test-pathogen-repo-build-aws-batch.outputs.aws-batch-job-id }} + secrets: inherit + lint: runs-on: ubuntu-latest steps: