Skip to content

Fetch and ingest

Fetch and ingest #1668

name: Fetch and ingest
on:
schedule:
# Note times are in UTC, which is 1 or 2 hours behind CET depending on daylight savings.
#
# Note the actual runs might be late.
# Numerous people were confused, about that, including me:
# - https://github.community/t/scheduled-action-running-consistently-late/138025/11
# - https://github.com/github/docs/issues/3059
#
# Note, '*' is a special character in YAML, so you have to quote this string.
#
# Docs:
# - https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#schedule
#
# Tool that deciphers this particular format of crontab string:
# - https://crontab.guru/
#
# Runs at 4pm UTC (12pm EDT) since curation by NCBI happens on the East Coast.
- cron: '0 16 * * *'
repository_dispatch:
types:
- fetch-and-ingest
# Manually triggered using GitHub's UI
workflow_dispatch:
inputs:
image:
description: 'Specific container image to use for build (will override the default of "nextstrain build")'
required: false
env:
NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
jobs:
fetch-and-ingest:
runs-on: ubuntu-latest
env:
GITHUB_RUN_ID: ${{ github.run_id }}
SLACK_CHANNELS: monkeypox-updates
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
steps:
- uses: actions/checkout@v3
- uses: nextstrain/.github/actions/setup-nextstrain-cli@master
with:
cli-version: ">=7.1.0"
python-version: "3.10"
- name: run_pipeline
run: |
nextstrain build \
--aws-batch \
--detach \
--no-download \
--cpus 32 \
--memory 64gib \
--env AWS_DEFAULT_REGION \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env GITHUB_RUN_ID \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
--env PAT_GITHUB_DISPATCH \
ingest \
--configfiles config/config.yaml config/optional.yaml
env:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
PAT_GITHUB_DISPATCH: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_WORKFLOW_DISPATCH }}
- name: notify_pipeline_failed
if: ${{ failure() }}
run: ./ingest/vendored/notify-on-job-fail Ingest nextstrain/monkeypox