Skip to content

Commit

Permalink
fixup: conform to pathogen-repo-guide
Browse files Browse the repository at this point in the history
Match the upload rules in nextstrain/pathogen-repo-guide#36
  • Loading branch information
j23414 committed Mar 14, 2024
1 parent a27da54 commit 08893be
Showing 1 changed file with 13 additions and 31 deletions.
44 changes: 13 additions & 31 deletions ingest/build-configs/nextstrain-automation/upload.smk
Original file line number Diff line number Diff line change
Expand Up @@ -18,39 +18,11 @@ send_notifications = (
)


def _get_upload_inputs(wildcards):
"""
If the file_to_upload has Slack notifications that depend on diffs with S3 files,
then we want the upload rule to run after the notification rule.
This function is mostly to keep track of which flag files to expect for
the rules in `slack_notifications.smk`, so it only includes flag files if
`send_notifications` is True.
"""
inputs = {
"file_to_upload": config["files_to_upload"][
wildcards.remote_file_name
],
}

if send_notifications:
flag_file = []

if file_to_upload == "data/genbank.ndjson":
flag_file = "data/notify/genbank-record-change.done"
elif file_to_upload == "results/metadata.tsv":
flag_file = "data/notify/metadata-diff.done"

inputs["notify_flag_file"] = flag_file

return inputs


rule upload_to_s3:
input:
unpack(_get_upload_inputs),
file_to_upload=lambda wildcards: config["files_to_upload"][wildcards.remote_file],
output:
"data/upload/s3/{remote_file_name}.done",
"results/upload/{remote_file}.upload",
params:
quiet="" if send_notifications else "--quiet",
s3_dst=config["s3_dst"],
Expand All @@ -60,6 +32,16 @@ rule upload_to_s3:
./vendored/upload-to-s3 \
{params.quiet} \
{input.file_to_upload:q} \
{params.s3_dst:q}/{wildcards.remote_file_name:q} \
{params.s3_dst:q}/{wildcards.remote_file:q} \
{params.cloudfront_domain} 2>&1 | tee {output}
"""


rule upload_all:
input:
uploads=[
f"results/upload/{remote_file}.upload"
for remote_file in config["files_to_upload"].keys()
],
output:
touch("results/upload_all.done")

0 comments on commit 08893be

Please sign in to comment.