Add / tweak bits to allow upload of ingested files to S3 [#2]

ingest/Snakefile

@@ -25,3 +25,8 @@ rule clean:
         """
         rm -rfv {params.targets}
         """
+
+# Import custom rules provided via the config.
+if "custom_rules" in config:
+  for rule_file in config["custom_rules"]:
+    include: rule_file

ingest/build-configs/nextstrain-automation/config.yaml

@@ -1,9 +1,5 @@
-# This configuration file should contain all required configuration
-# parameters for the ingest workflow to run with additional Nextstrain
-# automation rules.
-
-# Custom rules to run as part of the Nextstrain automated workflow The
-# paths should be relative to the ingest directory.
+# Custom rules to run as part of the Nextstrain automated workflow.
+# The paths should be relative to the ingest directory.
 custom_rules:
   - build-configs/nextstrain-automation/upload.smk
 
@@ -13,12 +9,11 @@ custom_rules:
 cloudfront_domain: "data.nextstrain.org"
 
 # Nextstrain AWS S3 Bucket with pathogen prefix
-s3_dst: "s3://nextstrain-data/files/workflows/seasonal-cov"
+s3_dst: "s3://nextstrain-data/files/workflows/yellow-fever"
 
 # Mapping of files to upload
 files_to_upload:
   ncbi.ndjson.zst: data/ncbi.ndjson
   metadata.tsv.zst: results/metadata.tsv
   sequences.fasta.zst: results/sequences.fasta
   alignments.fasta.zst: results/alignment.fasta
-  translations.zip: results/translations.zip

ingest/build-configs/nextstrain-automation/upload.smk

@@ -28,7 +28,7 @@ rule upload_to_s3:
         s3_dst=config["s3_dst"],
         cloudfront_domain=config["cloudfront_domain"],
     shell:
-        """
+        r"""
         ./vendored/upload-to-s3 \
             {params.quiet} \
             {input.file_to_upload:q} \
@@ -39,6 +39,9 @@ rule upload_to_s3:
 
 rule upload_all:
     input:
-        uploads=[f"results/upload/{remote_file}.upload" for remote_file in config["files_to_upload"].keys()],
+        uploads=[
+            f"results/upload/{remote_file}.upload"
+            for remote_file in config["files_to_upload"].keys()
+        ],
     output:
         touch("results/upload_all.done"),