Replace join metadata and clades script with csvtk and tsv append

ingest/bin/ndjson-to-tsv-and-fasta

@@ -37,7 +37,8 @@ if __name__ == '__main__':
                 args.metadata_columns,
                 restval="",
                 extrasaction='ignore',
-                delimiter='\t'
+                delimiter='\t',
+                lineterminator='\n',
             )
             metadata_csv.writeheader()
 

ingest/config/config.yaml

@@ -66,3 +66,10 @@ transform:
     'authors',
     'institution'
   ]
+
+# Params for Nextclade related rules
+nextclade:
+  # Field to use as the sequence ID in the Nextclade file
+  id_field: 'seqName'
+  # Fields from a Nextclade file to be renamed (if desired) and appended to a metadata file
+  field_map: 'source-data/nextclade-field-map.tsv'

ingest/source-data/nextclade-field-map.tsv

@@ -0,0 +1,16 @@
+key	value
+seqName	seqName
+clade	clade
+outbreak	outbreak
+lineage	lineage
+coverage	coverage
+totalMissing	missing_data
+totalSubstitutions	divergence
+totalNonACGTNs	nonACGTN
+qc.missingData.status	QC_missing_data
+qc.mixedSites.status	QC_mixed_sites
+qc.privateMutations.status	QC_rare_mutations
+qc.frameShifts.status	QC_frame_shifts
+qc.stopCodons.status	QC_stop_codons
+frameShifts	frame_shifts
+isReverseComplement	is_reverse_complement

ingest/workflow/snakemake_rules/nextclade.smk

@@ -56,15 +56,31 @@ rule join_metadata_clades:
     input:
         nextclade="data/nextclade.tsv",
         metadata="data/metadata_raw.tsv",
+        nextclade_field_map=config["nextclade"]["field_map"],
     output:
-        "data/metadata.tsv",
+        metadata="data/metadata.tsv",
     params:
         id_field=config["transform"]["id_field"],
+        nextclade_id_field=config["nextclade"]["id_field"],
     shell:
         """
-        python3 bin/join-metadata-and-clades.py \
-                --id-field {params.id_field} \
-                --metadata {input.metadata} \
-                --nextclade {input.nextclade} \
-                -o {output}
+        export SUBSET_FIELDS=`awk 'NR>1 {{print $1}}' {input.nextclade_field_map} | tr '\n' ',' | sed 's/,$//g'`
+
+        csvtk -tl cut -f $SUBSET_FIELDS \
+            {input.nextclade} \
+        | csvtk -tl rename2 \
+            -F \
+            -f '*' \
+            -p '(.+)' \
+            -r '{{kv}}' \
+            -k {input.nextclade_field_map} \
+        | tsv-join -H \
+            --filter-file - \
+            --key-fields {params.nextclade_id_field} \
+            --data-fields {params.id_field} \
+            --append-fields '*' \
+            --write-all ? \
+            {input.metadata} \
+        | tsv-select -H --exclude {params.nextclade_id_field} \
+            > {output.metadata}
         """