Merge pull request #176 from nextstrain/use-pathogen-embed

Add t-SNE embeddings and clusters to workflow
nextstrain · Aug 29, 2024 · 08efd02 · 08efd02
2 parents 89f6cfd + 654b438
commit 08efd02
Show file tree

Hide file tree

Showing 17 changed files with 336 additions and 17 deletions.
diff --git a/config/h1n1pdm/auspice_config.json b/config/h1n1pdm/auspice_config.json
@@ -121,6 +121,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -141,7 +156,8 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",

diff --git a/config/h3n2/auspice_config.json b/config/h3n2/auspice_config.json
@@ -131,6 +131,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -151,7 +166,8 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",

diff --git a/config/vic/auspice_config.json b/config/vic/auspice_config.json
@@ -106,6 +106,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -126,7 +141,8 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",

diff --git a/profiles/ci/builds.yaml b/profiles/ci/builds.yaml
@@ -39,6 +39,10 @@ recency:
   date_bin_labels: ["last week", "last month", "last quarter"]
   upper_bin_label: older
 
+embedding:
+  # Set lower value of perplexity when sample size is small.
+  perplexity: 9
+
 builds:
     "ci_build":
       lineage: h3n2
@@ -51,6 +55,7 @@ builds:
       enable_titer_models: true
       enable_lbi: true
       enable_glycosylation: true
+      enable_embeddings: true
       titer_collections:
         - name: cdc_cell_fra
           data: "example_data/cdc_h3n2_cell_fra_titers.tsv"

diff --git a/profiles/nextflu-private.yaml b/profiles/nextflu-private.yaml
@@ -185,6 +185,7 @@ builds:
     enable_lbi: true
     enable_titer_models: true
     enable_measurements: true
+    enable_embeddings: true
     min_date: "2Y"
     reference_min_date: "6Y"
     max_date: "4W"
@@ -218,6 +219,7 @@ builds:
     enable_forecasts: true
     enable_titer_models: true
     enable_measurements: true
+    enable_embeddings: true
     min_date: "2Y"
     reference_min_date: "6Y"
     max_date: "4W"
@@ -246,6 +248,7 @@ builds:
     enable_lbi: true
     enable_titer_models: true
     enable_measurements: true
+    enable_embeddings: true
     min_date: "2Y"
     reference_min_date: "6Y"
     max_date: "4W"

diff --git a/profiles/nextflu-private/h1n1pdm/ha/auspice_config.json b/profiles/nextflu-private/h1n1pdm/ha/auspice_config.json
@@ -344,6 +344,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -370,7 +385,8 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",

diff --git a/profiles/nextflu-private/h1n1pdm/na/auspice_config.json b/profiles/nextflu-private/h1n1pdm/na/auspice_config.json
@@ -154,6 +154,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -175,12 +190,13 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",
     "map",
     "entropy",
     "frequencies"
   ]
-}
+}
diff --git a/profiles/nextflu-private/h3n2/ha/auspice_config.json b/profiles/nextflu-private/h3n2/ha/auspice_config.json
@@ -496,6 +496,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -526,7 +541,8 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",
@@ -535,4 +551,4 @@
     "entropy",
     "frequencies"
   ]
-}
+}
diff --git a/profiles/nextflu-private/h3n2/na/auspice_config.json b/profiles/nextflu-private/h3n2/na/auspice_config.json
@@ -188,6 +188,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -209,12 +224,13 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",
     "map",
     "entropy",
     "frequencies"
   ]
-}
+}
diff --git a/profiles/nextflu-private/vic/ha/auspice_config.json b/profiles/nextflu-private/vic/ha/auspice_config.json
@@ -230,6 +230,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -256,7 +271,8 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",

diff --git a/profiles/nextflu-private/vic/na/auspice_config.json b/profiles/nextflu-private/vic/na/auspice_config.json
@@ -132,6 +132,21 @@
       "key": "year_month",
       "title": "Year/month",
       "type": "categorical"
+    },
+    {
+      "key": "tsne_x",
+      "title": "t-SNE 1",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_y",
+      "title": "t-SNE 2",
+      "type": "continuous"
+    },
+    {
+      "key": "tsne_cluster",
+      "title": "t-SNE cluster",
+      "type": "categorical"
     }
   ],
   "geo_resolutions": [
@@ -153,12 +168,13 @@
     "submitting_lab",
     "recency",
     "epiweek",
-    "year_month"
+    "year_month",
+    "tsne_cluster"
   ],
   "panels": [
     "tree",
     "map",
     "entropy",
     "frequencies"
   ]
-}
+}
diff --git a/profiles/nextstrain-public.yaml b/profiles/nextstrain-public.yaml
@@ -70,6 +70,7 @@ array-builds:
       enable_glycosylation: true
       enable_lbi: true
       enable_titer_models: true
+      enable_embeddings: true
       include: "'config/{lineage}/reference_strains.txt'"
       exclude: "'config/{lineage}/outliers.txt'"
       titer_collections:
@@ -162,6 +163,7 @@ array-builds:
       enable_glycosylation: true
       enable_lbi: true
       enable_titer_models: true
+      enable_embeddings: true
       include: "'config/{lineage}/reference_strains.txt'"
       exclude: "'config/{lineage}/outliers.txt'"
       titer_collections:
@@ -192,6 +194,7 @@ array-builds:
       enable_lbi: true
       enable_titer_models: true
       enable_forecasts: false
+      enable_embeddings: true
       include: "'config/{lineage}/reference_strains.txt'"
       exclude: "'config/{lineage}/outliers.txt'"
       titer_collections: