Use CSET, not airtable ids, in groups

See also georgetown-cset/airtable-etl#9
georgetown-cset · May 16, 2024 · cf91198 · cf91198
1 parent fcfdbd3
commit cf91198
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 1 deletion.
diff --git a/...able_configs/company_annotation/main.json → ..._configs/company_annotation/000_main.json b/...able_configs/company_annotation/main.json → ..._configs/company_annotation/000_main.json
@@ -4,6 +4,7 @@
   "schema_name": "organizations",
   "merge_query": "default_merge",
   "production_table": "organizations",
+  "include_airtable_id": true,
   "column_map": {
     "Organization": "EXCLUDE",
     "Name": "name",

diff --git a/company_linkage/airtable_configs/company_annotation/groups.json b/company_linkage/airtable_configs/company_annotation/groups.json
@@ -2,7 +2,7 @@
   "airtable_table": "Groups",
   "airtable_base": "appqtfVHf5tIRsfkI",
   "schema_name": "groups",
-  "merge_query": "default_merge",
+  "merge_query": "groups_merge",
   "production_table": "groups",
   "column_map": {
     "Group": "name",

diff --git a/company_linkage/airtable_queries/groups_merge.sql b/company_linkage/airtable_queries/groups_merge.sql
@@ -0,0 +1,10 @@
+SELECT
+  {{ params.staging_table_name }}.name,
+  organizations.CSET_id
+FROM
+  {{ params.staging_dataset }}.{{ params.staging_table_name }}
+LEFT JOIN
+  UNNEST(CSET_ids) AS airtable_cset_id
+LEFT JOIN
+  `{{ params.staging_dataset }}.000_main_raw` as organizations
+ON organizations.airtable_id = airtable_cset_id
diff --git a/company_linkage/airtable_schemas/company_annotation/organizations.json b/company_linkage/airtable_schemas/company_annotation/organizations.json
@@ -11,6 +11,12 @@
     "name": "CSET_id",
     "type": "INTEGER"
   },
+  {
+    "description": "Airtable's identifier for the organization",
+    "mode": "NULLABLE",
+    "name": "airtable_id",
+    "type": "STRING"
+  },
   {
     "description": "Country where the organization is located",
     "mode": "NULLABLE",