feat: support input database without patient extensions

Features:
- The core__patient no longer requires the presence of race/ethnicity
  extensions to build (it now uses the deep-schema checking code)

Bug Fixes:
- If there are both detailed and ombCategory values for an extension,
  no longer report both - just prefer the ombCategory version.

cumulus_library/studies/core/builder_patient.py

@@ -18,9 +18,49 @@
 class PatientBuilder(BaseTableBuilder):
     display_text = "Creating Patient tables..."
 
+    @staticmethod
+    def make_extension_query(
+        schema: str,
+        cursor: databases.DatabaseCursor,
+        parser: databases.DatabaseParser,
+        name: str,
+        url: str,
+    ) -> str:
+        has_extensions = sql_utils.is_field_present(
+            schema=schema,
+            cursor=cursor,
+            parser=parser,
+            source_table="patient",
+            source_col="extension",
+            expected={
+                "extension": {
+                    "url": {},
+                    "valueCoding": sql_utils.CODING,
+                },
+                "url": {},
+            },
+        )
+        if has_extensions:
+            config = sql_utils.ExtensionConfig(
+                source_table="patient",
+                source_id="id",
+                target_table=f"core__patient_ext_{name}",
+                target_col_prefix=name,
+                fhir_extension=url,
+                ext_systems=["ombCategory", "detailed"],
+                is_array=True,
+            )
+            return base_templates.get_extension_denormalize_query(config)
+        else:
+            return base_templates.get_ctas_empty_query(
+                schema_name=schema,
+                table_name=f"core__patient_ext_{name}",
+                table_cols=["id", "system", f"{name}_code", f"{name}_display"],
+            )
+
     def prepare_queries(
         self,
-        cursor: object,
+        cursor: databases.DatabaseCursor,
         schema: str,
         *args,
         parser: databases.DatabaseParser = None,
@@ -41,18 +81,13 @@ def prepare_queries(
                 "fhirpath": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity",
             },
         ]
-
         for extension in extension_types:
-            config = sql_utils.ExtensionConfig(
-                source_table="patient",
-                source_id="id",
-                target_table=f"core__patient_ext_{extension['name']}",
-                target_col_prefix=extension["name"],
-                fhir_extension=extension["fhirpath"],
-                ext_systems=["ombCategory", "detailed", "text"],
-                is_array=True,
+            self.queries.append(
+                self.make_extension_query(
+                    schema, cursor, parser, extension["name"], extension["fhirpath"]
+                )
             )
-            self.queries.append(base_templates.get_extension_denormalize_query(config))
+
         validated_schema = sql_utils.validate_schema(
             cursor, schema, expected_table_cols, parser
         )

cumulus_library/studies/core/reference_sql/builder_condition.sql

@@ -419,7 +419,6 @@ CREATE TABLE core__condition AS
 WITH temp_condition AS (
     SELECT
         c.id,
-        c.category,
         c.subject.reference AS subject_ref,
         c.encounter.reference AS encounter_ref,
         cca.code,
@@ -431,9 +430,7 @@ WITH temp_condition AS (
         date_trunc('month', date(from_iso8601_timestamp(c."recordedDate")))
             AS recordedDate_month,
         date_trunc('year', date(from_iso8601_timestamp(c."recordedDate")))
-            AS recordedDate_year,
-        c.verificationStatus,
-        c.clinicalStatus
+            AS recordedDate_year
     FROM condition AS c
     LEFT JOIN core__condition_codable_concepts_all AS cca ON c.id = cca.id
 )

cumulus_library/studies/core/reference_sql/builder_documentreference.sql

@@ -153,7 +153,6 @@ CREATE TABLE core__documentreference AS
 WITH temp_documentreference AS (
     SELECT DISTINCT
         dr.id,
-        dr.type,
         dr.status,
         dr.docStatus,
         dr.context,

cumulus_library/studies/core/reference_sql/builder_encounter.sql

@@ -273,9 +273,9 @@ CREATE TABLE IF NOT EXISTS "main"."core__encounter_dn_servicetype"
 AS (
     SELECT * FROM (
         VALUES
-        (cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
+        (cast(NULL AS varchar),cast(NULL AS bigint),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
     )
-        AS t ("id","code","code_system","display")
+        AS t ("id","row","code","code_system","display")
     WHERE 1 = 0 -- ensure empty table
 );
 
@@ -285,9 +285,9 @@ CREATE TABLE IF NOT EXISTS "main"."core__encounter_dn_priority"
 AS (
     SELECT * FROM (
         VALUES
-        (cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
+        (cast(NULL AS varchar),cast(NULL AS bigint),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
     )
-        AS t ("id","code","code_system","display")
+        AS t ("id","row","code","code_system","display")
     WHERE 1 = 0 -- ensure empty table
 );
 
@@ -571,9 +571,9 @@ temp_encounter_nullable AS (
     SELECT DISTINCT
         e.id,
         e.status,
-        e.class,
+        e.class.code AS class_code,
+        e.class.system AS class_code_system,
         e.subject.reference AS subject_ref,
-        e.period,
         date(from_iso8601_timestamp(e.period.start)) AS period_start,
         date_trunc('day', date(from_iso8601_timestamp(e."period"."end")))
             AS period_end_day,
@@ -598,7 +598,8 @@ temp_encounter AS (
     SELECT DISTINCT
         e.id,
         e.status,
-        e.class,
+        e.class_code,
+        e.class_code_system,
         e.subject_ref,
         e.period_start,
         e.period_start_day,
@@ -664,7 +665,7 @@ SELECT DISTINCT
     concat('Encounter/', e.id) AS encounter_ref
 FROM temp_encounter AS e
 LEFT JOIN core__fhir_mapping_expected_act_encounter_code_v3 AS eac
-    ON e.class.code = eac.found
+    ON e.class_code = eac.found
 LEFT JOIN core__fhir_act_encounter_code_v3 AS ac ON eac.expected = ac.code
 INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref
 WHERE

cumulus_library/studies/core/reference_sql/builder_medicationrequest.sql

@@ -100,7 +100,6 @@ WITH temp_mr AS (
         date(from_iso8601_timestamp(mr.authoredOn)) AS authoredOn,
         date_trunc('month', date(from_iso8601_timestamp(mr."authoredOn")))
             AS authoredOn_month,
-        cast(NULL as varchar) AS display,
         mr.reportedBoolean,
         mr.dosageInstruction,
         mr.subject.reference AS subject_ref,

cumulus_library/studies/core/reference_sql/builder_observation.sql

@@ -244,9 +244,9 @@ CREATE TABLE IF NOT EXISTS "main"."core__observation_component_valuecodeableconc
 AS (
     SELECT * FROM (
         VALUES
-        (cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
+        (cast(NULL AS varchar),cast(NULL AS bigint),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
     )
-        AS t ("id","code","code_system","display")
+        AS t ("id","row","code","code_system","display")
     WHERE 1 = 0 -- ensure empty table
 );
 
@@ -256,9 +256,9 @@ CREATE TABLE IF NOT EXISTS "main"."core__observation_dn_interpretation"
 AS (
     SELECT * FROM (
         VALUES
-        (cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
+        (cast(NULL AS varchar),cast(NULL AS bigint),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
     )
-        AS t ("id","code","code_system","display")
+        AS t ("id","row","code","code_system","display")
     WHERE 1 = 0 -- ensure empty table
 );
 
@@ -304,9 +304,9 @@ CREATE TABLE IF NOT EXISTS "main"."core__observation_dn_dataabsentreason"
 AS (
     SELECT * FROM (
         VALUES
-        (cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
+        (cast(NULL AS varchar),cast(NULL AS bigint),cast(NULL AS varchar),cast(NULL AS varchar),cast(NULL AS varchar))
     )
-        AS t ("id","code","code_system","display")
+        AS t ("id","row","code","code_system","display")
     WHERE 1 = 0 -- ensure empty table
 );
 
@@ -319,6 +319,8 @@ WITH temp_observation AS (
     SELECT
         o.id,
         o.status,
+        o.encounter.reference AS encounter_ref,
+        o.subject.reference AS subject_ref,
         o.valueString,
         o.valueQuantity.value AS valueQuantity_value,
         o.valueQuantity.comparator AS valueQuantity_comparator,
@@ -345,9 +347,7 @@ WITH temp_observation AS (
         odvcc.display AS valueCodeableConcept_display,
         odda.code AS dataAbsentReason_code,
         odda.code_system AS dataAbsentReason_code_system,
-        odda.display AS dataAbsentReason_display,
-        o.subject.reference AS subject_ref,
-        o.encounter.reference AS encounter_ref
+        odda.display AS dataAbsentReason_display
     FROM observation AS o
     LEFT JOIN core__observation_dn_category AS odcat ON o.id = odcat.id
     LEFT JOIN core__observation_dn_code AS odc ON o.id = odc.id
@@ -375,7 +375,7 @@ SELECT
     valueCodeableConcept_display,
     valueQuantity_value,
     valueQuantity_comparator,
-    valueQuantity_unit,valueQuantity_code_system AS valueQuantity_system, -- old alias
+    valueQuantity_unit,
     valueQuantity_code_system,
     valueQuantity_code,
     valueString,

cumulus_library/studies/core/reference_sql/builder_patient.sql

@@ -43,23 +43,6 @@ CREATE TABLE core__patient_ext_race AS (
             AND ext_child.ext.valuecoding.display != ''
     ),
 
-    system_text AS (
-        SELECT DISTINCT
-            s.id AS id,
-            '2' AS priority,
-            'text' AS system, -- noqa: RF04
-            ext_child.ext.valuecoding.code AS race_code,
-            ext_child.ext.valuecoding.display AS race_display
-        FROM
-            patient AS s,
-            UNNEST(s.extension) AS ext_parent (ext),
-            UNNEST(ext_parent.ext.extension) AS ext_child (ext)
-        WHERE
-            ext_parent.ext.url = 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-race'
-            AND ext_child.ext.url = 'text'
-            AND ext_child.ext.valuecoding.display != ''
-    ),
-
     union_table AS (
         SELECT
             id,
@@ -76,14 +59,6 @@ CREATE TABLE core__patient_ext_race AS (
             race_code,
             race_display
         FROM system_detailed
-        UNION
-        SELECT
-            id,
-            priority,
-            system,
-            race_code,
-            race_display
-        FROM system_text
 
         ORDER BY id, priority
     )
@@ -118,7 +93,7 @@ CREATE TABLE core__patient_ext_race AS (
             ) AS race_display,
             ROW_NUMBER()
                 OVER (
-                    PARTITION BY id, system
+                    PARTITION BY id
                     ORDER BY priority ASC
                 ) AS available_priority
         FROM union_table
@@ -166,23 +141,6 @@ CREATE TABLE core__patient_ext_ethnicity AS (
             AND ext_child.ext.valuecoding.display != ''
     ),
 
-    system_text AS (
-        SELECT DISTINCT
-            s.id AS id,
-            '2' AS priority,
-            'text' AS system, -- noqa: RF04
-            ext_child.ext.valuecoding.code AS ethnicity_code,
-            ext_child.ext.valuecoding.display AS ethnicity_display
-        FROM
-            patient AS s,
-            UNNEST(s.extension) AS ext_parent (ext),
-            UNNEST(ext_parent.ext.extension) AS ext_child (ext)
-        WHERE
-            ext_parent.ext.url = 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity'
-            AND ext_child.ext.url = 'text'
-            AND ext_child.ext.valuecoding.display != ''
-    ),
-
     union_table AS (
         SELECT
             id,
@@ -199,14 +157,6 @@ CREATE TABLE core__patient_ext_ethnicity AS (
             ethnicity_code,
             ethnicity_display
         FROM system_detailed
-        UNION
-        SELECT
-            id,
-            priority,
-            system,
-            ethnicity_code,
-            ethnicity_display
-        FROM system_text
 
         ORDER BY id, priority
     )
@@ -241,7 +191,7 @@ CREATE TABLE core__patient_ext_ethnicity AS (
             ) AS ethnicity_display,
             ROW_NUMBER()
                 OVER (
-                    PARTITION BY id, system
+                    PARTITION BY id
                     ORDER BY priority ASC
                 ) AS available_priority
         FROM union_table

cumulus_library/template_sql/extension_denormalize.sql.jinja

@@ -72,7 +72,7 @@ CREATE TABLE {{ target_table }} AS (
             {%- endif %}
             ROW_NUMBER()
                 OVER (
-                    PARTITION BY id, system
+                    PARTITION BY id
                     ORDER BY priority ASC
                 ) AS available_priority
         FROM union_table

cumulus_library/template_sql/sql_utils.py

@@ -203,10 +203,13 @@ def is_field_populated(
     parser: databases.DatabaseParser,
     source_table: str,
     hierarchy: list[tuple],
-    expected: list | None = None,
+    expected: list | dict | None = None,
 ) -> bool:
     """Traverses a complex field and determines if it exists and has data
 
+    Non-core studies that rely on the core tables shouldn't need this method.
+    This is just to examine the weird and wonderful world of the raw FHIR tables.
+
     :keyword schema: The schema/database name
     :keyword cursor: a PEP-249 compliant database cursor
     :keyword source_table: The table to query against
@@ -217,7 +220,7 @@ def is_field_populated(
         If none, we assume it is a CodeableConcept.
     :returns: a boolean indicating if valid data is present.
     """
-    if not _check_schema_if_exists(
+    if not is_field_present(
         schema=schema,
         cursor=cursor,
         parser=parser,
@@ -256,14 +259,14 @@ def is_field_populated(
     return True
 
 
-def _check_schema_if_exists(
+def is_field_present(
     *,
     schema: str,
     cursor: databases.DatabaseCursor,
     parser: databases.DatabaseParser,
     source_table: str,
     source_col: str,
-    expected: str | None = None,
+    expected: list | dict | None = None,
 ) -> bool:
     """Validation check for a column existing, and having the expected schema