Convert concept & coding builders to new schema approach

- Remove a lot of codeable concept expected values, when passed to a
  jinja file. The concept builders handle that themselves.
- Move validate_schema() from core_templates to sql_utils
- Have the concept & coding builders use validate_schema to check for
  field existence, and pass down the full expected field schema in
  places that need it.
- Add encounter `class` system validation (should be v3-ActCode)
- Remove deprecated valueQuantity_system, as no one should be using it
  right now.

cumulus_library/studies/core/builder_condition.py

@@ -4,13 +4,10 @@
 
 expected_table_cols = {
     "condition": {
-        "category": [],
-        "clinicalstatus": [],
         "id": [],
         "recordedDate": [],
-        "verificationstatus": [],
-        "subject": ["reference", "display", "type"],
-        "encounter": ["reference", "display", "type"],
+        "subject": sql_utils.REFERENCE,
+        "encounter": sql_utils.REFERENCE,
     }
 }
 
@@ -85,7 +82,7 @@ def prepare_queries(
         **kwargs,
     ):
         self.denormalize_codes()
-        validated_schema = core_templates.validate_schema(
+        validated_schema = sql_utils.validate_schema(
             cursor, schema, expected_table_cols, parser
         )
         self.queries.append(

cumulus_library/studies/core/builder_documentreference.py

@@ -5,13 +5,11 @@
 expected_table_cols = {
     "documentreference": {
         "id": [],
-        "type": [],
         "status": [],
         "date": [],
         "docStatus": [],
-        "subject": ["reference"],
-        "context": {"encounter": ["reference"], "period": ["start"]},
-        "category": [],
+        "subject": sql_utils.REFERENCE,
+        "context": {"encounter": sql_utils.REFERENCE, "period": ["start"]},
     }
 }
 
@@ -30,6 +28,7 @@ def prepare_queries(
         self.queries = sql_utils.denormalize_complex_objects(
             schema,
             cursor,
+            parser,
             [
                 sql_utils.CodeableConceptConfig(
                     source_table="documentreference",
@@ -59,10 +58,11 @@ def prepare_queries(
                     source_id="id",
                     column_hierarchy=[("content", list), ("format", dict)],
                     target_table="core__documentreference_dn_format",
+                    expected={"format": sql_utils.CODING},
                 ),
             ],
         )
-        validated_schema = core_templates.validate_schema(
+        validated_schema = sql_utils.validate_schema(
             cursor, schema, expected_table_cols, parser
         )
         self.queries.append(

cumulus_library/studies/core/builder_encounter.py

@@ -6,24 +6,14 @@
 
 expected_table_cols = {
     "encounter": {
+        "id": [],
         "status": [],
         "period": [
             "start",
             "end",
         ],
-        "class": [
-            "code",
-            "system",
-            "display",
-            "userSelected",
-            "version",
-        ],
-        "subject": [
-            "reference",
-            "display",
-            "type",
-        ],
-        "id": [],
+        "class": sql_utils.CODING,
+        "subject": sql_utils.REFERENCE,
     },
     "etl__completion": {
         "group_name": [],
@@ -47,7 +37,7 @@ def __post_init__(self):
 class CoreEncounterBuilder(base_table_builder.BaseTableBuilder):
     display_text = "Creating Encounter tables..."
 
-    def denormalize_codes(self, schema, cursor):
+    def denormalize_codes(self, schema, cursor, parser):
         code_configs = [
             EncConfig(
                 column_hierarchy=[("type", list)],
@@ -110,11 +100,11 @@ def denormalize_codes(self, schema, cursor):
                     ("hospitalization", dict),
                     ("dischargedisposition", dict),
                 ],
-                filter_priority=False,
+                expected={"dischargedisposition": sql_utils.CODEABLE_CONCEPT},
             ),
         ]
         self.queries += sql_utils.denormalize_complex_objects(
-            schema, cursor, code_configs
+            schema, cursor, parser, code_configs
         )
 
     def prepare_queries(
@@ -128,8 +118,9 @@ def prepare_queries(
         self.denormalize_codes(
             schema,
             cursor,
+            parser,
         )
-        validated_schema = core_templates.validate_schema(
+        validated_schema = sql_utils.validate_schema(
             cursor, schema, expected_table_cols, parser
         )
         self.queries += [

cumulus_library/studies/core/builder_medication.py

@@ -1,14 +1,14 @@
 """ Module for generating core medication table"""
 
-from cumulus_library import base_table_builder, base_utils
+from cumulus_library import base_table_builder, base_utils, databases
 from cumulus_library.studies.core.core_templates import core_templates
 from cumulus_library.template_sql import base_templates, sql_utils
 
 
 class MedicationBuilder(base_table_builder.BaseTableBuilder):
     display_text = "Creating Medication table..."
 
-    def _check_data_in_fields(self, cursor, schema: str):
+    def _check_data_in_fields(self, cursor, parser, schema: str):
         """Validates whether either observed medication source is present
 
         We opt to not use the core_templates.utils based version of
@@ -37,6 +37,7 @@ def _check_data_in_fields(self, cursor, schema: str):
                 source_table=table,
                 hierarchy=[(inline_col, dict), ("coding", list)],
                 cursor=cursor,
+                parser=parser,
             )
             if data_types["inline"]:
                 query = base_templates.get_column_datatype_query(
@@ -56,8 +57,9 @@ def _check_data_in_fields(self, cursor, schema: str):
                 schema=schema,
                 source_table=table,
                 hierarchy=[("medicationreference", dict), ("reference", dict)],
-                expected=["reference"],
+                expected=sql_utils.REFERENCE,
                 cursor=cursor,
+                parser=parser,
             ):
                 return data_types, has_userselected
 
@@ -83,15 +85,22 @@ def _check_data_in_fields(self, cursor, schema: str):
 
             return data_types, has_userselected
 
-    def prepare_queries(self, cursor: object, schema: str, *args, **kwargs) -> dict:
+    def prepare_queries(
+        self,
+        cursor: databases.DatabaseCursor,
+        schema: str,
+        parser: databases.DatabaseParser = None,
+        *args,
+        **kwargs,
+    ) -> dict:
         """Constructs queries related to condition codeableConcept
 
         :param cursor: A database cursor object
         :param schema: the schema/db name, matching the cursor
-
+        :param parser: A database parser
         """
         medication_datasources, has_userselected = self._check_data_in_fields(
-            cursor, schema
+            cursor, parser, schema
         )
         if (
             medication_datasources["inline"]

cumulus_library/studies/core/builder_medicationrequest.py

@@ -14,10 +14,8 @@
         "intent": [],
         "authoredOn": [],
         "reportedBoolean": [],
-        "category": ["code", "system", "display"],
-        "medicationCodeableConcept": ["code", "system", "display"],
-        "subject": ["reference"],
-        "encounter": ["reference"],
+        "subject": sql_utils.REFERENCE,
+        "encounter": sql_utils.REFERENCE,
         "dosageInstruction": ["text"],
     }
 }
@@ -54,9 +52,9 @@ def prepare_queries(
             ),
         ]
         self.queries += sql_utils.denormalize_complex_objects(
-            schema, cursor, code_sources
+            schema, cursor, parser, code_sources
         )
-        validated_schema = core_templates.validate_schema(
+        validated_schema = sql_utils.validate_schema(
             cursor, schema, expected_table_cols, parser
         )
         self.queries.append(

cumulus_library/studies/core/builder_observation.py

@@ -9,30 +9,15 @@
 expected_table_cols = {
     "observation": {
         "id": [],
-        "category": [],
         "component": {
-            "code": [],
-            "dataAbsentReason": [],
             "valueQuantity": ["code", "comparator", "system", "unit", "value"],
         },
         "status": [],
-        "code": [],
-        "interpretation": [],
-        "referenceRange": [
-            "low",
-            "high",
-            "normalValue",
-            "type",
-            "appliesTo",
-            "age",
-            "text",
-        ],
         "effectiveDateTime": [],
-        "valueQuantity": ["value", "comparator", "unit", "system", "code"],
-        "valueCodeableConcept": [],
+        "valueQuantity": ["code", "comparator", "system", "unit", "value"],
         "valueString": [],
-        "subject": ["reference"],
-        "encounter": ["reference"],
+        "subject": sql_utils.REFERENCE,
+        "encounter": sql_utils.REFERENCE,
     }
 }
 
@@ -73,18 +58,22 @@ def prepare_queries(
             ObsConfig(
                 is_public=True,
                 column_hierarchy=[("component", list), ("code", dict)],
+                expected={"code": sql_utils.CODEABLE_CONCEPT},
             ),
             ObsConfig(
                 is_public=True,
                 column_hierarchy=[("component", list), ("dataabsentreason", dict)],
+                expected={"dataabsentreason": sql_utils.CODEABLE_CONCEPT},
             ),
             ObsConfig(
                 is_public=True,
                 column_hierarchy=[("component", list), ("interpretation", list)],
+                expected={"interpretation": sql_utils.CODEABLE_CONCEPT},
             ),
             ObsConfig(
                 is_public=True,
                 column_hierarchy=[("component", list), ("valuecodeableconcept", dict)],
+                expected={"valuecodeableconcept": sql_utils.CODEABLE_CONCEPT},
             ),
             ObsConfig(
                 column_hierarchy=[("interpretation", list)], filter_priority=False
@@ -100,9 +89,9 @@ def prepare_queries(
         ]
 
         self.queries += sql_utils.denormalize_complex_objects(
-            schema, cursor, code_sources
+            schema, cursor, parser, code_sources
         )
-        validated_schema = core_templates.validate_schema(
+        validated_schema = sql_utils.validate_schema(
             cursor, schema, expected_table_cols, parser
         )
         self.queries += [

cumulus_library/studies/core/builder_patient.py

@@ -9,7 +9,7 @@
     "patient": {
         "id": [],
         "gender": [],
-        "address": [],
+        "address": {"postalCode": {}},
         "birthDate": [],
     }
 }
@@ -53,7 +53,7 @@ def prepare_queries(
                 is_array=True,
             )
             self.queries.append(base_templates.get_extension_denormalize_query(config))
-        validated_schema = core_templates.validate_schema(
+        validated_schema = sql_utils.validate_schema(
             cursor, schema, expected_table_cols, parser
         )
         self.queries.append(

cumulus_library/studies/core/core_templates/condition.sql.jinja

@@ -17,7 +17,6 @@ WITH temp_condition AS (
                 'condition',
                 'c',
                 [
-                    'category',
                     ('subject', 'reference', 'subject_ref'),
                     ('encounter', 'reference', 'encounter_ref'),
                 ], 
@@ -54,9 +53,7 @@ WITH temp_condition AS (
                 ],
                 schema
             ) 
-        }},
-        c.verificationStatus,
-        c.clinicalStatus
+        }}
     FROM condition AS c
     LEFT JOIN core__condition_codable_concepts_all AS cca ON c.id = cca.id
 )

cumulus_library/studies/core/core_templates/core_templates.py

@@ -12,14 +12,5 @@ def get_core_template(
 ) -> str:
     """Extracts code system details as a standalone table"""
     return base_templates.get_base_template(
-        target_table, path=pathlib.Path(__file__).parent, schema=schema, config=config
+        target_table, path=PATH, schema=schema, config=config
     )
-
-
-def validate_schema(cursor: object, schema: str, expected_table_cols, parser):
-    validated_schema = {}
-    for table, cols in expected_table_cols.items():
-        query = base_templates.get_column_datatype_query(schema, table, cols.keys())
-        table_schema = cursor.execute(query).fetchall()
-        validated_schema[table] = parser.validate_table_schema(cols, table_schema)
-    return validated_schema

cumulus_library/studies/core/core_templates/documentreference.sql.jinja

@@ -16,10 +16,9 @@ WITH temp_documentreference AS (
                 'documentreference',
                 'dr',
                 [
-                    'type',
                     'status',
                     'docStatus',
-                    'context',                
+                    'context',
                     ('subject', 'reference', 'subject_ref'),
                 ], 
                 schema

cumulus_library/studies/core/core_templates/encounter.sql.jinja

@@ -26,9 +26,9 @@ temp_encounter_nullable AS (
                 'e',
                 [
                     'status',
-                    'class', 
+                    ('class', 'code', 'class_code'),
+                    ('class', 'system', 'class_code_system'),
                     ('subject', 'reference', 'subject_ref'),
-                    'period',
                 ], 
                 schema
             ) 
@@ -46,7 +46,7 @@ temp_encounter_nullable AS (
                 'encounter',
                 'e',
                 [
-                    ('period','end', 'period_end_day', 'day'),
+                    ('period', 'end', 'period_end_day', 'day'),
                     ('period', 'start', 'period_start_day', 'day'),
                     ('period', 'start', 'period_start_week', 'week'),
                     ('period', 'start', 'period_start_month', 'month'),
@@ -68,7 +68,8 @@ temp_encounter AS (
     SELECT DISTINCT
         e.id,
         e.status,
-        e.class,
+        e.class_code,
+        e.class_code_system,
         e.subject_ref,
         e.period_start,
         e.period_start_day,
@@ -134,7 +135,8 @@ SELECT DISTINCT
     concat('Encounter/', e.id) AS encounter_ref
 FROM temp_encounter AS e
 LEFT JOIN core__fhir_mapping_expected_act_encounter_code_v3 AS eac
-    ON e.class.code = eac.found
+    ON e.class_code = eac.found
+    AND e.class_code_system = 'http://terminology.hl7.org/CodeSystem/v3-ActCode'
 LEFT JOIN core__fhir_act_encounter_code_v3 AS ac ON eac.expected = ac.code
 INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref
 WHERE

cumulus_library/studies/core/core_templates/medicationrequest.sql.jinja

@@ -34,7 +34,6 @@ WITH temp_mr AS (
 
                 'mr',
                 [
-                    'display',
                     'reportedBoolean',
                     'dosageInstruction',
                     ('subject', 'reference', 'subject_ref'),