smart-on-fhir · dogversioning · Mar 15, 2024 · Mar 14, 2024 · Mar 14, 2024 · Mar 15, 2024
diff --git a/.sqlfluffignore b/.sqlfluffignore
@@ -1,4 +1,5 @@
 /scratch/
+reference_sql/
 # This is a common destination for debugging sql generation
 output.sql
 
@@ -10,4 +11,4 @@ encounter.sql.jinja
 # This file has namespace collisions with vars in other sqlfluff templates.
 # Ignoring for now - could be addressed with an in-folder .sqlfluff config
 # or by a refactor of variable names
-count.sql.jinja
+count.sql.jinja
diff --git a/cumulus_library/.sqlfluff b/cumulus_library/.sqlfluff
@@ -23,16 +23,61 @@ load_macros_from_path = cumulus_library/template_sql/shared_macros,cumulus_libra
 code_systems = ["http://snomed.info/sct", "http://hl7.org/fhir/sid/icd-10-cm"]
 col_type_list = ["a string","b string"]
 columns = ['a','b']
-cc_columns = [{"name": "baz", "is_array": True}, {"name": "foobar", "is_array": False}]
+cc_columns = 
+    [
+        {"name": "baz", "is_array": True}, 
+        {"name": "foobar", "is_array": False}
+    ]
 cc_column = 'code'
-code_system_tables = [{table_name":"hasarray","column_name":"acol","is_bare_coding":False,"is_array":True, "has_data": True},{"table_name":"noarray","column_name":"col","is_bare_coding":False,"is_array":False, "has_data": True}{"table_name":"bare","column_name":"bcol","is_bare_coding":True,"is_array":False, "has_data": True},{"table_name":"empty","column_name":"empty","is_bare_coding":False,"is_array":False, "has_data": False}]
+code_system_tables = 
+    [
+        {
+            table_name":"hasarray",
+            "column_name":"acol",
+            "is_bare_coding":False,
+            "is_array":True, 
+            "has_data": True
+        },
+        {
+            "table_name":"noarray",
+            "column_name":"col",
+            "is_bare_coding":False,
+            "is_array":False, 
+            "has_data": True
+        },
+        {
+            "table_name":"bare",
+            "column_name":"bcol",
+            "is_bare_coding":True,
+            "is_array":False, 
+            "has_data": True
+        },
+        {
+            "table_name":"empty",
+            "column_name":"empty",
+            "is_bare_coding":False,
+            "is_array":False,
+            "has_data": False
+        }
+    ]
 column_name = 'bar'
 column_names = ['foo', 'bar']
 conditions = ["1 > 0", "1 < 2"]
-config = {"medication_datasources" : {"by_contained_ref" : True, "by_external_ref" : True}, 'has_userselected': False}
+config = 
+    {
+        "medication_datasources" : {
+            "by_contained_ref" : True, 
+            "by_external_ref" : True
+        }, 
+        'has_userselected': False
+    }
 count_ref = count_ref
 count_table = count_table
-dataset = [["foo","foo"],["bar","bar"]]
+dataset = 
+    [
+        ["foo","foo"],
+        ["bar","bar"]
+    ]
 dependent_variable = is_flu
 ext_systems = ["omb", "text"]
 field = 'column_name'
@@ -41,15 +86,116 @@ filter_resource = True
 fhir_extension = condition
 fhir_resource = patient
 id = 'id'
-join_cols_by_table = { "join_table": { "join_id": "enc_ref","included_cols": [["a"], ["b", "c"]]}}
+join_cols_by_table = 
+    { 
+        "join_table": { 
+            "join_id": "enc_ref",
+            "included_cols": [["a"], ["b", "c"]]
+        }
+    }
 join_id = subject_ref
 neg_source_table = neg_source_table
 output_table_name = 'created_table'
 prefix = Test
 primary_ref = encounter_ref
 pos_source_table = pos_source_table
 schema_name = test_schema
-schema = {'condition': {'category': {'coding': True, 'code': True, 'display': True, 'system': True, 'userSelected': True, 'version': True, 'text': True}, 'clinicalstatus': {'coding': True, 'code': True, 'display': True, 'system': True, 'userSelected': True, 'version': True, 'text': True}, 'id': True, 'recordeddate': True, 'verificationstatus': {'coding': True, 'code': True, 'display': True, 'system': True, 'userSelected': True, 'version': True, 'text': True}, 'subject': {'reference': True, 'display': False, 'type': True}, 'encounter': {'reference': True, 'display': False, 'type': True}}, 'documentreference': {'id': True, 'type': True, 'status': True, 'docstatus': True, 'context': {'period': True, 'start': True}, 'subject': {'reference': True}}, 'encounter': {'status': True, 'period': {'start': True, 'end': False}, 'class': {'code': True, 'system': True, 'display': False, 'userSelected': True, 'version': True}, 'subject': {'reference': True, 'display': False, 'type': True}, 'id': True}, 'medicationrequest': {'id': True, 'status': True, 'intent': True, 'authoredon': True, 'category': {'code': True, 'system': True, 'display': False}, 'subject': {'reference': True}}, 'observation': {'id': True, 'category': {'coding': True, 'code': True, 'display': True, 'system': True, 'text': True}, 'status': True, 'code': {'coding': True, 'code': True, 'display': True, 'system': True, 'text': True}, 'interpretation': {'coding': True, 'code': True, 'display': True, 'system': True, 'text': True}, 'referencerange': {'low': False, 'high': False, 'normalvalue': False, 'type': False, 'appliesto': False, 'age': False, 'text': True}, 'effectivedatetime': True, 'valuequantity': {'value': True, 'comparator': False, 'unit': False, 'system': False, 'code': False}, 'valuecodeableconcept': {'coding': True, 'code': True, 'display': True, 'system': True}, 'subject': {'reference': True}, 'encounter': {'reference': True}}, 'patient': {'id': True, 'gender': True, 'address': True, 'birthdate': True}}
+schema = 
+    {
+        'condition': {
+            'category': {
+                'coding': True, 'code': True, 'display': True, 'system': True, 'userSelected': True, 'version': True, 'text': True
+            },
+            'clinicalstatus': {
+                'coding': True, 'code': True, 'display': True, 'system': True, 'userSelected': True, 'version': True, 'text': True
+            },
+            'id': True, 
+            'recordeddate': True, 
+            'verificationstatus': {
+                'coding': True, 'code': True, 'display': True, 'system': True, 'userSelected': True, 'version': True, 'text': True},
+            'subject': {
+                'reference': True, 'display': False, 'type': True
+            },
+            'encounter': {
+                'reference': True, 'display': False, 'type': True
+            }
+        }, 
+        'documentreference': {
+            'id': True, 
+            'type': True, 
+            'status': True, 
+            'docstatus': True, 
+            'context': {
+                'period': True, 'start': True
+            }, 
+            'subject': {
+                'reference': True
+            }
+        }, 
+        'encounter': {
+            'status': True, 
+            'period': {
+                'start': True, 'end': False
+            }, 
+            'class': {
+                'code': True, 'system': True, 'display': False, 'userSelected': True, 'version': True
+            }, 
+            'subject': {
+                'reference': True, 'display': False, 'type': True
+            }, 
+            'id': True
+        }, 
+        'medicationrequest': {
+            'id': True, 
+            'status': True, 
+            'intent': True, 
+            'authoredon': True, 
+            'category': {
+                'code': True, 'system': True, 'display': False
+            }, 
+            'subject': {
+                'reference': True
+            }, 
+            'encounter': {
+                'reference': True
+            }
+        }, 
+        'observation': {
+            'id': True, 
+            'category': {
+                'coding': True, 'code': True, 'display': True, 'system': True, 'text': True
+            },
+            'status': True,
+            'code': {
+                'coding': True, 'code': True, 'display': True, 'system': True, 'text': True
+            }, 
+            'interpretation': {
+                'coding': True, 'code': True, 'display': True, 'system': True, 'text': True
+            }, 
+            'referencerange': {
+                'low': False, 'high': False, 'normalvalue': False, 'type': False, 'appliesto': False, 'age': False, 'text': True
+            }, 
+            'effectivedatetime': True, 
+            'valuequantity': {
+                'value': True, 'comparator': False, 'unit': False, 'system': False, 'code': False
+            }, 
+            'valuecodeableconcept': {
+                'coding': True, 'code': True, 'display': True, 'system': True
+            }, 
+            'subject': {
+                'reference': True
+            }, 
+            'encounter': {
+                'reference': True
+            }
+        }, 
+        'patient': {
+            'id': True, 
+            'gender': True, 
+            'address': True, 
+            'birthdate': True
+        }
+    }
 source_table = source_table
 source_id = source_id
 table_cols = ["a","b"]
@@ -60,7 +206,19 @@ table_suffix = 2024_01_01_11_11_11
 target_col_prefix = prefix
 target_table = target_table
 type_casts={"b": "VARCHAR"}
-unnests = [{"source col": "g", "table_alias": "i", "row_alias":"j"}, {"source col": "k", "table_alias": "l", "row_alias":"m"},]
+unnests = 
+    [
+        {
+            "source col": "g", 
+            "table_alias": "i", 
+            "row_alias":"j"
+        }, 
+        {
+            "source col": "k", 
+            "table_alias": "l", 
+            "row_alias":"m"
+        },
+    ]
 view_cols = ["c","d"]
 view_name = test_view
 view_or_table_name = test_view_or_table

diff --git a/cumulus_library/studies/core/builder_medicationrequest.py b/cumulus_library/studies/core/builder_medicationrequest.py
@@ -5,36 +5,27 @@
 
 from cumulus_library import base_table_builder, databases
 from cumulus_library.studies.core.core_templates import core_templates
-from cumulus_library.template_sql import base_templates, sql_utils
+from cumulus_library.template_sql import sql_utils
 
 expected_table_cols = {
     "medicationrequest": {
         "id": [],
         "status": [],
         "intent": [],
         "authoredon": [],
+        "reportedboolean": [],
         "category": ["code", "system", "display"],
+        "medicationcodeableconcept": ["code", "system", "display"],
         "subject": ["reference"],
+        "encounter": ["reference"],
+        "dosageinstruction": ["text"],
     }
 }
 
 
 class MedicationRequestBuilder(base_table_builder.BaseTableBuilder):
     display_text = "Creating MedicationRequest tables..."
 
-    def denormalize_codes(self):
-        preferred_config = sql_utils.CodeableConceptConfig(
-            source_table="medicationrequest",
-            source_id="id",
-            column_name="category",
-            is_array=True,
-            target_table="core__medicationrequest_dn_category",
-            filter_priority=False,
-        )
-        self.queries.append(
-            base_templates.get_codeable_concept_denormalize_query(preferred_config)
-        )
-
     def prepare_queries(
         self,
         cursor: object,
@@ -48,7 +39,23 @@ def prepare_queries(
         :param cursor: A database cursor object
         :param schema: the schema/db name, matching the cursor
         """
-        self.denormalize_codes()
+        code_sources = [
+            sql_utils.CodeableConceptConfig(
+                source_table="medicationrequest",
+                source_id="id",
+                column_name="category",
+                is_array=True,
+                target_table="core__medicationrequest_dn_category",
+            ),
+            sql_utils.CodeableConceptConfig(
+                source_table="medicationrequest",
+                source_id="id",
+                column_name="medicationcodeableconcept",
+                is_array=False,
+                target_table="core__medicationrequest_dn_medication",
+            ),
+        ]
+        self.queries += sql_utils.denormalize_codes(schema, cursor, code_sources)
         validated_schema = core_templates.validate_schema(
             cursor, schema, expected_table_cols, parser
         )

diff --git a/cumulus_library/studies/core/core_templates/medication.sql.jinja b/cumulus_library/studies/core/core_templates/medication.sql.jinja
@@ -121,7 +121,7 @@ CREATE TABLE core__medication AS (
 
     mre_ref_id AS (
         SELECT DISTINCT
-            mr.id AS id,
+            mr.id,
             mr.encounter,
             mr.subject,
             substring(mr.medicationreference.reference, 12) AS medication_id
@@ -148,7 +148,7 @@ CREATE TABLE core__medication AS (
             mmn.id,
             mmn.encounter.reference AS encounter_ref,
             mmn.subject.reference AS patient_ref,
-            t.r.code AS code,
+            t.r.code,
             mmn.code.text AS display,
             t.r.system AS code_system,
             FALSE AS userselected

diff --git a/cumulus_library/studies/core/core_templates/medicationrequest.sql.jinja b/cumulus_library/studies/core/core_templates/medicationrequest.sql.jinja
@@ -6,7 +6,11 @@ WITH temp_mr AS (
         {{- utils.basic_cols(
                 'medicationrequest',
                 'mr',
-                ['id', 'status', 'intent']
+                [
+                    'id', 
+                    'status', 
+                    'intent',
+                ]
             ) 
         }},
         {{- utils.date_cols_from_str(
@@ -20,41 +24,49 @@ WITH temp_mr AS (
                 'medicationrequest',
                 'mr',
                 [
-                    ('authoredon', 'month')
+                    ('authoredon', 'month'),
                 ],
                 schema
             ) 
         }},
         {{- utils.nullable_cols(
                 'medicationrequest',
+
                 'mr',
                 [
                     'display',
-                    ('subject', 'reference', 'subject_ref')
+                    'reportedboolean',
+                    'dosageinstruction',
+                    ('subject', 'reference', 'subject_ref'),
+                    ('encounter', 'reference', 'encounter_ref'),
                 ],
                 schema
         ) -}},
-        cm.code AS rx_code,
-        cm.code_system AS rx_code_system,
-        cm.display AS rx_display,
         mrc.code AS category_code,
-        mrc.code_system AS category_code_system
+        mrc.code_system AS category_code_system,
+        mrm.code AS medication_code,
+        mrm.code_system AS medication_code_system,
+        mrm.display AS medication_display
     FROM medicationrequest AS mr
-    INNER JOIN core__medication AS cm ON mr.id = cm.id
     LEFT JOIN core__medicationrequest_dn_category AS mrc ON mr.id = mrc.id
-    WHERE cm.code_system = 'http://www.nlm.nih.gov/research/umls/rxnorm'
+    LEFT JOIN core__medicationrequest_dn_medication AS mrm ON mr.id = mrm.id
+    WHERE mrm.code_system = 'http://www.nlm.nih.gov/research/umls/rxnorm'
 )
 
 SELECT
-    id,
-    status,
-    intent,
-    authoredon,
-    authoredon_month,
-    category_code,
-    category_code_system,
-    rx_code_system,
-    rx_code,
-    rx_display,
-    subject_ref
-FROM temp_mr
+    mr.id,
+    mr.status,
+    mr.intent,
+    mr.category_code,
+    mr.category_code_system,
+    mr.reportedboolean,
+    mr.medication_code_system,
+    mr.medication_code,
+    mr.medication_display,
+    mr.authoredon,
+    mr.authoredon_month,
+    dose_row.dose_col.text AS doseageinstruction_text,
+    mr.subject_ref,
+    mr.encounter_ref
+FROM temp_mr AS mr,
+    UNNEST(dosageinstruction) AS dose_row (dose_col)
diff --git a/cumulus_library/studies/core/count_core.py b/cumulus_library/studies/core/count_core.py
@@ -93,7 +93,7 @@ def count_core_encounter_service(self, duration: str = "month"):
     def count_core_medicationrequest(self, duration: str = "month"):
         table_name = self.get_table_name("count_medicationrequest", duration=duration)
         from_table = self.get_table_name("medicationrequest")
-        cols = ["status", "intent", f"authoredon_{duration}", "rx_display"]
+        cols = ["status", "intent", f"authoredon_{duration}", "medication_display"]
         return self.count_medicationrequest(table_name, from_table, cols)
 
     def count_core_observation_lab(self, duration: str = "month"):