Add AllergyIntolerance support to core study

.gitattributes

@@ -0,0 +1,2 @@
+# Ignore bsv files, else GitHub will detect the Bluespec language
+*.bsv linguist-vendored

.github/pull_request_template.md

@@ -1,5 +1,7 @@
 
 ### Checklist
-- [ ] Consider if documentation (like in `docs/`, or if you've changed the structure of a table) needs to be updated
+- [ ] Consider if documentation in `docs/` needs to be updated
+  - If you've changed the structure of a table, you may need to run `generate-md`
+  - If you've added/removed `core` study fields that not in US Core, update our list of those in `core-study-details.md`
 - [ ] Consider if tests should be added
-- [ ] Update template repo if there are changes to study configuration
+- [ ] Update template repo if there are changes to study configuration in `manifest.toml`

cumulus_library/.sqlfluff

@@ -105,6 +105,20 @@ remote_location = s3://bucket/study/data/
 schema_name = test_schema
 schema = 
     {
+        'allergyintolerance': {
+            'category': True,
+            'criticality': True,
+            'encounter': {
+                'reference': True,
+            },
+            'id': True,
+            'patient': {
+                'reference': True,
+            },
+            'reaction': {
+                'severity': True,
+            },
+        },
         'condition': {
             'category': {
                 'coding': True, 'code': True, 'display': True, 'system': True, 'userSelected': True, 'version': True, 'text': True

cumulus_library/__init__.py

@@ -6,4 +6,4 @@
 from .study_manifest import StudyManifest
 
 __all__ = ["BaseTableBuilder", "CountsBuilder", "StudyConfig", "StudyManifest"]
-__version__ = "3.0.0"
+__version__ = "3.1.0"

cumulus_library/statistics/counts.py

@@ -6,6 +6,9 @@
 from cumulus_library import base_table_builder, errors, study_manifest
 from cumulus_library.statistics.statistics_templates import counts_templates
 
+# Defined here for easy overriding by tests
+DEFAULT_MIN_SUBJECT = 10
+
 
 class CountsBuilder(base_table_builder.BaseTableBuilder):
     """Extends BaseTableBuilder for counts-related use cases"""
@@ -39,13 +42,17 @@ def get_table_name(self, table_name: str, duration=None) -> str:
         else:
             return f"{self.study_prefix}__{table_name}"
 
-    def get_where_clauses(self, clause: list | str | None = None, min_subject: int = 10) -> str:
+    def get_where_clauses(
+        self, clause: list | str | None = None, min_subject: int | None = None
+    ) -> str:
         """Convenience method for constructing arbitrary where clauses.
 
         :param clause: either a string or a list of sql where statements
         :param min_subject: if clause is none, the bin size for a cnt_subject_ref filter
             (deprecated, use count_[fhir_resource](min_subject) instead)
         """
+        if min_subject is None:
+            min_subject = DEFAULT_MIN_SUBJECT
         if clause is None:
             return [f"cnt_subject_ref >= {min_subject}"]
         elif isinstance(clause, str):
@@ -79,22 +86,52 @@ def get_count_query(
                 "where_clauses",
                 "fhir_resource",
                 "filter_resource",
+                "patient_link",
             ]:
                 raise errors.CountsBuilderError(f"count_query received unexpected key: {key}")
+        if "min_subject" in kwargs and kwargs["min_subject"] is None:
+            kwargs["min_subject"] = DEFAULT_MIN_SUBJECT
         return counts_templates.get_count_query(table_name, source_table, table_cols, **kwargs)
 
     # ----------------------------------------------------------------------
     # The following function all wrap get_count_query as convenience methods.
     # We're not trying to be overly clever about this to persist the docstrings as the
     # primary interface that study authors would see when using these functions.
 
+    def count_allergyintolerance(
+        self,
+        table_name: str,
+        source_table: str,
+        table_cols: list,
+        where_clauses: list | None = None,
+        min_subject: int | None = None,
+    ) -> str:
+        """wrapper method for constructing allergyintolerance counts tables
+
+        :param table_name: The name of the table to create. Must start with study prefix
+        :param source_table: The table to create counts data from
+        :param table_cols: The columns from the source table to add to the count table
+        :param where_clauses: An array of where clauses to use for filtering the data
+        :param min_subject: An integer setting the minimum bin size for inclusion
+            (default: 10)
+        """
+        return self.get_count_query(
+            table_name,
+            source_table,
+            table_cols,
+            where_clauses=where_clauses,
+            min_subject=min_subject,
+            fhir_resource="allergyintolerance",
+            patient_link="patient_ref",
+        )
+
     def count_condition(
         self,
         table_name: str,
         source_table: str,
         table_cols: list,
         where_clauses: list | None = None,
-        min_subject: int = 10,
+        min_subject: int | None = None,
     ) -> str:
         """wrapper method for constructing condition counts tables
 
@@ -112,7 +149,7 @@ def count_condition(
             where_clauses=where_clauses,
             min_subject=min_subject,
             fhir_resource="condition",
-            filter_resource="encounter",
+            filter_resource=True,
         )
 
     def count_documentreference(
@@ -121,7 +158,7 @@ def count_documentreference(
         source_table: str,
         table_cols: list,
         where_clauses: list | None = None,
-        min_subject: int = 10,
+        min_subject: int | None = None,
     ) -> str:
         """wrapper method for constructing documentreference counts tables
 
@@ -139,7 +176,7 @@ def count_documentreference(
             where_clauses=where_clauses,
             min_subject=min_subject,
             fhir_resource="documentreference",
-            filter_resource="encounter",
+            filter_resource=True,
         )
 
     def count_encounter(
@@ -148,7 +185,7 @@ def count_encounter(
         source_table: str,
         table_cols: list,
         where_clauses: list | None = None,
-        min_subject: int = 10,
+        min_subject: int | None = None,
     ) -> str:
         """wrapper method for constructing encounter counts tables
 
@@ -174,7 +211,7 @@ def count_medicationrequest(
         source_table: str,
         table_cols: list,
         where_clauses: list | None = None,
-        min_subject: int = 10,
+        min_subject: int | None = None,
     ) -> str:
         """wrapper method for constructing medicationrequests counts tables
 
@@ -200,7 +237,7 @@ def count_observation(
         source_table: str,
         table_cols: list,
         where_clauses: list | None = None,
-        min_subject: int = 10,
+        min_subject: int | None = None,
     ) -> str:
         """wrapper method for constructing observation counts tables
 
@@ -226,7 +263,7 @@ def count_patient(
         source_table: str,
         table_cols: list,
         where_clauses: list | None = None,
-        min_subject: int = 10,
+        min_subject: int | None = None,
     ) -> str:
         """wrapper method for constructing patient counts tables
 

cumulus_library/statistics/statistics_templates/count.sql.jinja

@@ -2,14 +2,13 @@
 {#- TODO: move secondary/tertiary resource declaration upstream to
 statistics/counts.py level -#}
 {%- macro secondary_resource(fhir_resource) -%}
-{%- if fhir_resource in ('encounter', 'condition') -%}
+{%- if fhir_resource in ('documentreference', 'encounter', 'observation') -%}
+{{ fhir_resource }}_ref
+{%- elif fhir_resource == 'condition' -%}
+{#- count condition by encounter, because they are often duplicated for billing purposes -#}
 encounter_ref
-{%- elif fhir_resource == 'documentreference' -%}
-documentreference_ref
-{%- elif fhir_resource == 'observation'-%}
-observation_ref
 {%- else -%}
-{#- implicit null -#}
+{#- implicit null, meaning "count by patient" -#}
 {%- endif -%}
 {%- endmacro -%}
 
@@ -65,7 +64,7 @@ CREATE TABLE {{ table_name }} AS (
     {%- if filter_resource %}
     filtered_table AS (
         SELECT
-            s.subject_ref,
+            s.{{ patient_link }},
             {%- if secondary %}
             s.{{ secondary }},
             {%- endif -%}
@@ -102,19 +101,19 @@ CREATE TABLE {{ table_name }} AS (
         {%- endif -%}
 
         {#- resource specific filtering conditions -#}
-        {%- if fhir_resource == 'documentreference' and filter_resource %}
+        {%- if fhir_resource == 'documentreference' %}
         WHERE (s.status = 'current')
         AND (s.docStatus IS null OR s.docStatus IN ('final', 'amended'))
-        {%- elif fhir_resource == 'encounter' and filter_resource %}
+        {%- elif fhir_resource == 'encounter' %}
         WHERE s.status = 'finished'
-        {%- elif fhir_resource == 'observation' and filter_resource %}
+        {%- elif fhir_resource == 'observation' %}
         WHERE (s.status = 'final' OR s.status= 'amended')
         {%- endif %}
     ),
     {% endif %}
     null_replacement AS (
         SELECT
-            subject_ref,
+            {{ patient_link }},
             {%- if secondary %}
             {{ secondary }},
             {%- endif -%}
@@ -177,7 +176,7 @@ CREATE TABLE {{ table_name }} AS (
 
     powerset AS (
         SELECT
-            count(DISTINCT subject_ref) AS cnt_subject_ref,
+            count(DISTINCT {{ patient_link }}) AS cnt_subject_ref,
             {{-cols_delineated_list(table_cols, fhir_resource)}},
             concat_ws(
                 '-',

cumulus_library/statistics/statistics_templates/counts_templates.py

@@ -15,6 +15,7 @@ class CountableFhirResource(Enum):
     ids.
     """
 
+    ALLERGYINTOLERANCE = "allergyintolerance"
     CONDITION = "condition"
     DOCUMENTREFERENCE = "documentreference"
     ENCOUNTER = "encounter"
@@ -40,6 +41,7 @@ def get_count_query(
     where_clauses: list | None = None,
     fhir_resource: str | None = None,
     filter_resource: bool | None = True,
+    patient_link: str = "subject_ref",
 ) -> str:
     """Generates count tables for generating study outputs"""
     path = Path(__file__).parent
@@ -67,6 +69,7 @@ def get_count_query(
         where_clauses=where_clauses,
         fhir_resource=fhir_resource,
         filter_resource=filter_resource,
+        patient_link=patient_link,
     )
     # workaround for conflicting sqlfluff enforcement
     return query.replace("-- noqa: disable=LT02\n", "")

cumulus_library/studies/core/builder_allergyintolerance.py

@@ -0,0 +1,66 @@
+import cumulus_library
+from cumulus_library.studies.core.core_templates import core_templates
+from cumulus_library.template_sql import sql_utils
+
+expected_table_cols = {
+    "allergyintolerance": {
+        "id": [],
+        "type": [],
+        "category": [],
+        "criticality": [],
+        "recordedDate": [],
+        "patient": sql_utils.REFERENCE,
+        "encounter": sql_utils.REFERENCE,
+        "reaction": ["severity"],
+    }
+}
+
+
+class CoreAllergyIntoleranceBuilder(cumulus_library.BaseTableBuilder):
+    display_text = "Creating AllergyIntolerance tables..."
+
+    def prepare_queries(self, *args, config: cumulus_library.StudyConfig, **kwargs):
+        code_sources = [
+            sql_utils.CodeableConceptConfig(
+                source_table="allergyintolerance",
+                column_hierarchy=[("clinicalStatus", dict)],
+                target_table="core__allergyintolerance_dn_clinical_status",
+                filter_priority=True,
+                code_systems=[
+                    # Restrict to just this required binding system
+                    "http://terminology.hl7.org/CodeSystem/allergyintolerance-clinical",
+                ],
+            ),
+            sql_utils.CodeableConceptConfig(
+                source_table="allergyintolerance",
+                column_hierarchy=[("verificationStatus", dict)],
+                target_table="core__allergyintolerance_dn_verification_status",
+                filter_priority=True,
+                code_systems=[
+                    # Restrict to just this required binding system
+                    "http://terminology.hl7.org/CodeSystem/allergyintolerance-verification",
+                ],
+            ),
+            sql_utils.CodeableConceptConfig(
+                source_table="allergyintolerance",
+                column_hierarchy=[("code", dict)],
+                target_table="core__allergyintolerance_dn_code",
+            ),
+            sql_utils.CodeableConceptConfig(
+                source_table="allergyintolerance",
+                column_hierarchy=[("reaction", list), ("substance", dict)],
+                target_table="core__allergyintolerance_dn_reaction_substance",
+                expected={"substance": sql_utils.CODEABLE_CONCEPT},
+            ),
+            sql_utils.CodeableConceptConfig(
+                source_table="allergyintolerance",
+                column_hierarchy=[("reaction", list), ("manifestation", list)],
+                target_table="core__allergyintolerance_dn_reaction_manifestation",
+                expected={"manifestation": sql_utils.CODEABLE_CONCEPT},
+            ),
+        ]
+        self.queries += sql_utils.denormalize_complex_objects(config.db, code_sources)
+        validated_schema = sql_utils.validate_schema(config.db, expected_table_cols)
+        self.queries.append(
+            core_templates.get_core_template("allergyintolerance", validated_schema)
+        )