core: drop 2016 cutoff from core tables

cumulus_library/studies/core/core_templates/allergyintolerance.sql.jinja

@@ -134,5 +134,4 @@ LEFT JOIN core__allergyintolerance_dn_code AS dn_code ON ta.id = dn_code.id
 LEFT JOIN core__allergyintolerance_dn_clinical_status AS dn_cstat ON ta.id = dn_cstat.id
 LEFT JOIN core__allergyintolerance_dn_verification_status AS dn_vstat
     ON ta.id = dn_vstat.id
-LEFT JOIN temp_category AS tcat ON ta.id = tcat.id
-WHERE ta.recordedDate BETWEEN date('2016-01-01') AND current_date;
+LEFT JOIN temp_category AS tcat ON ta.id = tcat.id;

cumulus_library/studies/core/core_templates/condition.sql.jinja

@@ -80,5 +80,4 @@ SELECT
 FROM temp_condition AS tc
 LEFT JOIN core__condition_dn_category AS cdc ON tc.id = cdc.id
 LEFT JOIN core__condition_dn_clinical_status AS cdcs ON tc.id = cdcs.id
-LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id
-WHERE tc.recordedDate BETWEEN date('2016-01-01') AND current_date;
+LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id;

cumulus_library/studies/core/core_templates/documentreference.sql.jinja

@@ -89,5 +89,4 @@ SELECT DISTINCT
     te.encounter_ref,
     concat('DocumentReference/', tdr.id) AS documentreference_ref
 FROM temp_documentreference AS tdr
-LEFT JOIN temp_encounters AS te ON tdr.id = te.id
-WHERE date(tdr.author_day) BETWEEN date('2016-06-01') AND current_date;
+LEFT JOIN temp_encounters AS te ON tdr.id = te.id;

cumulus_library/studies/core/core_templates/encounter.sql.jinja

@@ -137,6 +137,4 @@ FROM temp_encounter AS e
 LEFT JOIN core__fhir_mapping_expected_act_encounter_code_v3 AS eac
     ON e.class_code = eac.found
 LEFT JOIN core__fhir_act_encounter_code_v3 AS ac ON eac.expected = ac.code
-INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref
-WHERE
-    e.period_start_day BETWEEN date('2016-06-01') AND current_date;
+INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref;

cumulus_library/studies/core/core_templates/observation.sql.jinja

@@ -90,8 +90,4 @@ SELECT
     subject_ref,
     encounter_ref,
     concat('Observation/', id) AS observation_ref
-FROM temp_observation
-WHERE
-    effectiveDateTime_day BETWEEN date(
-        from_iso8601_timestamp('2016-06-01')
-    ) AND current_date;
+FROM temp_observation;

cumulus_library/studies/core/reference_sql/builder_allergyintolerance.sql

@@ -336,5 +336,4 @@ LEFT JOIN core__allergyintolerance_dn_code AS dn_code ON ta.id = dn_code.id
 LEFT JOIN core__allergyintolerance_dn_clinical_status AS dn_cstat ON ta.id = dn_cstat.id
 LEFT JOIN core__allergyintolerance_dn_verification_status AS dn_vstat
     ON ta.id = dn_vstat.id
-LEFT JOIN temp_category AS tcat ON ta.id = tcat.id
-WHERE ta.recordedDate BETWEEN date('2016-01-01') AND current_date;
+LEFT JOIN temp_category AS tcat ON ta.id = tcat.id;

cumulus_library/studies/core/reference_sql/builder_condition.sql

@@ -485,5 +485,4 @@ SELECT
 FROM temp_condition AS tc
 LEFT JOIN core__condition_dn_category AS cdc ON tc.id = cdc.id
 LEFT JOIN core__condition_dn_clinical_status AS cdcs ON tc.id = cdcs.id
-LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id
-WHERE tc.recordedDate BETWEEN date('2016-01-01') AND current_date;
+LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id;

cumulus_library/studies/core/reference_sql/builder_documentreference.sql

@@ -214,5 +214,4 @@ SELECT DISTINCT
     te.encounter_ref,
     concat('DocumentReference/', tdr.id) AS documentreference_ref
 FROM temp_documentreference AS tdr
-LEFT JOIN temp_encounters AS te ON tdr.id = te.id
-WHERE date(tdr.author_day) BETWEEN date('2016-06-01') AND current_date;
+LEFT JOIN temp_encounters AS te ON tdr.id = te.id;

cumulus_library/studies/core/reference_sql/builder_encounter.sql

@@ -705,9 +705,7 @@ FROM temp_encounter AS e
 LEFT JOIN core__fhir_mapping_expected_act_encounter_code_v3 AS eac
     ON e.class_code = eac.found
 LEFT JOIN core__fhir_act_encounter_code_v3 AS ac ON eac.expected = ac.code
-INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref
-WHERE
-    e.period_start_day BETWEEN date('2016-06-01') AND current_date;
+INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref;
 
 -- ###########################################################
 

cumulus_library/studies/core/reference_sql/builder_observation.sql

@@ -403,11 +403,7 @@ SELECT
     subject_ref,
     encounter_ref,
     concat('Observation/', id) AS observation_ref
-FROM temp_observation
-WHERE
-    effectiveDateTime_day BETWEEN date(
-        from_iso8601_timestamp('2016-06-01')
-    ) AND current_date;
+FROM temp_observation;
 
 -- ###########################################################
 

tests/core/test_core_allergy.py

@@ -229,17 +229,6 @@ def test_core_allergy_many_cases(tmp_path):
     assert expected_set == dict_set_from_list(rows)
 
 
-def test_core_allergy_date_cutoff(tmp_path):
-    """Verify that we ignore rows before 2016"""
-    testbed = testbed_utils.LocalTestbed(tmp_path)
-    testbed.add_allergy_intolerance("Old", recorded="2015")
-    testbed.add_allergy_intolerance("New", recorded="2016")
-
-    con = testbed.build()
-    df = con.sql("SELECT id FROM core__allergyintolerance").df()
-    assert ["New"] == list(df.id)
-
-
 @mock.patch("cumulus_library.builders.counts.DEFAULT_MIN_SUBJECT", new=1)
 def test_core_count_allergy_intolerance_by_patient(tmp_path):
     """Verify that our basic allergy count is per-patient"""

tests/regression/reference/core__count_allergyintolerance_month.csv

@@ -1,33 +1,172 @@
 "cnt","category","recordeddate_month","code_display","reaction_manifestation_display"
-20,,,,
-19,,,,"cumulus__none"
-17,,,,"Eruption of skin (disorder)"
-17,"environment",,,
-16,,,"Animal dander (substance)",
-16,"food",,,
-16,"environment",,,"cumulus__none"
-16,"environment",,"Animal dander (substance)",
-15,,,"Mold (organism)",
-15,,,"House dust mite (organism)",
-15,,,"House dust mite (organism)","cumulus__none"
-15,"environment",,"Mold (organism)",
-15,"environment",,"House dust mite (organism)",
-15,"environment",,"House dust mite (organism)","cumulus__none"
-13,,,,"Rhinoconjunctivitis (disorder)"
-12,,,,"Wheal (finding)"
-12,,,,"Cough (finding)"
-12,,,"Tree pollen (substance)",
-12,,,"Tree pollen (substance)","cumulus__none"
-12,,,"Grass pollen (substance)",
-12,,,"Grass pollen (substance)","cumulus__none"
-12,"food",,,"Eruption of skin (disorder)"
-12,"environment",,,"Rhinoconjunctivitis (disorder)"
-12,"environment",,,"Eruption of skin (disorder)"
-12,"environment",,"Tree pollen (substance)",
-12,"environment",,"Tree pollen (substance)","cumulus__none"
-12,"environment",,"Grass pollen (substance)",
-12,"environment",,"Grass pollen (substance)","cumulus__none"
-11,,,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
-11,,,"Animal dander (substance)","Eruption of skin (disorder)"
-11,"environment",,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
-11,"environment",,"Animal dander (substance)","Eruption of skin (disorder)"
+172,,,,
+139,,,,"cumulus__none"
+132,"food",,,
+124,"environment",,,
+115,,,,"Eruption of skin (disorder)"
+111,,,,"Wheal (finding)"
+108,"environment",,,"cumulus__none"
+104,,,"Mold (organism)",
+104,"environment",,"Mold (organism)",
+100,,,"Animal dander (substance)",
+100,"environment",,"Animal dander (substance)",
+90,,,,"Rhinoconjunctivitis (disorder)"
+87,,,"Tree pollen (substance)",
+87,,,"Tree pollen (substance)","cumulus__none"
+87,"environment",,"Tree pollen (substance)",
+87,"environment",,"Tree pollen (substance)","cumulus__none"
+85,"environment",,,"Rhinoconjunctivitis (disorder)"
+84,,,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
+84,"environment",,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
+80,,,"Grass pollen (substance)",
+80,,,"Grass pollen (substance)","cumulus__none"
+80,"environment",,"Grass pollen (substance)",
+80,"environment",,"Grass pollen (substance)","cumulus__none"
+79,,,"House dust mite (organism)",
+79,,,"House dust mite (organism)","cumulus__none"
+79,"environment",,"House dust mite (organism)",
+79,"environment",,"House dust mite (organism)","cumulus__none"
+78,"food",,,"Eruption of skin (disorder)"
+77,"medication",,,
+72,"food",,,"Wheal (finding)"
+70,,,,"Dyspnea (finding)"
+65,,,,"Allergic angioedema (disorder)"
+62,"environment",,,"Eruption of skin (disorder)"
+58,"food",,,"Dyspnea (finding)"
+57,,,"Animal dander (substance)","Eruption of skin (disorder)"
+57,"food",,,"Allergic angioedema (disorder)"
+57,"environment",,"Animal dander (substance)","Eruption of skin (disorder)"
+55,"environment",,,"Wheal (finding)"
+52,,,,"Cough (finding)"
+51,,,,"Itching (finding)"
+50,"food",,,"cumulus__none"
+46,,,"Mold (organism)","cumulus__none"
+46,"environment",,"Mold (organism)","cumulus__none"
+44,,,"Shellfish (substance)",
+44,"food",,,"Itching (finding)"
+44,"food",,"Shellfish (substance)",
+43,"medication",,,"cumulus__none"
+42,,,"Animal dander (substance)","Wheal (finding)"
+42,"environment",,"Animal dander (substance)","Wheal (finding)"
+41,,,,"Anaphylaxis (disorder)"
+38,,,"Wheat (substance)",
+38,,,"Wheat (substance)","cumulus__none"
+38,"food",,"Wheat (substance)",
+38,"food",,"Wheat (substance)","cumulus__none"
+37,,,"Bee venom (substance)",
+37,"environment",,"Bee venom (substance)",
+36,,,"Shellfish (substance)","Eruption of skin (disorder)"
+36,"food",,,"Anaphylaxis (disorder)"
+36,"food",,"Shellfish (substance)","Eruption of skin (disorder)"
+34,,,"Aspirin",
+34,"medication",,"Aspirin",
+32,,,,"Allergic skin rash"
+32,,,"Mold (organism)","Allergic skin rash"
+32,,,"Fish (substance)",
+32,,,"Eggs (edible) (substance)",
+32,"food",,,"Cough (finding)"
+32,"food",,"Fish (substance)",
+32,"food",,"Eggs (edible) (substance)",
+32,"environment",,,"Allergic skin rash"
+32,"environment",,"Mold (organism)","Allergic skin rash"
+30,,,,"Finding of vomiting (finding)"
+30,,,,"Diarrhea (finding)"
+30,,,"Fish (substance)","Eruption of skin (disorder)"
+30,"food",,,"Finding of vomiting (finding)"
+30,"food",,,"Diarrhea (finding)"
+30,"food",,"Fish (substance)","Eruption of skin (disorder)"
+28,,,"Latex (substance)",
+28,,,"Fish (substance)","Dyspnea (finding)"
+28,"food",,"Fish (substance)","Dyspnea (finding)"
+28,"environment",,"Latex (substance)",
+27,,,"Bee venom (substance)","cumulus__none"
+27,"environment",,"Bee venom (substance)","cumulus__none"
+26,,,"Tree nut (substance)",
+26,,,"Tree nut (substance)","Wheal (finding)"
+26,,,"Tree nut (substance)","Eruption of skin (disorder)"
+26,,,"Tree nut (substance)","Allergic angioedema (disorder)"
+26,,,"Peanut (substance)",
+26,"food",,"Tree nut (substance)",
+26,"food",,"Tree nut (substance)","Wheal (finding)"
+26,"food",,"Tree nut (substance)","Eruption of skin (disorder)"
+26,"food",,"Tree nut (substance)","Allergic angioedema (disorder)"
+26,"food",,"Peanut (substance)",
+24,,,"Tree nut (substance)","Itching (finding)"
+24,,,"Shellfish (substance)","Wheal (finding)"
+24,"medication",,,"Wheal (finding)"
+24,"food",,"Tree nut (substance)","Itching (finding)"
+24,"food",,"Shellfish (substance)","Wheal (finding)"
+23,,,"Shellfish (substance)","Allergic angioedema (disorder)"
+23,,,"Lisinopril",
+23,"medication",,"Lisinopril",
+23,"food",,"Shellfish (substance)","Allergic angioedema (disorder)"
+22,,,"Peanut (substance)","Wheal (finding)"
+22,,,"Cow's milk (substance)",
+22,,,"Animal dander (substance)","Cough (finding)"
+22,"food",,"Peanut (substance)","Wheal (finding)"
+22,"food",,"Cow's milk (substance)",
+22,"environment",,,"Dyspnea (finding)"
+22,"environment",,,"Cough (finding)"
+22,"environment",,"Animal dander (substance)","Cough (finding)"
+21,,,"Shellfish (substance)","Itching (finding)"
+21,,,"Latex (substance)","Wheal (finding)"
+21,"food",,,"Rhinoconjunctivitis (disorder)"
+21,"food",,"Shellfish (substance)","Itching (finding)"
+21,"environment",,"Latex (substance)","Wheal (finding)"
+19,,,"Tree nut (substance)","Anaphylaxis (disorder)"
+19,,,"Aspirin","cumulus__none"
+19,,,"Animal dander (substance)","Dyspnea (finding)"
+19,"medication",,"Aspirin","cumulus__none"
+19,"food",,"Tree nut (substance)","Anaphylaxis (disorder)"
+19,"environment",,"Animal dander (substance)","Dyspnea (finding)"
+18,,,"Shellfish (substance)","Diarrhea (finding)"
+18,,,"Shellfish (substance)","Cough (finding)"
+18,,,"Penicillin V",
+18,,,"Lisinopril","cumulus__none"
+18,"medication",,"Penicillin V",
+18,"medication",,"Lisinopril","cumulus__none"
+18,"food",,"Shellfish (substance)","Diarrhea (finding)"
+18,"food",,"Shellfish (substance)","Cough (finding)"
+17,,,"Cow's milk (substance)","Wheal (finding)"
+17,"food",,"Cow's milk (substance)","Wheal (finding)"
+16,,,"Peanut (substance)","Allergic angioedema (disorder)"
+16,"medication",,,"Eruption of skin (disorder)"
+16,"food",,"Peanut (substance)","Allergic angioedema (disorder)"
+15,,,"Shellfish (substance)","Anaphylaxis (disorder)"
+15,,,"Fish (substance)","Rhinoconjunctivitis (disorder)"
+15,"food",,"Shellfish (substance)","Anaphylaxis (disorder)"
+15,"food",,"Fish (substance)","Rhinoconjunctivitis (disorder)"
+14,,,,"Sneezing"
+14,,,"Shellfish (substance)","Dyspnea (finding)"
+14,,,"Mold (organism)","Sneezing"
+14,,,"Eggs (edible) (substance)","cumulus__none"
+14,"food",,"Shellfish (substance)","Dyspnea (finding)"
+14,"food",,"Eggs (edible) (substance)","cumulus__none"
+14,"environment",,,"Sneezing"
+14,"environment",,"Mold (organism)","Sneezing"
+13,,,,"Nose running"
+13,,,"Mold (organism)","Nose running"
+13,,,"Eggs (edible) (substance)","Wheal (finding)"
+13,"food",,"Eggs (edible) (substance)","Wheal (finding)"
+13,"environment",,,"Nose running"
+13,"environment",,"Mold (organism)","Nose running"
+12,,,,"Wheezing (finding)"
+12,,,,"Nausea (finding)"
+12,,,"Tree nut (substance)","Finding of vomiting (finding)"
+12,,,"Tree nut (substance)","Dyspnea (finding)"
+12,,,"Sulfamethoxazole / Trimethoprim",
+12,,,"Shellfish (substance)","Nausea (finding)"
+12,"medication",,"Sulfamethoxazole / Trimethoprim",
+12,"food",,,"Wheezing (finding)"
+12,"food",,,"Nausea (finding)"
+12,"food",,"Tree nut (substance)","Finding of vomiting (finding)"
+12,"food",,"Tree nut (substance)","Dyspnea (finding)"
+12,"food",,"Shellfish (substance)","Nausea (finding)"
+11,,,"Tree nut (substance)","Cough (finding)"
+11,"food",,"Tree nut (substance)","Cough (finding)"
+10,,,"Peanut (substance)","Eruption of skin (disorder)"
+10,,,"Peanut (substance)","Dyspnea (finding)"
+10,,,"Bee venom (substance)","Eruption of skin (disorder)"
+10,"food",,"Peanut (substance)","Eruption of skin (disorder)"
+10,"food",,"Peanut (substance)","Dyspnea (finding)"
+10,"environment",,"Bee venom (substance)","Eruption of skin (disorder)"