Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

core: drop 2016 cutoff from core tables #304

Merged
merged 1 commit into from
Oct 3, 2024
Merged

core: drop 2016 cutoff from core tables #304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions cumulus_library/studies/core/core_templates/allergyintolerance.sql.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,5 +134,4 @@ LEFT JOIN core__allergyintolerance_dn_code AS dn_code ON ta.id = dn_code.id
LEFT JOIN core__allergyintolerance_dn_clinical_status AS dn_cstat ON ta.id = dn_cstat.id
LEFT JOIN core__allergyintolerance_dn_verification_status AS dn_vstat
ON ta.id = dn_vstat.id
LEFT JOIN temp_category AS tcat ON ta.id = tcat.id
WHERE ta.recordedDate BETWEEN date('2016-01-01') AND current_date;
LEFT JOIN temp_category AS tcat ON ta.id = tcat.id;
3 changes: 1 addition & 2 deletions cumulus_library/studies/core/core_templates/condition.sql.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,5 +80,4 @@ SELECT
FROM temp_condition AS tc
LEFT JOIN core__condition_dn_category AS cdc ON tc.id = cdc.id
LEFT JOIN core__condition_dn_clinical_status AS cdcs ON tc.id = cdcs.id
LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id
WHERE tc.recordedDate BETWEEN date('2016-01-01') AND current_date;
LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id;
3 changes: 1 addition & 2 deletions cumulus_library/studies/core/core_templates/documentreference.sql.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,5 +89,4 @@ SELECT DISTINCT
te.encounter_ref,
concat('DocumentReference/', tdr.id) AS documentreference_ref
FROM temp_documentreference AS tdr
LEFT JOIN temp_encounters AS te ON tdr.id = te.id
WHERE date(tdr.author_day) BETWEEN date('2016-06-01') AND current_date;
LEFT JOIN temp_encounters AS te ON tdr.id = te.id;
4 changes: 1 addition & 3 deletions cumulus_library/studies/core/core_templates/encounter.sql.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,4 @@ FROM temp_encounter AS e
LEFT JOIN core__fhir_mapping_expected_act_encounter_code_v3 AS eac
ON e.class_code = eac.found
LEFT JOIN core__fhir_act_encounter_code_v3 AS ac ON eac.expected = ac.code
INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref
WHERE
e.period_start_day BETWEEN date('2016-06-01') AND current_date;
INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref;
6 changes: 1 addition & 5 deletions cumulus_library/studies/core/core_templates/observation.sql.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,4 @@ SELECT
subject_ref,
encounter_ref,
concat('Observation/', id) AS observation_ref
FROM temp_observation
WHERE
effectiveDateTime_day BETWEEN date(
from_iso8601_timestamp('2016-06-01')
) AND current_date;
FROM temp_observation;
3 changes: 1 addition & 2 deletions cumulus_library/studies/core/reference_sql/builder_allergyintolerance.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,5 +336,4 @@ LEFT JOIN core__allergyintolerance_dn_code AS dn_code ON ta.id = dn_code.id
LEFT JOIN core__allergyintolerance_dn_clinical_status AS dn_cstat ON ta.id = dn_cstat.id
LEFT JOIN core__allergyintolerance_dn_verification_status AS dn_vstat
ON ta.id = dn_vstat.id
LEFT JOIN temp_category AS tcat ON ta.id = tcat.id
WHERE ta.recordedDate BETWEEN date('2016-01-01') AND current_date;
LEFT JOIN temp_category AS tcat ON ta.id = tcat.id;
3 changes: 1 addition & 2 deletions cumulus_library/studies/core/reference_sql/builder_condition.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,5 +485,4 @@ SELECT
FROM temp_condition AS tc
LEFT JOIN core__condition_dn_category AS cdc ON tc.id = cdc.id
LEFT JOIN core__condition_dn_clinical_status AS cdcs ON tc.id = cdcs.id
LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id
WHERE tc.recordedDate BETWEEN date('2016-01-01') AND current_date;
LEFT JOIN core__condition_dn_verification_status AS cdvs ON tc.id = cdvs.id;
3 changes: 1 addition & 2 deletions cumulus_library/studies/core/reference_sql/builder_documentreference.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,5 +214,4 @@ SELECT DISTINCT
te.encounter_ref,
concat('DocumentReference/', tdr.id) AS documentreference_ref
FROM temp_documentreference AS tdr
LEFT JOIN temp_encounters AS te ON tdr.id = te.id
WHERE date(tdr.author_day) BETWEEN date('2016-06-01') AND current_date;
LEFT JOIN temp_encounters AS te ON tdr.id = te.id;
4 changes: 1 addition & 3 deletions cumulus_library/studies/core/reference_sql/builder_encounter.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -705,9 +705,7 @@ FROM temp_encounter AS e
LEFT JOIN core__fhir_mapping_expected_act_encounter_code_v3 AS eac
ON e.class_code = eac.found
LEFT JOIN core__fhir_act_encounter_code_v3 AS ac ON eac.expected = ac.code
INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref
WHERE
e.period_start_day BETWEEN date('2016-06-01') AND current_date;
INNER JOIN core__patient AS p ON e.subject_ref = p.subject_ref;

-- ###########################################################

Expand Down
6 changes: 1 addition & 5 deletions cumulus_library/studies/core/reference_sql/builder_observation.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,11 +403,7 @@ SELECT
subject_ref,
encounter_ref,
concat('Observation/', id) AS observation_ref
FROM temp_observation
WHERE
effectiveDateTime_day BETWEEN date(
from_iso8601_timestamp('2016-06-01')
) AND current_date;
FROM temp_observation;

-- ###########################################################

Expand Down
11 changes: 0 additions & 11 deletions tests/core/test_core_allergy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,17 +229,6 @@ def test_core_allergy_many_cases(tmp_path):
assert expected_set == dict_set_from_list(rows)


def test_core_allergy_date_cutoff(tmp_path):
"""Verify that we ignore rows before 2016"""
testbed = testbed_utils.LocalTestbed(tmp_path)
testbed.add_allergy_intolerance("Old", recorded="2015")
testbed.add_allergy_intolerance("New", recorded="2016")

con = testbed.build()
df = con.sql("SELECT id FROM core__allergyintolerance").df()
assert ["New"] == list(df.id)


@mock.patch("cumulus_library.builders.counts.DEFAULT_MIN_SUBJECT", new=1)
def test_core_count_allergy_intolerance_by_patient(tmp_path):
"""Verify that our basic allergy count is per-patient"""
Expand Down
203 changes: 171 additions & 32 deletions tests/regression/reference/core__count_allergyintolerance_month.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,33 +1,172 @@
"cnt","category","recordeddate_month","code_display","reaction_manifestation_display"
20,,,,
19,,,,"cumulus__none"
17,,,,"Eruption of skin (disorder)"
17,"environment",,,
16,,,"Animal dander (substance)",
16,"food",,,
16,"environment",,,"cumulus__none"
16,"environment",,"Animal dander (substance)",
15,,,"Mold (organism)",
15,,,"House dust mite (organism)",
15,,,"House dust mite (organism)","cumulus__none"
15,"environment",,"Mold (organism)",
15,"environment",,"House dust mite (organism)",
15,"environment",,"House dust mite (organism)","cumulus__none"
13,,,,"Rhinoconjunctivitis (disorder)"
12,,,,"Wheal (finding)"
12,,,,"Cough (finding)"
12,,,"Tree pollen (substance)",
12,,,"Tree pollen (substance)","cumulus__none"
12,,,"Grass pollen (substance)",
12,,,"Grass pollen (substance)","cumulus__none"
12,"food",,,"Eruption of skin (disorder)"
12,"environment",,,"Rhinoconjunctivitis (disorder)"
12,"environment",,,"Eruption of skin (disorder)"
12,"environment",,"Tree pollen (substance)",
12,"environment",,"Tree pollen (substance)","cumulus__none"
12,"environment",,"Grass pollen (substance)",
12,"environment",,"Grass pollen (substance)","cumulus__none"
11,,,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
11,,,"Animal dander (substance)","Eruption of skin (disorder)"
11,"environment",,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
11,"environment",,"Animal dander (substance)","Eruption of skin (disorder)"
172,,,,
139,,,,"cumulus__none"
132,"food",,,
124,"environment",,,
115,,,,"Eruption of skin (disorder)"
111,,,,"Wheal (finding)"
108,"environment",,,"cumulus__none"
104,,,"Mold (organism)",
104,"environment",,"Mold (organism)",
100,,,"Animal dander (substance)",
100,"environment",,"Animal dander (substance)",
90,,,,"Rhinoconjunctivitis (disorder)"
87,,,"Tree pollen (substance)",
87,,,"Tree pollen (substance)","cumulus__none"
87,"environment",,"Tree pollen (substance)",
87,"environment",,"Tree pollen (substance)","cumulus__none"
85,"environment",,,"Rhinoconjunctivitis (disorder)"
84,,,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
84,"environment",,"Animal dander (substance)","Rhinoconjunctivitis (disorder)"
80,,,"Grass pollen (substance)",
80,,,"Grass pollen (substance)","cumulus__none"
80,"environment",,"Grass pollen (substance)",
80,"environment",,"Grass pollen (substance)","cumulus__none"
79,,,"House dust mite (organism)",
79,,,"House dust mite (organism)","cumulus__none"
79,"environment",,"House dust mite (organism)",
79,"environment",,"House dust mite (organism)","cumulus__none"
78,"food",,,"Eruption of skin (disorder)"
77,"medication",,,
72,"food",,,"Wheal (finding)"
70,,,,"Dyspnea (finding)"
65,,,,"Allergic angioedema (disorder)"
62,"environment",,,"Eruption of skin (disorder)"
58,"food",,,"Dyspnea (finding)"
57,,,"Animal dander (substance)","Eruption of skin (disorder)"
57,"food",,,"Allergic angioedema (disorder)"
57,"environment",,"Animal dander (substance)","Eruption of skin (disorder)"
55,"environment",,,"Wheal (finding)"
52,,,,"Cough (finding)"
51,,,,"Itching (finding)"
50,"food",,,"cumulus__none"
46,,,"Mold (organism)","cumulus__none"
46,"environment",,"Mold (organism)","cumulus__none"
44,,,"Shellfish (substance)",
44,"food",,,"Itching (finding)"
44,"food",,"Shellfish (substance)",
43,"medication",,,"cumulus__none"
42,,,"Animal dander (substance)","Wheal (finding)"
42,"environment",,"Animal dander (substance)","Wheal (finding)"
41,,,,"Anaphylaxis (disorder)"
38,,,"Wheat (substance)",
38,,,"Wheat (substance)","cumulus__none"
38,"food",,"Wheat (substance)",
38,"food",,"Wheat (substance)","cumulus__none"
37,,,"Bee venom (substance)",
37,"environment",,"Bee venom (substance)",
36,,,"Shellfish (substance)","Eruption of skin (disorder)"
36,"food",,,"Anaphylaxis (disorder)"
36,"food",,"Shellfish (substance)","Eruption of skin (disorder)"
34,,,"Aspirin",
34,"medication",,"Aspirin",
32,,,,"Allergic skin rash"
32,,,"Mold (organism)","Allergic skin rash"
32,,,"Fish (substance)",
32,,,"Eggs (edible) (substance)",
32,"food",,,"Cough (finding)"
32,"food",,"Fish (substance)",
32,"food",,"Eggs (edible) (substance)",
32,"environment",,,"Allergic skin rash"
32,"environment",,"Mold (organism)","Allergic skin rash"
30,,,,"Finding of vomiting (finding)"
30,,,,"Diarrhea (finding)"
30,,,"Fish (substance)","Eruption of skin (disorder)"
30,"food",,,"Finding of vomiting (finding)"
30,"food",,,"Diarrhea (finding)"
30,"food",,"Fish (substance)","Eruption of skin (disorder)"
28,,,"Latex (substance)",
28,,,"Fish (substance)","Dyspnea (finding)"
28,"food",,"Fish (substance)","Dyspnea (finding)"
28,"environment",,"Latex (substance)",
27,,,"Bee venom (substance)","cumulus__none"
27,"environment",,"Bee venom (substance)","cumulus__none"
26,,,"Tree nut (substance)",
26,,,"Tree nut (substance)","Wheal (finding)"
26,,,"Tree nut (substance)","Eruption of skin (disorder)"
26,,,"Tree nut (substance)","Allergic angioedema (disorder)"
26,,,"Peanut (substance)",
26,"food",,"Tree nut (substance)",
26,"food",,"Tree nut (substance)","Wheal (finding)"
26,"food",,"Tree nut (substance)","Eruption of skin (disorder)"
26,"food",,"Tree nut (substance)","Allergic angioedema (disorder)"
26,"food",,"Peanut (substance)",
24,,,"Tree nut (substance)","Itching (finding)"
24,,,"Shellfish (substance)","Wheal (finding)"
24,"medication",,,"Wheal (finding)"
24,"food",,"Tree nut (substance)","Itching (finding)"
24,"food",,"Shellfish (substance)","Wheal (finding)"
23,,,"Shellfish (substance)","Allergic angioedema (disorder)"
23,,,"Lisinopril",
23,"medication",,"Lisinopril",
23,"food",,"Shellfish (substance)","Allergic angioedema (disorder)"
22,,,"Peanut (substance)","Wheal (finding)"
22,,,"Cow's milk (substance)",
22,,,"Animal dander (substance)","Cough (finding)"
22,"food",,"Peanut (substance)","Wheal (finding)"
22,"food",,"Cow's milk (substance)",
22,"environment",,,"Dyspnea (finding)"
22,"environment",,,"Cough (finding)"
22,"environment",,"Animal dander (substance)","Cough (finding)"
21,,,"Shellfish (substance)","Itching (finding)"
21,,,"Latex (substance)","Wheal (finding)"
21,"food",,,"Rhinoconjunctivitis (disorder)"
21,"food",,"Shellfish (substance)","Itching (finding)"
21,"environment",,"Latex (substance)","Wheal (finding)"
19,,,"Tree nut (substance)","Anaphylaxis (disorder)"
19,,,"Aspirin","cumulus__none"
19,,,"Animal dander (substance)","Dyspnea (finding)"
19,"medication",,"Aspirin","cumulus__none"
19,"food",,"Tree nut (substance)","Anaphylaxis (disorder)"
19,"environment",,"Animal dander (substance)","Dyspnea (finding)"
18,,,"Shellfish (substance)","Diarrhea (finding)"
18,,,"Shellfish (substance)","Cough (finding)"
18,,,"Penicillin V",
18,,,"Lisinopril","cumulus__none"
18,"medication",,"Penicillin V",
18,"medication",,"Lisinopril","cumulus__none"
18,"food",,"Shellfish (substance)","Diarrhea (finding)"
18,"food",,"Shellfish (substance)","Cough (finding)"
17,,,"Cow's milk (substance)","Wheal (finding)"
17,"food",,"Cow's milk (substance)","Wheal (finding)"
16,,,"Peanut (substance)","Allergic angioedema (disorder)"
16,"medication",,,"Eruption of skin (disorder)"
16,"food",,"Peanut (substance)","Allergic angioedema (disorder)"
15,,,"Shellfish (substance)","Anaphylaxis (disorder)"
15,,,"Fish (substance)","Rhinoconjunctivitis (disorder)"
15,"food",,"Shellfish (substance)","Anaphylaxis (disorder)"
15,"food",,"Fish (substance)","Rhinoconjunctivitis (disorder)"
14,,,,"Sneezing"
14,,,"Shellfish (substance)","Dyspnea (finding)"
14,,,"Mold (organism)","Sneezing"
14,,,"Eggs (edible) (substance)","cumulus__none"
14,"food",,"Shellfish (substance)","Dyspnea (finding)"
14,"food",,"Eggs (edible) (substance)","cumulus__none"
14,"environment",,,"Sneezing"
14,"environment",,"Mold (organism)","Sneezing"
13,,,,"Nose running"
13,,,"Mold (organism)","Nose running"
13,,,"Eggs (edible) (substance)","Wheal (finding)"
13,"food",,"Eggs (edible) (substance)","Wheal (finding)"
13,"environment",,,"Nose running"
13,"environment",,"Mold (organism)","Nose running"
12,,,,"Wheezing (finding)"
12,,,,"Nausea (finding)"
12,,,"Tree nut (substance)","Finding of vomiting (finding)"
12,,,"Tree nut (substance)","Dyspnea (finding)"
12,,,"Sulfamethoxazole / Trimethoprim",
12,,,"Shellfish (substance)","Nausea (finding)"
12,"medication",,"Sulfamethoxazole / Trimethoprim",
12,"food",,,"Wheezing (finding)"
12,"food",,,"Nausea (finding)"
12,"food",,"Tree nut (substance)","Finding of vomiting (finding)"
12,"food",,"Tree nut (substance)","Dyspnea (finding)"
12,"food",,"Shellfish (substance)","Nausea (finding)"
11,,,"Tree nut (substance)","Cough (finding)"
11,"food",,"Tree nut (substance)","Cough (finding)"
10,,,"Peanut (substance)","Eruption of skin (disorder)"
10,,,"Peanut (substance)","Dyspnea (finding)"
10,,,"Bee venom (substance)","Eruption of skin (disorder)"
10,"food",,"Peanut (substance)","Eruption of skin (disorder)"
10,"food",,"Peanut (substance)","Dyspnea (finding)"
10,"environment",,"Bee venom (substance)","Eruption of skin (disorder)"
Binary file modified tests/regression/reference/core__count_allergyintolerance_month.parquet
View file
Open in desktop
Binary file not shown.
Loading
Loading