Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generic populated field detection #196

Merged
merged 8 commits into from
Mar 22, 2024
Merged

Generic populated field detection #196

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f694f72
Generic populated field detection
dogversioning Mar 18, 2024
dae38e3
column hierarchy rework
dogversioning Mar 21, 2024
f965d00
ref sql regen
dogversioning Mar 21, 2024
81ad97a
discovery cleanup
dogversioning Mar 21, 2024
b521457
trimmed unnessecary line in sql_utils
dogversioning Mar 21, 2024
4e2ceaa
Light cleanup
dogversioning Mar 21, 2024
dcd9de1
much ado about names
dogversioning Mar 21, 2024
9b71dad
tweaks to nested field handling
dogversioning Mar 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cumulus_library/.sqlfluff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ join_cols_by_table =
join_id = subject_ref
neg_source_table = neg_source_table
output_table_name = 'created_table'
parent_field = 'parent'
prefix = Test
primary_ref = encounter_ref
pos_source_table = pos_source_table
Expand Down
6 changes: 2 additions & 4 deletions cumulus_library/studies/core/builder_condition.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@ def denormalize_codes(self):
preferred_config = sql_utils.CodeableConceptConfig(
source_table="condition",
source_id="id",
column_name="code",
is_array=False,
column_hierarchy=[("code", dict)],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah this consolidation of arguments makes sense, yeah.

nit: maybe now that there's less overload on the word, you could just use columns instead of column_hierarchy which is easy to say or type wrong. 😄

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i don't like columns because ordering is important and not implicit in that name. open to other ideas.

target_table="core__condition_codable_concepts_display",
filter_priority=True,
code_systems=[
Expand All @@ -69,8 +68,7 @@ def denormalize_codes(self):
all_config = sql_utils.CodeableConceptConfig(
source_table="condition",
source_id="id",
column_name="code",
is_array=False,
column_hierarchy=[("code", dict)],
target_table="core__condition_codable_concepts_all",
filter_priority=False,
)
Expand Down
13 changes: 5 additions & 8 deletions cumulus_library/studies/core/builder_documentreference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,14 @@ def prepare_queries(
parser: databases.DatabaseParser = None,
**kwargs,
):
self.queries += sql_utils.denormalize_codes(
self.queries = sql_utils.denormalize_complex_objects(
schema,
cursor,
[
sql_utils.CodeableConceptConfig(
source_table="documentreference",
source_id="id",
column_name="type",
is_array=False,
column_hierarchy=[("type", dict)],
target_table="core__documentreference_dn_type",
),
# TODO: The US core profile allows an extensible code for category, but
Expand All @@ -47,19 +46,17 @@ def prepare_queries(
sql_utils.CodeableConceptConfig(
source_table="documentreference",
source_id="id",
column_name="category",
is_array=True,
column_hierarchy=[("category", list)],
filter_priority=True,
target_table="core__documentreference_dn_category",
code_systems=[
"http://hl7.org/fhir/us/core/ValueSet/us-core-documentreference-category"
],
),
sql_utils.CodeableConceptConfig(
sql_utils.CodingConfig(
source_table="documentreference",
source_id="id",
column_name="content.format",
is_array=False,
column_hierarchy=[("format", dict), ("content", dict)],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is flipped, right? If so, you should probably add a unit test that would catch that.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

well a test would have caught it if the test data had been correct, so that should be addressed now.

target_table="core__documentreference_dn_format",
),
],
Expand Down
24 changes: 12 additions & 12 deletions cumulus_library/studies/core/builder_encounter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class EncConfig(sql_utils.CodeableConceptConfig):
source_table: str = "encounter"

def __post_init__(self):
self.target_table = f"core__encounter_dn_{self.column_name.split('.')[-1]}"
self.target_table = f"core__encounter_dn_{self.column_hierarchy[-1][0]}"


class CoreEncounterBuilder(base_table_builder.BaseTableBuilder):
Expand All @@ -44,8 +44,7 @@ class CoreEncounterBuilder(base_table_builder.BaseTableBuilder):
def denormalize_codes(self, schema, cursor):
code_configs = [
EncConfig(
column_name="type",
is_array=True,
column_hierarchy=[("type", list)],
filter_priority=True,
code_systems=[
"http://terminology.hl7.org/CodeSystem/encounter-type",
Expand All @@ -62,8 +61,7 @@ def denormalize_codes(self, schema, cursor):
],
),
EncConfig(
column_name="servicetype",
is_array=False,
column_hierarchy=[("servicetype", dict)],
filter_priority=True,
code_systems=[
"http://terminology.hl7.org/CodeSystem/service-type",
Expand All @@ -76,8 +74,7 @@ def denormalize_codes(self, schema, cursor):
],
),
EncConfig(
column_name="priority",
is_array=False,
column_hierarchy=[("priority", dict)],
filter_priority=True,
code_systems=[
"http://terminology.hl7.org/CodeSystem/v3-ActPriority",
Expand All @@ -89,8 +86,7 @@ def denormalize_codes(self, schema, cursor):
],
),
EncConfig(
column_name="reasoncode",
is_array=True,
column_hierarchy=[("reasoncode", list)],
filter_priority=True,
code_systems=[
"http://terminology.hl7.org/CodeSystem/v3-ActPriority",
Expand All @@ -104,12 +100,16 @@ def denormalize_codes(self, schema, cursor):
],
),
EncConfig(
column_name="hospitalization.dischargedisposition",
is_array=False,
column_hierarchy=[
("hospitalization", dict),
("dischargedisposition", dict),
],
filter_priority=False,
),
]
self.queries += sql_utils.denormalize_codes(schema, cursor, code_configs)
self.queries += sql_utils.denormalize_complex_objects(
schema, cursor, code_configs
)

def prepare_queries(
self,
Expand Down
41 changes: 16 additions & 25 deletions cumulus_library/studies/core/builder_medication.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,23 @@ def _check_data_in_fields(self, cursor, schema: str):
}

table = "medicationrequest"
base_col = "medicationcodeableconcept"
inline_col = "medicationcodeableconcept"
with base_utils.get_progress_bar(transient=True) as progress:
task = progress.add_task(
"Detecting available medication sources...",
total=7,
total=3,
)

# inline medications from FHIR medication
data_types["inline"] = sql_utils.is_codeable_concept_populated(
schema, table, base_col, cursor
data_types["inline"] = sql_utils.is_field_populated(
schema=schema,
source_table=table,
hierarchy=[(inline_col, dict), ("coding", list)],
dogversioning marked this conversation as resolved.
Show resolved Hide resolved
cursor=cursor,
)
if data_types["inline"]:
query = base_templates.get_column_datatype_query(
schema, table, [base_col]
schema, table, [inline_col]
)
cursor.execute(query)
progress.advance(task)
Expand All @@ -47,27 +50,15 @@ def _check_data_in_fields(self, cursor, schema: str):
has_userselected = True
else:
has_userselected = False
# Validating presence of FHIR medication requests
query = base_templates.get_is_table_not_empty_query(
"medicationrequest", "medicationreference"
)
cursor.execute(query)
progress.advance(task)
if cursor.fetchone() is None:
return data_types, has_userselected
query = base_templates.get_column_datatype_query(
schema, "medicationrequest", ["medicationreference"]
)
cursor.execute(query)
progress.advance(task)
if "reference" not in cursor.fetchone()[0]:
return data_types, has_userselected
query = base_templates.get_is_table_not_empty_query(
"medicationrequest", "medicationreference.reference"
)
cursor.execute(query)
progress.advance(task)
if cursor.fetchone() is None:
# Validating presence of FHIR medication requests
if not sql_utils.is_field_populated(
schema=schema,
source_table=table,
hierarchy=[("medicationreference", dict), ("reference", dict)],
expected=["reference"],
cursor=cursor,
):
return data_types, has_userselected

# checking med ref contents for our two linkage cases
Expand Down
10 changes: 5 additions & 5 deletions cumulus_library/studies/core/builder_medicationrequest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,19 +43,19 @@ def prepare_queries(
sql_utils.CodeableConceptConfig(
source_table="medicationrequest",
source_id="id",
column_name="category",
is_array=True,
column_hierarchy=[("category", list)],
target_table="core__medicationrequest_dn_category",
),
sql_utils.CodeableConceptConfig(
source_table="medicationrequest",
source_id="id",
column_name="medicationcodeableconcept",
is_array=False,
column_hierarchy=[("medicationcodeableconcept", dict)],
target_table="core__medicationrequest_dn_medication",
),
]
self.queries += sql_utils.denormalize_codes(schema, cursor, code_sources)
self.queries += sql_utils.denormalize_complex_objects(
schema, cursor, code_sources
)
validated_schema = core_templates.validate_schema(
cursor, schema, expected_table_cols, parser
)
Expand Down
18 changes: 9 additions & 9 deletions cumulus_library/studies/core/builder_observation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class ObsConfig(sql_utils.CodeableConceptConfig):
source_table: str = "observation"

def __post_init__(self):
self.target_table = f"core__observation_dn_{self.column_name}"
self.target_table = f"core__observation_dn_{self.column_hierarchy[-1][0]}"


class ObservationBuilder(base_table_builder.BaseTableBuilder):
Expand All @@ -57,24 +57,24 @@ def prepare_queries(
:param schema: the schema/db name, matching the cursor
"""
code_sources = [
ObsConfig(column_name="category", is_array=True, filter_priority=False),
ObsConfig(column_name="code", is_array=False, filter_priority=False),
ObsConfig(column_hierarchy=[("category", list)], filter_priority=False),
ObsConfig(column_hierarchy=[("code", dict)], filter_priority=False),
ObsConfig(
column_name="interpretation", is_array=True, filter_priority=False
column_hierarchy=[("interpretation", list)], filter_priority=False
),
ObsConfig(
column_name="valuecodeableconcept",
is_array=False,
column_hierarchy=[("valuecodeableconcept", dict)],
filter_priority=False,
),
ObsConfig(
column_name="dataabsentreason",
is_array=False,
column_hierarchy=[("dataabsentreason", dict)],
filter_priority=False,
),
]

self.queries += sql_utils.denormalize_codes(schema, cursor, code_sources)
self.queries += sql_utils.denormalize_complex_objects(
schema, cursor, code_sources
)
validated_schema = core_templates.validate_schema(
cursor, schema, expected_table_cols, parser
)
Expand Down
12 changes: 6 additions & 6 deletions cumulus_library/studies/core/builder_patient.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ def prepare_queries(

for extension in extension_types:
config = sql_utils.ExtensionConfig(
"patient",
"id",
f"core__patient_ext_{extension['name']}",
extension["name"],
extension["fhirpath"],
["ombCategory", "detailed", "text"],
source_table="patient",
source_id="id",
target_table=f"core__patient_ext_{extension['name']}",
target_col_prefix=extension["name"],
fhir_extension=extension["fhirpath"],
ext_systems=["ombCategory", "detailed", "text"],
is_array=True,
)
self.queries.append(base_templates.get_extension_denormalize_query(config))
Expand Down
32 changes: 16 additions & 16 deletions cumulus_library/studies/core/reference_sql/builder_condition.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ CREATE TABLE core__condition_codable_concepts_display AS (
SELECT DISTINCT
s.id AS id,
'0' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand All @@ -27,8 +27,8 @@ CREATE TABLE core__condition_codable_concepts_display AS (
SELECT DISTINCT
s.id AS id,
'1' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand All @@ -41,8 +41,8 @@ CREATE TABLE core__condition_codable_concepts_display AS (
SELECT DISTINCT
s.id AS id,
'2' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand All @@ -55,8 +55,8 @@ CREATE TABLE core__condition_codable_concepts_display AS (
SELECT DISTINCT
s.id AS id,
'3' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand All @@ -69,8 +69,8 @@ CREATE TABLE core__condition_codable_concepts_display AS (
SELECT DISTINCT
s.id AS id,
'4' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand All @@ -83,8 +83,8 @@ CREATE TABLE core__condition_codable_concepts_display AS (
SELECT DISTINCT
s.id AS id,
'5' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand All @@ -97,8 +97,8 @@ CREATE TABLE core__condition_codable_concepts_display AS (
SELECT DISTINCT
s.id AS id,
'6' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand Down Expand Up @@ -201,8 +201,8 @@ CREATE TABLE core__condition_codable_concepts_all AS (
system_code_0 AS (
SELECT DISTINCT
s.id AS id,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
condition AS s,
Expand Down
8 changes: 4 additions & 4 deletions cumulus_library/studies/core/reference_sql/builder_documentreference.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ CREATE TABLE core__documentreference_dn_type AS (
system_type_0 AS (
SELECT DISTINCT
s.id AS id,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
documentreference AS s,
Expand Down Expand Up @@ -47,8 +47,8 @@ CREATE TABLE core__documentreference_dn_category AS (
SELECT DISTINCT
s.id AS id,
'0' AS priority,
u.codeable_concept.code AS code,
u.codeable_concept.display AS display,
u.codeable_concept.code,
u.codeable_concept.display,
u.codeable_concept.system AS code_system
FROM
documentreference AS s,
Expand Down
Loading
Loading