From bc2563231ca515d771ccdd7e17fe30450ac1d993 Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Fri, 9 Feb 2024 11:14:55 -0500 Subject: [PATCH 1/4] Added archival mode --- cumulus_library/cli.py | 34 ++- cumulus_library/cli_parser.py | 21 +- .../core/core_templates/encounter.sql.jinja | 1 - .../core/reference_sql/builder_condition.sql | 110 +++++++- .../builder_documentreference.sql | 17 +- .../core/reference_sql/builder_encounter.sql | 242 ++++++++++++++++-- .../core/reference_sql/builder_medication.sql | 8 +- .../builder_medicationrequest.sql | 11 +- .../reference_sql/builder_observation.sql | 19 +- .../core/reference_sql/builder_patient.sql | 12 +- .../reference_sql/builder_prereq_tables.sql | 8 +- .../studies/core/reference_sql/count_core.sql | 8 +- cumulus_library/study_parser.py | 56 +++- tests/conftest.py | 2 +- tests/test_cli.py | 61 +++-- tests/test_data/core/core__encounter.txt | 100 ++++---- 16 files changed, 543 insertions(+), 167 deletions(-) diff --git a/cumulus_library/cli.py b/cumulus_library/cli.py index df63b6ab..35fcd330 100755 --- a/cumulus_library/cli.py +++ b/cumulus_library/cli.py @@ -193,7 +193,9 @@ def clean_and_build_all(self, study_dict: dict, stats_build: bool) -> None: self.clean_and_build_study(study_dict[key], stats_build=stats_build) ### Data exporters - def export_study(self, target: pathlib.Path, data_path: pathlib.Path) -> None: + def export_study( + self, target: pathlib.Path, data_path: pathlib.Path, archive: bool + ) -> None: """Exports aggregates defined in a manifest :param target: A path to the study directory @@ -201,12 +203,12 @@ def export_study(self, target: pathlib.Path, data_path: pathlib.Path) -> None: if data_path is None: sys.exit("Missing destination - please provide a path argument.") studyparser = study_parser.StudyManifestParser(target, data_path) - studyparser.export_study(self.db, data_path) + studyparser.export_study(self.db, self.schema_name, data_path, archive) - def export_all(self, study_dict: dict, data_path: pathlib.Path): + def export_all(self, study_dict: dict, data_path: pathlib.Path, archive: bool): """Exports all defined count tables to disk""" for key in study_dict.keys(): - self.export_study(study_dict[key], data_path) + self.export_study(study_dict[key], data_path, archive) def generate_study_sql( self, @@ -294,6 +296,7 @@ def get_studies_by_manifest_path(path: pathlib.Path) -> dict: def run_cli(args: dict): """Controls which library tasks are run based on CLI arguments""" + console = rich.console.Console() if args["action"] == "create": create_template(args["create_dir"]) @@ -307,7 +310,7 @@ def run_cli(args: dict): runner = StudyRunner(db_backend, data_path=args.get("data_path")) if args.get("verbose"): runner.verbose = True - print("Testing connection to database...") + console.print("[italic] Connecting to database...") runner.cursor.execute("SHOW DATABASES") study_dict = get_study_dict(args["study_dir"]) if "prefix" not in args.keys(): @@ -344,11 +347,28 @@ def run_cli(args: dict): ) elif args["action"] == "export": + if args["archive"]: + warning_text = ( + "🚨[bold red] This will export all study tables [/bold red]🚨" + "\n\nDepending on your study definition, this data may contain " + "data that would be characterized as a [italic]limited data " + "set[/italic], primarily dates, on a per patient level.\n\n" + "[bold]By doing this, you are assuming the responsibility for " + "meeting your organization's security requirements for " + "storing this data in a secure manager.[/bold]\n\n" + "Type Y to proceed, or any other value to quit.\n" + ) + console.print(warning_text) + response = input() + if response.lower() != "y": + sys.exit() if "all" in args["target"]: - runner.export_all(study_dict, args["data_path"]) + runner.export_all(study_dict, args["data_path"], args["archive"]) else: for target in args["target"]: - runner.export_study(study_dict[target], args["data_path"]) + runner.export_study( + study_dict[target], args["data_path"], args["archive"] + ) elif args["action"] == "generate-sql": for target in args["target"]: diff --git a/cumulus_library/cli_parser.py b/cumulus_library/cli_parser.py index 7ecdddef..876b3d1b 100644 --- a/cumulus_library/cli_parser.py +++ b/cumulus_library/cli_parser.py @@ -127,21 +127,6 @@ def get_parser() -> argparse.ArgumentParser: dest="action", ) - # Study creation - - create = actions.add_parser( - "create", help="Create a study instance from a template" - ) - create.add_argument( - "create_dir", - default="./", - nargs="?", - help=( - "The the directory the study will be created in. Default is " - "the current directory." - ), - ) - # Database cleaning clean = actions.add_parser( @@ -204,7 +189,11 @@ def get_parser() -> argparse.ArgumentParser: add_data_path_argument(export) add_verbose_argument(export) add_db_config(export) - + export.add_argument( + "--archive", + action="store_true", + help="Generates archive of :all: study tables, ignoring manifest export list.", + ) # Aggregator upload upload = actions.add_parser( diff --git a/cumulus_library/studies/core/core_templates/encounter.sql.jinja b/cumulus_library/studies/core/core_templates/encounter.sql.jinja index 6f907d15..b2adeaf8 100644 --- a/cumulus_library/studies/core/core_templates/encounter.sql.jinja +++ b/cumulus_library/studies/core/core_templates/encounter.sql.jinja @@ -83,7 +83,6 @@ temp_encounter AS ( SELECT DISTINCT e.id, - e.class AS enc_class, ac.code AS enc_class_code, ac.display AS enc_class_display, e.status, diff --git a/cumulus_library/studies/core/reference_sql/builder_condition.sql b/cumulus_library/studies/core/reference_sql/builder_condition.sql index e5b5c926..bd305475 100644 --- a/cumulus_library/studies/core/reference_sql/builder_condition.sql +++ b/cumulus_library/studies/core/reference_sql/builder_condition.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### @@ -20,7 +20,7 @@ CREATE TABLE core__condition_codable_concepts_display AS ( condition AS s, UNNEST(s.code.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://snomed.info/sct' + u.codeable_concept.system LIKE 'http://snomed.info/sct' ), --noqa: LT07 system_code_1 AS ( @@ -34,7 +34,7 @@ CREATE TABLE core__condition_codable_concepts_display AS ( condition AS s, UNNEST(s.code.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://hl7.org/fhir/sid/icd-10-cm' + u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-10-cm' ), --noqa: LT07 system_code_2 AS ( @@ -48,7 +48,63 @@ CREATE TABLE core__condition_codable_concepts_display AS ( condition AS s, UNNEST(s.code.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://hl7.org/fhir/sid/icd-9-cm' + u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-9-cm' + ), --noqa: LT07 + + system_code_3 AS ( + SELECT DISTINCT + s.id AS id, + '3' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + condition AS s, + UNNEST(s.code.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-9-cm/diagnosis' + ), --noqa: LT07 + + system_code_4 AS ( + SELECT DISTINCT + s.id AS id, + '4' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + condition AS s, + UNNEST(s.code.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'urn:oid:1.2.840.114350.1.13.71.2.7.2.728286' + ), --noqa: LT07 + + system_code_5 AS ( + SELECT DISTINCT + s.id AS id, + '5' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + condition AS s, + UNNEST(s.code.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'urn:oid:1.2.840.114350.1.13.71.2.7.4.698084.10375' + ), --noqa: LT07 + + system_code_6 AS ( + SELECT DISTINCT + s.id AS id, + '6' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + condition AS s, + UNNEST(s.code.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'http://terminology.hl7.org/CodeSystem/data-absent-reason' ), --noqa: LT07 union_table AS ( @@ -75,6 +131,39 @@ CREATE TABLE core__condition_codable_concepts_display AS ( code, display FROM system_code_2 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_code_3 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_code_4 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_code_5 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_code_6 + ), partitioned_table AS ( @@ -127,6 +216,7 @@ CREATE TABLE core__condition_codable_concepts_all AS ( code, display FROM system_code_0 + ) SELECT id, @@ -154,11 +244,11 @@ WITH temp_condition AS ( cca.code_system, cca.display, date(from_iso8601_timestamp(c.recordeddate)) AS recordeddate, - date_trunc('week', date(from_iso8601_timestamp(c.recordeddate))) + date_trunc('week', date(from_iso8601_timestamp(c."recordeddate"))) AS recordeddate_week, - date_trunc('month', date(from_iso8601_timestamp(c.recordeddate))) + date_trunc('month', date(from_iso8601_timestamp(c."recordeddate"))) AS recordeddate_month, - date_trunc('year', date(from_iso8601_timestamp(c.recordeddate))) + date_trunc('year', date(from_iso8601_timestamp(c."recordeddate"))) AS recordeddate_year FROM condition AS c LEFT JOIN core__condition_codable_concepts_all AS cca ON c.id = cca.id diff --git a/cumulus_library/studies/core/reference_sql/builder_documentreference.sql b/cumulus_library/studies/core/reference_sql/builder_documentreference.sql index 248f3baa..c727a0ac 100644 --- a/cumulus_library/studies/core/reference_sql/builder_documentreference.sql +++ b/cumulus_library/studies/core/reference_sql/builder_documentreference.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### @@ -27,6 +27,7 @@ CREATE TABLE core__documentreference_dn_type AS ( code, display FROM system_type_0 + ) SELECT id, @@ -51,13 +52,13 @@ WITH temp_documentreference AS ( dr.context, dr.subject.reference AS subject_ref, dr.context.period.start AS author_date, - date_trunc('day', date(from_iso8601_timestamp(dr.context.period.start))) + date_trunc('day', date(from_iso8601_timestamp(dr."context"."period"."start"))) AS author_day, - date_trunc('week', date(from_iso8601_timestamp(dr.context.period.start))) + date_trunc('week', date(from_iso8601_timestamp(dr."context"."period"."start"))) AS author_week, - date_trunc('month', date(from_iso8601_timestamp(dr.context.period.start))) + date_trunc('month', date(from_iso8601_timestamp(dr."context"."period"."start"))) AS author_month, - date_trunc('year', date(from_iso8601_timestamp(dr.context.period.start))) + date_trunc('year', date(from_iso8601_timestamp(dr."context"."period"."start"))) AS author_year, cdrt.code, cdrt.code_system, diff --git a/cumulus_library/studies/core/reference_sql/builder_encounter.sql b/cumulus_library/studies/core/reference_sql/builder_encounter.sql index b4909fd2..798e1757 100644 --- a/cumulus_library/studies/core/reference_sql/builder_encounter.sql +++ b/cumulus_library/studies/core/reference_sql/builder_encounter.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### @@ -21,7 +21,7 @@ CREATE TABLE core__encounter_dn_type AS ( UNNEST(s.type) AS cc (cc_row), UNNEST(cc.cc_row.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://terminology.hl7.org/CodeSystem/encounter-type' + u.codeable_concept.system LIKE 'http://terminology.hl7.org/CodeSystem/encounter-type' ), --noqa: LT07 system_type_1 AS ( @@ -36,7 +36,7 @@ CREATE TABLE core__encounter_dn_type AS ( UNNEST(s.type) AS cc (cc_row), UNNEST(cc.cc_row.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://terminology.hl7.org/CodeSystem/v2-0004' + u.codeable_concept.system LIKE 'http://terminology.hl7.org/CodeSystem/v2-0004' ), --noqa: LT07 system_type_2 AS ( @@ -51,7 +51,7 @@ CREATE TABLE core__encounter_dn_type AS ( UNNEST(s.type) AS cc (cc_row), UNNEST(cc.cc_row.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'urn:oid:2.16.840.1.113883.4.642.3.248' + u.codeable_concept.system LIKE 'urn:oid:2.16.840.1.113883.4.642.3.248' ), --noqa: LT07 system_type_3 AS ( @@ -66,7 +66,82 @@ CREATE TABLE core__encounter_dn_type AS ( UNNEST(s.type) AS cc (cc_row), UNNEST(cc.cc_row.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://snomed.info/sct' + u.codeable_concept.system LIKE 'http://snomed.info/sct' + ), --noqa: LT07 + + system_type_4 AS ( + SELECT DISTINCT + s.id AS id, + '4' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.type) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'https://fhir.cerner.com/%/codeSet/71' + ), --noqa: LT07 + + system_type_5 AS ( + SELECT DISTINCT + s.id AS id, + '5' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.type) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.10110' + ), --noqa: LT07 + + system_type_6 AS ( + SELECT DISTINCT + s.id AS id, + '6' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.type) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.18875' + ), --noqa: LT07 + + system_type_7 AS ( + SELECT DISTINCT + s.id AS id, + '7' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.type) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'urn:oid:1.2.840.114350.1.13.71.2.7.10.698084.30' + ), --noqa: LT07 + + system_type_8 AS ( + SELECT DISTINCT + s.id AS id, + '8' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.type) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'urn:oid:1.2.840.114350.1.13.71.2.7.2.808267' ), --noqa: LT07 union_table AS ( @@ -101,6 +176,47 @@ CREATE TABLE core__encounter_dn_type AS ( code, display FROM system_type_3 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_type_4 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_type_5 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_type_6 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_type_7 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_type_8 + ), partitioned_table AS ( @@ -169,7 +285,7 @@ CREATE TABLE core__encounter_dn_reasoncode AS ( UNNEST(s.reasoncode) AS cc (cc_row), UNNEST(cc.cc_row.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://terminology.hl7.org/CodeSystem/v3-ActPriority' + u.codeable_concept.system LIKE 'http://terminology.hl7.org/CodeSystem/v3-ActPriority' ), --noqa: LT07 system_reasoncode_1 AS ( @@ -184,7 +300,67 @@ CREATE TABLE core__encounter_dn_reasoncode AS ( UNNEST(s.reasoncode) AS cc (cc_row), UNNEST(cc.cc_row.coding) AS u (codeable_concept) WHERE - u.codeable_concept.system = 'http://snomed.info/sct' + u.codeable_concept.system LIKE 'http://snomed.info/sct' + ), --noqa: LT07 + + system_reasoncode_2 AS ( + SELECT DISTINCT + s.id AS id, + '2' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.reasoncode) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-10-cm' + ), --noqa: LT07 + + system_reasoncode_3 AS ( + SELECT DISTINCT + s.id AS id, + '3' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.reasoncode) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'http://hl7.org/fhir/sid/icd-9-cm' + ), --noqa: LT07 + + system_reasoncode_4 AS ( + SELECT DISTINCT + s.id AS id, + '4' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.reasoncode) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'https://fhir.cerner.com/%/nomenclature' + ), --noqa: LT07 + + system_reasoncode_5 AS ( + SELECT DISTINCT + s.id AS id, + '5' AS priority, + u.codeable_concept.code AS code, + u.codeable_concept.display AS display, + u.codeable_concept.system AS code_system + FROM + encounter AS s, + UNNEST(s.reasoncode) AS cc (cc_row), + UNNEST(cc.cc_row.coding) AS u (codeable_concept) + WHERE + u.codeable_concept.system LIKE 'urn:oid:1.2.840.114350.1.13.71.2.7.2.728286' ), --noqa: LT07 union_table AS ( @@ -203,6 +379,39 @@ CREATE TABLE core__encounter_dn_reasoncode AS ( code, display FROM system_reasoncode_1 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_reasoncode_2 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_reasoncode_3 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_reasoncode_4 + UNION + SELECT + id, + priority, + code_system, + code, + display + FROM system_reasoncode_5 + ), partitioned_table AS ( @@ -243,15 +452,15 @@ WITH temp_encounter_nullable AS ( e.subject.reference AS subject_ref, e.period, date(from_iso8601_timestamp(e.period.start)) AS period_start, - cast(NULL AS date) AS period_end_day - , - date_trunc('day', date(from_iso8601_timestamp(e.period.start))) + date_trunc('day', date(from_iso8601_timestamp(e."period"."end"))) + AS period_end_day, + date_trunc('day', date(from_iso8601_timestamp(e."period"."start"))) AS period_start_day, - date_trunc('week', date(from_iso8601_timestamp(e.period.start))) + date_trunc('week', date(from_iso8601_timestamp(e."period"."start"))) AS period_start_week, - date_trunc('month', date(from_iso8601_timestamp(e.period.start))) + date_trunc('month', date(from_iso8601_timestamp(e."period"."start"))) AS period_start_month, - date_trunc('year', date(from_iso8601_timestamp(e.period.start))) + date_trunc('year', date(from_iso8601_timestamp(e."period"."start"))) AS period_start_year FROM encounter AS e ), @@ -286,7 +495,6 @@ temp_encounter AS ( SELECT DISTINCT e.id, - e.class AS enc_class, ac.code AS enc_class_code, ac.display AS enc_class_display, e.status, diff --git a/cumulus_library/studies/core/reference_sql/builder_medication.sql b/cumulus_library/studies/core/reference_sql/builder_medication.sql index fdaf55d1..56124abb 100644 --- a/cumulus_library/studies/core/reference_sql/builder_medication.sql +++ b/cumulus_library/studies/core/reference_sql/builder_medication.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### diff --git a/cumulus_library/studies/core/reference_sql/builder_medicationrequest.sql b/cumulus_library/studies/core/reference_sql/builder_medicationrequest.sql index b7c2a5d0..538a29a8 100644 --- a/cumulus_library/studies/core/reference_sql/builder_medicationrequest.sql +++ b/cumulus_library/studies/core/reference_sql/builder_medicationrequest.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### @@ -28,6 +28,7 @@ CREATE TABLE core__medicationrequest_dn_category AS ( code, display FROM system_category_0 + ) SELECT id, @@ -49,7 +50,7 @@ WITH temp_mr AS ( mr.status, mr.intent, date(from_iso8601_timestamp(mr.authoredon)) AS authoredon, - date_trunc('month', date(from_iso8601_timestamp(mr.authoredon))) + date_trunc('month', date(from_iso8601_timestamp(mr."authoredon"))) AS authoredon_month, NULL AS display, mr.subject.reference AS subject_ref, diff --git a/cumulus_library/studies/core/reference_sql/builder_observation.sql b/cumulus_library/studies/core/reference_sql/builder_observation.sql index 83e28bb6..3de37900 100644 --- a/cumulus_library/studies/core/reference_sql/builder_observation.sql +++ b/cumulus_library/studies/core/reference_sql/builder_observation.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### @@ -28,6 +28,7 @@ CREATE TABLE core__observation_dn_category AS ( code, display FROM system_category_0 + ) SELECT id, @@ -61,6 +62,7 @@ CREATE TABLE core__observation_dn_code AS ( code, display FROM system_code_0 + ) SELECT id, @@ -105,6 +107,7 @@ CREATE TABLE core__observation_dn_valuecodeableconcept AS ( code, display FROM system_valuecodeableconcept_0 + ) SELECT id, @@ -124,13 +127,13 @@ WITH temp_observation AS ( SELECT o.id, o.status, - date_trunc('day', date(from_iso8601_timestamp(o.effectivedatetime))) + date_trunc('day', date(from_iso8601_timestamp(o."effectivedatetime"))) AS effectivedatetime_day, - date_trunc('week', date(from_iso8601_timestamp(o.effectivedatetime))) + date_trunc('week', date(from_iso8601_timestamp(o."effectivedatetime"))) AS effectivedatetime_week, - date_trunc('month', date(from_iso8601_timestamp(o.effectivedatetime))) + date_trunc('month', date(from_iso8601_timestamp(o."effectivedatetime"))) AS effectivedatetime_month, - date_trunc('year', date(from_iso8601_timestamp(o.effectivedatetime))) + date_trunc('year', date(from_iso8601_timestamp(o."effectivedatetime"))) AS effectivedatetime_year, odc.code AS observation_code, odc.code_system AS observation_code_system, diff --git a/cumulus_library/studies/core/reference_sql/builder_patient.sql b/cumulus_library/studies/core/reference_sql/builder_patient.sql index b60ea2a0..b7919526 100644 --- a/cumulus_library/studies/core/reference_sql/builder_patient.sql +++ b/cumulus_library/studies/core/reference_sql/builder_patient.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### @@ -84,6 +84,7 @@ CREATE TABLE core__patient_ext_race AS ( race_code, race_display FROM system_text + ORDER BY id, priority ) @@ -206,6 +207,7 @@ CREATE TABLE core__patient_ext_ethnicity AS ( ethnicity_code, ethnicity_display FROM system_text + ORDER BY id, priority ) @@ -267,7 +269,7 @@ WITH temp_patient AS ( ) SELECT DISTINCT - tp.id AS id, + tp.id, tp.gender, tp.birthdate, CASE diff --git a/cumulus_library/studies/core/reference_sql/builder_prereq_tables.sql b/cumulus_library/studies/core/reference_sql/builder_prereq_tables.sql index 6bede737..bb9524f4 100644 --- a/cumulus_library/studies/core/reference_sql/builder_prereq_tables.sql +++ b/cumulus_library/studies/core/reference_sql/builder_prereq_tables.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### diff --git a/cumulus_library/studies/core/reference_sql/count_core.sql b/cumulus_library/studies/core/reference_sql/count_core.sql index b495455c..6097c103 100644 --- a/cumulus_library/studies/core/reference_sql/count_core.sql +++ b/cumulus_library/studies/core/reference_sql/count_core.sql @@ -1,8 +1,8 @@ -- noqa: disable=all --- This sql was autogenerated as a reference example using the library CLI. --- Its format is tied to the specific database it was run against, and it may not --- be correct for all databases. Use the CLI's build option to derive the best SQL --- for your dataset. +-- This sql was autogenerated as a reference example using the library +-- CLI.Its format is tied to the specific database it was run against, +-- and it may not be correct for all databases. Use the CLI's build +-- option to derive the best SQL for your dataset. -- ########################################################### diff --git a/cumulus_library/study_parser.py b/cumulus_library/study_parser.py index 7b1a4aa7..390e7c0b 100644 --- a/cumulus_library/study_parser.py +++ b/cumulus_library/study_parser.py @@ -1,10 +1,13 @@ """ Contains classes for loading study data based on manifest.toml files """ import csv +import datetime import importlib.util import inspect import pathlib +import shutil import sys +import zipfile import toml from rich.progress import Progress, TaskID, track @@ -665,22 +668,36 @@ def _execute_build_queries( # Database exporting functions def export_study( - self, db: databases.DatabaseBackend, data_path: pathlib.Path - ) -> list: + self, + db: databases.DatabaseBackend, + schema_name: str, + data_path: pathlib.Path, + archive: bool, + ) -> None: """Exports csvs/parquet extracts of tables listed in export_list - :param db: A database backend - :returns: list of executed queries (for unit testing only) + :param schema_name: the schema/database to target + :param data_path: the path to the place on disk to save data + :param archive: If true, get all study data and zip with timestamp """ + table_list = [] + path = pathlib.Path(f"{data_path}/{self.get_study_prefix()}/") self.reset_counts_exports() - queries = [] + if archive: + table_query = base_templates.get_show_tables( + schema_name, f"{self.get_study_prefix()}__" + ) + result = db.cursor().execute(table_query).fetchall() + for row in result: + table_list.append(row[0]) + else: + table_list = self.get_export_table_list() for table in track( - self.get_export_table_list(), - description=f"Exporting {self.get_study_prefix()} counts...", + table_list, + description=f"Exporting {self.get_study_prefix()} data...", ): - query = f"select * from {table}" + query = f"SELECT * FROM {table}" dataframe = db.execute_as_pandas(query) - path = pathlib.Path(f"{data_path}/{self.get_study_prefix()}/") path.mkdir(parents=True, exist_ok=True) dataframe = dataframe.sort_values( by=list(dataframe.columns), ascending=False, na_position="first" @@ -688,6 +705,21 @@ def export_study( dataframe.to_csv( f"{path}/{table}.csv", index=False, quoting=csv.QUOTE_MINIMAL ) - dataframe.to_parquet(f"{path}/{table}.parquet", index=False) - queries.append(query) - return queries + if not archive: + dataframe.to_parquet(f"{path}/{table}.parquet", index=False) + if archive: + file_list = [file for file in path.glob("**/*") if file.is_file()] + timestamp = ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + with zipfile.ZipFile( + f"{data_path}/{self.get_study_prefix()}_{timestamp}.zip", + "w", + zipfile.ZIP_DEFLATED, + ) as f: + for file in file_list: + f.write(file, file.relative_to(path)) + file.unlink() + shutil.rmtree(path) diff --git a/tests/conftest.py b/tests/conftest.py index 5b953fe7..0a25d4a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,7 +72,7 @@ def duckdb_args(args: list, tmp_path, stats=False): "duckdb", "--database", f"{tmp_path}/duck.db", - f"{tmp_path}/counts", + f"{tmp_path}/export", ] return [*args, "--db-type", "duckdb", "--database", f"{tmp_path}/duck.db"] diff --git a/tests/test_cli.py b/tests/test_cli.py index c03e0118..01acfc4b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,8 +2,13 @@ import builtins import glob +import io import os +import pathlib import shutil +import sys +import zipfile +from contextlib import contextmanager from contextlib import nullcontext as does_not_raise from pathlib import Path from unittest import mock @@ -35,6 +40,14 @@ def open(self, *args, **kwargs): return self.builtin_open(*args, **kwargs) +@contextmanager +def mock_stdin(value: str): + stdin = sys.stdin + sys.stdin = value + yield + sys.stdin = stdin + + @mock.patch.dict( os.environ, clear=True, @@ -308,6 +321,39 @@ def test_cli_executes_queries(tmp_path, build_args, export_args, expected_tables assert any(export_table in x for x in csv_files) +@mock.patch.dict( + os.environ, + clear=True, +) +@pytest.mark.parametrize( + "args,input_txt,expects_files, raises", + [ + (["export", "-t", "core", "--archive"], "Y", True, does_not_raise()), + (["export", "-t", "core", "--archive"], "N", True, pytest.raises(SystemExit)), + ], +) +def test_cli_export_archive(tmp_path, args, input_txt, expects_files, raises): + with raises: + with mock_stdin(io.StringIO(input_txt)): + build_args = duckdb_args(["build", "-t", "core"], tmp_path) + stats_mock = pathlib.Path(f"{tmp_path}/export/core/stats") + stats_mock.mkdir(parents=True, exist_ok=True) + with open(stats_mock / "test.txt", "w") as f: + f.write("test") + export_args = duckdb_args(args, tmp_path) + cli.main(cli_args=build_args) + cli.main(cli_args=export_args) + export_paths = list(pathlib.Path(f"{tmp_path}/export").glob("*")) + assert len(export_paths) == 1 + archive = zipfile.ZipFile(export_paths[0]) + for file in [ + "core__encounter.csv", + "core__count_encounter_type_month.csv", + "stats/test.txt", + ]: + assert file in archive.namelist() + + @mock.patch.dict( os.environ, clear=True, @@ -397,21 +443,6 @@ def test_cli_stats_rebuild(tmp_path): assert len(expected) == 2 -@mock.patch.dict( - os.environ, - clear=True, -) -def test_cli_creates_study(tmp_path): - cli.main(cli_args=["create", f"{tmp_path}/studydir/"]) - with open( - "./cumulus_library/studies/template/manifest.toml", encoding="UTF-8" - ) as file: - source = toml.load(file) - with open(f"{tmp_path}/studydir/manifest.toml", encoding="UTF-8") as file: - target = toml.load(file) - assert source == target - - @mock.patch.dict( os.environ, clear=True, diff --git a/tests/test_data/core/core__encounter.txt b/tests/test_data/core/core__encounter.txt index a4871e67..1ac5c0ef 100644 --- a/tests/test_data/core/core__encounter.txt +++ b/tests/test_data/core/core__encounter.txt @@ -1,50 +1,50 @@ -('029d1814-d7bf-0624-524d-7ccda5f320f6', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 15, datetime.date(2018, 6, 3), datetime.date(2018, 6, 3), datetime.date(2018, 5, 28), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/e80dda2c-a260-dbf7-3167-bf0945f3a91d', 'Encounter/029d1814-d7bf-0624-524d-7ccda5f320f6', 'female', 'black or african american', 'not hispanic or latino', '672') -('02eb4e14-1a6f-d968-2c26-c0cf5023afe0', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 57, datetime.date(2018, 6, 7), datetime.date(2018, 6, 7), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/c7ad408d-fcae-b54a-eb1d-26d48f7a5f84', 'Encounter/02eb4e14-1a6f-d968-2c26-c0cf5023afe0', 'female', 'white', 'not hispanic or latino', '660') -('03e34b19-2889-b828-792d-2a83400c55be', {'id': None, 'code': 'EMER', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'EMER', 'emergency', 'finished', None, None, None, None, None, None, None, None, 20, datetime.date(2018, 7, 19), datetime.date(2018, 7, 19), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/ad3ed58a-5645-af0a-eeca-ab543123a8aa', 'Encounter/03e34b19-2889-b828-792d-2a83400c55be', 'female', 'white', 'not hispanic or latino', '662') -('11381dc6-0e06-da55-0735-d1e7bbf8bb35', {'id': None, 'code': 'EMER', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'EMER', 'emergency', 'finished', None, None, None, None, None, None, None, None, 8, datetime.date(2018, 7, 15), datetime.date(2018, 7, 15), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/dffa62dc-8ec2-1cd6-ee75-f9156a5283fe', 'Encounter/11381dc6-0e06-da55-0735-d1e7bbf8bb35', 'male', 'white', 'hispanic or latino', '672') -('1154d05c-8727-9373-4224-25b9fdba9ab3', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 25, datetime.date(2018, 6, 14), datetime.date(2018, 6, 14), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/24906e2c-e556-71dc-23d9-3e1d5fb08986', 'Encounter/1154d05c-8727-9373-4224-25b9fdba9ab3', 'female', 'white', 'hispanic or latino', '671') -('1d679c3a-2765-5e13-e2a3-4bd76a898fc6', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 50, datetime.date(2018, 6, 15), datetime.date(2018, 6, 15), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/cb8c0665-30ee-479c-8994-d29f1a6848b0', 'Encounter/1d679c3a-2765-5e13-e2a3-4bd76a898fc6', 'male', 'white', 'not hispanic or latino', '662') -('299b6495-3fe7-8db3-c494-6e1ce8b7986d', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 30, datetime.date(2018, 6, 2), datetime.date(2018, 6, 2), datetime.date(2018, 5, 28), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/8877ef1f-7cd7-3242-d7f0-73cf3f7165f4', 'Encounter/299b6495-3fe7-8db3-c494-6e1ce8b7986d', 'male', 'black or african american', 'hispanic or latino', '660') -('2b1ee164-6c87-420d-a9e2-6c235ebeef71', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 15, datetime.date(2018, 6, 8), datetime.date(2018, 6, 8), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/6a883108-7b87-120b-d163-d369336e04e5', 'Encounter/2b1ee164-6c87-420d-a9e2-6c235ebeef71', 'male', 'white', 'not hispanic or latino', '672') -('2f55edb9-a906-0b40-e183-89b1d65d1aa1', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 16, datetime.date(2018, 7, 21), datetime.date(2018, 7, 21), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/0734762a-9db6-22fc-b595-c3e472bb2a9a', 'Encounter/2f55edb9-a906-0b40-e183-89b1d65d1aa1', 'female', 'white', 'not hispanic or latino', '672') -('32d0ae2d-1be8-9e90-a4da-4c222abd88a9', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 72, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/17fde357-dcc9-af8b-a8d3-4bd213afeb22', 'Encounter/32d0ae2d-1be8-9e90-a4da-4c222abd88a9', 'male', 'white', 'not hispanic or latino', '662') -('366df7d3-2ea9-db22-7cdd-60fa8a5c45ca', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 66, datetime.date(2018, 6, 15), datetime.date(2018, 6, 15), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/ad239efc-637c-e428-c829-b87e700d5446', 'Encounter/366df7d3-2ea9-db22-7cdd-60fa8a5c45ca', 'male', 'black or african american', 'not hispanic or latino', '678') -('37604257-be1a-120f-81ee-336f81603f92', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 48, datetime.date(2018, 7, 16), datetime.date(2018, 7, 16), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/50f8b42e-17a6-e932-8546-da94004c597c', 'Encounter/37604257-be1a-120f-81ee-336f81603f92', 'male', 'white', 'not hispanic or latino', '666') -('4b03a408-6694-88e3-0e63-3ee464ecd6cd', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 73, datetime.date(2018, 7, 11), datetime.date(2018, 7, 11), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/5ce2e599-fb6e-9b4d-3c2e-87310619b957', 'Encounter/4b03a408-6694-88e3-0e63-3ee464ecd6cd', 'male', 'white', 'not hispanic or latino', '660') -('4c4d0730-201f-5b75-c657-8d0de09cc28f', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 60, datetime.date(2018, 6, 14), datetime.date(2018, 6, 14), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/267fc42d-cd9e-8527-1f9e-887fe7776147', 'Encounter/4c4d0730-201f-5b75-c657-8d0de09cc28f', 'female', 'white', 'not hispanic or latino', '676') -('5c3450fb-12f0-08f3-6e4d-8a5e433e19a4', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 46, datetime.date(2018, 6, 21), datetime.date(2018, 6, 21), datetime.date(2018, 6, 18), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/a46846ca-3f95-2cbb-3a9d-5eae150a0273', 'Encounter/5c3450fb-12f0-08f3-6e4d-8a5e433e19a4', 'female', 'white', 'not hispanic or latino', '660') -('5c994000-aa78-2be5-e6cf-99f230d50c2f', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 81, datetime.date(2018, 7, 13), datetime.date(2018, 7, 13), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/9eaa056b-1efc-0cc8-70ff-62c8f704cc13', 'Encounter/5c994000-aa78-2be5-e6cf-99f230d50c2f', 'male', 'white', 'not hispanic or latino', '672') -('6565ef5c-30b9-8697-6ca6-2b77894d5437', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 19, datetime.date(2018, 7, 19), datetime.date(2018, 7, 19), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/d3c0274f-f42b-8d2b-15f2-82331e723383', 'Encounter/6565ef5c-30b9-8697-6ca6-2b77894d5437', 'female', 'white', 'not hispanic or latino', '668') -('65f8fdca-a949-80a0-8072-094e9aaee474', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '10509002', 'http://snomed.info/sct', 9, datetime.date(2018, 6, 17), datetime.date(2018, 6, 17), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/7f941bcc-e7b2-99e1-585f-129d0ef1c13d', 'Encounter/65f8fdca-a949-80a0-8072-094e9aaee474', 'female', 'white', 'not hispanic or latino', '674') -('683b04eb-663a-849f-715f-4ccd70bf1524', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 12, datetime.date(2018, 7, 26), datetime.date(2018, 7, 26), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/cc9fb8e2-fe52-b72a-aebb-9d10260f121b', 'Encounter/683b04eb-663a-849f-715f-4ccd70bf1524', 'female', 'white', 'not hispanic or latino', '660') -('6a952afd-3be5-e27e-9e05-5a1e085e34d0', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '444814009', 'http://snomed.info/sct', 16, datetime.date(2018, 7, 21), datetime.date(2018, 7, 21), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/f9399f0d-5401-09f3-d4ff-89b1aa51b9c8', 'Encounter/6a952afd-3be5-e27e-9e05-5a1e085e34d0', 'male', 'white', 'not hispanic or latino', '674') -('75312bd2-d5ac-c62e-c9df-0004783725c7', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 11, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/3ae095ec-8fe0-133b-36d4-8785a6ad8df3', 'Encounter/75312bd2-d5ac-c62e-c9df-0004783725c7', 'female', 'white', 'not hispanic or latino', '662') -('75b68644-535d-bdc1-4c31-aa9fe7e1822f', {'id': None, 'code': 'EMER', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'EMER', 'emergency', 'finished', None, None, None, None, None, None, '55680006', 'http://snomed.info/sct', 25, datetime.date(2018, 6, 30), datetime.date(2018, 6, 30), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/82b8a670-4700-30e8-24a0-b83efa3c5e0a', 'Encounter/75b68644-535d-bdc1-4c31-aa9fe7e1822f', 'female', 'white', 'not hispanic or latino', '672') -('79d8f213-7847-646b-8a66-5da208cc1c27', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 77, datetime.date(2018, 7, 14), datetime.date(2018, 7, 14), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/26a3984f-b2a8-e67f-7abc-ff147a0e6e35', 'Encounter/79d8f213-7847-646b-8a66-5da208cc1c27', 'male', 'white', 'not hispanic or latino', '674') -('83d0d564-3bbf-48eb-7445-bd2b81130671', {'id': None, 'code': 'EMER', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'EMER', 'emergency', 'finished', None, None, None, None, None, None, '55680006', 'http://snomed.info/sct', 69, datetime.date(2018, 6, 16), datetime.date(2018, 6, 17), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/16be855b-ece2-8b96-1ef9-a4d93adf3289', 'Encounter/83d0d564-3bbf-48eb-7445-bd2b81130671', 'male', 'white', 'not hispanic or latino', '660') -('8ff1dc01-5a28-b2d8-3b42-4b7a7d539970', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 55, datetime.date(2018, 7, 15), datetime.date(2018, 7, 15), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/c1bfec36-dc2c-afc8-c767-3d35ed2bf6f0', 'Encounter/8ff1dc01-5a28-b2d8-3b42-4b7a7d539970', 'female', 'white', 'not hispanic or latino', '662') -('91f94a9d-69a7-e30a-cd1a-68c52dc01e70', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 22, datetime.date(2018, 6, 29), datetime.date(2018, 6, 29), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/a28be3e1-a6bd-7df4-fc81-1140848f8453', 'Encounter/91f94a9d-69a7-e30a-cd1a-68c52dc01e70', 'female', 'white', 'not hispanic or latino', '662') -('98d4bd14-d78e-debb-e7dc-2df7786aedf3', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 89, datetime.date(2018, 7, 9), datetime.date(2018, 7, 9), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/e455ca3f-fc16-6ffc-297a-adc27e2db183', 'Encounter/98d4bd14-d78e-debb-e7dc-2df7786aedf3', 'male', 'white', 'hispanic or latino', '667') -('aa890974-162f-5906-dc71-2bb6d2185314', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 50, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/c7c5c028-f1fb-962b-8db2-dfd4c6d6b02a', 'Encounter/aa890974-162f-5906-dc71-2bb6d2185314', 'male', 'white', 'not hispanic or latino', '672') -('aabb3ac3-c4a3-f613-9507-63280adb9209', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 0, datetime.date(2018, 7, 31), datetime.date(2018, 7, 31), datetime.date(2018, 7, 30), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/2a0e1946-acf6-5a7e-9399-a9cbc4730199', 'Encounter/aabb3ac3-c4a3-f613-9507-63280adb9209', 'female', 'white', 'not hispanic or latino', '675') -('b5974881-ae62-ddd6-b905-8c86c1ca9e33', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '271737000', 'http://snomed.info/sct', 41, datetime.date(2018, 6, 25), datetime.date(2018, 6, 25), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/47c37c92-5932-9cfe-66be-208556780fe0', 'Encounter/b5974881-ae62-ddd6-b905-8c86c1ca9e33', 'male', 'white', 'not hispanic or latino', '675') -('b864bcd8-14e0-8bec-b7cc-f8629d91470e', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 15, datetime.date(2018, 6, 28), datetime.date(2018, 6, 28), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/c20e5afd-30df-ac3d-6684-cc29438a9bc4', 'Encounter/b864bcd8-14e0-8bec-b7cc-f8629d91470e', 'female', 'white', 'not hispanic or latino', '662') -('ba84689e-2f9f-7cea-af1f-d69ffdd3a3eb', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 39, datetime.date(2018, 6, 15), datetime.date(2018, 6, 15), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/7bf52d54-0d2a-265a-15aa-eeed7aaf4af6', 'Encounter/ba84689e-2f9f-7cea-af1f-d69ffdd3a3eb', 'female', 'white', 'not hispanic or latino', '670') -('bca7cabc-b2fc-8a08-c69b-5bc0afa20d80', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '271737000', 'http://snomed.info/sct', 28, datetime.date(2018, 7, 13), datetime.date(2018, 7, 13), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/3627adb8-f741-acf3-2dd6-10f3bcbe1077', 'Encounter/bca7cabc-b2fc-8a08-c69b-5bc0afa20d80', 'female', 'white', 'not hispanic or latino', '674') -('beb26500-4ccd-ce0a-44f6-74f44be5fafe', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 14, datetime.date(2018, 6, 7), datetime.date(2018, 6, 7), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/149de67a-2809-59a8-bfa2-36df509021dc', 'Encounter/beb26500-4ccd-ce0a-44f6-74f44be5fafe', 'female', 'white', 'not hispanic or latino', '667') -('c4605953-3103-ede6-e0c0-e0588e6f019e', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 61, datetime.date(2018, 7, 27), datetime.date(2018, 7, 27), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/a5b171a7-9b28-20e7-86a7-936ecbf55f36', 'Encounter/c4605953-3103-ede6-e0c0-e0588e6f019e', 'female', 'white', 'not hispanic or latino', '675') -('c6ec2350-43d4-abab-2e84-4d2aadb337a7', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 53, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/26baae20-c8c5-003a-ab6b-ebcc49be20db', 'Encounter/c6ec2350-43d4-abab-2e84-4d2aadb337a7', 'male', 'white', 'not hispanic or latino', '678') -('ca45bbef-ef2a-3b3c-ea5e-76bcd5865780', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '72892002', 'http://snomed.info/sct', 48, datetime.date(2018, 6, 12), datetime.date(2018, 6, 12), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/9b17654f-a902-3d56-9000-4ade3dd3059f', 'Encounter/ca45bbef-ef2a-3b3c-ea5e-76bcd5865780', 'female', 'white', 'hispanic or latino', '672') -('d2782687-6885-037c-957d-579fbd681d2a', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 54, datetime.date(2018, 6, 13), datetime.date(2018, 6, 13), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/3cf7af45-2bee-aa9c-d524-40b487149d60', 'Encounter/d2782687-6885-037c-957d-579fbd681d2a', 'male', 'white', 'not hispanic or latino', '000') -('d5f342b7-017c-f2e7-8697-5a038c91518e', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '72892002', 'http://snomed.info/sct', 20, datetime.date(2018, 7, 28), datetime.date(2018, 7, 28), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/2da1bee2-2bc2-4511-84e1-42860310e2fb', 'Encounter/d5f342b7-017c-f2e7-8697-5a038c91518e', 'female', 'white', 'not hispanic or latino', '660') -('d735c414-9dd3-c9b1-285c-8da79a7fbbdf', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 61, datetime.date(2018, 7, 29), datetime.date(2018, 7, 29), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/7cef0e6f-9aea-4079-dfc6-18a96454708e', 'Encounter/d735c414-9dd3-c9b1-285c-8da79a7fbbdf', 'female', 'white', 'not hispanic or latino', '662') -('d73ed087-e0ae-78e0-7a05-1bac1060c476', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 45, datetime.date(2018, 7, 10), datetime.date(2018, 7, 10), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/51032f44-d514-26e9-3e85-e956561c076e', 'Encounter/d73ed087-e0ae-78e0-7a05-1bac1060c476', 'female', 'white', 'not hispanic or latino', '667') -('dc5ed645-3979-e765-3e03-6ba2173027c3', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 90, datetime.date(2018, 7, 7), datetime.date(2018, 7, 7), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/ac91b90d-97e4-4fc5-41cd-036bac49e6e8', 'Encounter/dc5ed645-3979-e765-3e03-6ba2173027c3', 'male', 'white', 'not hispanic or latino', '671') -('de9d67de-6ae3-32f7-20f2-e719ae23a9a3', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 3, datetime.date(2018, 7, 30), datetime.date(2018, 7, 30), datetime.date(2018, 7, 30), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/2858705f-5af1-9869-4d94-894e09a9f99a', 'Encounter/de9d67de-6ae3-32f7-20f2-e719ae23a9a3', 'male', 'white', 'not hispanic or latino', '672') -('e5dabcb6-1d7a-7467-dbba-b864d0d5f5b0', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 49, datetime.date(2018, 7, 28), datetime.date(2018, 7, 28), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/0b052286-9534-99a8-8d5e-06c2c04a7df7', 'Encounter/e5dabcb6-1d7a-7467-dbba-b864d0d5f5b0', 'female', 'white', 'not hispanic or latino', '662') -('e613f29d-7505-6f2e-a1f5-bfbec300752d', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '88805009', 'http://snomed.info/sct', 84, datetime.date(2018, 6, 29), datetime.date(2018, 6, 29), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/1c498b42-61fd-6341-69c3-053f6e4fe404', 'Encounter/e613f29d-7505-6f2e-a1f5-bfbec300752d', 'female', 'white', 'not hispanic or latino', '667') -('e922a884-7039-a171-a65e-78051fe7afe6', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 1, datetime.date(2018, 6, 13), datetime.date(2018, 6, 13), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/8022fbbe-aaa4-056c-d0f5-ec074bf0656c', 'Encounter/e922a884-7039-a171-a65e-78051fe7afe6', 'male', 'white', 'not hispanic or latino', '665') -('ed151e04-3dd6-8cb7-a3e5-777c8a8667f1', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '195662009', 'http://snomed.info/sct', 57, datetime.date(2018, 6, 6), datetime.date(2018, 6, 6), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/19158de4-66a2-f70f-e3bf-4396b312c8f1', 'Encounter/ed151e04-3dd6-8cb7-a3e5-777c8a8667f1', 'female', 'white', 'not hispanic or latino', '000') -('f2752dd7-1bf1-739d-dd8c-40122d0b63bc', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '161665007', 'http://snomed.info/sct', 77, datetime.date(2018, 6, 1), datetime.date(2018, 6, 2), datetime.date(2018, 5, 28), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/9c8d8539-0b1e-73e2-b64f-83f1ea601fa4', 'Encounter/f2752dd7-1bf1-739d-dd8c-40122d0b63bc', 'male', 'white', 'not hispanic or latino', '674') -('f964be66-3fcd-95c8-0021-71c7d24c91b7', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 63, datetime.date(2018, 6, 5), datetime.date(2018, 6, 5), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/a5bc08ea-9462-c4f5-1bd2-ff342598ac99', 'Encounter/f964be66-3fcd-95c8-0021-71c7d24c91b7', 'female', 'white', 'not hispanic or latino', '661') -('fd0754a4-e96d-cba7-b3c0-77697a09c86e', {'id': None, 'code': 'AMB', 'display': None, 'system': 'http://terminology.hl7.org/CodeSystem/v3-ActCode', 'userSelected': None, 'version': None}, 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 100, datetime.date(2018, 7, 10), datetime.date(2018, 7, 10), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/6385ddd7-2639-6505-3789-0521b8f66c8b', 'Encounter/fd0754a4-e96d-cba7-b3c0-77697a09c86e', 'female', 'white', 'not hispanic or latino', '672') +('029d1814-d7bf-0624-524d-7ccda5f320f6', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 15, datetime.date(2018, 6, 3), datetime.date(2018, 6, 3), datetime.date(2018, 5, 28), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/e80dda2c-a260-dbf7-3167-bf0945f3a91d', 'Encounter/029d1814-d7bf-0624-524d-7ccda5f320f6', 'female', 'black or african american', 'not hispanic or latino', '672') +('02eb4e14-1a6f-d968-2c26-c0cf5023afe0', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 57, datetime.date(2018, 6, 7), datetime.date(2018, 6, 7), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/c7ad408d-fcae-b54a-eb1d-26d48f7a5f84', 'Encounter/02eb4e14-1a6f-d968-2c26-c0cf5023afe0', 'female', 'white', 'not hispanic or latino', '660') +('03e34b19-2889-b828-792d-2a83400c55be', 'EMER', 'emergency', 'finished', None, None, None, None, None, None, None, None, 20, datetime.date(2018, 7, 19), datetime.date(2018, 7, 19), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/ad3ed58a-5645-af0a-eeca-ab543123a8aa', 'Encounter/03e34b19-2889-b828-792d-2a83400c55be', 'female', 'white', 'not hispanic or latino', '662') +('11381dc6-0e06-da55-0735-d1e7bbf8bb35', 'EMER', 'emergency', 'finished', None, None, None, None, None, None, None, None, 8, datetime.date(2018, 7, 15), datetime.date(2018, 7, 15), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/dffa62dc-8ec2-1cd6-ee75-f9156a5283fe', 'Encounter/11381dc6-0e06-da55-0735-d1e7bbf8bb35', 'male', 'white', 'hispanic or latino', '672') +('1154d05c-8727-9373-4224-25b9fdba9ab3', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 25, datetime.date(2018, 6, 14), datetime.date(2018, 6, 14), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/24906e2c-e556-71dc-23d9-3e1d5fb08986', 'Encounter/1154d05c-8727-9373-4224-25b9fdba9ab3', 'female', 'white', 'hispanic or latino', '671') +('1d679c3a-2765-5e13-e2a3-4bd76a898fc6', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 50, datetime.date(2018, 6, 15), datetime.date(2018, 6, 15), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/cb8c0665-30ee-479c-8994-d29f1a6848b0', 'Encounter/1d679c3a-2765-5e13-e2a3-4bd76a898fc6', 'male', 'white', 'not hispanic or latino', '662') +('299b6495-3fe7-8db3-c494-6e1ce8b7986d', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 30, datetime.date(2018, 6, 2), datetime.date(2018, 6, 2), datetime.date(2018, 5, 28), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/8877ef1f-7cd7-3242-d7f0-73cf3f7165f4', 'Encounter/299b6495-3fe7-8db3-c494-6e1ce8b7986d', 'male', 'black or african american', 'hispanic or latino', '660') +('2b1ee164-6c87-420d-a9e2-6c235ebeef71', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 15, datetime.date(2018, 6, 8), datetime.date(2018, 6, 8), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/6a883108-7b87-120b-d163-d369336e04e5', 'Encounter/2b1ee164-6c87-420d-a9e2-6c235ebeef71', 'male', 'white', 'not hispanic or latino', '672') +('2f55edb9-a906-0b40-e183-89b1d65d1aa1', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 16, datetime.date(2018, 7, 21), datetime.date(2018, 7, 21), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/0734762a-9db6-22fc-b595-c3e472bb2a9a', 'Encounter/2f55edb9-a906-0b40-e183-89b1d65d1aa1', 'female', 'white', 'not hispanic or latino', '672') +('32d0ae2d-1be8-9e90-a4da-4c222abd88a9', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 72, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/17fde357-dcc9-af8b-a8d3-4bd213afeb22', 'Encounter/32d0ae2d-1be8-9e90-a4da-4c222abd88a9', 'male', 'white', 'not hispanic or latino', '662') +('366df7d3-2ea9-db22-7cdd-60fa8a5c45ca', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 66, datetime.date(2018, 6, 15), datetime.date(2018, 6, 15), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/ad239efc-637c-e428-c829-b87e700d5446', 'Encounter/366df7d3-2ea9-db22-7cdd-60fa8a5c45ca', 'male', 'black or african american', 'not hispanic or latino', '678') +('37604257-be1a-120f-81ee-336f81603f92', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 48, datetime.date(2018, 7, 16), datetime.date(2018, 7, 16), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/50f8b42e-17a6-e932-8546-da94004c597c', 'Encounter/37604257-be1a-120f-81ee-336f81603f92', 'male', 'white', 'not hispanic or latino', '666') +('4b03a408-6694-88e3-0e63-3ee464ecd6cd', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 73, datetime.date(2018, 7, 11), datetime.date(2018, 7, 11), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/5ce2e599-fb6e-9b4d-3c2e-87310619b957', 'Encounter/4b03a408-6694-88e3-0e63-3ee464ecd6cd', 'male', 'white', 'not hispanic or latino', '660') +('4c4d0730-201f-5b75-c657-8d0de09cc28f', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 60, datetime.date(2018, 6, 14), datetime.date(2018, 6, 14), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/267fc42d-cd9e-8527-1f9e-887fe7776147', 'Encounter/4c4d0730-201f-5b75-c657-8d0de09cc28f', 'female', 'white', 'not hispanic or latino', '676') +('5c3450fb-12f0-08f3-6e4d-8a5e433e19a4', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 46, datetime.date(2018, 6, 21), datetime.date(2018, 6, 21), datetime.date(2018, 6, 18), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/a46846ca-3f95-2cbb-3a9d-5eae150a0273', 'Encounter/5c3450fb-12f0-08f3-6e4d-8a5e433e19a4', 'female', 'white', 'not hispanic or latino', '660') +('5c994000-aa78-2be5-e6cf-99f230d50c2f', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 81, datetime.date(2018, 7, 13), datetime.date(2018, 7, 13), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/9eaa056b-1efc-0cc8-70ff-62c8f704cc13', 'Encounter/5c994000-aa78-2be5-e6cf-99f230d50c2f', 'male', 'white', 'not hispanic or latino', '672') +('6565ef5c-30b9-8697-6ca6-2b77894d5437', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 19, datetime.date(2018, 7, 19), datetime.date(2018, 7, 19), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/d3c0274f-f42b-8d2b-15f2-82331e723383', 'Encounter/6565ef5c-30b9-8697-6ca6-2b77894d5437', 'female', 'white', 'not hispanic or latino', '668') +('65f8fdca-a949-80a0-8072-094e9aaee474', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '10509002', 'http://snomed.info/sct', 9, datetime.date(2018, 6, 17), datetime.date(2018, 6, 17), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/7f941bcc-e7b2-99e1-585f-129d0ef1c13d', 'Encounter/65f8fdca-a949-80a0-8072-094e9aaee474', 'female', 'white', 'not hispanic or latino', '674') +('683b04eb-663a-849f-715f-4ccd70bf1524', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 12, datetime.date(2018, 7, 26), datetime.date(2018, 7, 26), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/cc9fb8e2-fe52-b72a-aebb-9d10260f121b', 'Encounter/683b04eb-663a-849f-715f-4ccd70bf1524', 'female', 'white', 'not hispanic or latino', '660') +('6a952afd-3be5-e27e-9e05-5a1e085e34d0', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '444814009', 'http://snomed.info/sct', 16, datetime.date(2018, 7, 21), datetime.date(2018, 7, 21), datetime.date(2018, 7, 16), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/f9399f0d-5401-09f3-d4ff-89b1aa51b9c8', 'Encounter/6a952afd-3be5-e27e-9e05-5a1e085e34d0', 'male', 'white', 'not hispanic or latino', '674') +('75312bd2-d5ac-c62e-c9df-0004783725c7', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 11, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/3ae095ec-8fe0-133b-36d4-8785a6ad8df3', 'Encounter/75312bd2-d5ac-c62e-c9df-0004783725c7', 'female', 'white', 'not hispanic or latino', '662') +('75b68644-535d-bdc1-4c31-aa9fe7e1822f', 'EMER', 'emergency', 'finished', None, None, None, None, None, None, '55680006', 'http://snomed.info/sct', 25, datetime.date(2018, 6, 30), datetime.date(2018, 6, 30), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/82b8a670-4700-30e8-24a0-b83efa3c5e0a', 'Encounter/75b68644-535d-bdc1-4c31-aa9fe7e1822f', 'female', 'white', 'not hispanic or latino', '672') +('79d8f213-7847-646b-8a66-5da208cc1c27', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 77, datetime.date(2018, 7, 14), datetime.date(2018, 7, 14), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/26a3984f-b2a8-e67f-7abc-ff147a0e6e35', 'Encounter/79d8f213-7847-646b-8a66-5da208cc1c27', 'male', 'white', 'not hispanic or latino', '674') +('83d0d564-3bbf-48eb-7445-bd2b81130671', 'EMER', 'emergency', 'finished', None, None, None, None, None, None, '55680006', 'http://snomed.info/sct', 69, datetime.date(2018, 6, 16), datetime.date(2018, 6, 17), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/16be855b-ece2-8b96-1ef9-a4d93adf3289', 'Encounter/83d0d564-3bbf-48eb-7445-bd2b81130671', 'male', 'white', 'not hispanic or latino', '660') +('8ff1dc01-5a28-b2d8-3b42-4b7a7d539970', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 55, datetime.date(2018, 7, 15), datetime.date(2018, 7, 15), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/c1bfec36-dc2c-afc8-c767-3d35ed2bf6f0', 'Encounter/8ff1dc01-5a28-b2d8-3b42-4b7a7d539970', 'female', 'white', 'not hispanic or latino', '662') +('91f94a9d-69a7-e30a-cd1a-68c52dc01e70', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 22, datetime.date(2018, 6, 29), datetime.date(2018, 6, 29), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/a28be3e1-a6bd-7df4-fc81-1140848f8453', 'Encounter/91f94a9d-69a7-e30a-cd1a-68c52dc01e70', 'female', 'white', 'not hispanic or latino', '662') +('98d4bd14-d78e-debb-e7dc-2df7786aedf3', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 89, datetime.date(2018, 7, 9), datetime.date(2018, 7, 9), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/e455ca3f-fc16-6ffc-297a-adc27e2db183', 'Encounter/98d4bd14-d78e-debb-e7dc-2df7786aedf3', 'male', 'white', 'hispanic or latino', '667') +('aa890974-162f-5906-dc71-2bb6d2185314', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 50, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/c7c5c028-f1fb-962b-8db2-dfd4c6d6b02a', 'Encounter/aa890974-162f-5906-dc71-2bb6d2185314', 'male', 'white', 'not hispanic or latino', '672') +('aabb3ac3-c4a3-f613-9507-63280adb9209', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 0, datetime.date(2018, 7, 31), datetime.date(2018, 7, 31), datetime.date(2018, 7, 30), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/2a0e1946-acf6-5a7e-9399-a9cbc4730199', 'Encounter/aabb3ac3-c4a3-f613-9507-63280adb9209', 'female', 'white', 'not hispanic or latino', '675') +('b5974881-ae62-ddd6-b905-8c86c1ca9e33', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '271737000', 'http://snomed.info/sct', 41, datetime.date(2018, 6, 25), datetime.date(2018, 6, 25), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/47c37c92-5932-9cfe-66be-208556780fe0', 'Encounter/b5974881-ae62-ddd6-b905-8c86c1ca9e33', 'male', 'white', 'not hispanic or latino', '675') +('b864bcd8-14e0-8bec-b7cc-f8629d91470e', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 15, datetime.date(2018, 6, 28), datetime.date(2018, 6, 28), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/c20e5afd-30df-ac3d-6684-cc29438a9bc4', 'Encounter/b864bcd8-14e0-8bec-b7cc-f8629d91470e', 'female', 'white', 'not hispanic or latino', '662') +('ba84689e-2f9f-7cea-af1f-d69ffdd3a3eb', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 39, datetime.date(2018, 6, 15), datetime.date(2018, 6, 15), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/7bf52d54-0d2a-265a-15aa-eeed7aaf4af6', 'Encounter/ba84689e-2f9f-7cea-af1f-d69ffdd3a3eb', 'female', 'white', 'not hispanic or latino', '670') +('bca7cabc-b2fc-8a08-c69b-5bc0afa20d80', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '271737000', 'http://snomed.info/sct', 28, datetime.date(2018, 7, 13), datetime.date(2018, 7, 13), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/3627adb8-f741-acf3-2dd6-10f3bcbe1077', 'Encounter/bca7cabc-b2fc-8a08-c69b-5bc0afa20d80', 'female', 'white', 'not hispanic or latino', '674') +('beb26500-4ccd-ce0a-44f6-74f44be5fafe', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 14, datetime.date(2018, 6, 7), datetime.date(2018, 6, 7), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/149de67a-2809-59a8-bfa2-36df509021dc', 'Encounter/beb26500-4ccd-ce0a-44f6-74f44be5fafe', 'female', 'white', 'not hispanic or latino', '667') +('c4605953-3103-ede6-e0c0-e0588e6f019e', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 61, datetime.date(2018, 7, 27), datetime.date(2018, 7, 27), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/a5b171a7-9b28-20e7-86a7-936ecbf55f36', 'Encounter/c4605953-3103-ede6-e0c0-e0588e6f019e', 'female', 'white', 'not hispanic or latino', '675') +('c6ec2350-43d4-abab-2e84-4d2aadb337a7', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 53, datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/26baae20-c8c5-003a-ab6b-ebcc49be20db', 'Encounter/c6ec2350-43d4-abab-2e84-4d2aadb337a7', 'male', 'white', 'not hispanic or latino', '678') +('ca45bbef-ef2a-3b3c-ea5e-76bcd5865780', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '72892002', 'http://snomed.info/sct', 48, datetime.date(2018, 6, 12), datetime.date(2018, 6, 12), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/9b17654f-a902-3d56-9000-4ade3dd3059f', 'Encounter/ca45bbef-ef2a-3b3c-ea5e-76bcd5865780', 'female', 'white', 'hispanic or latino', '672') +('d2782687-6885-037c-957d-579fbd681d2a', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 54, datetime.date(2018, 6, 13), datetime.date(2018, 6, 13), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/3cf7af45-2bee-aa9c-d524-40b487149d60', 'Encounter/d2782687-6885-037c-957d-579fbd681d2a', 'male', 'white', 'not hispanic or latino', '000') +('d5f342b7-017c-f2e7-8697-5a038c91518e', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '72892002', 'http://snomed.info/sct', 20, datetime.date(2018, 7, 28), datetime.date(2018, 7, 28), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/2da1bee2-2bc2-4511-84e1-42860310e2fb', 'Encounter/d5f342b7-017c-f2e7-8697-5a038c91518e', 'female', 'white', 'not hispanic or latino', '660') +('d735c414-9dd3-c9b1-285c-8da79a7fbbdf', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 61, datetime.date(2018, 7, 29), datetime.date(2018, 7, 29), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/7cef0e6f-9aea-4079-dfc6-18a96454708e', 'Encounter/d735c414-9dd3-c9b1-285c-8da79a7fbbdf', 'female', 'white', 'not hispanic or latino', '662') +('d73ed087-e0ae-78e0-7a05-1bac1060c476', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 45, datetime.date(2018, 7, 10), datetime.date(2018, 7, 10), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/51032f44-d514-26e9-3e85-e956561c076e', 'Encounter/d73ed087-e0ae-78e0-7a05-1bac1060c476', 'female', 'white', 'not hispanic or latino', '667') +('dc5ed645-3979-e765-3e03-6ba2173027c3', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 90, datetime.date(2018, 7, 7), datetime.date(2018, 7, 7), datetime.date(2018, 7, 2), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/ac91b90d-97e4-4fc5-41cd-036bac49e6e8', 'Encounter/dc5ed645-3979-e765-3e03-6ba2173027c3', 'male', 'white', 'not hispanic or latino', '671') +('de9d67de-6ae3-32f7-20f2-e719ae23a9a3', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 3, datetime.date(2018, 7, 30), datetime.date(2018, 7, 30), datetime.date(2018, 7, 30), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/2858705f-5af1-9869-4d94-894e09a9f99a', 'Encounter/de9d67de-6ae3-32f7-20f2-e719ae23a9a3', 'male', 'white', 'not hispanic or latino', '672') +('e5dabcb6-1d7a-7467-dbba-b864d0d5f5b0', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 49, datetime.date(2018, 7, 28), datetime.date(2018, 7, 28), datetime.date(2018, 7, 23), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/0b052286-9534-99a8-8d5e-06c2c04a7df7', 'Encounter/e5dabcb6-1d7a-7467-dbba-b864d0d5f5b0', 'female', 'white', 'not hispanic or latino', '662') +('e613f29d-7505-6f2e-a1f5-bfbec300752d', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '88805009', 'http://snomed.info/sct', 84, datetime.date(2018, 6, 29), datetime.date(2018, 6, 29), datetime.date(2018, 6, 25), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/1c498b42-61fd-6341-69c3-053f6e4fe404', 'Encounter/e613f29d-7505-6f2e-a1f5-bfbec300752d', 'female', 'white', 'not hispanic or latino', '667') +('e922a884-7039-a171-a65e-78051fe7afe6', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 1, datetime.date(2018, 6, 13), datetime.date(2018, 6, 13), datetime.date(2018, 6, 11), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/8022fbbe-aaa4-056c-d0f5-ec074bf0656c', 'Encounter/e922a884-7039-a171-a65e-78051fe7afe6', 'male', 'white', 'not hispanic or latino', '665') +('ed151e04-3dd6-8cb7-a3e5-777c8a8667f1', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '195662009', 'http://snomed.info/sct', 57, datetime.date(2018, 6, 6), datetime.date(2018, 6, 6), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/19158de4-66a2-f70f-e3bf-4396b312c8f1', 'Encounter/ed151e04-3dd6-8cb7-a3e5-777c8a8667f1', 'female', 'white', 'not hispanic or latino', '000') +('f2752dd7-1bf1-739d-dd8c-40122d0b63bc', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '161665007', 'http://snomed.info/sct', 77, datetime.date(2018, 6, 1), datetime.date(2018, 6, 2), datetime.date(2018, 5, 28), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/9c8d8539-0b1e-73e2-b64f-83f1ea601fa4', 'Encounter/f2752dd7-1bf1-739d-dd8c-40122d0b63bc', 'male', 'white', 'not hispanic or latino', '674') +('f964be66-3fcd-95c8-0021-71c7d24c91b7', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, '431857002', 'http://snomed.info/sct', 63, datetime.date(2018, 6, 5), datetime.date(2018, 6, 5), datetime.date(2018, 6, 4), datetime.date(2018, 6, 1), datetime.date(2018, 1, 1), 'Patient/a5bc08ea-9462-c4f5-1bd2-ff342598ac99', 'Encounter/f964be66-3fcd-95c8-0021-71c7d24c91b7', 'female', 'white', 'not hispanic or latino', '661') +('fd0754a4-e96d-cba7-b3c0-77697a09c86e', 'AMB', 'ambulatory', 'finished', None, None, None, None, None, None, None, None, 100, datetime.date(2018, 7, 10), datetime.date(2018, 7, 10), datetime.date(2018, 7, 9), datetime.date(2018, 7, 1), datetime.date(2018, 1, 1), 'Patient/6385ddd7-2639-6505-3789-0521b8f66c8b', 'Encounter/fd0754a4-e96d-cba7-b3c0-77697a09c86e', 'female', 'white', 'not hispanic or latino', '672') From abd2ba5cdef3327d0cb6bb6bee3924ed00472a48 Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Fri, 9 Feb 2024 11:37:36 -0500 Subject: [PATCH 2/4] test tweaks --- cumulus_library/study_parser.py | 6 +++++- tests/test_cli.py | 2 +- tests/test_study_parser.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cumulus_library/study_parser.py b/cumulus_library/study_parser.py index 390e7c0b..bdf4c47b 100644 --- a/cumulus_library/study_parser.py +++ b/cumulus_library/study_parser.py @@ -673,12 +673,13 @@ def export_study( schema_name: str, data_path: pathlib.Path, archive: bool, - ) -> None: + ) -> list: """Exports csvs/parquet extracts of tables listed in export_list :param db: A database backend :param schema_name: the schema/database to target :param data_path: the path to the place on disk to save data :param archive: If true, get all study data and zip with timestamp + :returns: a list of queries, (only for unit tests) """ table_list = [] path = pathlib.Path(f"{data_path}/{self.get_study_prefix()}/") @@ -692,6 +693,7 @@ def export_study( table_list.append(row[0]) else: table_list = self.get_export_table_list() + queries = [] for table in track( table_list, description=f"Exporting {self.get_study_prefix()} data...", @@ -707,6 +709,7 @@ def export_study( ) if not archive: dataframe.to_parquet(f"{path}/{table}.parquet", index=False) + queries.append(queries) if archive: file_list = [file for file in path.glob("**/*") if file.is_file()] timestamp = ( @@ -723,3 +726,4 @@ def export_study( f.write(file, file.relative_to(path)) file.unlink() shutil.rmtree(path) + return queries diff --git a/tests/test_cli.py b/tests/test_cli.py index 01acfc4b..37356982 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -316,7 +316,7 @@ def test_cli_executes_queries(tmp_path, build_args, export_args, expected_tables with open(f"{manifest_dir}/manifest.toml", encoding="UTF-8") as file: config = toml.load(file) - csv_files = glob.glob(f"{tmp_path}/counts/{build_args[2]}/*.csv") + csv_files = glob.glob(f"{tmp_path}/export/{build_args[2]}/*.csv") for export_table in config["export_config"]["export_list"]: assert any(export_table in x for x in csv_files) diff --git a/tests/test_study_parser.py b/tests/test_study_parser.py index f6af4c29..67f426c1 100644 --- a/tests/test_study_parser.py +++ b/tests/test_study_parser.py @@ -332,6 +332,6 @@ def test_export_study(tmp_path, mock_db_core): f"{Path(__file__).parent.parent}/cumulus_library/studies/core", data_path=f"{tmp_path}/export", ) - parser.export_study(mock_db_core, f"{tmp_path}/export") + parser.export_study(mock_db_core, None, f"{tmp_path}/export", False) for file in Path(f"{tmp_path}/export").glob("*.*"): assert file in parser.get_export_table_list() From f586e0ad81adc754a1cd444b6936290a3213a662 Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Tue, 13 Feb 2024 09:01:26 -0500 Subject: [PATCH 3/4] PR feedback --- cumulus_library/study_parser.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cumulus_library/study_parser.py b/cumulus_library/study_parser.py index bdf4c47b..2672d379 100644 --- a/cumulus_library/study_parser.py +++ b/cumulus_library/study_parser.py @@ -689,8 +689,7 @@ def export_study( schema_name, f"{self.get_study_prefix()}__" ) result = db.cursor().execute(table_query).fetchall() - for row in result: - table_list.append(row[0]) + table_list = [row[0] for row in result] else: table_list = self.get_export_table_list() queries = [] @@ -707,8 +706,7 @@ def export_study( dataframe.to_csv( f"{path}/{table}.csv", index=False, quoting=csv.QUOTE_MINIMAL ) - if not archive: - dataframe.to_parquet(f"{path}/{table}.parquet", index=False) + dataframe.to_parquet(f"{path}/{table}.parquet", index=False) queries.append(queries) if archive: file_list = [file for file in path.glob("**/*") if file.is_file()] From df7335ded4e81cdc49c980381ab89c18717948e2 Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Tue, 13 Feb 2024 09:16:23 -0500 Subject: [PATCH 4/4] moved zip to utils --- cumulus_library/base_utils.py | 17 +++++++++++++++++ cumulus_library/study_parser.py | 23 +++-------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cumulus_library/base_utils.py b/cumulus_library/base_utils.py index 213b9880..da59eecd 100644 --- a/cumulus_library/base_utils.py +++ b/cumulus_library/base_utils.py @@ -3,6 +3,8 @@ import datetime import json import os +import shutil +import zipfile from contextlib import contextmanager from rich import progress @@ -82,3 +84,18 @@ def get_tablename_safe_iso_timestamp() -> str: iso_timestamp = get_utc_datetime().isoformat() safe_timestamp = iso_timestamp.replace(":", "_").replace("-", "_").replace("+", "_") return safe_timestamp + + +def zip_dir(read_path, write_path, archive_name): + """Moves a directory to an archive""" + file_list = [file for file in read_path.glob("**/*") if file.is_file()] + timestamp = get_utc_datetime().isoformat().replace("+00:00", "Z") + with zipfile.ZipFile( + f"{write_path}/{archive_name}_{timestamp}.zip", + "w", + zipfile.ZIP_DEFLATED, + ) as f: + for file in file_list: + f.write(file, file.relative_to(read_path)) + file.unlink() + shutil.rmtree(read_path) diff --git a/cumulus_library/study_parser.py b/cumulus_library/study_parser.py index 2672d379..30f66164 100644 --- a/cumulus_library/study_parser.py +++ b/cumulus_library/study_parser.py @@ -1,13 +1,10 @@ """ Contains classes for loading study data based on manifest.toml files """ import csv -import datetime import importlib.util import inspect import pathlib -import shutil import sys -import zipfile import toml from rich.progress import Progress, TaskID, track @@ -681,8 +678,6 @@ def export_study( :param archive: If true, get all study data and zip with timestamp :returns: a list of queries, (only for unit tests) """ - table_list = [] - path = pathlib.Path(f"{data_path}/{self.get_study_prefix()}/") self.reset_counts_exports() if archive: table_query = base_templates.get_show_tables( @@ -692,7 +687,9 @@ def export_study( table_list = [row[0] for row in result] else: table_list = self.get_export_table_list() + queries = [] + path = pathlib.Path(f"{data_path}/{self.get_study_prefix()}/") for table in track( table_list, description=f"Exporting {self.get_study_prefix()} data...", @@ -709,19 +706,5 @@ def export_study( dataframe.to_parquet(f"{path}/{table}.parquet", index=False) queries.append(queries) if archive: - file_list = [file for file in path.glob("**/*") if file.is_file()] - timestamp = ( - datetime.datetime.now(datetime.timezone.utc) - .isoformat() - .replace("+00:00", "Z") - ) - with zipfile.ZipFile( - f"{data_path}/{self.get_study_prefix()}_{timestamp}.zip", - "w", - zipfile.ZIP_DEFLATED, - ) as f: - for file in file_list: - f.write(file, file.relative_to(path)) - file.unlink() - shutil.rmtree(path) + base_utils.zip_dir(path, data_path, self.get_study_prefix()) return queries