From 88eed269a5e7d691b7ac36a69e490b909984f9ee Mon Sep 17 00:00:00 2001 From: Matt Garber Date: Mon, 29 Jul 2024 15:42:41 -0400 Subject: [PATCH] ruff format, noqa jinja autoescape --- cumulus_library/actions/importer.py | 12 ++----- cumulus_library/actions/uploader.py | 7 ++-- cumulus_library/base_utils.py | 4 +-- cumulus_library/cli.py | 36 +++++-------------- cumulus_library/cli_parser.py | 20 +++-------- cumulus_library/databases/athena.py | 17 +++------ cumulus_library/databases/base.py | 4 +-- cumulus_library/databases/duckdb.py | 4 +-- cumulus_library/databases/utils.py | 10 ++---- cumulus_library/protected_table_builder.py | 6 ++-- cumulus_library/schema/typesystem.py | 12 ++----- cumulus_library/schema/valueset.py | 4 +-- cumulus_library/statistics/counts.py | 16 +++------ .../statistics_templates/counts_templates.py | 8 ++--- .../studies/core/builder_condition.py | 7 ++-- .../studies/core/builder_documentreference.py | 4 +-- .../studies/core/builder_medicationrequest.py | 4 +-- .../studies/core/builder_observation.py | 4 +-- .../studies/core/builder_patient.py | 8 ++--- .../core/core_templates/core_templates.py | 4 +-- cumulus_library/studies/core/count_core.py | 16 +++------ .../studies/discovery/code_detection.py | 11 ++---- cumulus_library/study_manifest.py | 4 +-- .../template_sql/base_templates.py | 6 ++-- tests/conftest.py | 8 ++--- tests/core/test_core.py | 4 +-- tests/core/test_core_completion.py | 4 +-- tests/core/test_core_meds.py | 3 +- tests/core/test_core_observation.py | 24 ++++--------- tests/core/test_core_patient.py | 8 ++--- tests/test_actions.py | 32 +++++------------ tests/test_base_templates.py | 4 +-- tests/test_cli.py | 23 ++++-------- tests/test_counts_builder.py | 12 ++----- tests/test_counts_templates.py | 4 +-- .../local_template.py | 4 +-- .../study_python_local_template/module1.py | 4 +-- tests/test_data/study_python_valid/module1.py | 4 +-- tests/test_databases.py | 4 +-- tests/test_duckdb.py | 14 ++------ tests/test_logging.py | 8 ++--- tests/test_psm.py | 6 +--- tests/test_psm_templates.py | 8 ++--- tests/test_study_parser.py | 9 ++--- tests/testbed_utils.py | 16 +++------ 45 files changed, 110 insertions(+), 321 deletions(-) diff --git a/cumulus_library/actions/importer.py b/cumulus_library/actions/importer.py index 8278ab41..151ddfe1 100644 --- a/cumulus_library/actions/importer.py +++ b/cumulus_library/actions/importer.py @@ -11,9 +11,7 @@ def _create_table_from_parquet(archive, file, study_name, config): try: - parquet_path = pathlib.Path( - archive.extract(file), path=tempfile.TemporaryFile() - ) + parquet_path = pathlib.Path(archive.extract(file), path=tempfile.TemporaryFile()) # While convenient to access, this exposes us to panda's type system, # which is messy - this could be optionally be replaced by pyarrow if it # becomes problematic. @@ -56,13 +54,9 @@ def import_archive(config: base_utils.StudyConfig, *, archive_path: pathlib.Path files = archive.namelist() files = [file for file in files if file.endswith(".parquet")] except zipfile.BadZipFile as e: - raise errors.StudyImportError( - f"File {archive_path} is not a valid archive." - ) from e + raise errors.StudyImportError(f"File {archive_path} is not a valid archive.") from e if not any("__" in file for file in files): - raise errors.StudyImportError( - f"File {archive_path} contains non-study parquet files." - ) + raise errors.StudyImportError(f"File {archive_path} contains non-study parquet files.") study_name = files[0].split("__")[0] for file in files[1:]: if file.split("__")[0] != study_name: diff --git a/cumulus_library/actions/uploader.py b/cumulus_library/actions/uploader.py index 7b29e5df..ad684e66 100644 --- a/cumulus_library/actions/uploader.py +++ b/cumulus_library/actions/uploader.py @@ -65,8 +65,7 @@ def upload_files(args: dict): """Wrapper to prep files & console output""" if args["data_path"] is None: sys.exit( - "No data directory provided - please provide a path to your" - "study export folder." + "No data directory provided - please provide a path to your" "study export folder." ) file_paths = list(args["data_path"].glob("**/*.parquet")) if args["target"]: @@ -79,9 +78,7 @@ def upload_files(args: dict): if not args["user"] or not args["id"]: sys.exit("user/id not provided, please pass --user and --id") try: - meta_version = next( - filter(lambda x: str(x).endswith("__meta_version.parquet"), file_paths) - ) + meta_version = next(filter(lambda x: str(x).endswith("__meta_version.parquet"), file_paths)) version = str(read_parquet(meta_version)["data_package_version"][0]) file_paths.remove(meta_version) except StopIteration: diff --git a/cumulus_library/base_utils.py b/cumulus_library/base_utils.py index 062ca654..adf06955 100644 --- a/cumulus_library/base_utils.py +++ b/cumulus_library/base_utils.py @@ -124,9 +124,7 @@ def zip_dir(read_path, write_path, archive_name): shutil.rmtree(read_path) -def update_query_if_schema_specified( - query: str, manifest: study_manifest.StudyManifest -): +def update_query_if_schema_specified(query: str, manifest: study_manifest.StudyManifest): if manifest and manifest.get_dedicated_schema(): # External queries in athena require a schema to be specified already, so # rather than splitting and ending up with a table name like diff --git a/cumulus_library/cli.py b/cumulus_library/cli.py index 74063f22..39bd55e5 100755 --- a/cumulus_library/cli.py +++ b/cumulus_library/cli.py @@ -98,9 +98,7 @@ def clean_and_build_study( """ manifest = study_manifest.StudyManifest(target, self.data_path) try: - builder.run_protected_table_builder( - config=self.get_config(manifest), manifest=manifest - ) + builder.run_protected_table_builder(config=self.get_config(manifest), manifest=manifest) if not continue_from: log_utils.log_transaction( config=self.get_config(manifest), @@ -111,9 +109,7 @@ def clean_and_build_study( config=self.get_config(manifest), manifest=manifest, ) - builder.run_table_builder( - config=self.get_config(manifest), manifest=manifest - ) + builder.run_table_builder(config=self.get_config(manifest), manifest=manifest) else: log_utils.log_transaction( @@ -127,9 +123,7 @@ def clean_and_build_study( manifest=manifest, continue_from=continue_from, ) - builder.run_counts_builders( - config=self.get_config(manifest), manifest=manifest - ) + builder.run_counts_builders(config=self.get_config(manifest), manifest=manifest) builder.run_statistics_builders( config=self.get_config(manifest), manifest=manifest, @@ -170,9 +164,7 @@ def run_matching_table_builder( ) ### Data exporters - def export_study( - self, target: pathlib.Path, data_path: pathlib.Path, archive: bool - ) -> None: + def export_study(self, target: pathlib.Path, data_path: pathlib.Path, archive: bool) -> None: """Exports aggregates defined in a manifest :param target: A path to the study directory @@ -199,9 +191,7 @@ def generate_study_sql( :keyword builder: Specify a single builder to generate sql from """ manifest = study_manifest.StudyManifest(target) - file_generator.run_generate_sql( - config=self.get_config(manifest), manifest=manifest - ) + file_generator.run_generate_sql(config=self.get_config(manifest), manifest=manifest) def generate_study_markdown( self, @@ -212,9 +202,7 @@ def generate_study_markdown( :param target: A path to the study directory """ manifest = study_manifest.StudyManifest(target) - file_generator.run_generate_markdown( - config=self.get_config(manifest), manifest=manifest - ) + file_generator.run_generate_markdown(config=self.get_config(manifest), manifest=manifest) def get_abs_path(path: str) -> pathlib.Path: @@ -324,9 +312,7 @@ def run_cli(args: dict): elif args["action"] == "build": for target in args["target"]: if args["builder"]: - runner.run_matching_table_builder( - study_dict[target], args["builder"] - ) + runner.run_matching_table_builder(study_dict[target], args["builder"]) else: runner.clean_and_build_study( study_dict[target], @@ -350,9 +336,7 @@ def run_cli(args: dict): if response.lower() != "y": sys.exit() for target in args["target"]: - runner.export_study( - study_dict[target], args["data_path"], args["archive"] - ) + runner.export_study(study_dict[target], args["data_path"], args["archive"]) elif args["action"] == "import": for archive in args["archive_path"]: @@ -361,9 +345,7 @@ def run_cli(args: dict): elif args["action"] == "generate-sql": for target in args["target"]: - runner.generate_study_sql( - study_dict[target], builder=args["builder"] - ) + runner.generate_study_sql(study_dict[target], builder=args["builder"]) elif args["action"] == "generate-md": for target in args["target"]: diff --git a/cumulus_library/cli_parser.py b/cumulus_library/cli_parser.py index 0c4ac67e..a61d5712 100644 --- a/cumulus_library/cli_parser.py +++ b/cumulus_library/cli_parser.py @@ -228,9 +228,7 @@ def get_parser() -> argparse.ArgumentParser: # Database export - export = actions.add_parser( - "export", help="Generates files on disk from Athena tables/views" - ) + export = actions.add_parser("export", help="Generates files on disk from Athena tables/views") add_custom_option(export) add_target_argument(export) add_study_dir_argument(export) @@ -245,9 +243,7 @@ def get_parser() -> argparse.ArgumentParser: # Database import - importer = actions.add_parser( - "import", help="Recreates a study from an exported archive" - ) + importer = actions.add_parser("import", help="Recreates a study from an exported archive") add_db_config(importer) add_verbose_argument(importer) importer.add_argument( @@ -258,15 +254,11 @@ def get_parser() -> argparse.ArgumentParser: ) # Aggregator upload - upload = actions.add_parser( - "upload", help="Bulk uploads data to Cumulus aggregator" - ) + upload = actions.add_parser("upload", help="Bulk uploads data to Cumulus aggregator") add_data_path_argument(upload) add_target_argument(upload) - upload.add_argument( - "--id", help="Site ID. Default is value of CUMULUS_AGGREGATOR_ID" - ) + upload.add_argument("--id", help="Site ID. Default is value of CUMULUS_AGGREGATOR_ID") upload.add_argument( "--preview", default=False, @@ -281,9 +273,7 @@ def get_parser() -> argparse.ArgumentParser: ), default="https://aggregator.smartcumulus.org/upload/", ) - upload.add_argument( - "--user", help="Cumulus user. Default is value of CUMULUS_AGGREGATOR_USER" - ) + upload.add_argument("--user", help="Cumulus user. Default is value of CUMULUS_AGGREGATOR_USER") # Generate a study's template-driven sql diff --git a/cumulus_library/databases/athena.py b/cumulus_library/databases/athena.py index 2bed8f37..6ee0bb44 100644 --- a/cumulus_library/databases/athena.py +++ b/cumulus_library/databases/athena.py @@ -76,10 +76,7 @@ def col_parquet_types_from_pandas(self, field_types: list) -> list: match field: case numpy.dtypes.ObjectDType(): output.append("STRING") - case ( - pandas.core.arrays.integer.Int64Dtype() - | numpy.dtypes.Int64DType() - ): + case pandas.core.arrays.integer.Int64Dtype() | numpy.dtypes.Int64DType(): output.append("INT") case numpy.dtypes.Float64DType(): output.append("DOUBLE") @@ -114,9 +111,7 @@ def col_pyarrow_types_from_sql(self, columns: list[tuple]) -> list: case "timestamp": output.append((column[0], pyarrow.timestamp("s"))) case _: - raise errors.CumulusLibraryError( - f"Unsupported SQL type '{column[1]}' found." - ) + raise errors.CumulusLibraryError(f"Unsupported SQL type '{column[1]}' found.") return output def upload_file( @@ -134,18 +129,14 @@ def upload_file( s3_path = wg_conf["OutputLocation"] bucket = "/".join(s3_path.split("/")[2:3]) key_prefix = "/".join(s3_path.split("/")[3:]) - encryption_type = wg_conf.get("EncryptionConfiguration", {}).get( - "EncryptionOption", {} - ) + encryption_type = wg_conf.get("EncryptionConfiguration", {}).get("EncryptionOption", {}) if encryption_type != "SSE_KMS": raise errors.AWSError( f"Bucket {bucket} has unexpected encryption type {encryption_type}." "AWS KMS encryption is expected for Cumulus buckets" ) kms_arn = wg_conf.get("EncryptionConfiguration", {}).get("KmsKey", None) - s3_key = ( - f"{key_prefix}cumulus_user_uploads/{self.schema_name}/" f"{study}/{topic}" - ) + s3_key = f"{key_prefix}cumulus_user_uploads/{self.schema_name}/" f"{study}/{topic}" if not remote_filename: remote_filename = file.name diff --git a/cumulus_library/databases/base.py b/cumulus_library/databases/base.py index a2a7bd29..376d582c 100644 --- a/cumulus_library/databases/base.py +++ b/cumulus_library/databases/base.py @@ -92,9 +92,7 @@ def _recursively_validate( return output - def validate_table_schema( - self, expected: dict[str, list], schema: list[tuple] - ) -> dict: + def validate_table_schema(self, expected: dict[str, list], schema: list[tuple]) -> dict: """Public interface for investigating if fields are in a table schema. expected is a dictionary of string column names to *something*: diff --git a/cumulus_library/databases/duckdb.py b/cumulus_library/databases/duckdb.py index 4ce1cdab..c483183c 100644 --- a/cumulus_library/databases/duckdb.py +++ b/cumulus_library/databases/duckdb.py @@ -94,9 +94,7 @@ def insert_tables(self, tables: dict[str, pyarrow.Table]) -> None: self.connection.register(name, table) @staticmethod - def _compat_array_join( - value: list[str | None] | None, delimiter: str | None - ) -> str | None: + def _compat_array_join(value: list[str | None] | None, delimiter: str | None) -> str | None: if value is None: return None if delimiter is None or delimiter == "None": diff --git a/cumulus_library/databases/utils.py b/cumulus_library/databases/utils.py index 301b1cd5..6fd11318 100644 --- a/cumulus_library/databases/utils.py +++ b/cumulus_library/databases/utils.py @@ -61,11 +61,7 @@ def read_ndjson_dir(path: str) -> dict[str, pyarrow.Table]: "etl__completion_encounters", ] for metadata_table in metadata_tables: - rows = list( - cumulus_fhir_support.read_multiline_json_from_dir( - f"{path}/{metadata_table}" - ) - ) + rows = list(cumulus_fhir_support.read_multiline_json_from_dir(f"{path}/{metadata_table}")) if rows: # Auto-detecting the schema works for these simple tables all_tables[metadata_table] = pyarrow.Table.from_pylist(rows) @@ -113,8 +109,6 @@ def create_db_backend(args: dict[str, str]) -> (base.DatabaseBackend, str): if args.get("load_ndjson_dir"): sys.exit("Loading an ndjson dir is not supported with --db-type=athena.") else: - raise errors.CumulusLibraryError( - f"'{db_config.db_type}' is not a supported database." - ) + raise errors.CumulusLibraryError(f"'{db_config.db_type}' is not a supported database.") return (backend, schema_name) diff --git a/cumulus_library/protected_table_builder.py b/cumulus_library/protected_table_builder.py index 78bd8650..85ee2fa4 100644 --- a/cumulus_library/protected_table_builder.py +++ b/cumulus_library/protected_table_builder.py @@ -51,12 +51,10 @@ def prepare_queries( else: db_schema = config.schema transactions = ( - f"{manifest.get_study_prefix()}" - f"__{enums.ProtectedTables.TRANSACTIONS.value}" + f"{manifest.get_study_prefix()}" f"__{enums.ProtectedTables.TRANSACTIONS.value}" ) statistics = ( - f"{manifest.get_study_prefix()}" - f"__{enums.ProtectedTables.STATISTICS.value}" + f"{manifest.get_study_prefix()}" f"__{enums.ProtectedTables.STATISTICS.value}" ) self.queries.append( base_templates.get_ctas_empty_query( diff --git a/cumulus_library/schema/typesystem.py b/cumulus_library/schema/typesystem.py index 296f7099..f64852fa 100644 --- a/cumulus_library/schema/typesystem.py +++ b/cumulus_library/schema/typesystem.py @@ -16,16 +16,10 @@ class Structure(FHIR): Patient = "http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient" Encounter = "http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter" - DocumentReference = ( - "http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference" - ) + DocumentReference = "http://hl7.org/fhir/us/core/StructureDefinition/us-core-documentreference" Condition = "http://hl7.org/fhir/condition-definitions.html" - ObservationLab = ( - "http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab" - ) - ObservationValue = ( - "http://hl7.org/fhir/observation-definitions.html#Observation.value_x_" - ) + ObservationLab = "http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab" + ObservationValue = "http://hl7.org/fhir/observation-definitions.html#Observation.value_x_" VitalSign = "http://hl7.org/fhir/us/vitals/ImplementationGuide/hl7.fhir.us.vitals" diff --git a/cumulus_library/schema/valueset.py b/cumulus_library/schema/valueset.py index 4c5ed990..5ece5ff3 100644 --- a/cumulus_library/schema/valueset.py +++ b/cumulus_library/schema/valueset.py @@ -26,9 +26,7 @@ class ValueSet(Enum): DocumentType = "http://hl7.org/fhir/ValueSet/c80-doc-typecodes" ObservationCode = "http://hl7.org/fhir/ValueSet/observation-codes" ObservationCategory = "http://hl7.org/fhir/ValueSet/observation-category" - ObservationInterpretation = ( - "http://hl7.org/fhir/ValueSet/observation-interpretation" - ) + ObservationInterpretation = "http://hl7.org/fhir/ValueSet/observation-interpretation" def __init__(self, url: str): self.url = url diff --git a/cumulus_library/statistics/counts.py b/cumulus_library/statistics/counts.py index a973642c..48d6bbe2 100644 --- a/cumulus_library/statistics/counts.py +++ b/cumulus_library/statistics/counts.py @@ -39,9 +39,7 @@ def get_table_name(self, table_name: str, duration=None) -> str: else: return f"{self.study_prefix}__{table_name}" - def get_where_clauses( - self, clause: list | str | None = None, min_subject: int = 10 - ) -> str: + def get_where_clauses(self, clause: list | str | None = None, min_subject: int = 10) -> str: """Convenience method for constructing arbitrary where clauses. :param clause: either a string or a list of sql where statements @@ -55,9 +53,7 @@ def get_where_clauses( elif isinstance(clause, list): return clause else: - raise errors.CountsBuilderError( - f"get_where_clauses invalid clause {clause}" - ) + raise errors.CountsBuilderError(f"get_where_clauses invalid clause {clause}") def get_count_query( self, table_name: str, source_table: str, table_cols: list, **kwargs @@ -84,12 +80,8 @@ def get_count_query( "fhir_resource", "filter_resource", ]: - raise errors.CountsBuilderError( - f"count_query received unexpected key: {key}" - ) - return counts_templates.get_count_query( - table_name, source_table, table_cols, **kwargs - ) + raise errors.CountsBuilderError(f"count_query received unexpected key: {key}") + return counts_templates.get_count_query(table_name, source_table, table_cols, **kwargs) # ---------------------------------------------------------------------- # The following function all wrap get_count_query as convenience methods. diff --git a/cumulus_library/statistics/statistics_templates/counts_templates.py b/cumulus_library/statistics/statistics_templates/counts_templates.py index e743b029..7e1320d6 100644 --- a/cumulus_library/statistics/statistics_templates/counts_templates.py +++ b/cumulus_library/statistics/statistics_templates/counts_templates.py @@ -52,13 +52,9 @@ def get_count_query( for item in table_cols: # TODO: remove check after cutover if isinstance(item, list): - table_col_classed.append( - CountColumn(name=item[0], db_type=item[1], alias=item[2]) - ) + table_col_classed.append(CountColumn(name=item[0], db_type=item[1], alias=item[2])) else: - table_col_classed.append( - CountColumn(name=item, db_type="varchar", alias=None) - ) + table_col_classed.append(CountColumn(name=item, db_type="varchar", alias=None)) table_cols = table_col_classed query = base_templates.get_base_template( diff --git a/cumulus_library/studies/core/builder_condition.py b/cumulus_library/studies/core/builder_condition.py index 8ab22e58..b9954a72 100644 --- a/cumulus_library/studies/core/builder_condition.py +++ b/cumulus_library/studies/core/builder_condition.py @@ -69,8 +69,7 @@ def denormalize_codes(self): ), ] self.queries += [ - base_templates.get_codeable_concept_denormalize_query(config) - for config in configs + base_templates.get_codeable_concept_denormalize_query(config) for config in configs ] def prepare_queries( @@ -81,6 +80,4 @@ def prepare_queries( ): self.denormalize_codes() validated_schema = sql_utils.validate_schema(config.db, expected_table_cols) - self.queries.append( - core_templates.get_core_template("condition", validated_schema) - ) + self.queries.append(core_templates.get_core_template("condition", validated_schema)) diff --git a/cumulus_library/studies/core/builder_documentreference.py b/cumulus_library/studies/core/builder_documentreference.py index 94a2a531..c58221c0 100644 --- a/cumulus_library/studies/core/builder_documentreference.py +++ b/cumulus_library/studies/core/builder_documentreference.py @@ -59,6 +59,4 @@ def prepare_queries( ], ) validated_schema = sql_utils.validate_schema(config.db, expected_table_cols) - self.queries.append( - core_templates.get_core_template("documentreference", validated_schema) - ) + self.queries.append(core_templates.get_core_template("documentreference", validated_schema)) diff --git a/cumulus_library/studies/core/builder_medicationrequest.py b/cumulus_library/studies/core/builder_medicationrequest.py index ce5a3e17..466dd1d2 100644 --- a/cumulus_library/studies/core/builder_medicationrequest.py +++ b/cumulus_library/studies/core/builder_medicationrequest.py @@ -44,6 +44,4 @@ def prepare_queries( ] self.queries += sql_utils.denormalize_complex_objects(config.db, code_sources) validated_schema = sql_utils.validate_schema(config.db, expected_table_cols) - self.queries.append( - core_templates.get_core_template("medicationrequest", validated_schema) - ) + self.queries.append(core_templates.get_core_template("medicationrequest", validated_schema)) diff --git a/cumulus_library/studies/core/builder_observation.py b/cumulus_library/studies/core/builder_observation.py index d1d50f98..d6b24515 100644 --- a/cumulus_library/studies/core/builder_observation.py +++ b/cumulus_library/studies/core/builder_observation.py @@ -72,9 +72,7 @@ def prepare_queries( column_hierarchy=[("component", list), ("valuecodeableconcept", dict)], expected={"valuecodeableconcept": sql_utils.CODEABLE_CONCEPT}, ), - ObsConfig( - column_hierarchy=[("interpretation", list)], filter_priority=False - ), + ObsConfig(column_hierarchy=[("interpretation", list)], filter_priority=False), ObsConfig( column_hierarchy=[("valuecodeableconcept", dict)], filter_priority=False, diff --git a/cumulus_library/studies/core/builder_patient.py b/cumulus_library/studies/core/builder_patient.py index fc4b256e..aaddb913 100644 --- a/cumulus_library/studies/core/builder_patient.py +++ b/cumulus_library/studies/core/builder_patient.py @@ -76,12 +76,8 @@ def prepare_queries( ] for extension in extension_types: self.queries.append( - self.make_extension_query( - config.db, extension["name"], extension["fhirpath"] - ) + self.make_extension_query(config.db, extension["name"], extension["fhirpath"]) ) validated_schema = sql_utils.validate_schema(config.db, expected_table_cols) - self.queries.append( - core_templates.get_core_template("patient", validated_schema) - ) + self.queries.append(core_templates.get_core_template("patient", validated_schema)) diff --git a/cumulus_library/studies/core/core_templates/core_templates.py b/cumulus_library/studies/core/core_templates/core_templates.py index 8d11a325..c915538b 100644 --- a/cumulus_library/studies/core/core_templates/core_templates.py +++ b/cumulus_library/studies/core/core_templates/core_templates.py @@ -11,6 +11,4 @@ def get_core_template( config: dict | None = None, ) -> str: """Extracts code system details as a standalone table""" - return base_templates.get_base_template( - target_table, path=PATH, schema=schema, config=config - ) + return base_templates.get_base_template(target_table, path=PATH, schema=schema, config=config) diff --git a/cumulus_library/studies/core/count_core.py b/cumulus_library/studies/core/count_core.py index 7c7eaf93..bfed77a1 100644 --- a/cumulus_library/studies/core/count_core.py +++ b/cumulus_library/studies/core/count_core.py @@ -40,9 +40,7 @@ def count_core_encounter(self, duration: str | None = None): return self.count_encounter(table_name, from_table, cols) - def _count_core_encounter_type( - self, table_name: str, cols: list, duration: str | None = None - ): + def _count_core_encounter_type(self, table_name: str, cols: list, duration: str | None = None): """ Encounter Type information is for every visit, and therefore this SQL should be precise in which fields to select (This is a BIG query). @@ -67,9 +65,7 @@ def count_core_encounter_all_types(self, duration: str | None = None): "serviceType_display", "priority_display", ] - return self._count_core_encounter_type( - "count_encounter_all_types", cols, duration - ) + return self._count_core_encounter_type("count_encounter_all_types", cols, duration) # The following encounter tables all count on one specific code type @@ -79,15 +75,11 @@ def count_core_encounter_enc_type(self, duration: str = "month"): def count_core_encounter_priority(self, duration: str = "month"): cols = ["class_display", "priority_display"] - return self._count_core_encounter_type( - "count_encounter_priority", cols, duration - ) + return self._count_core_encounter_type("count_encounter_priority", cols, duration) def count_core_encounter_service(self, duration: str = "month"): cols = ["class_display", "serviceType_display"] - return self._count_core_encounter_type( - "count_encounter_service", cols, duration - ) + return self._count_core_encounter_type("count_encounter_service", cols, duration) def count_core_medicationrequest(self, duration: str = "month"): table_name = self.get_table_name("count_medicationrequest", duration=duration) diff --git a/cumulus_library/studies/discovery/code_detection.py b/cumulus_library/studies/discovery/code_detection.py index 36bd22fc..5a7f2217 100644 --- a/cumulus_library/studies/discovery/code_detection.py +++ b/cumulus_library/studies/discovery/code_detection.py @@ -28,9 +28,7 @@ def _check_codes_in_fields(self, code_sources: list[dict], database) -> dict: total=len(code_sources), ) for code_source in code_sources: - code_source["has_data"] = self._check_coding_against_db( - code_source, database - ) + code_source["has_data"] = self._check_coding_against_db(code_source, database) progress.advance(task) return code_sources @@ -50,8 +48,7 @@ def prepare_queries( for code_definition in code_definitions.code_list: if not required_keys.issubset(code_definition): raise KeyError( - "Expected table_name and column_hierarchy keys in " - f"{code_definition!s}" + "Expected table_name and column_hierarchy keys in " f"{code_definition!s}" ) code_source = { "has_data": False, @@ -60,7 +57,5 @@ def prepare_queries( code_source[key] = code_definition[key] code_sources.append(code_source) code_sources = self._check_codes_in_fields(code_sources, config.db) - query = discovery_templates.get_system_pairs( - "discovery__code_sources", code_sources - ) + query = discovery_templates.get_system_pairs("discovery__code_sources", code_sources) self.queries.append(query) diff --git a/cumulus_library/study_manifest.py b/cumulus_library/study_manifest.py index 42a028a3..bd46a38d 100644 --- a/cumulus_library/study_manifest.py +++ b/cumulus_library/study_manifest.py @@ -37,9 +37,7 @@ def load_study_manifest(self, study_path: pathlib.Path) -> None: try: with open(f"{study_path}/manifest.toml", "rb") as file: config = tomllib.load(file) - if not config.get("study_prefix") or not isinstance( - config["study_prefix"], str - ): + if not config.get("study_prefix") or not isinstance(config["study_prefix"], str): raise errors.StudyManifestParsingError( f"Invalid prefix in manifest at {study_path}" ) diff --git a/cumulus_library/template_sql/base_templates.py b/cumulus_library/template_sql/base_templates.py index 28e12267..2a92d8bf 100644 --- a/cumulus_library/template_sql/base_templates.py +++ b/cumulus_library/template_sql/base_templates.py @@ -34,7 +34,7 @@ def get_base_template(filename_stem: str, path: pathlib.Path | None = None, **kw if path: macro_paths.append(path) loader = jinja2.FileSystemLoader(macro_paths) - env = jinja2.Environment(loader=loader, autoescape=True).from_string(template) + env = jinja2.Environment(loader=loader).from_string(template) # noqa: S701 return env.render(**kwargs, db_type=db_config.db_type) @@ -62,7 +62,7 @@ def get_codeable_concept_denormalize_query( :param config: a CodableConeptConfig """ - if config.column_hierarchy > 2: + if len(config.column_hierarchy) > 2: raise errors.CumulusLibraryError( "base_templates.get_codeable_concept_denormalize_query expects a column " f"hierarchy of length 2 or less, but received: {config.column_hierarchy}" @@ -102,7 +102,7 @@ def get_coding_denormalize_query( :param config: a CodingConfig """ - if config.column_hierarchy > 2: + if len(config.column_hierarchy) > 2: raise errors.CumulusLibraryError( "base_templates.get_coding_denormalize_query expects a column " f"hierarchy of length 2, but received: {config.column_hierarchy}" diff --git a/tests/conftest.py b/tests/conftest.py index 0eeab46d..f2eb60c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -100,9 +100,7 @@ def update_nested_obj(id_path, obj, i): obj[id_path[0]] = obj[id_path[0]] + str(i) else: if isinstance(obj, list): - obj[0][id_path[0]] = update_nested_obj( - id_path[1:], obj[0][id_path[0]], i - ) + obj[0][id_path[0]] = update_nested_obj(id_path[1:], obj[0][id_path[0]], i) else: obj[id_path[0]] = update_nested_obj(id_path[1:], obj[id_path[0]], i) @@ -131,9 +129,7 @@ def update_nested_obj(id_path, obj, i): # workaround for pandas/null/boolean casting issues for null_bool_col in ["multipleBirthBoolean"]: if null_bool_col in output_df.columns: - output_df[null_bool_col] = output_df[null_bool_col].replace( - {0.0: False} - ) + output_df[null_bool_col] = output_df[null_bool_col].replace({0.0: False}) output_df = output_df.replace({numpy.nan: None}) write_path = Path(str(target_dir) + f"/{key}/{filepath.name}") diff --git a/tests/core/test_core.py b/tests/core/test_core.py index f965d5a2..9645f486 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -72,9 +72,7 @@ def test_core_count_missing_data(tmp_path): ) cursor = testbed.build() - table_rows, cols = conftest.get_sorted_table_data( - cursor, "core__count_encounter_month" - ) + table_rows, cols = conftest.get_sorted_table_data(cursor, "core__count_encounter_month") # For regenerating data if needed # with open( # f"./tests/test_data/core/core__count_encounter_month_missing_data.txt", diff --git a/tests/core/test_core_completion.py b/tests/core/test_core_completion.py index b6945c35..325454c6 100644 --- a/tests/core/test_core_completion.py +++ b/tests/core/test_core_completion.py @@ -119,9 +119,7 @@ def test_completion_multiple_groups(tmp_path): testbed.add_encounter("Incomplete") # early observation load (G1) testbed.add_etl_completion_encounters(group="G1", ids=["Incomplete"], time="2021") - testbed.add_etl_completion_encounters( - group="G2", ids=["Complete", "Incomplete"], time="2021" - ) + testbed.add_etl_completion_encounters(group="G2", ids=["Complete", "Incomplete"], time="2021") testbed.add_etl_completion_encounters(group="G3", ids=["Complete"], time="2021") testbed.add_etl_completion_encounters( group="Unreferenced", ids=["Complete", "Incomplete"], time="2021" diff --git a/tests/core/test_core_meds.py b/tests/core/test_core_meds.py index a50bbcdd..f7e2491e 100644 --- a/tests/core/test_core_meds.py +++ b/tests/core/test_core_meds.py @@ -58,8 +58,7 @@ def test_core_medreq_only_rxnorm(tmp_path, codings, expected): testbed.add_medication_request("A", codings=codings) con = testbed.build() codes = con.sql( - "SELECT medication_code FROM core__medicationrequest " - "ORDER BY medication_code" + "SELECT medication_code FROM core__medicationrequest " "ORDER BY medication_code" ).fetchall() assert [x[0] for x in codes] == expected diff --git a/tests/core/test_core_observation.py b/tests/core/test_core_observation.py index d5c768a9..726ae119 100644 --- a/tests/core/test_core_observation.py +++ b/tests/core/test_core_observation.py @@ -33,9 +33,7 @@ def test_core_observation_component_low_schema(tmp_path): # Spot check some tables (a basic one, then the custom weird valuequantity one) rows = con.sql("SELECT id, row FROM core__observation_component_code").fetchall() assert 0 == len(rows) - rows = con.sql( - "SELECT id FROM core__observation_component_valuequantity" - ).fetchall() + rows = con.sql("SELECT id FROM core__observation_component_valuequantity").fetchall() assert 0 == len(rows) _assert_valuequantity_schema(con) @@ -55,9 +53,7 @@ def test_core_observation_component_valuequantity_low_schema(tmp_path): ], ) con = testbed.build() # most importantly, this shouldn't blow up - rows = con.sql( - "SELECT id FROM core__observation_component_valuequantity" - ).fetchall() + rows = con.sql("SELECT id FROM core__observation_component_valuequantity").fetchall() assert 1 == len(rows) _assert_valuequantity_schema(con) @@ -143,9 +139,7 @@ def test_core_observation_component(tmp_path): ) con = testbed.build() - df = con.sql( - "SELECT * FROM core__observation_component_code ORDER BY id, row, code" - ).df() + df = con.sql("SELECT * FROM core__observation_component_code ORDER BY id, row, code").df() rows = json.loads(df.to_json(orient="records")) assert [ { @@ -191,8 +185,7 @@ def test_core_observation_component(tmp_path): ] == rows df = con.sql( - "SELECT * FROM core__observation_component_dataabsentreason " - "ORDER BY id, row, code" + "SELECT * FROM core__observation_component_dataabsentreason " "ORDER BY id, row, code" ).df() rows = json.loads(df.to_json(orient="records")) assert [ @@ -223,8 +216,7 @@ def test_core_observation_component(tmp_path): ] == rows df = con.sql( - "SELECT * FROM core__observation_component_interpretation " - "ORDER BY id, row, code" + "SELECT * FROM core__observation_component_interpretation " "ORDER BY id, row, code" ).df() rows = json.loads(df.to_json(orient="records")) assert [ @@ -271,8 +263,7 @@ def test_core_observation_component(tmp_path): ] == rows df = con.sql( - "SELECT * FROM core__observation_component_valuecodeableconcept " - "ORDER BY id, row, code" + "SELECT * FROM core__observation_component_valuecodeableconcept " "ORDER BY id, row, code" ).df() rows = json.loads(df.to_json(orient="records")) assert [ @@ -295,8 +286,7 @@ def test_core_observation_component(tmp_path): ] == rows df = con.sql( - "SELECT * FROM core__observation_component_valuequantity " - "ORDER BY id, row, code" + "SELECT * FROM core__observation_component_valuequantity " "ORDER BY id, row, code" ).df() rows = json.loads(df.to_json(orient="records")) assert [ diff --git a/tests/core/test_core_patient.py b/tests/core/test_core_patient.py index 31bc15e5..38a604de 100644 --- a/tests/core/test_core_patient.py +++ b/tests/core/test_core_patient.py @@ -157,14 +157,10 @@ def test_core_patient_addresses(tmp_path, addresses, expected): ), ], ) -def test_core_patient_extensions( - tmp_path, extensions, expected_ethnicity, expected_race -): +def test_core_patient_extensions(tmp_path, extensions, expected_ethnicity, expected_race): """Verify that we grab race & ethnicity correctly""" testbed = testbed_utils.LocalTestbed(tmp_path, with_patient=False) testbed.add_patient("A", extension=extensions) con = testbed.build() - displays = con.sql( - "SELECT ethnicity_display, race_display FROM core__patient" - ).fetchall() + displays = con.sql("SELECT ethnicity_display, race_display FROM core__patient").fetchall() assert [(expected_ethnicity, expected_race)] == displays diff --git a/tests/test_actions.py b/tests/test_actions.py index 24dcc501..d9441bec 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -66,9 +66,7 @@ def test_clean_study(mock_db_config, verbose, prefix, confirm, stats, target, ra "AS SELECT 'study_valid' as study_name, " "'study_valid__123' AS table_name" ) - mock_db_config.db.cursor().execute( - "CREATE TABLE study_valid__123 (test int)" - ) + mock_db_config.db.cursor().execute("CREATE TABLE study_valid__123 (test int)") mock_db_config.db.cursor().execute( "CREATE VIEW study_valid__456 AS SELECT * FROM study_valid__123" ) @@ -123,9 +121,7 @@ def test_run_protected_table_builder(mock_db_config, study_path, stats): .execute("SELECT distinct(table_name) FROM information_schema.tables ") .fetchall() ) - assert ( - f"{manifest.get_study_prefix()}__{enums.ProtectedTables.TRANSACTIONS.value}", - ) in tables + assert (f"{manifest.get_study_prefix()}__{enums.ProtectedTables.TRANSACTIONS.value}",) in tables if stats: assert ( f"{manifest.get_study_prefix()}__{enums.ProtectedTables.STATISTICS.value}", @@ -345,13 +341,9 @@ def test_import_study(tmp_path, mock_db_config): (tmp_path / "archive/test__table.parquet").unlink() (tmp_path / "archive/test__table.csv").unlink() mock_db_config.schema = "schema_name" - importer.import_archive( - config=mock_db_config, archive_path=tmp_path / "archive/test.zip" - ) + importer.import_archive(config=mock_db_config, archive_path=tmp_path / "archive/test.zip") assert len(list((tmp_path / "archive").glob("*"))) == 1 - test_table = ( - mock_db_config.db.cursor().execute("SELECT * FROM test__table").fetchall() - ) + test_table = mock_db_config.db.cursor().execute("SELECT * FROM test__table").fetchall() assert test_table == [ ("a", 1, 1.1, True, datetime.datetime(2024, 1, 1, 0, 0)), ("b", 2, 2.2, False, datetime.datetime(2024, 1, 1, 0, 0)), @@ -364,13 +356,9 @@ def test_import_study(tmp_path, mock_db_config): with pytest.raises(errors.StudyImportError): with open(tmp_path / "archive/empty.zip", "w"): pass - importer.import_archive( - config=mock_db_config, archive_path=tmp_path / "archive/empty.zip" - ) + importer.import_archive(config=mock_db_config, archive_path=tmp_path / "archive/empty.zip") with pytest.raises(errors.StudyImportError): - importer.import_archive( - config=mock_db_config, archive_path=tmp_path / "duck.db" - ) + importer.import_archive(config=mock_db_config, archive_path=tmp_path / "duck.db") with pytest.raises(errors.StudyImportError): df.to_parquet(tmp_path / "archive/test__table.parquet") df.to_parquet(tmp_path / "archive/other_test__table.parquet") @@ -467,9 +455,7 @@ def test_generate_sql(mock_path, mock_db_config, tmp_path): study_path=pathlib.Path(f"{tmp_path}/study_python_valid/") ) file_generator.run_generate_sql(manifest=manifest, config=mock_db_config) - files = list( - pathlib.Path(f"{tmp_path}/study_python_valid/reference_sql/").glob("*") - ) + files = list(pathlib.Path(f"{tmp_path}/study_python_valid/reference_sql/").glob("*")) files = [str(x) for x in files] assert len(files) == 2 assert "module1.sql" in ",".join(files) @@ -494,9 +480,7 @@ def test_generate_md(mock_path, mock_db_config, tmp_path): ) builder.run_table_builder(config=mock_db_config, manifest=manifest) file_generator.run_generate_markdown(config=mock_db_config, manifest=manifest) - with open( - f"{tmp_path}/study_python_valid/study_python_valid_generated.md" - ) as f: + with open(f"{tmp_path}/study_python_valid/study_python_valid_generated.md") as f: generated_md = f.read() expected_table = """### study_python_valid__table diff --git a/tests/test_base_templates.py b/tests/test_base_templates.py index 8db1cdf5..a99e3aae 100644 --- a/tests/test_base_templates.py +++ b/tests/test_base_templates.py @@ -307,9 +307,7 @@ def test_get_create_table_from_union(): (["table_a"], [], {}, "", pytest.raises(errors.CumulusLibraryError)), ], ) -def test_create_view_from_tables( - tables, table_aliases, column_aliases, expected, raises -): +def test_create_view_from_tables(tables, table_aliases, column_aliases, expected, raises): with raises: query = base_templates.get_create_view_from_tables( view_name="view", diff --git a/tests/test_cli.py b/tests/test_cli.py index 7abf3b13..1d7dfd21 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -221,8 +221,7 @@ def test_generate_md(mock_path, tmp_path): cli.main(cli_args=args) test_file = f"{tmp_path}/study_python_valid/study_python_valid_generated.md" ref_file = ( - pathlib.Path(__file__).resolve().parent - / "test_data/study_python_valid_generated.md" + pathlib.Path(__file__).resolve().parent / "test_data/study_python_valid_generated.md" ) assert filecmp.cmp(test_file, ref_file, shallow=True) @@ -403,9 +402,7 @@ def test_clean(mock_path, tmp_path, args, expected, raises): # pylint: disable= ), ], ) -def test_cli_executes_queries( - tmp_path, build_args, export_args, expected_tables, raises -): +def test_cli_executes_queries(tmp_path, build_args, export_args, expected_tables, raises): with raises: build_args = duckdb_args(build_args, tmp_path) cli.main(cli_args=build_args) @@ -430,9 +427,7 @@ def test_cli_executes_queries( if len(build_args) == 9: manifest_dir = cli.get_study_dict([])[build_args[2]] else: - manifest_dir = cli.get_study_dict([cli.get_abs_path(build_args[4])])[ - build_args[2] - ] + manifest_dir = cli.get_study_dict([cli.get_abs_path(build_args[4])])[build_args[2]] with open(f"{manifest_dir}/manifest.toml", "rb") as file: config = tomllib.load(file) @@ -616,12 +611,8 @@ def test_cli_upload_studies(mock_glob, args, status, login_error, raises): def test_cli_upload_filter(mock_upload_data, mock_glob, args, calls): mock_glob.side_effect = [ [ - Path( - str(Path(__file__).parent) + "/test_data/count_synthea_patient.parquet" - ), - Path( - str(Path(__file__).parent) + "/other_data/count_synthea_patient.parquet" - ), + Path(str(Path(__file__).parent) + "/test_data/count_synthea_patient.parquet"), + Path(str(Path(__file__).parent) + "/other_data/count_synthea_patient.parquet"), ], ] cli.main(cli_args=args) @@ -650,9 +641,7 @@ def test_cli_umls_parsing(mock_config, mode, tmp_path): @mock.patch.dict(os.environ, clear=True) def test_cli_single_builder(tmp_path): - cli.main( - cli_args=duckdb_args(["build", "--builder=patient", "--target=core"], tmp_path) - ) + cli.main(cli_args=duckdb_args(["build", "--builder=patient", "--target=core"], tmp_path)) db = databases.DuckDatabaseBackend(f"{tmp_path}/duck.db") tables = {x[0] for x in db.cursor().execute("show tables").fetchall()} assert { diff --git a/tests/test_counts_builder.py b/tests/test_counts_builder.py index 71e04c39..7bd95ac0 100644 --- a/tests/test_counts_builder.py +++ b/tests/test_counts_builder.py @@ -87,12 +87,8 @@ def test_get_where_clauses(clause, min_subject, expected, raises): ("table", "source", None, {}, pytest.raises(errors.CountsBuilderError)), ], ) -@mock.patch( - "cumulus_library.statistics.statistics_templates.counts_templates.get_count_query" -) -def test_get_count_query( - mock_count, table_name, source_table, table_cols, kwargs, raises -): +@mock.patch("cumulus_library.statistics.statistics_templates.counts_templates.get_count_query") +def test_get_count_query(mock_count, table_name, source_table, table_cols, kwargs, raises): with raises: builder = counts.CountsBuilder(study_prefix=TEST_PREFIX) builder.get_count_query(table_name, source_table, table_cols, **kwargs) @@ -159,9 +155,7 @@ def test_get_count_query( ), ], ) -@mock.patch( - "cumulus_library.statistics.statistics_templates.counts_templates.get_count_query" -) +@mock.patch("cumulus_library.statistics.statistics_templates.counts_templates.get_count_query") def test_count_wrappers( mock_count, table_name, diff --git a/tests/test_counts_templates.py b/tests/test_counts_templates.py index 73866d2c..b9c06964 100644 --- a/tests/test_counts_templates.py +++ b/tests/test_counts_templates.py @@ -200,9 +200,7 @@ ], ) def test_count_query(expected, kwargs): - query = counts_templates.get_count_query( - "test_table", "test_source", ["age", "sex"], **kwargs - ) + query = counts_templates.get_count_query("test_table", "test_source", ["age", "sex"], **kwargs) # Snippet for getting updated template output # with open("output.sql", "w") as f: # f.write(query) diff --git a/tests/test_data/study_python_local_template/local_template.py b/tests/test_data/study_python_local_template/local_template.py index 8991cee6..0786760b 100644 --- a/tests/test_data/study_python_local_template/local_template.py +++ b/tests/test_data/study_python_local_template/local_template.py @@ -10,6 +10,4 @@ def get_local_template( schema: dict[dict[bool]] | None = None, config: dict | None = None, ) -> str: - return base_templates.get_base_template( - target_table, path=PATH, schema=schema, config=config - ) + return base_templates.get_base_template(target_table, path=PATH, schema=schema, config=config) diff --git a/tests/test_data/study_python_local_template/module1.py b/tests/test_data/study_python_local_template/module1.py index 52f38b6a..2b1a28b6 100644 --- a/tests/test_data/study_python_local_template/module1.py +++ b/tests/test_data/study_python_local_template/module1.py @@ -7,6 +7,4 @@ class ModuleOneRunner(BaseTableBuilder): display_text = "module1" def prepare_queries(self, *args, **kwargs): - self.queries.append( - local_template.get_local_template("table", config={"field": "foo"}) - ) + self.queries.append(local_template.get_local_template("table", config={"field": "foo"})) diff --git a/tests/test_data/study_python_valid/module1.py b/tests/test_data/study_python_valid/module1.py index 5db22e8e..c09a1f2f 100644 --- a/tests/test_data/study_python_valid/module1.py +++ b/tests/test_data/study_python_valid/module1.py @@ -5,6 +5,4 @@ class ModuleOneRunner(BaseTableBuilder): display_text = "module1" def prepare_queries(self, *args, **kwargs): - self.queries.append( - "CREATE TABLE IF NOT EXISTS study_python_valid__table (test int);" - ) + self.queries.append("CREATE TABLE IF NOT EXISTS study_python_valid__table (test int);") diff --git a/tests/test_databases.py b/tests/test_databases.py index 439305ce..86d5c8a4 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -481,9 +481,7 @@ def test_duckdb_to_utf8(mock_db): def test_duckdb_date(mock_db, data, field_type, raises): with raises: cursor = mock_db.cursor() - field = cursor.execute( - f"SELECT date(CAST('{data}' AS {field_type})) AS field" - ).fetchall() + field = cursor.execute(f"SELECT date(CAST('{data}' AS {field_type})) AS field").fetchall() assert field == [(datetime.date(2000, 1, 1),)] diff --git a/tests/test_duckdb.py b/tests/test_duckdb.py index a6975915..876c8581 100644 --- a/tests/test_duckdb.py +++ b/tests/test_duckdb.py @@ -64,11 +64,7 @@ def test_duckdb_core_build_and_export(tmp_path): ) def test_duckdb_from_iso8601_timestamp(timestamp, expected): db = databases.DuckDatabaseBackend(":memory:") - parsed = ( - db.cursor() - .execute(f"select from_iso8601_timestamp('{timestamp}')") - .fetchone()[0] - ) + parsed = db.cursor().execute(f"select from_iso8601_timestamp('{timestamp}')").fetchone()[0] assert parsed == expected @@ -94,9 +90,7 @@ def test_duckdb_load_ndjson_dir(tmp_path): ) expected_good_count = len({f for f, v in filenames.items() if v}) - found_ids = { - row[0] for row in db.cursor().execute("select id from patient").fetchall() - } + found_ids = {row[0] for row in db.cursor().execute("select id from patient").fetchall()} found_good = {row_id for row_id in found_ids if row_id.startswith("Good")} found_bad = found_ids - found_good assert len(found_good) == expected_good_count @@ -149,9 +143,7 @@ def test_duckdb_table_schema(): actual_schema = db.cursor().execute(query).fetchall() # Validate that schema against what we were looking for - validated_schema = db.parser().validate_table_schema( - target_schema, actual_schema - ) + validated_schema = db.parser().validate_table_schema(target_schema, actual_schema) # Note the all mixed-case results. # These are guaranteed to be the same case as the expected/target schema. expected_schema = { diff --git a/tests/test_logging.py b/tests/test_logging.py index 000c607e..0c619a17 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -159,9 +159,7 @@ def test_migrate_transactions_athena(mock_pyathena): status="debug", message="message", ) - expected = ( - "ALTER TABLE test.study_valid__lib_transactions" " ADD COLUMNS(message string)" - ) + expected = "ALTER TABLE test.study_valid__lib_transactions" " ADD COLUMNS(message string)" call_args = mock_pyathena.return_value.cursor.return_value.execute.call_args_list assert expected == call_args[2][0][0] @@ -204,9 +202,7 @@ def test_migrate_transactions_athena(mock_pyathena): ), ], ) -def test_statistics( - mock_db, schema, study, table_type, table_name, view_type, raises, expects -): +def test_statistics(mock_db, schema, study, table_type, table_name, view_type, raises, expects): with raises: cursor = mock_db.cursor() table = sql_utils.StatisticsTable() diff --git a/tests/test_psm.py b/tests/test_psm.py index d0b46706..72287cf2 100644 --- a/tests/test_psm.py +++ b/tests/test_psm.py @@ -87,11 +87,7 @@ def test_psm_create( drop_table=True, table_suffix=safe_timestamp, ) - df = ( - builder.config.db.cursor() - .execute("select * from psm_test__psm_encounter_covariate") - .df() - ) + df = builder.config.db.cursor().execute("select * from psm_test__psm_encounter_covariate").df() ed_series = df["example_diagnosis"].value_counts() assert ed_series.iloc[0] == neg_set diff --git a/tests/test_psm_templates.py b/tests/test_psm_templates.py index a48e5d1f..0f39ab56 100644 --- a/tests/test_psm_templates.py +++ b/tests/test_psm_templates.py @@ -48,13 +48,9 @@ ), ], ) -def test_get_distinct_ids( - columns, source_table, join_id, filter_table, expected, raises -): +def test_get_distinct_ids(columns, source_table, join_id, filter_table, expected, raises): with raises: - query = psm_templates.get_distinct_ids( - columns, source_table, join_id, filter_table - ) + query = psm_templates.get_distinct_ids(columns, source_table, join_id, filter_table) assert query == expected diff --git a/tests/test_study_parser.py b/tests/test_study_parser.py index 3debcc03..e22571c3 100644 --- a/tests/test_study_parser.py +++ b/tests/test_study_parser.py @@ -73,19 +73,14 @@ def test_manifest_data(mock_load, mock_open, manifest_key, raises): if expected["sql_config"]["file_names"] is None: assert manifest.get_sql_file_list() == [] else: - assert ( - manifest.get_sql_file_list() == expected["sql_config"]["file_names"] - ) + assert manifest.get_sql_file_list() == expected["sql_config"]["file_names"] else: assert manifest.get_sql_file_list() == [] if "export_config" in expected.keys(): if expected["export_config"]["export_list"] is None: assert manifest.get_export_table_list() == [] else: - assert ( - manifest.get_export_table_list() - == expected["export_config"]["export_list"] - ) + assert manifest.get_export_table_list() == expected["export_config"]["export_list"] else: assert manifest.get_export_table_list() == [] diff --git a/tests/testbed_utils.py b/tests/testbed_utils.py index a5289c4b..f034c715 100644 --- a/tests/testbed_utils.py +++ b/tests/testbed_utils.py @@ -57,9 +57,7 @@ def add_condition(self, row_id: str, recorded: str = "2020", **kwargs) -> None: }, ) - def add_document_reference( - self, row_id: str, start: str = "2020", **kwargs - ) -> None: + def add_document_reference(self, row_id: str, start: str = "2020", **kwargs) -> None: """Adds a DocumentReference with all the SQL-required fields filled out""" context = kwargs.pop("context", {}) period = context.setdefault("period", {}) @@ -74,9 +72,7 @@ def add_document_reference( }, ) - def add_encounter( - self, row_id: str, patient: str = "A", start: str = "2020", **kwargs - ) -> None: + def add_encounter(self, row_id: str, patient: str = "A", start: str = "2020", **kwargs) -> None: """Adds an Encounter with all the SQL-required fields filled out""" period = kwargs.pop("period", {}) period["start"] = start @@ -127,9 +123,7 @@ def add_etl_completion( }, ) - def add_etl_completion_encounters( - self, *, group: str, ids: Iterable[str], time: str - ) -> None: + def add_etl_completion_encounters(self, *, group: str, ids: Iterable[str], time: str) -> None: """Adds rows to etl__completion_encounters""" if len(time) == 4: # allow just a year as a shorthand @@ -175,9 +169,7 @@ def add_medication_request( } ] case "external": - kwargs["medicationReference"] = { - "reference": f"Medication/med-{row_id}" - } + kwargs["medicationReference"] = {"reference": f"Medication/med-{row_id}"} self.add( "medication", {