diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bab99b78..e091dca0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: | @@ -54,7 +54,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Install linters run: | python -m pip install --upgrade pip @@ -76,7 +76,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/cumulus_library/actions/builder.py b/cumulus_library/actions/builder.py index 1fdccf75..e4362a6e 100644 --- a/cumulus_library/actions/builder.py +++ b/cumulus_library/actions/builder.py @@ -5,8 +5,8 @@ import inspect import pathlib import sys +import tomllib -import toml from rich.progress import Progress, TaskID from cumulus_library import ( @@ -202,8 +202,8 @@ def run_statistics_builders( # across the board safe_timestamp = base_utils.get_tablename_safe_iso_timestamp() toml_path = pathlib.Path(f"{manifest._study_path}/{file}") - with open(toml_path, encoding="UTF-8") as file: - stats_config = toml.load(file) + with open(toml_path, "rb") as file: + stats_config = tomllib.load(file) config_type = stats_config["config_type"] target_table = stats_config["target_table"] diff --git a/cumulus_library/base_utils.py b/cumulus_library/base_utils.py index 53c190e2..4308e211 100644 --- a/cumulus_library/base_utils.py +++ b/cumulus_library/base_utils.py @@ -105,7 +105,7 @@ def get_progress_bar(**kwargs) -> progress.Progress: def get_utc_datetime() -> datetime.datetime: - return datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) + return datetime.datetime.now(datetime.UTC).replace(microsecond=0) def get_tablename_safe_iso_timestamp() -> str: diff --git a/cumulus_library/statistics/psm.py b/cumulus_library/statistics/psm.py index 93939412..bd7a450d 100644 --- a/cumulus_library/statistics/psm.py +++ b/cumulus_library/statistics/psm.py @@ -4,13 +4,13 @@ import os import pathlib import sys +import tomllib import warnings from dataclasses import dataclass import matplotlib.pyplot as plt import pandas import seaborn as sns -import toml from psmpy import PsmPy # these imports are mimicing PsmPy imports for re-implemented functions @@ -61,8 +61,8 @@ def __init__(self, toml_config_path: str, data_path: pathlib.Path, **kwargs): self.toml_path = toml_config_path self.data_path = data_path try: - with open(self.toml_path, encoding="UTF-8") as file: - toml_config = toml.load(file) + with open(self.toml_path, "rb") as file: + toml_config = tomllib.load(file) except OSError: sys.exit(f"PSM configuration not found at {self.toml_path}") diff --git a/cumulus_library/study_manifest.py b/cumulus_library/study_manifest.py index b9879bb5..2de9f509 100644 --- a/cumulus_library/study_manifest.py +++ b/cumulus_library/study_manifest.py @@ -1,8 +1,7 @@ """Class for loading study configuration data from manifest.toml files""" import pathlib - -import toml +import tomllib from cumulus_library import errors @@ -36,8 +35,8 @@ def load_study_manifest(self, study_path: pathlib.Path) -> None: :raises StudyManifestParsingError: the manifest.toml is malformed or missing. """ try: - with open(f"{study_path}/manifest.toml", encoding="UTF-8") as file: - config = toml.load(file) + with open(f"{study_path}/manifest.toml", "rb") as file: + config = tomllib.load(file) if not config.get("study_prefix") or not isinstance( config["study_prefix"], str ): @@ -50,7 +49,7 @@ def load_study_manifest(self, study_path: pathlib.Path) -> None: raise errors.StudyManifestFilesystemError( f"Missing or invalid manifest found at {study_path}" ) from e - except toml.TomlDecodeError as e: + except tomllib.TOMLDecodeError as e: # just unify the error classes for convenience of catching them raise errors.StudyManifestParsingError(str(e)) from e @@ -75,7 +74,7 @@ def get_sql_file_list(self, continue_from: str | None = None) -> list[str] | Non :returns: An array of sql files from the manifest, or None if not found. """ sql_config = self._study_config.get("sql_config", {}) - sql_files = sql_config.get("file_names", []) + sql_files = sql_config.get("file_names", []) or [] if continue_from: for pos, file in enumerate(sql_files): if continue_from.replace(".sql", "") == file.replace(".sql", ""): @@ -118,7 +117,7 @@ def get_export_table_list(self) -> list[str] | None: :returns: An array of tables to export from the manifest, or None if not found. """ export_config = self._study_config.get("export_config", {}) - export_table_list = export_config.get("export_list", []) + export_table_list = export_config.get("export_list", []) or [] for table in export_table_list: if not table.startswith(f"{self.get_study_prefix()}__"): raise errors.StudyManifestParsingError( diff --git a/pyproject.toml b/pyproject.toml index bcd54842..4d98a54a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cumulus-library" -requires-python = ">= 3.10" +requires-python = ">= 3.11" dependencies = [ "ctakesclient >= 1.3", "cumulus-fhir-support >= 1.2", @@ -16,7 +16,6 @@ dependencies = [ "rich >= 13.2", "sqlfluff >= 3", "sqlparse >0.4", - "toml >= 0.10" ] description = "Clinical study SQL generation for data derived from bulk FHIR" readme = "README.md" diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 14e6ffe9..f965d5a2 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -1,9 +1,9 @@ """unit tests for counts generation""" import datetime # noqa: F401 +import tomllib import pytest -import toml from tests import conftest, testbed_utils @@ -102,7 +102,8 @@ def test_core_count_missing_data(tmp_path): def test_core_counts_exported(mock_db_core): - manifest = toml.load(f"{conftest.LIBRARY_ROOT}/studies/core/manifest.toml") + with open(f"{conftest.LIBRARY_ROOT}/studies/core/manifest.toml", "rb") as f: + manifest = tomllib.load(f) manifest["export_config"]["export_list"].remove("core__meta_version") manifest["export_config"]["export_list"].remove("core__meta_date") count_tables = ( diff --git a/tests/test_cli.py b/tests/test_cli.py index 30cbf9e5..9c8b2802 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,6 +8,7 @@ import pathlib import shutil import sys +import tomllib import zipfile from contextlib import contextmanager from contextlib import nullcontext as does_not_raise @@ -18,7 +19,6 @@ import pandas import pytest import responses -import toml from cumulus_library import cli, databases, errors from tests.conftest import duckdb_args @@ -434,8 +434,8 @@ def test_cli_executes_queries( build_args[2] ] - with open(f"{manifest_dir}/manifest.toml", encoding="UTF-8") as file: - config = toml.load(file) + with open(f"{manifest_dir}/manifest.toml", "rb") as file: + config = tomllib.load(file) csv_files = glob.glob(f"{tmp_path}/export/{build_args[2]}/*.csv") export_tables = config["export_config"]["export_list"] for export_table in export_tables: diff --git a/tests/test_data/parser_mock_data.py b/tests/test_data/parser_mock_data.py index 9a164290..cd3b9b12 100644 --- a/tests/test_data/parser_mock_data.py +++ b/tests/test_data/parser_mock_data.py @@ -1,11 +1,9 @@ """test util for accessing mock toml configs""" -import toml - def get_mock_toml(key: str): """convenience method for in-mem toml object""" - return toml.dumps(mock_manifests[key]) + return mock_manifests[key] mock_manifests = { diff --git a/tests/test_data/study_python_counts_valid/manifest.toml b/tests/test_data/study_python_counts_valid/manifest.toml index a791e4fb..021d38f5 100644 --- a/tests/test_data/study_python_counts_valid/manifest.toml +++ b/tests/test_data/study_python_counts_valid/manifest.toml @@ -1,4 +1,6 @@ study_prefix = "study_python_counts_valid" [counts_builder_config] -file_names = ["module1.py", "module1.py", "module2.py"] +file_names = [ + "module1.py", "module1.py", "module2.py" +] diff --git a/tests/test_study_parser.py b/tests/test_study_parser.py index a6a200fa..844ea660 100644 --- a/tests/test_study_parser.py +++ b/tests/test_study_parser.py @@ -58,34 +58,33 @@ def test_load_manifest(manifest_path, expected, raises): ("invalid_none", pytest.raises(TypeError)), ], ) -def test_manifest_data(manifest_key, raises): - with mock.patch( - "builtins.open", mock.mock_open(read_data=get_mock_toml(manifest_key)) - ): - with raises: - if manifest_key == "invalid_none": - parser = study_manifest.StudyManifest() - else: - parser = study_manifest.StudyManifest("./path") - expected = mock_manifests[manifest_key] - assert parser.get_study_prefix() == expected["study_prefix"] - if "sql_config" in expected.keys(): - if expected["sql_config"]["file_names"] is None: - assert parser.get_sql_file_list() == [] - else: - assert ( - parser.get_sql_file_list() - == expected["sql_config"]["file_names"] - ) +@mock.patch("builtins.open") +@mock.patch("tomllib.load") +def test_manifest_data(mock_load, mock_open, manifest_key, raises): + mock_load.return_value = get_mock_toml(manifest_key) + with raises: + if manifest_key == "invalid_none": + manifest = study_manifest.StudyManifest() + else: + manifest = study_manifest.StudyManifest("./path") + expected = mock_manifests[manifest_key] + assert manifest.get_study_prefix() == expected["study_prefix"] + if "sql_config" in expected.keys(): + if expected["sql_config"]["file_names"] is None: + assert manifest.get_sql_file_list() == [] else: - assert parser.get_sql_file_list() == [] - if "export_config" in expected.keys(): - if expected["export_config"]["export_list"] is None: - assert parser.get_export_table_list() == [] - else: - assert ( - parser.get_export_table_list() - == expected["export_config"]["export_list"] - ) + assert ( + manifest.get_sql_file_list() == expected["sql_config"]["file_names"] + ) + else: + assert manifest.get_sql_file_list() == [] + if "export_config" in expected.keys(): + if expected["export_config"]["export_list"] is None: + assert manifest.get_export_table_list() == [] else: - assert parser.get_export_table_list() == [] + assert ( + manifest.get_export_table_list() + == expected["export_config"]["export_list"] + ) + else: + assert manifest.get_export_table_list() == []