Uprev to 3.11

.github/workflows/ci.yaml

@@ -23,7 +23,7 @@ jobs:
       - name: Set up Python 
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Install dependencies
         run: |
@@ -54,7 +54,7 @@ jobs:
       - name: Set up Python 
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install linters
         run: |
           python -m pip install --upgrade pip
@@ -76,7 +76,7 @@ jobs:
       - name: Set up Python 
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

cumulus_library/actions/builder.py

@@ -5,8 +5,8 @@
 import inspect
 import pathlib
 import sys
+import tomllib
 
-import toml
 from rich.progress import Progress, TaskID
 
 from cumulus_library import (
@@ -202,8 +202,8 @@ def run_statistics_builders(
         # across the board
         safe_timestamp = base_utils.get_tablename_safe_iso_timestamp()
         toml_path = pathlib.Path(f"{manifest._study_path}/{file}")
-        with open(toml_path, encoding="UTF-8") as file:
-            stats_config = toml.load(file)
+        with open(toml_path, "rb") as file:
+            stats_config = tomllib.load(file)
             config_type = stats_config["config_type"]
             target_table = stats_config["target_table"]
 

cumulus_library/base_utils.py

@@ -105,7 +105,7 @@ def get_progress_bar(**kwargs) -> progress.Progress:
 
 
 def get_utc_datetime() -> datetime.datetime:
-    return datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0)
+    return datetime.datetime.now(datetime.UTC).replace(microsecond=0)
 
 
 def get_tablename_safe_iso_timestamp() -> str:

cumulus_library/statistics/psm.py

@@ -4,13 +4,13 @@
 import os
 import pathlib
 import sys
+import tomllib
 import warnings
 from dataclasses import dataclass
 
 import matplotlib.pyplot as plt
 import pandas
 import seaborn as sns
-import toml
 from psmpy import PsmPy
 
 # these imports are mimicing PsmPy imports for re-implemented functions
@@ -61,8 +61,8 @@ def __init__(self, toml_config_path: str, data_path: pathlib.Path, **kwargs):
         self.toml_path = toml_config_path
         self.data_path = data_path
         try:
-            with open(self.toml_path, encoding="UTF-8") as file:
-                toml_config = toml.load(file)
+            with open(self.toml_path, "rb") as file:
+                toml_config = tomllib.load(file)
 
         except OSError:
             sys.exit(f"PSM configuration not found at {self.toml_path}")

cumulus_library/study_manifest.py

@@ -1,8 +1,7 @@
 """Class for loading study configuration data from manifest.toml files"""
 
 import pathlib
-
-import toml
+import tomllib
 
 from cumulus_library import errors
 
@@ -36,8 +35,8 @@ def load_study_manifest(self, study_path: pathlib.Path) -> None:
         :raises StudyManifestParsingError: the manifest.toml is malformed or missing.
         """
         try:
-            with open(f"{study_path}/manifest.toml", encoding="UTF-8") as file:
-                config = toml.load(file)
+            with open(f"{study_path}/manifest.toml", "rb") as file:
+                config = tomllib.load(file)
                 if not config.get("study_prefix") or not isinstance(
                     config["study_prefix"], str
                 ):
@@ -50,7 +49,7 @@ def load_study_manifest(self, study_path: pathlib.Path) -> None:
             raise errors.StudyManifestFilesystemError(
                 f"Missing or invalid manifest found at {study_path}"
             ) from e
-        except toml.TomlDecodeError as e:
+        except tomllib.TOMLDecodeError as e:
             # just unify the error classes for convenience of catching them
             raise errors.StudyManifestParsingError(str(e)) from e
 
@@ -75,7 +74,7 @@ def get_sql_file_list(self, continue_from: str | None = None) -> list[str] | Non
         :returns: An array of sql files from the manifest, or None if not found.
         """
         sql_config = self._study_config.get("sql_config", {})
-        sql_files = sql_config.get("file_names", [])
+        sql_files = sql_config.get("file_names", []) or []
         if continue_from:
             for pos, file in enumerate(sql_files):
                 if continue_from.replace(".sql", "") == file.replace(".sql", ""):
@@ -118,7 +117,7 @@ def get_export_table_list(self) -> list[str] | None:
         :returns: An array of tables to export from the manifest, or None if not found.
         """
         export_config = self._study_config.get("export_config", {})
-        export_table_list = export_config.get("export_list", [])
+        export_table_list = export_config.get("export_list", []) or []
         for table in export_table_list:
             if not table.startswith(f"{self.get_study_prefix()}__"):
                 raise errors.StudyManifestParsingError(

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cumulus-library"
-requires-python = ">= 3.10"
+requires-python = ">= 3.11"
 dependencies = [
     "ctakesclient >= 1.3",
     "cumulus-fhir-support >= 1.2",
@@ -16,7 +16,6 @@ dependencies = [
     "rich >= 13.2",
     "sqlfluff  >= 3",
     "sqlparse >0.4",
-    "toml >= 0.10"
 ]
 description = "Clinical study SQL generation for data derived from bulk FHIR"
 readme = "README.md"

tests/core/test_core.py

@@ -1,9 +1,9 @@
 """unit tests for counts generation"""
 
 import datetime  # noqa: F401
+import tomllib
 
 import pytest
-import toml
 
 from tests import conftest, testbed_utils
 
@@ -102,7 +102,8 @@ def test_core_count_missing_data(tmp_path):
 
 
 def test_core_counts_exported(mock_db_core):
-    manifest = toml.load(f"{conftest.LIBRARY_ROOT}/studies/core/manifest.toml")
+    with open(f"{conftest.LIBRARY_ROOT}/studies/core/manifest.toml", "rb") as f:
+        manifest = tomllib.load(f)
     manifest["export_config"]["export_list"].remove("core__meta_version")
     manifest["export_config"]["export_list"].remove("core__meta_date")
     count_tables = (

tests/test_cli.py

@@ -8,6 +8,7 @@
 import pathlib
 import shutil
 import sys
+import tomllib
 import zipfile
 from contextlib import contextmanager
 from contextlib import nullcontext as does_not_raise
@@ -18,7 +19,6 @@
 import pandas
 import pytest
 import responses
-import toml
 
 from cumulus_library import cli, databases, errors
 from tests.conftest import duckdb_args
@@ -434,8 +434,8 @@ def test_cli_executes_queries(
                     build_args[2]
                 ]
 
-            with open(f"{manifest_dir}/manifest.toml", encoding="UTF-8") as file:
-                config = toml.load(file)
+            with open(f"{manifest_dir}/manifest.toml", "rb") as file:
+                config = tomllib.load(file)
             csv_files = glob.glob(f"{tmp_path}/export/{build_args[2]}/*.csv")
             export_tables = config["export_config"]["export_list"]
             for export_table in export_tables:

tests/test_data/parser_mock_data.py

@@ -1,11 +1,9 @@
 """test util for accessing mock toml configs"""
 
-import toml
-
 
 def get_mock_toml(key: str):
     """convenience method for in-mem toml object"""
-    return toml.dumps(mock_manifests[key])
+    return mock_manifests[key]
 
 
 mock_manifests = {

tests/test_data/study_python_counts_valid/manifest.toml

@@ -1,4 +1,6 @@
 study_prefix = "study_python_counts_valid"
 
 [counts_builder_config]
-file_names = ["module1.py", "module1.py", "module2.py"]
+file_names = [
+    "module1.py", "module1.py", "module2.py"
+]

tests/test_study_parser.py

@@ -58,34 +58,33 @@ def test_load_manifest(manifest_path, expected, raises):
         ("invalid_none", pytest.raises(TypeError)),
     ],
 )
-def test_manifest_data(manifest_key, raises):
-    with mock.patch(
-        "builtins.open", mock.mock_open(read_data=get_mock_toml(manifest_key))
-    ):
-        with raises:
-            if manifest_key == "invalid_none":
-                parser = study_manifest.StudyManifest()
-            else:
-                parser = study_manifest.StudyManifest("./path")
-            expected = mock_manifests[manifest_key]
-            assert parser.get_study_prefix() == expected["study_prefix"]
-            if "sql_config" in expected.keys():
-                if expected["sql_config"]["file_names"] is None:
-                    assert parser.get_sql_file_list() == []
-                else:
-                    assert (
-                        parser.get_sql_file_list()
-                        == expected["sql_config"]["file_names"]
-                    )
+@mock.patch("builtins.open")
+@mock.patch("tomllib.load")
+def test_manifest_data(mock_load, mock_open, manifest_key, raises):
+    mock_load.return_value = get_mock_toml(manifest_key)
+    with raises:
+        if manifest_key == "invalid_none":
+            manifest = study_manifest.StudyManifest()
+        else:
+            manifest = study_manifest.StudyManifest("./path")
+        expected = mock_manifests[manifest_key]
+        assert manifest.get_study_prefix() == expected["study_prefix"]
+        if "sql_config" in expected.keys():
+            if expected["sql_config"]["file_names"] is None:
+                assert manifest.get_sql_file_list() == []
             else:
-                assert parser.get_sql_file_list() == []
-            if "export_config" in expected.keys():
-                if expected["export_config"]["export_list"] is None:
-                    assert parser.get_export_table_list() == []
-                else:
-                    assert (
-                        parser.get_export_table_list()
-                        == expected["export_config"]["export_list"]
-                    )
+                assert (
+                    manifest.get_sql_file_list() == expected["sql_config"]["file_names"]
+                )
+        else:
+            assert manifest.get_sql_file_list() == []
+        if "export_config" in expected.keys():
+            if expected["export_config"]["export_list"] is None:
+                assert manifest.get_export_table_list() == []
             else:
-                assert parser.get_export_table_list() == []
+                assert (
+                    manifest.get_export_table_list()
+                    == expected["export_config"]["export_list"]
+                )
+        else:
+            assert manifest.get_export_table_list() == []