diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index ee2cb2e..bcb5a31 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -13,13 +13,13 @@ jobs: id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: master - - name: Set up Python 3.10 - uses: actions/setup-python@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Install setuptools run: | diff --git a/.github/workflows/run_tox.yml b/.github/workflows/run_tox.yml index a768730..0d39187 100644 --- a/.github/workflows/run_tox.yml +++ b/.github/workflows/run_tox.yml @@ -7,11 +7,11 @@ jobs: name: pre-commit runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.10' - - uses: pre-commit/action@v3.0.0 + python-version: '3.12' + - uses: pre-commit/action@v3.0.1 test: needs: pre-commit @@ -19,12 +19,12 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f03988a..80eae02 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: check-ast - id: check-builtin-literals @@ -13,17 +13,34 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.1.5 + rev: v0.8.2 hooks: - id: ruff - # - id: ruff-format + name: ruff unused imports + # F401 [*] {name} imported but unused + args: [ "--select", "F401", "--extend-exclude", "__init__.py", "--fix"] + + - id: ruff + # I001 [*] Import block is un-sorted or un-formatted + # UP035 [*] Import from {target} instead: {names} + # Q000 [*] Double quote found but single quotes preferred + # Q001 [*] Double quote multiline found but single quotes preferred + args: [ "--select", "I001,UP035,Q000,Q001", "--fix"] + - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - id: rst-backticks + - repo: https://github.com/JelleZijlstra/autotyping + rev: 24.9.0 + hooks: + - id: autotyping + types: [python] + args: [--safe] + + - repo: meta hooks: - id: check-hooks-apply diff --git a/.ruff.toml b/.ruff.toml index 038b024..fc84935 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,75 +1,106 @@ - -line-length = 120 indent-width = 4 +line-length = 120 + +target-version = "py310" -target-version = "py38" - -# https://docs.astral.sh/ruff/settings/#ignore-init-module-imports -ignore-init-module-imports = true -src = ["src", "test"] -extend-exclude = ["__init__.py"] - -select = [ - "E", "W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w - "I", # https://docs.astral.sh/ruff/rules/#isort-i - "UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up - - "A", # https://docs.astral.sh/ruff/rules/#flake8-builtins-a - "ASYNC", # https://docs.astral.sh/ruff/rules/#flake8-async-async - "C4", # https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4 - "EM", # https://docs.astral.sh/ruff/rules/#flake8-errmsg-em - "FIX", # https://docs.astral.sh/ruff/rules/#flake8-fixme-fix - "INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp - "ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc - "PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie - "PT", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt - "PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth - "RET", # https://docs.astral.sh/ruff/rules/#flake8-return-ret - "SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim - "SLOT", # https://docs.astral.sh/ruff/rules/#flake8-slots-slot - "T10", # https://docs.astral.sh/ruff/rules/#flake8-debugger-t10 - "TCH", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch - "TD", # https://docs.astral.sh/ruff/rules/#flake8-todos-td - - "TRY", # https://docs.astral.sh/ruff/rules/#tryceratops-try - "FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly - "PERF", # https://docs.astral.sh/ruff/rules/#perflint-perf - "RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf - - "PL", # https://docs.astral.sh/ruff/rules/#pylint-pl +src = [ + "src", + "tests" ] + +[lint] +select = ["ALL"] + ignore = [ - "A003", # https://docs.astral.sh/ruff/rules/builtin-attribute-shadowing/ + "D", # https://docs.astral.sh/ruff/rules/#pydocstyle-d + "T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20 + "DTZ", # https://docs.astral.sh/ruff/rules/#flake8-datetimez-dtz + "SLF", # https://docs.astral.sh/ruff/rules/#flake8-self-slf + "RET501", # https://docs.astral.sh/ruff/rules/unnecessary-return-none/#unnecessary-return-none-ret501 "TRY400", # https://docs.astral.sh/ruff/rules/error-instead-of-exception/ - "PLR1711", # https://docs.astral.sh/ruff/rules/useless-return/ + + # https://docs.astral.sh/ruff/rules/#flake8-builtins-a + "A003", # Python builtin is shadowed by class attribute {name} from {row} + + # https://docs.astral.sh/ruff/rules/#pyflakes-f + "F401", # {name} imported but unused; consider using importlib.util.find_spec to test for availability + + # https://docs.astral.sh/ruff/rules/#flake8-bandit-s + "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes + + # https://docs.astral.sh/ruff/rules/#pyupgrade-up + "UP038", # Use X | Y in {} call instead of (X, Y) + + # https://docs.astral.sh/ruff/rules/#flake8-annotations-ann + "ANN101", # Missing type annotation for {name} in method + "ANN102", # Missing type annotation for {name} in classmethod + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in {name} + + # https://docs.astral.sh/ruff/rules/#flake8-blind-except-ble + "BLE001", # Do not catch blind exception: {name} + + # https://docs.astral.sh/ruff/rules/#flake8-raise-rse + "RSE102", # Unnecessary parentheses on raised exception + + # https://docs.astral.sh/ruff/rules/#flake8-commas-com + "COM812", # Trailing comma missing + "COM819", # Trailing comma prohibited + + # https://docs.astral.sh/ruff/rules/#warning-w_1 + "PLW0603", # Using the global statement to update {name} is discouraged + + # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g + "G004", # Logging statement uses f-string + + # https://docs.astral.sh/ruff/rules/#refactor-r + "PLR1711", # Useless return statement at end of function + + # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf + "RUF005", # Consider {expression} instead of concatenation + + # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt + "PT007", # Wrong values type in @pytest.mark.parametrize expected {values} of {row} ] [format] -# Use single quotes for non-triple-quoted strings. quote-style = "single" -[lint.per-file-ignores] -"docs/*" = [ - "A001", # A001 Variable `copyright` is shadowing a Python builtin - "E402", # E402 Module level import not at top of file - "INP001", # INP001 File `FILE_NAME` is part of an implicit namespace package. Add an `__init__.py`. -] +# https://docs.astral.sh/ruff/settings/#lintflake8-quotes +[lint.flake8-quotes] +inline-quotes = "single" +multiline-quotes = "single" -"tests/*" = [ - "INP001", # INP001 File `FILE_NAME` is part of an implicit namespace package. Add an `__init__.py`. - "ISC002", # ISC002 Implicitly concatenated string literals over multiple lines - "PLR2004", # PLR2004 Magic value used in comparison, consider replacing 5 with a constant variable + +[lint.flake8-builtins] +builtins-ignorelist = ["id", "input"] + + +# https://docs.astral.sh/ruff/settings/#lintisort +[lint.isort] +lines-after-imports = 2 # https://docs.astral.sh/ruff/settings/#lint_isort_lines-after-imports + + +[lint.per-file-ignores] +"docs/conf.py" = [ + "INP001", # File `conf.py` is part of an implicit namespace package. Add an `__init__.py`. + "A001", # Variable `copyright` is shadowing a Python builtin + "PTH118", # `os.path.join()` should be replaced by `Path` with `/` operator + "PTH100", # `os.path.abspath()` should be replaced by `Path.resolve()` ] "setup.py" = ["PTH123"] -"src/easyconfig/yaml/from_model.py" = ["PLR0911"] # PLR0911 Too many return statements (7 > 6) +"tests/*" = [ + "ANN", # https://docs.astral.sh/ruff/rules/#flake8-annotations-ann + # https://docs.astral.sh/ruff/rules/#flake8-bandit-s + "S101", # Use of assert detected -[lint.isort] -# https://docs.astral.sh/ruff/settings/#isort-lines-after-imports -lines-after-imports = 2 + # https://docs.astral.sh/ruff/rules/#refactor-r + "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable + "PLR0913", # Too many arguments in function definition ({c_args} > {max_args}) +] diff --git a/docs/class_reference.rst b/docs/class_reference.rst index 3f42633..53f9fab 100644 --- a/docs/class_reference.rst +++ b/docs/class_reference.rst @@ -22,3 +22,10 @@ Subscription .. autoclass:: easyconfig.config_objs.ConfigObjSubscription :members: + + +PreProcess +====================================== + +.. autoclass:: easyconfig.pre_process.PreProcess + :members: diff --git a/docs/requirements.txt b/docs/requirements.txt index ff8bf35..c0c5a5d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # Packages required to build the documentation -sphinx == 7.2.6 -sphinx-autodoc-typehints == 1.25.2 -sphinx_rtd_theme == 2.0.0 -sphinx-exec-code == 0.12 +sphinx == 8.1.3 +sphinx-autodoc-typehints == 2.5.0 +sphinx_rtd_theme == 3.0.2 +sphinx-exec-code == 0.14 diff --git a/docs/usage.rst b/docs/usage.rst index 0116069..f99deb0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -77,11 +77,13 @@ Nested example with the convenience base classes from easyconfig. # ------------ hide: stop ------------- -Description and comments +Default file generation -------------------------------------- It's possible to specify a description through the pydantic ``Field``. The description will be created as a comment in the .yml file. -Note that the comments will be aligned properly +Note that the comments will be aligned properly. +With the ``in_file`` argument it's possible to skip entries from appearing in the default file +(e.g. for advanced settings). When added manually to the file these values will still be loaded as expected. .. exec_code:: :language_output: yaml @@ -94,6 +96,7 @@ Note that the comments will be aligned properly class MySimpleAppConfig(AppBaseModel): retries: int = Field(5, description='Amount of retries on error') url: str = Field('localhost', description='Url used for connection') + advanced: str = Field('something advanced', in_file=False) port: int = 443 @@ -125,7 +128,7 @@ yaml file .. exec_code:: :language_output: yaml - :hide_code: + :hide: a = """ env_var: "${MY_USER}" @@ -145,7 +148,7 @@ yaml file .. exec_code:: :language_output: yaml - :hide_code: + :hide: :caption_output: After expansion @@ -222,3 +225,46 @@ This is especially useful feature if the application allows dynamic reloading of # This will trigger the callback CONFIG.load_config_file('/my/configuration/file.yml') # ------------ skip: stop ------------- + + + +Preprocessing +-------------------------------------- +With preprocessing it's possible to introduce changes in a non-breaking way + + +.. exec_code:: + :language_output: yaml + + from pydantic import Field + from easyconfig import AppBaseModel, BaseModel, create_app_config + + + class HttpConfig(BaseModel): + url: str = 'localhost' + port: int = 443 + retries: int = 3 + timeout: int = 0 + + + class MySimpleAppConfig(AppBaseModel): + http: HttpConfig = HttpConfig() + + + CONFIG = create_app_config(MySimpleAppConfig()) + + # Setup preprocessing, these are the migration steps from the old format + preprocess = CONFIG.load_preprocess + preprocess.rename_entry(['server'], 'http') + preprocess.move_entry(['wait time'], ['http', 'timeout']) + preprocess.set_log_func(print) # This should normally be logger.info or logger.debug + + # Load some old legacy format where http was still named server + CONFIG.load_config_dict({ + 'server': { # this entry will be renamed to http + 'retries': 5 + }, + 'wait time': 10 # this entry will be moved to http.timeout + }) + + print(f'timeout: {CONFIG.http.timeout}') diff --git a/readme.md b/readme.md index a9bcede..50fd254 100644 --- a/readme.md +++ b/readme.md @@ -40,6 +40,13 @@ That way the users can have some guidance how to change the program behaviour. It's possible to use environment variable or files for expansion. Easyconfig will load all values # Changelog +#### 0.4.0 (2024-01-10) +- Minimum required python version is now 3.10 +- Added preprocessor to so it's possible to move and deprecate configuration entries +- Added property to get the loaded configuration file +- Many fixes +- Updated CI and code linters + #### 0.3.2 (2024-01-10) - Updated CI and code linters diff --git a/requirements.txt b/requirements.txt index fa5c3e0..24c5d28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,9 @@ -r requirements_setup.txt # testing dependencies -pytest == 7.4.4 -pre-commit == 3.5.0 +pre-commit == 4.0.1 +pytest == 8.3.4 # linter -ruff == 0.1.11 +ruff == 0.8.2 +pur == 7.3.2 diff --git a/setup.py b/setup.py index f1d608f..d1d2a36 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,17 @@ -import typing from pathlib import Path from setuptools import find_packages, setup def load_version() -> str: - version: typing.Dict[str, str] = {} + version: dict[str, str] = {} with open('src/easyconfig/__version__.py') as fp: exec(fp.read(), version) assert version['__version__'], version return version['__version__'] -def load_req() -> typing.List[str]: +def load_req() -> list[str]: with open('requirements_setup.txt') as f: return f.readlines() @@ -20,7 +19,7 @@ def load_req() -> typing.List[str]: __version__ = load_version() print(f'Version: {__version__}') -print('') +print() # When we run tox tests we don't have these files available, so we skip them readme = Path(__file__).with_name('readme.md') @@ -46,18 +45,17 @@ def load_req() -> typing.List[str]: package_data={'easyconfig': ['py.typed']}, packages=find_packages('src', exclude=['tests*']), install_requires=load_req(), - python_requires='>=3.8', + python_requires='>=3.10', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3 :: Only', 'Topic :: Software Development :: Libraries', ], diff --git a/src/easyconfig/__const__.py b/src/easyconfig/__const__.py index 349bd4a..da7ff52 100644 --- a/src/easyconfig/__const__.py +++ b/src/easyconfig/__const__.py @@ -5,7 +5,7 @@ class _MissingType(Enum): MISSING_OBJ = object() - def __str__(self): + def __str__(self) -> str: return '' diff --git a/src/easyconfig/__init__.py b/src/easyconfig/__init__.py index d6c58d8..da2e0ca 100644 --- a/src/easyconfig/__init__.py +++ b/src/easyconfig/__init__.py @@ -1,10 +1,12 @@ from easyconfig import errors from easyconfig.__version__ import __version__ + # isort: split from easyconfig.models import AppBaseModel, AppBaseSettings, AppConfigMixin, BaseModel, BaseSettings, ConfigMixin + # isort: split from easyconfig.create_app_config import create_app_config diff --git a/src/easyconfig/__version__.py b/src/easyconfig/__version__.py index 73e3bb4..abeeedb 100644 --- a/src/easyconfig/__version__.py +++ b/src/easyconfig/__version__.py @@ -1 +1 @@ -__version__ = '0.3.2' +__version__ = '0.4.0' diff --git a/src/easyconfig/config_objs/__init__.py b/src/easyconfig/config_objs/__init__.py index 6cbf532..a5873dd 100644 --- a/src/easyconfig/config_objs/__init__.py +++ b/src/easyconfig/config_objs/__init__.py @@ -1,6 +1,7 @@ from .subscription import ConfigObjSubscription, SubscriptionParent + # isort: split from .app_config import AppConfig -from .object_config import ConfigObj, HINT_CONFIG_OBJ, HINT_CONFIG_OBJ_TYPE +from .object_config import ConfigObj diff --git a/src/easyconfig/config_objs/app_config.py b/src/easyconfig/config_objs/app_config.py index aff39f4..e70ddcb 100644 --- a/src/easyconfig/config_objs/app_config.py +++ b/src/easyconfig/config_objs/app_config.py @@ -2,12 +2,13 @@ from io import StringIO from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final from easyconfig.__const__ import MISSING, MISSING_TYPE from easyconfig.config_objs.object_config import ConfigObj from easyconfig.errors import FileDefaultsNotSetError from easyconfig.expansion import expand_obj +from easyconfig.pre_process import PreProcess from easyconfig.yaml import CommentedMap, cmap_from_model, write_aligned_yaml, yaml_rt @@ -17,13 +18,29 @@ class AppConfig(ConfigObj): - def __init__(self, model: BaseModel, path: tuple[str, ...] = ('__root__',), parent: MISSING_TYPE | Self = MISSING): - super().__init__(model, path, parent) + def __init__(self, model: BaseModel, path: tuple[str, ...] = ('__root__',), + parent: MISSING_TYPE | Self = MISSING, file_defaults: BaseModel | None = None, **kwargs) -> None: + super().__init__(model, path, parent, **kwargs) - self._file_defaults: BaseModel | None = None + self._file_defaults: Final = file_defaults + self._preprocess: Final = PreProcess(self._file_defaults) self._file_path: Path | None = None - def set_file_path(self, path: Path | str): + @property + def loaded_file_path(self) -> Path: + """Path to the loaded configuration file""" + + if self._file_path is None: + msg = 'No file loaded' + raise ValueError(msg) + return self._file_path + + @property + def load_preprocess(self) -> PreProcess: + """A preprocessor which can be used to preprocess the configuration data before it is loaded""" + return self._preprocess + + def set_file_path(self, path: Path | str) -> None: """Set the path to the configuration file. If no file extension is specified ``.yml`` will be automatically appended. @@ -39,12 +56,14 @@ def set_file_path(self, path: Path | str): if not self._file_path.suffix: self._file_path = self._file_path.with_suffix('.yml') - def load_config_dict(self, cfg: dict, /, expansion: bool = True): + def load_config_dict(self, cfg: dict, /, expansion: bool = True) -> Self: """Load the configuration from a dictionary :param cfg: config dict which will be loaded :param expansion: Expand ${...} in strings """ + self._preprocess.run(cfg) + if expansion: expand_obj(cfg) @@ -55,7 +74,7 @@ def load_config_dict(self, cfg: dict, /, expansion: bool = True): self._set_values(model_obj) return self - def load_config_file(self, path: Path | str | None = None, expansion: bool = True): + def load_config_file(self, path: Path | str | None = None, *, expansion: bool = True) -> Self: """Load configuration from a yaml file. If the file does not exist a default file will be created :param path: Path to file diff --git a/src/easyconfig/config_objs/object_config.py b/src/easyconfig/config_objs/object_config.py index 12dffb7..f17b5e8 100644 --- a/src/easyconfig/config_objs/object_config.py +++ b/src/easyconfig/config_objs/object_config.py @@ -1,7 +1,10 @@ +from __future__ import annotations + from inspect import getmembers, isfunction -from typing import TYPE_CHECKING, Any, Callable, Dict, Final, List, Tuple, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Final from pydantic import BaseModel +from typing_extensions import Self from easyconfig import AppConfigMixin from easyconfig.__const__ import MISSING, MISSING_TYPE @@ -10,35 +13,36 @@ if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + from pydantic.fields import FieldInfo - import easyconfig -HINT_CONFIG_OBJ = TypeVar('HINT_CONFIG_OBJ', bound='ConfigObj') -HINT_CONFIG_OBJ_TYPE = Type[HINT_CONFIG_OBJ] +def should_be_copied(o: object) -> bool: + return isfunction(o) or isinstance(o, property) + -NO_COPY = [n for n, o in getmembers(AppConfigMixin) if isfunction(o)] +NO_COPY = tuple(n for n, o in getmembers(AppConfigMixin) if should_be_copied(o)) class ConfigObj: - def __init__( - self, - model: BaseModel, - path: Tuple[str, ...] = ('__root__',), - parent: Union[MISSING_TYPE, HINT_CONFIG_OBJ] = MISSING, - ): + def __init__(self, model: BaseModel, path: tuple[str, ...] = ('__root__',), + parent: MISSING_TYPE | ConfigObj = MISSING, **kwargs) -> None: + super().__init__(**kwargs) + self._obj_parent: Final = parent self._obj_path: Final = path self._obj_model_class: Final = model.__class__ - self._obj_model_fields: Dict[str, FieldInfo] = model.model_fields - self._obj_model_private_attrs: List[str] = list(model.__private_attributes__.keys()) + self._obj_model_fields: dict[str, FieldInfo] = model.model_fields + self._obj_model_private_attrs: list[str] = list(model.__private_attributes__.keys()) - self._obj_keys: Tuple[str, ...] = () - self._obj_values: Dict[str, Any] = {} - self._obj_children: Dict[str, Union[HINT_CONFIG_OBJ, Tuple[HINT_CONFIG_OBJ, ...]]] = {} + self._obj_keys: tuple[str, ...] = () + self._obj_values: dict[str, Any] = {} + self._obj_children: dict[str, ConfigObj | tuple[ConfigObj, ...]] = {} - self._obj_subscriptions: List[SubscriptionParent] = [] + self._obj_subscriptions: list[SubscriptionParent] = [] self._last_model: BaseModel = model @@ -47,25 +51,22 @@ def _full_obj_path(self) -> str: return '.'.join(self._obj_path) @classmethod - def from_model( - cls, - model: BaseModel, - path: Tuple[str, ...] = ('__root__',), - parent: Union[MISSING_TYPE, HINT_CONFIG_OBJ] = MISSING, - ): + def from_model(cls, model: BaseModel, path: tuple[str, ...] = ('__root__',), + parent: MISSING_TYPE | ConfigObj = MISSING, **kwargs) -> Self: + # Copy functions from the class definition to the child class functions = {} for name, member in getmembers(model.__class__): - if not name.startswith('_') and name not in NO_COPY and isfunction(member): + if not name.startswith('_') and name not in NO_COPY and should_be_copied(member): functions[name] = member # Create a new class that pulls down the user defined functions if there are any # It's not possible to attach the functions to the existing class instance if functions: new_cls = type(f'{model.__class__.__name__}{cls.__name__}', (cls,), functions) - ret = new_cls(model, path, parent) + ret = new_cls(model, path, parent, **kwargs) else: - ret = cls(model, path, parent) + ret = cls(model, path, parent, **kwargs) # Set the values or create corresponding subclasses keys = [] @@ -77,10 +78,10 @@ def from_model( keys.append(key) if isinstance(value, BaseModel): - ret._obj_children[key] = attrib = cls.from_model(value, path=(*path, key), parent=ret) + ret._obj_children[key] = attrib = ConfigObj.from_model(value, path=(*path, key), parent=ret) elif isinstance(value, tuple) and all(isinstance(x, BaseModel) for x in value): ret._obj_children[key] = attrib = tuple( - cls.from_model(o, path=(*path, key, str(i)), parent=ret) for i, o in enumerate(value) + ConfigObj.from_model(o, path=(*path, key, str(i)), parent=ret) for i, o in enumerate(value) ) else: ret._obj_values[key] = attrib = value @@ -145,7 +146,7 @@ def _set_values(self, obj: BaseModel) -> bool: return propagate - def __repr__(self): + def __repr__(self) -> str: return f'<{self.__class__.__name__} {self._full_obj_path}>' # def __getattr__(self, item): @@ -155,9 +156,17 @@ def __repr__(self): # ------------------------------------------------------------------------------------------------------------------ # Match class signature with the Mixin Classes # ------------------------------------------------------------------------------------------------------------------ - def subscribe_for_changes( - self, func: Callable[[], Any], propagate: bool = False, on_next_load: bool = True - ) -> 'easyconfig.config_objs.ConfigObjSubscription': + @property + def loaded_file_path(self) -> Path: + """Path to the loaded configuration file""" + + obj = self + while obj._obj_path != ('__root__',): + obj = obj._obj_parent + return obj.loaded_file_path + + def subscribe_for_changes(self, func: Callable[[], Any], *, + propagate: bool = False, on_next_load: bool = True) -> ConfigObjSubscription: """When a value in this container changes the passed function will be called. :param func: function which will be called @@ -176,18 +185,34 @@ def subscribe_for_changes( self._obj_subscriptions.append(sub) return ConfigObjSubscription(sub, target) + # ----------------------------------------------------- + # pydantic 1 + @classmethod + def parse_obj(cls, *args: Any, **kwargs: Any): + raise FunctionCallNotAllowedError() + + @classmethod + def parse_raw(cls, *args: Any, **kwargs: Any): + raise FunctionCallNotAllowedError() + + @classmethod + def parse_file(cls, *args: Any, **kwargs: Any): + raise FunctionCallNotAllowedError() + @classmethod - def parse_obj(cls, *args, **kwargs): + def from_orm(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() + # ----------------------------------------------------- + # pydantic 2 @classmethod - def parse_raw(cls, *args, **kwargs): + def model_validate_strings(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() @classmethod - def parse_file(cls, *args, **kwargs): + def model_validate(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() @classmethod - def from_orm(cls, *args, **kwargs): + def model_validate_json(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() diff --git a/src/easyconfig/config_objs/subscription.py b/src/easyconfig/config_objs/subscription.py index a4c925b..9d5a46c 100644 --- a/src/easyconfig/config_objs/subscription.py +++ b/src/easyconfig/config_objs/subscription.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Callable, Final, Optional +from collections.abc import Callable +from typing import TYPE_CHECKING, Final from easyconfig.errors import SubscriptionAlreadyCanceledError @@ -14,9 +15,9 @@ def __init__( cfg_obj: 'easyconfig.config_objs.HINT_CONFIG_OBJ', propagate: bool = False, on_next: bool = False, - ): - self.func: Optional[Callable] = func - self.cfg_obj: Optional['easyconfig.config_objs.HINT_CONFIG_OBJ'] = cfg_obj + ) -> None: + self.func: Callable | None = func + self.cfg_obj: easyconfig.config_objs.HINT_CONFIG_OBJ | None = cfg_obj self.propagate: Final = propagate self.on_next: bool = on_next @@ -35,15 +36,15 @@ def notify(self, value_changed: bool) -> bool: # don't propagate first load value return False - def cancel(self): + def cancel(self) -> None: self.cfg_obj._obj_subscriptions.remove(self) self.func = None self.cfg_obj = None class ConfigObjSubscription: - def __init__(self, sub_obj: 'SubscriptionParent', target: str): - self._sub_obj: Optional['SubscriptionParent'] = sub_obj + def __init__(self, sub_obj: 'SubscriptionParent', target: str) -> None: + self._sub_obj: SubscriptionParent | None = sub_obj self._sub_target: Final = target def cancel(self): @@ -56,5 +57,5 @@ def cancel(self): raise SubscriptionAlreadyCanceledError(msg) parent.cancel() - def __str__(self): + def __str__(self) -> str: return f'<{self.__class__.__name__} {self._sub_target}>' diff --git a/src/easyconfig/create_app_config.py b/src/easyconfig/create_app_config.py index abc554f..2d19ab1 100644 --- a/src/easyconfig/create_app_config.py +++ b/src/easyconfig/create_app_config.py @@ -1,5 +1,6 @@ +from collections.abc import Callable, Iterable from inspect import isfunction -from typing import Any, Callable, Dict, FrozenSet, Iterable, Optional, TypeVar, Union +from typing import Any, TypeAlias, TypeVar from pydantic import BaseModel @@ -9,11 +10,11 @@ TYPE_WRAPPED = TypeVar('TYPE_WRAPPED', bound=BaseModel) -TYPE_DEFAULTS = Union[BaseModel, Dict[str, Any]] +TYPE_DEFAULTS: TypeAlias = BaseModel | dict[str, Any] # noinspection PyProtectedMember -def check_field_args(model: AppConfig, allowed: FrozenSet[str]): +def check_field_args(model: AppConfig, allowed: frozenset[str]) -> None: """Check extra args of pydantic fields""" # Model fields @@ -38,8 +39,8 @@ def check_field_args(model: AppConfig, allowed: FrozenSet[str]): def get_file_values( - model: BaseModel, file_values: Union[MISSING_TYPE, None, TYPE_DEFAULTS, Callable[[], TYPE_DEFAULTS]] = MISSING -) -> Optional[BaseModel]: + model: BaseModel, file_values: MISSING_TYPE | None | TYPE_DEFAULTS | Callable[[], TYPE_DEFAULTS] = MISSING +) -> BaseModel | None: # Implicit default if file_values is MISSING: file_values = model @@ -61,12 +62,13 @@ def get_file_values( def create_app_config( model: TYPE_WRAPPED, - file_values: Union[MISSING_TYPE, None, TYPE_DEFAULTS, Callable[[], TYPE_DEFAULTS]] = MISSING, - validate_file_values=True, - check_field_extra_args: Optional[Iterable[str]] = (ARG_NAME_IN_FILE,), + file_values: MISSING_TYPE | None | TYPE_DEFAULTS | Callable[[], TYPE_DEFAULTS] = MISSING, *, + validate_file_values: bool = True, + check_field_extra_args: Iterable[str] | None = (ARG_NAME_IN_FILE,), ) -> TYPE_WRAPPED: - app_cfg = AppConfig.from_model(model) - app_cfg._file_defaults = get_file_values(model, file_values) + + file_defaults = get_file_values(model, file_values) + app_cfg = AppConfig.from_model(model, file_defaults=file_defaults) # ensure that the extra args have no typos if check_field_extra_args is not None: diff --git a/src/easyconfig/errors/__init__.py b/src/easyconfig/errors/__init__.py index f3ed672..f54970a 100644 --- a/src/easyconfig/errors/__init__.py +++ b/src/easyconfig/errors/__init__.py @@ -1,3 +1,8 @@ -from .errors import DuplicateSubscriptionError, ExtraKwArgsNotAllowedError, \ - FileDefaultsNotSetError, FunctionCallNotAllowedError, SubscriptionAlreadyCanceledError +from .errors import ( + DuplicateSubscriptionError, + ExtraKwArgsNotAllowedError, + FileDefaultsNotSetError, + FunctionCallNotAllowedError, + SubscriptionAlreadyCanceledError, +) from .handler import set_exception_handler diff --git a/src/easyconfig/errors/errors.py b/src/easyconfig/errors/errors.py index 6ec3a22..73f946b 100644 --- a/src/easyconfig/errors/errors.py +++ b/src/easyconfig/errors/errors.py @@ -19,7 +19,7 @@ class FileDefaultsNotSetError(EasyConfigError): class FunctionCallNotAllowedError(EasyConfigError): - def __init__(self): + def __init__(self) -> None: super().__init__('Call "load_config_dict" or "load_config_file" on the app config instance!') diff --git a/src/easyconfig/errors/handler.py b/src/easyconfig/errors/handler.py index 5dea4d6..8ab78d7 100644 --- a/src/easyconfig/errors/handler.py +++ b/src/easyconfig/errors/handler.py @@ -1,4 +1,5 @@ -from typing import Any, Callable +from collections.abc import Callable +from typing import Any def default_exception_handler(e: Exception): @@ -8,10 +9,10 @@ def default_exception_handler(e: Exception): HANDLER: Callable[[Exception], Any] = default_exception_handler -def set_exception_handler(handler: Callable[[Exception], Any]): - global HANDLER # noqa: PLW0603 +def set_exception_handler(handler: Callable[[Exception], Any]) -> None: + global HANDLER HANDLER = handler -def process_exception(e: Exception): +def process_exception(e: Exception) -> None: HANDLER(e) diff --git a/src/easyconfig/expansion/expand.py b/src/easyconfig/expansion/expand.py index 8e3e588..18a5f15 100644 --- a/src/easyconfig/expansion/expand.py +++ b/src/easyconfig/expansion/expand.py @@ -17,7 +17,7 @@ RE_ESCAPED = re.compile(RE_REPLACE.pattern.replace(r'(? tuple[str, str]: +def read_value(key: str, /, loc: ExpansionLocation) -> tuple[str, str]: if is_path(key): name, value = read_file_contents(key, loc) else: @@ -40,12 +40,11 @@ def expand_text(text: str, /, loc: ExpansionLocation): while m := RE_REPLACE.search(text, pos_start): m_start = m.start() m_end = m.end() - full_match = m_start == 0 and m_end == len(text) raw_value = m.group('value') raw_value = raw_value.replace('$}', '}') - name, value = read_value(raw_value, full_match, loc=loc) + name, value = read_value(raw_value, loc=loc) # value is valid value = expand_text(value, loc=loc.expand_value(name)) diff --git a/src/easyconfig/models/__init__.py b/src/easyconfig/models/__init__.py index 83b0e4b..257a209 100644 --- a/src/easyconfig/models/__init__.py +++ b/src/easyconfig/models/__init__.py @@ -1,6 +1,7 @@ from .app import AppConfigMixin from .config import ConfigMixin + # isort: split # Convenience Classes with sensible defaults diff --git a/src/easyconfig/models/app.py b/src/easyconfig/models/app.py index 026a623..078cec9 100644 --- a/src/easyconfig/models/app.py +++ b/src/easyconfig/models/app.py @@ -2,29 +2,41 @@ from typing import TYPE_CHECKING +from typing_extensions import Self + from easyconfig.models.config import ConfigMixin if TYPE_CHECKING: from pathlib import Path + from easyconfig.pre_process import PreProcess + class AppConfigMixin(ConfigMixin): - def set_file_path(self, path: Path | str): + @property + def loaded_file_path(self) -> Path: + """Path to the loaded configuration file""" + + @property + def load_preprocess(self) -> PreProcess: + """A preprocessor which can be used to preprocess the configuration data before it is loaded""" + + def set_file_path(self, path: Path | str) -> None: """Set the path to the configuration file. If no file extension is specified ``.yml`` will be automatically appended. :param path: Path obj or str """ - def load_config_dict(self, cfg: dict, /, expansion: bool = True): + def load_config_dict(self, cfg: dict, /, expansion: bool = True) -> Self: """Load the configuration from a dictionary :param cfg: config dict which will be loaded :param expansion: Expand ${...} in strings """ - def load_config_file(self, path: Path | str | None = None, expansion: bool = True): + def load_config_file(self, path: Path | str | None = None, *, expansion: bool = True) -> Self: """Load configuration from a yaml file. If the file does not exist a default file will be created :param path: Path to file diff --git a/src/easyconfig/models/config.py b/src/easyconfig/models/config.py index 9469b88..2381f88 100644 --- a/src/easyconfig/models/config.py +++ b/src/easyconfig/models/config.py @@ -1,17 +1,24 @@ -from typing import TYPE_CHECKING, Any, Callable +from __future__ import annotations + +from typing import TYPE_CHECKING, Any from easyconfig.errors import FunctionCallNotAllowedError if TYPE_CHECKING: - import easyconfig - import easyconfig.config_objs + from collections.abc import Callable + from pathlib import Path + + from easyconfig.config_objs import ConfigObjSubscription class ConfigMixin: - def subscribe_for_changes( - self, func: Callable[[], Any], propagate: bool = False, on_next_load: bool = True - ) -> 'easyconfig.config_objs.ConfigObjSubscription': + @property + def loaded_file_path(self) -> Path: + """Path to the loaded configuration file""" + + def subscribe_for_changes(self, func: Callable[[], Any], *, + propagate: bool = False, on_next_load: bool = True) -> ConfigObjSubscription: """When a value in this container changes the passed function will be called. :param func: function which will be called @@ -20,18 +27,34 @@ def subscribe_for_changes( :return: object which can be used to cancel the subscription """ + # ----------------------------------------------------- + # pydantic 1 + @classmethod + def parse_obj(cls, *args: Any, **kwargs: Any): + raise FunctionCallNotAllowedError() + + @classmethod + def parse_raw(cls, *args: Any, **kwargs: Any): + raise FunctionCallNotAllowedError() + + @classmethod + def parse_file(cls, *args: Any, **kwargs: Any): + raise FunctionCallNotAllowedError() + @classmethod - def parse_obj(cls, *args, **kwargs): + def from_orm(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() + # ----------------------------------------------------- + # pydantic 2 @classmethod - def parse_raw(cls, *args, **kwargs): + def model_validate_strings(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() @classmethod - def parse_file(cls, *args, **kwargs): + def model_validate(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() @classmethod - def from_orm(cls, *args, **kwargs): + def model_validate_json(cls, *args: Any, **kwargs: Any): raise FunctionCallNotAllowedError() diff --git a/src/easyconfig/pre_process/__init__.py b/src/easyconfig/pre_process/__init__.py new file mode 100644 index 0000000..37c01a2 --- /dev/null +++ b/src/easyconfig/pre_process/__init__.py @@ -0,0 +1,4 @@ +from .delete_entry import DeleteEntryPreProcess +from .move_entry import MoveEntryPreProcess +from .pre_process import PreProcess +from .rename_entry import RenameEntryPreProcess diff --git a/src/easyconfig/pre_process/base.py b/src/easyconfig/pre_process/base.py new file mode 100644 index 0000000..9184729 --- /dev/null +++ b/src/easyconfig/pre_process/base.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from collections.abc import Callable, MutableMapping, MutableSequence +from typing import TYPE_CHECKING, Any, Final, NewType, TypeVar + +from easyconfig.yaml import cmap_from_model + + +if TYPE_CHECKING: + from pydantic import BaseModel + + +ContainingObj = NewType('ContainingObj', MutableSequence | MutableMapping) + +A = TypeVar('A', bound=ContainingObj) + + +class PathAccessor: + def __init__(self, path: tuple[str | int, ...]) -> None: + if not path: + msg = 'Path with at least one entry expected' + raise ValueError(msg) + + self.path: Final = path + + @property + def path_name(self) -> str: + return '.'.join(str(part) for part in self.path) + + @property + def containing_name(self) -> str: + return '.'.join(str(part) for part in self.path[:-1]) + + @property + def key_name(self) -> str: + return str(self.path[-1]) + + def get_containing_obj(self, root: MutableSequence | MutableMapping) -> ContainingObj | None: + path = self.path + + if len(path) <= 1: + return root + + obj = root + + try: + for part in path[:-1]: + obj = obj[part] + except (KeyError, IndexError): + return None + + if not isinstance(obj, (MutableSequence, MutableMapping)): + return None + + return obj + + def get_containing_obj_or_create_default(self, root: MutableSequence | MutableMapping, + default: BaseModel | None = None) -> ContainingObj | None: + if (dst_obj := self.get_containing_obj(root)) is not None: + return dst_obj + + if default is None: + return None + + default_yaml = cmap_from_model(default) + obj = root + + current_path: tuple[str, ...] = () + for part in self.path[:-1]: + current_path += (str(part),) + + if default_yaml is not None: + try: + default_yaml = default_yaml[part] + except (KeyError, IndexError): + default_yaml = None + + try: + obj = obj[part] + except (KeyError, IndexError): + if default_yaml is None: + return None + + if isinstance(default_yaml, list): + obj[part] = [] + elif isinstance(default_yaml, dict): + obj[part] = {} + else: + msg = f'Unsupported type {type(default_yaml)} at {".".join(current_path):s}' + raise TypeError(msg) from None + obj = obj[part] + + return obj + + def obj_exists(self, containing_obj: ContainingObj) -> bool: + try: + containing_obj[self.path[-1]] + except (KeyError, IndexError): + return False + + return True + + def set_obj(self, containing_obj: A, value: Any) -> A: + if self.obj_exists(containing_obj): + msg = f'Object {self.path_name:s} already exists' + raise ValueError(msg) + + containing_obj[self.path[-1]] = value + return containing_obj + + def pop_obj(self, containing_obj: ContainingObj) -> Any: + return containing_obj.pop(self.path[-1]) + + def __repr__(self) -> str: + return f'<{self.__class__.__name__} {self.path_name:s}>' + + +class PreProcessBase: + def run(self, obj: MutableSequence | MutableMapping, log_func: Callable[[str], Any] | None = None) -> None: + raise NotImplementedError() + + def __eq__(self, other) -> bool: + raise NotImplementedError() diff --git a/src/easyconfig/pre_process/delete_entry.py b/src/easyconfig/pre_process/delete_entry.py new file mode 100644 index 0000000..e1c10d8 --- /dev/null +++ b/src/easyconfig/pre_process/delete_entry.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, Final + +from typing_extensions import override + +from .base import PathAccessor, PreProcessBase + + +class DeleteEntryPreProcess(PreProcessBase): + def __init__(self, src: tuple[str | int, ...]) -> None: + self.dst: Final = PathAccessor(src) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DeleteEntryPreProcess): + return False + return self.dst == other.dst + + @override + def run(self, obj: dict | list, log_func: Callable[[str], Any] | None = None) -> None: + if (parent := self.dst.get_containing_obj(obj)) is None: + return None + + # If it doesn't exist we don't have to drop it + if not self.dst.obj_exists(parent): + return None + + self.dst.pop_obj(parent) + + if log_func is not None: + log_func(f'Entry "{self.dst.path_name:s}" was deleted') diff --git a/src/easyconfig/pre_process/move_entry.py b/src/easyconfig/pre_process/move_entry.py new file mode 100644 index 0000000..57ad208 --- /dev/null +++ b/src/easyconfig/pre_process/move_entry.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from typing_extensions import override + +from easyconfig.yaml import cmap_from_model + +from .base import PathAccessor, PreProcessBase + + +if TYPE_CHECKING: + from collections.abc import Callable, MutableMapping, MutableSequence + + from pydantic import BaseModel + + +class MoveEntryPreProcess(PreProcessBase): + def __init__(self, src: tuple[str | int, ...], dst: tuple[str | int, ...], + defaults: BaseModel | None = None) -> None: + self.src: Final = PathAccessor(src) + self.dst: Final = PathAccessor(dst) + self.default: Final = defaults + + # Validate the dst if we have a default so we catch e.g. typos + if self.default is not None: + yaml_defaults = cmap_from_model(self.default) + if self.dst.get_containing_obj(yaml_defaults) is None: + msg = f'Path "{self.dst.containing_name}" does not exist in default' + raise ValueError(msg) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MoveEntryPreProcess): + return False + return self.src == other.src and self.dst == other.dst + + @override + def run(self, obj: MutableSequence | MutableMapping, log_func: Callable[[str], Any] | None = None) -> None: + if (src_obj := self.src.get_containing_obj(obj)) is None: + return None + + if (dst_obj := self.dst.get_containing_obj_or_create_default(obj, self.default)) is None: + return None + + # Never overwrite something + if self.dst.obj_exists(dst_obj): + return None + + # Source has to exist if we want to move to it + if not self.src.obj_exists(src_obj): + return None + + self.dst.set_obj(dst_obj, self.src.pop_obj(src_obj)) + + if log_func is not None: + log_func(f'Entry "{self.src.path_name:s}" moved to "{self.dst.path_name:s}"') diff --git a/src/easyconfig/pre_process/pre_process.py b/src/easyconfig/pre_process/pre_process.py new file mode 100644 index 0000000..42e9765 --- /dev/null +++ b/src/easyconfig/pre_process/pre_process.py @@ -0,0 +1,83 @@ +from collections.abc import Callable, MutableMapping, MutableSequence +from typing import Any, Final, TypeAlias + +from pydantic import BaseModel +from typing_extensions import Self, override + +from easyconfig.pre_process.base import PreProcessBase + +from .delete_entry import DeleteEntryPreProcess +from .move_entry import MoveEntryPreProcess +from .rename_entry import RenameEntryPreProcess + + +PATH_INPUT_TYPE: TypeAlias = tuple[str | int, ...] | list[str | int] + + +def get_path_tuple(obj: PATH_INPUT_TYPE) -> tuple[str | int, ...]: + if isinstance(obj, list): + obj = tuple(obj) + + if not isinstance(obj, tuple): + msg = f'Must be tuple or list, is {type(obj)}' + raise TypeError(msg) + + for o in obj: + if not isinstance(o, (str, int)): + msg = f'Must be str or int, is {type(obj)}' + raise TypeError(msg) + return obj + + +class PreProcess(PreProcessBase): + def __init__(self, default: BaseModel | None = None) -> None: + self._operations: tuple[PreProcessBase, ...] = () + self._default: Final = default + self._log: Callable[[str], Any] | None = None + + def _add(self, obj: PreProcessBase) -> None: + for existing in self._operations: + if existing == obj: + msg = f'Operation {obj} already exists' + raise ValueError(msg) + self._operations += (obj,) + + def set_log_func(self, log_func: Callable[[str], Any] | None) -> Self: + """Set a log function that will be called for each operation that is executed""" + self._log = log_func + return self + + def move_entry(self, src: PATH_INPUT_TYPE, dst: PATH_INPUT_TYPE) -> Self: + """Move an entry to a different location in the configuration + + :param src: current path to the entry + :param dst: new path to the entry + :return: + """ + self._add(MoveEntryPreProcess(get_path_tuple(src), get_path_tuple(dst), defaults=self._default)) + return self + + def rename_entry(self, src: PATH_INPUT_TYPE, name: str) -> Self: + """Rename an entry in the configuration + + :param src: path to the entry + :param name: new name + """ + self._add(RenameEntryPreProcess(get_path_tuple(src), name)) + return self + + def delete_entry(self, path: PATH_INPUT_TYPE) -> Self: + """Delete an entry in the configuration + + :param path: path to the entry + """ + self._add(DeleteEntryPreProcess(get_path_tuple(path))) + return self + + @override + def run(self, obj: MutableSequence | MutableMapping, log_func: Callable[[str], Any] | None = None) -> None: + if log_func is None: + log_func = self._log + + for op in self._operations: + op.run(obj, log_func) diff --git a/src/easyconfig/pre_process/rename_entry.py b/src/easyconfig/pre_process/rename_entry.py new file mode 100644 index 0000000..db6b72d --- /dev/null +++ b/src/easyconfig/pre_process/rename_entry.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from typing_extensions import override + +from .base import PathAccessor, PreProcessBase + + +if TYPE_CHECKING: + from collections.abc import Callable + + +class RenameEntryPreProcess(PreProcessBase): + def __init__(self, src: tuple[str | int, ...], new_name: str) -> None: + self.src: Final = PathAccessor(src) + self.dst: Final = PathAccessor(src[:-1] + (new_name,)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, RenameEntryPreProcess): + return False + return self.src == other.src and self.dst == other.dst + + @override + def run(self, obj: dict | list, log_func: Callable[[str], Any] | None = None) -> None: + if (parent := self.src.get_containing_obj(obj)) is None: + return None + + # Never overwrite something + if self.dst.obj_exists(parent): + return None + + # Source has to exist if we want to move to it + if not self.src.obj_exists(parent): + return None + + self.dst.set_obj(parent, self.src.pop_obj(parent)) + + if log_func is not None: + c_name = self.src.containing_name + loc = f' in "{c_name}"' if c_name else '' + log_func(f'Entry "{self.src.key_name:s}" renamed to "{self.dst.key_name:s}"{loc:s}') diff --git a/src/easyconfig/yaml/__init__.py b/src/easyconfig/yaml/__init__.py index 52c619c..e2c2640 100644 --- a/src/easyconfig/yaml/__init__.py +++ b/src/easyconfig/yaml/__init__.py @@ -1,5 +1,6 @@ from easyconfig.yaml.yaml import CommentedMap, CommentedSeq, yaml_rt, yaml_safe + # isort: split from easyconfig.yaml.align import write_aligned_yaml diff --git a/src/easyconfig/yaml/align.py b/src/easyconfig/yaml/align.py index fe81f78..115abdf 100644 --- a/src/easyconfig/yaml/align.py +++ b/src/easyconfig/yaml/align.py @@ -1,12 +1,12 @@ from io import StringIO -from typing import Any, Optional, Tuple, Union +from typing import Any from ruamel.yaml import CommentToken from easyconfig.yaml import yaml_rt -def get_column(obj: Tuple[Any, Any, Optional[CommentToken], Any]): +def get_column(obj: tuple[Any, Any, CommentToken | None, Any]): if (token := obj[2]) is None: return 0 return token.column @@ -48,7 +48,7 @@ def align_comments(d, extra_indent=0): return None -def remove_none(obj: Union[dict]): +def remove_none(obj: dict) -> None: rem = [] for index, value in obj.items(): if isinstance(value, dict): diff --git a/src/easyconfig/yaml/from_model.py b/src/easyconfig/yaml/from_model.py index da6b4ff..88af029 100644 --- a/src/easyconfig/yaml/from_model.py +++ b/src/easyconfig/yaml/from_model.py @@ -48,6 +48,9 @@ def _get_yaml_value(obj, parent_model: BaseModel, *, skip_none=True, obj_name: s def cmap_from_model(model: BaseModel, skip_none=True) -> CommentedMap: cmap = CommentedMap() for obj_name, field in model.model_fields.items(): + if field.exclude is True: + continue + value = getattr(model, obj_name, MISSING) if value is MISSING or (skip_none and value is None): continue diff --git a/tests/helper/my_path.py b/tests/helper/my_path.py index ab1cc13..ed32948 100644 --- a/tests/helper/my_path.py +++ b/tests/helper/my_path.py @@ -1,7 +1,6 @@ from io import StringIO, TextIOWrapper from pathlib import Path as _Path from sys import version_info -from typing import Optional _path_type = type(_Path()) @@ -12,7 +11,7 @@ class Path(_path_type): _flavour = _path_type._flavour - def __init__(self, *args, does_exist: bool = True, initial_value: Optional[str] = None, **kwargs): + def __init__(self, *args, does_exist: bool = True, initial_value: str | None = None, **kwargs) -> None: super().__init__() if hasattr(Path, '_from_parts'): @@ -35,14 +34,14 @@ def __new__(cls, *args, **kwargs): else: - def __init__(self, *args, does_exist: bool = True, initial_value: Optional[str] = None, **kwargs): + def __init__(self, *args, does_exist: bool = True, initial_value: str | None = None, **kwargs) -> None: super().__init__(*args) # Own Path implementation self.does_exist: bool = does_exist self.contents = None self._create_buffer(initial_value) - def _create_buffer(self, initial_value: Optional[str] = None): + def _create_buffer(self, initial_value: str | None = None) -> None: self.contents = StringIO(initial_value) self.contents.close = lambda: None diff --git a/tests/helper/test_my_path.py b/tests/helper/test_my_path.py index 186fc51..0460a9a 100644 --- a/tests/helper/test_my_path.py +++ b/tests/helper/test_my_path.py @@ -1,7 +1,7 @@ from .my_path import Path -def test_my_path(): +def test_my_path() -> None: # Test some path operations a = Path('a') b = a / 'c' diff --git a/tests/test_app_creation.py b/tests/test_app_creation.py index 715a1f3..f42c57b 100644 --- a/tests/test_app_creation.py +++ b/tests/test_app_creation.py @@ -1,17 +1,18 @@ # ruff: noqa: RUF012 from enum import Enum -from typing import List import pytest from pydantic import BaseModel, Field, ValidationError from easyconfig import create_app_config +from easyconfig.config_objs import AppConfig, ConfigObj from easyconfig.errors import ExtraKwArgsNotAllowedError, FileDefaultsNotSetError +from easyconfig.models import AppBaseModel as EasyAppBaseModel from easyconfig.models import BaseModel as EasyBaseModel -def test_simple(): +def test_simple() -> None: class SimpleModel(BaseModel): a: int = Field(5, alias='aaa') @@ -22,7 +23,21 @@ class SimpleModel(BaseModel): create_app_config(SimpleModel(), {'aaa': 'asdf'}) -def test_default_yaml(): +def test_process() -> None: + class SimpleModel(BaseModel): + a: int = Field(5, alias='aaa') + + msgs = [] + + a = create_app_config(SimpleModel(aaa=99)) + a.load_preprocess.rename_entry(['zzz'], 'aaa').set_log_func(msgs.append) + a.load_config_dict({'zzz': 999}) + + assert a.a == 999 + assert msgs == ['Entry "zzz" renamed to "aaa"'] + + +def test_default_yaml() -> None: class SimpleModel(BaseModel): a: int = Field(5, alias='aaa') @@ -37,7 +52,7 @@ class SimpleModel(BaseModel): a.generate_default_yaml() -def test_callback_for_default(): +def test_callback_for_default() -> None: class SimpleModel(BaseModel): a: int = Field(5, alias='aaa') @@ -51,7 +66,7 @@ def get_default(): assert a._file_defaults.a == 999 -def test_extra_kwargs(): +def test_extra_kwargs() -> None: class SimpleModelOk(BaseModel): a: int = Field(5, alias='aaa', in_file=False) @@ -66,7 +81,7 @@ class SimpleModelErr(BaseModel): assert str(e.value) == 'Extra kwargs for field "a" of SimpleModelErr are not allowed: in__file' -def test_list_of_models(): +def test_list_of_models() -> None: class MyEnum(str, Enum): A = 'aa' @@ -76,7 +91,7 @@ class SimpleModel(EasyBaseModel): c: MyEnum = MyEnum.A class EncapModel(EasyBaseModel): - c: List[SimpleModel] = [] + c: list[SimpleModel] = [] create_app_config( EncapModel( @@ -86,3 +101,21 @@ class EncapModel(EasyBaseModel): ] ) ) + + +def test_path() -> None: + class SimpleModel(EasyBaseModel): + z: str = 'asdf' + + class ParentModel(EasyAppBaseModel): + b: SimpleModel = SimpleModel() + + a = create_app_config(ParentModel()) + + assert isinstance(a, AppConfig) + assert isinstance(a.b, ConfigObj) + + a._file_path = o = object() + + assert a.loaded_file_path is o + assert a.b.loaded_file_path is o diff --git a/tests/test_config_objs/test_config_obj.py b/tests/test_config_objs/test_config_obj.py index 6448821..aef3a69 100644 --- a/tests/test_config_objs/test_config_obj.py +++ b/tests/test_config_objs/test_config_obj.py @@ -5,7 +5,7 @@ from easyconfig.errors import FunctionCallNotAllowedError -def test__repr__(): +def test__repr__() -> None: class SimpleModel(BaseModel): a: int = 5 @@ -13,7 +13,7 @@ class SimpleModel(BaseModel): assert repr(o) == '' -def test_forbidden_calls(): +def test_forbidden_calls() -> None: class SimpleModel(BaseModel): a: int = 5 @@ -34,16 +34,22 @@ class SimpleModel(BaseModel): assert str(e.value) == 'Call "load_config_dict" or "load_config_file" on the app config instance!' -def test_attr_access(): +def test_attr_access() -> None: test_list = [] class SimpleModel(BaseModel): a: int = 5 - def append(self): + def append(self) -> None: test_list.append(True) + @property + def value_10(self) -> int: + return 10 + o = ConfigObj.from_model(SimpleModel()) - o.append() + o.append() assert test_list == [True] + + assert o.value_10 == 10 diff --git a/tests/test_config_objs/test_from_model.py b/tests/test_config_objs/test_from_model.py index 48c298f..f3daee3 100644 --- a/tests/test_config_objs/test_from_model.py +++ b/tests/test_config_objs/test_from_model.py @@ -1,12 +1,10 @@ -from typing import Tuple - from pydantic import BaseModel, PrivateAttr from easyconfig.config_objs import ConfigObj from easyconfig.models import ConfigMixin -def test_parse_values(): +def test_parse_values() -> None: class SimpleModel(BaseModel): a: int = 5 b: int = 6 @@ -16,7 +14,7 @@ class SimpleModel(BaseModel): assert o.b == 6 -def test_parse_submodels(): +def test_parse_submodels() -> None: class SubModel(BaseModel, ConfigMixin): a: int = 5 b: int = 6 @@ -43,12 +41,12 @@ class SimpleModel(BaseModel, ConfigMixin): assert obj._obj_path == ('__root__', 'b') -def test_parse_submodel_tupels(): +def test_parse_submodel_tupels() -> None: class SubModel(BaseModel, ConfigMixin): a: int = 5 class SimpleModel(BaseModel, ConfigMixin): - a: Tuple[SubModel, SubModel] = (SubModel(), SubModel(a=7)) + a: tuple[SubModel, SubModel] = (SubModel(), SubModel(a=7)) c: int = 3 o = ConfigObj.from_model(SimpleModel()) @@ -65,7 +63,7 @@ class SimpleModel(BaseModel, ConfigMixin): assert obj._obj_path == ('__root__', 'a', '1') -def test_func_call(): +def test_func_call() -> None: class SimpleModel(BaseModel): a: int = 5 b: int = 6 @@ -84,13 +82,13 @@ def set_vars(self): assert o.b == 2 -def test_private_attr(): +def test_private_attr() -> None: class SimpleModel(BaseModel): a: int = 1 _b: int = PrivateAttr() _c: int = PrivateAttr(3) - def set_vars(self): + def set_vars(self) -> None: self._b = 99 o = ConfigObj.from_model(SimpleModel()) diff --git a/tests/test_config_objs/test_subscriptions.py b/tests/test_config_objs/test_subscriptions.py index 574f3d8..8dd239b 100644 --- a/tests/test_config_objs/test_subscriptions.py +++ b/tests/test_config_objs/test_subscriptions.py @@ -6,12 +6,12 @@ from easyconfig.models import ConfigMixin -def test_sub_name(): +def test_sub_name() -> None: class SimpleModel(BaseModel, ConfigMixin): a: int = 5 b: int = 6 - def my_func(): + def my_func() -> None: pass o = ConfigObj.from_model(SimpleModel()) @@ -20,7 +20,7 @@ def my_func(): assert str(sub) == '' -def test_sub_simple(): +def test_sub_simple() -> None: class SimpleModel(BaseModel, ConfigMixin): a: int = 5 b: int = 6 @@ -39,7 +39,7 @@ class SimpleModel(BaseModel, ConfigMixin): m.assert_called_once_with() -def test_sub_sub_no_propagate(): +def test_sub_sub_no_propagate() -> None: class SubModel(BaseModel, ConfigMixin): a: int = 5 b: int = 6 @@ -71,7 +71,7 @@ class SimpleModel(BaseModel, ConfigMixin): sub_parent.cancel() -def test_sub_sub_propagate(): +def test_sub_sub_propagate() -> None: class SubModel(BaseModel, ConfigMixin): a: int = 5 b: int = 6 diff --git a/tests/test_expansion/conftest.py b/tests/test_expansion/conftest.py index 000b67a..13330e4 100644 --- a/tests/test_expansion/conftest.py +++ b/tests/test_expansion/conftest.py @@ -3,7 +3,7 @@ from easyconfig.expansion import load_var as var_module -@pytest.fixture() +@pytest.fixture def envs(monkeypatch): env_dict = {} monkeypatch.setattr(var_module, 'environ', env_dict) diff --git a/tests/test_expansion/test_env.py b/tests/test_expansion/test_env.py index 12f3cd9..07fd920 100644 --- a/tests/test_expansion/test_env.py +++ b/tests/test_expansion/test_env.py @@ -4,7 +4,7 @@ from easyconfig.expansion.location import ExpansionLocation, log -def test_parse_env_key(): +def test_parse_env_key() -> None: assert parse_env_key('NAME') == ('NAME', None) assert parse_env_key('NAME:DEFAULT') == ('NAME', 'DEFAULT') assert parse_env_key('NAME:DEFAULT:MULTIPLE:COLON') == ('NAME', 'DEFAULT:MULTIPLE:COLON') @@ -13,7 +13,7 @@ def test_parse_env_key(): assert parse_env_key(':DEFAULT') == ('', 'DEFAULT') -def test_read_env_existing(envs: dict): +def test_read_env_existing(envs: dict) -> None: envs.update({'NAME': 'asdf'}) loc = ExpansionLocation(loc=(), stack=()) @@ -21,7 +21,7 @@ def test_read_env_existing(envs: dict): assert read_env_var('NAME:DEFAULT', loc=loc) == ('NAME', 'asdf') -def test_read_file_missing(caplog, envs): +def test_read_file_missing(caplog, envs) -> None: caplog.set_level(logging.DEBUG) loc = ExpansionLocation(loc=('key_1',), stack=()) diff --git a/tests/test_expansion/test_expansion.py b/tests/test_expansion/test_expansion.py index 9bece63..71904bd 100644 --- a/tests/test_expansion/test_expansion.py +++ b/tests/test_expansion/test_expansion.py @@ -5,7 +5,7 @@ from easyconfig.expansion.location import ExpansionLocation -def test_regex(): +def test_regex() -> None: assert RE_REPLACE.fullmatch('${}') assert RE_REPLACE.fullmatch('${asdf}') @@ -16,7 +16,7 @@ def test_regex(): assert RE_REPLACE.search('${${}').group(1) == '${' -def test_load_env(envs: dict): +def test_load_env(envs: dict) -> None: envs.update( {'NAME': 'asdf', 'RECURSE': 'Name: ${NAME}', 'TEST_$_DOLLAR': 'DOLLAR_WORKS', 'TEST_}_CURLY': 'CURLY_WORKS'} ) @@ -44,7 +44,7 @@ def test_load_env(envs: dict): assert expand_text('${TEST_$}_CURLY}', loc) == 'CURLY_WORKS' -def test_env_cyclic_reference(envs: dict): +def test_env_cyclic_reference(envs: dict) -> None: envs.update({'NAME': '${SELF}', 'SELF': 'Name: ${SELF}'}) with pytest.raises(CyclicEnvironmentVariableReferenceError) as e: @@ -53,7 +53,7 @@ def test_env_cyclic_reference(envs: dict): assert str(e.value) == 'Cyclic environment variable reference: NAME -> SELF -> SELF (at __root__.a)' -def test_expansion(envs: dict): +def test_expansion(envs: dict) -> None: envs.update({'NAME': 'ASDF'}) obj = {'a': {'b': ['${NAME}']}, 'b': '${MISSING:DEFAULT}'} diff --git a/tests/test_expansion/test_file.py b/tests/test_expansion/test_file.py index ea309b6..85f636f 100644 --- a/tests/test_expansion/test_file.py +++ b/tests/test_expansion/test_file.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize( 'txt', [pytest.param('c:/path/file', id='p1'), pytest.param('//server/share/path/file', id='p2')] ) -def test_is_path_win(txt: str): +def test_is_path_win(txt: str) -> None: assert is_path(txt) assert is_path(txt.upper()) @@ -18,7 +18,7 @@ def test_is_path_win(txt: str): assert is_path(txt.upper()) -def test_is_path(): +def test_is_path() -> None: # unix path assert is_path('/asdf') assert is_path('/asdf.txt') @@ -27,7 +27,7 @@ def test_is_path(): assert not is_path('asdf.txt') -def test_parse_path(): +def test_parse_path() -> None: assert parse_path_key('c:/path/file') == ('c:/path/file', None) assert parse_path_key('//server/share/path/file') == ('//server/share/path/file', None) assert parse_path_key('/asdf') == ('/asdf', None) @@ -44,7 +44,7 @@ def test_parse_path(): assert parse_path_key('/asdf:MY_DEFAULT:WITH:MORE') == ('/asdf', 'MY_DEFAULT:WITH:MORE') -def test_read_file_existing(caplog, tmp_path): +def test_read_file_existing(caplog, tmp_path) -> None: caplog.set_level(logging.DEBUG) loc = ExpansionLocation(loc=(), stack=()) @@ -60,7 +60,7 @@ def test_read_file_existing(caplog, tmp_path): assert value == 'asdf asdf' -def test_read_file_missing(caplog): +def test_read_file_missing(caplog) -> None: caplog.set_level(logging.DEBUG) loc = ExpansionLocation(loc=('key_1',), stack=()) diff --git a/tests/test_implementation/test_class_signatures.py b/tests/test_implementation/test_class_signatures.py index e255e9c..86899f8 100644 --- a/tests/test_implementation/test_class_signatures.py +++ b/tests/test_implementation/test_class_signatures.py @@ -15,7 +15,7 @@ (ConfigObj, ConfigMixin), ], ) -def test_signatures_match(mixin_cls, impl_cls): +def test_signatures_match(mixin_cls, impl_cls) -> None: """Ensure that the mixin and the implementation have the same signatures""" for name, value in inspect.getmembers(mixin_cls): @@ -23,6 +23,15 @@ def test_signatures_match(mixin_cls, impl_cls): continue impl = getattr(impl_cls, name) + if isinstance(value, property): + assert value.fset is None + assert value.fdel is None + value = value.fget + if isinstance(impl, property): + assert impl.fset is None + assert impl.fdel is None + impl = impl.fget + target_spec = inspect.getfullargspec(value) current_spec = inspect.getfullargspec(impl) diff --git a/tests/test_implementation/test_load_file.py b/tests/test_implementation/test_load_file.py index 7d924a9..bea5b86 100644 --- a/tests/test_implementation/test_load_file.py +++ b/tests/test_implementation/test_load_file.py @@ -1,13 +1,13 @@ import pytest -from helper import Path from pydantic import BaseModel as PydanticBaseModel from easyconfig import AppBaseModel, BaseModel from easyconfig.config_objs import AppConfig +from helper import Path @pytest.mark.parametrize('base_cls', [PydanticBaseModel, BaseModel, AppBaseModel]) -def test_mutate_simple(base_cls): +def test_mutate_simple(base_cls) -> None: class SimpleModel(base_cls): a: int = 5 b: int = 6 @@ -25,7 +25,7 @@ class SimpleModel(base_cls): @pytest.mark.parametrize('base_cls', [PydanticBaseModel, BaseModel, AppBaseModel]) -def test_mutate_nested(base_cls): +def test_mutate_nested(base_cls) -> None: class SubModel(base_cls): aa: int = 5 diff --git a/tests/test_pre_process/test_base.py b/tests/test_pre_process/test_base.py new file mode 100644 index 0000000..944086d --- /dev/null +++ b/tests/test_pre_process/test_base.py @@ -0,0 +1,44 @@ +import pytest + +from easyconfig.pre_process.base import PathAccessor + + +def test_containing_obj() -> None: + p = PathAccessor(('a', 'b')) + assert p.get_containing_obj({}) is None + + d = {} + assert p.get_containing_obj({'a': d}) is d + + p = PathAccessor(('a', 0, 'c', 'd')) + assert p.get_containing_obj({'a': ({'c': d},)}) is d + + +def test_obj_exists() -> None: + assert PathAccessor(('a', 'b')).obj_exists({}) is False + assert PathAccessor(('a', )).obj_exists({'a': 1}) is True + + assert PathAccessor((1, )).obj_exists([1]) is False + assert PathAccessor((0, )).obj_exists([1]) is True + + +def test_set_obj() -> None: + assert PathAccessor(('a', 'b')).set_obj({}, 1) == {'b': 1} + + with pytest.raises(ValueError) as e: + PathAccessor(('a', 'b')).set_obj({'b': 1}, 2) + assert str(e.value) == 'Object a.b already exists' + + +def test_pop_obj() -> None: + obj = {'b': 1} + assert PathAccessor(('a', 'b')).pop_obj(obj) == 1 + assert obj == {} + + with pytest.raises(KeyError) as e: + PathAccessor(('a', 'b')).pop_obj(obj) == 1 + assert str(e.value) == "'b'" + + obj = [0, 1, 2] + assert PathAccessor((1, )).pop_obj(obj) == 1 + assert obj == [0, 2] diff --git a/tests/test_pre_process/test_delete_entry.py b/tests/test_pre_process/test_delete_entry.py new file mode 100644 index 0000000..7f929ca --- /dev/null +++ b/tests/test_pre_process/test_delete_entry.py @@ -0,0 +1,52 @@ + +from easyconfig.pre_process import DeleteEntryPreProcess + + +def test_delete_root() -> None: + f = DeleteEntryPreProcess(('a',)) + + d = {} + f.run(d) + assert d == {} + + d = {'b': 1} + f.run(d) + assert d == {'b': 1} + + d = {'a': 1} + f.run(d) + assert d == {} + + d = {'a': {'c': 1}} + f.run(d) + assert d == {} + + d = {'b': {'c': 1}} + f.run(d) + assert d == {'b': {'c': 1}} + + +def test_delete_nested() -> None: + f = DeleteEntryPreProcess(('a', 'b')) + + d = {} + f.run(d) + assert d == {} + + d = {'b': 1} + f.run(d) + assert d == {'b': 1} + + d = {'a': 1} + f.run(d) + assert d == {'a': 1} + + msg = [] + d = {'a': {'b': 1}} + f.run(d, msg.append) + assert d == {'a': {}} + assert msg == ['Entry "a.b" was deleted'] + + d = {'a': {'c': 1}} + f.run(d) + assert d == {'a': {'c': 1}} diff --git a/tests/test_pre_process/test_move_entry.py b/tests/test_pre_process/test_move_entry.py new file mode 100644 index 0000000..cc25c75 --- /dev/null +++ b/tests/test_pre_process/test_move_entry.py @@ -0,0 +1,56 @@ +import pytest + +from easyconfig import BaseModel +from easyconfig.pre_process import MoveEntryPreProcess + + +def test_move() -> None: + f = MoveEntryPreProcess(('a',), ('b',)) + + d = {} + f.run(d) + assert d == {} + + d = {'b': 1} + f.run(d) + assert d == {'b': 1} + + d = {'a': 1} + f.run(d) + assert d == {'b': 1} + + d = {'a': {'c': 1}} + f.run(d) + assert d == {'b': {'c': 1}} + + msg = [] + f = MoveEntryPreProcess(('b',), ('a', 'd')) + d = {'a': {'c': 1}, 'b': 2} + f.run(d, msg.append) + assert d == {'a': {'c': 1, 'd': 2}} + assert msg == ['Entry "b" moved to "a.d"'] + + +def test_move_dst_does_not_exist() -> None: + class TestModelChildChild(BaseModel): + db: int = 4 + + class TestModelChild(BaseModel): + b: int = 3 + c: TestModelChildChild = TestModelChildChild() + + class TestModel(BaseModel): + a: TestModelChild = TestModelChild() + + f = MoveEntryPreProcess(('z',), ('a', 'b'), defaults=TestModel()) + d = {'z': 2} + f.run(d) + assert d == {'a': {'b': 2}} + + with pytest.raises(ValueError) as e: + MoveEntryPreProcess(('z',), ('b', 'b', 'b'), defaults=TestModel()) + assert str(e.value) == 'Path "b.b" does not exist in default' + + with pytest.raises(ValueError) as e: + MoveEntryPreProcess(('z',), ('a', 'd', 'b'), defaults=TestModel()) + assert str(e.value) == 'Path "a.d" does not exist in default' diff --git a/tests/test_pre_process/test_rename_entry.py b/tests/test_pre_process/test_rename_entry.py new file mode 100644 index 0000000..b553694 --- /dev/null +++ b/tests/test_pre_process/test_rename_entry.py @@ -0,0 +1,39 @@ +from easyconfig.pre_process import RenameEntryPreProcess + + +def test_rename() -> None: + f = RenameEntryPreProcess(('a',), 'b') + f.run({}) + + d = {'b': 1} + f.run(d) + assert d == {'b': 1} + + msg = [] + d = {'a': 1} + f.run(d, log_func=msg.append) + assert d == {'b': 1} + assert msg == ['Entry "a" renamed to "b"'] + + d = {'a': {'c': 1}} + f.run(d) + assert d == {'b': {'c': 1}} + + msg = [] + f = RenameEntryPreProcess(('a', 'd'), 'c') + d = {'a': {'d': 1}} + f.run(d, msg.append) + assert d == {'a': {'c': 1}} + assert msg == ['Entry "d" renamed to "c" in "a"'] + + +def test_not_found() -> None: + a = {} + RenameEntryPreProcess(('a',), 'b').run(a) + assert a == {} + + +def test_no_overwrite() -> None: + a = {'a': 1, 'b': 2} + RenameEntryPreProcess(('a',), 'b').run(a) + assert a == {'a': 1, 'b': 2} diff --git a/tests/test_pydantic/test_field_access.py b/tests/test_pydantic/test_field_access.py index 65d6589..80cb2b1 100644 --- a/tests/test_pydantic/test_field_access.py +++ b/tests/test_pydantic/test_field_access.py @@ -1,5 +1,3 @@ -from typing import List - from pydantic import BaseModel, Field, PrivateAttr from pydantic.fields import FieldInfo @@ -7,14 +5,14 @@ class MyDataSet: asdf = 'a' - def __init__(self, parent): + def __init__(self, parent) -> None: self.parent = parent class MyModel(BaseModel): _my_data: MyDataSet = PrivateAttr() - def __init__(self, **data): + def __init__(self, **data) -> None: super().__init__(**data) self._my_data = MyDataSet(self) @@ -22,10 +20,10 @@ def __init__(self, **data): class UserModel(MyModel): val_int: int val_str: str = 'test' - val_f: List[str] = Field(description='This key does this') + val_f: list[str] = Field(description='This key does this') -def test_get_model_desc(): +def test_get_model_desc() -> None: assert list(UserModel.model_fields.keys()) == [ 'val_int', 'val_str', @@ -38,7 +36,7 @@ def test_get_model_desc(): assert UserModel.model_fields['val_f'].description == 'This key does this' -def test_mutate(capsys): +def test_mutate(capsys) -> None: m = UserModel(val_int=1, val_f=['asdf']) assert isinstance(m, UserModel) @@ -60,7 +58,7 @@ def test_mutate(capsys): assert captured.out == "val_int: 88\nval_str: test\nval_f: ['asdf']\n" -def test_nested_models(): +def test_nested_models() -> None: class ChildModel(BaseModel): c_a: str = Field(3, description='desc c_a') c_b: int = Field(3, description='desc c_b') @@ -76,7 +74,7 @@ class ParentModel(BaseModel): assert 'b' in obj.model_fields -def test_env_access(): +def test_env_access() -> None: class ChildModel(BaseModel): c_a: str = Field(3, description='desc c_a', env='my_env_var') c_b: int = Field(3, description='desc c_b') @@ -90,7 +88,7 @@ class ParentModel(BaseModel): assert obj.b.model_fields['c_a'].json_schema_extra == {'env': 'my_env_var'} -def test_private_attr(): +def test_private_attr() -> None: class SimpleModel(BaseModel): _priv2: int = PrivateAttr(default=3) _priv3: int = PrivateAttr() diff --git a/tests/yaml/test_align.py b/tests/yaml/test_align.py index 85c04e0..3a69050 100644 --- a/tests/yaml/test_align.py +++ b/tests/yaml/test_align.py @@ -5,7 +5,7 @@ from easyconfig.yaml import CommentedMap, write_aligned_yaml, yaml_rt -@pytest.fixture() +@pytest.fixture def my_map(): top = CommentedMap() top['sub_key1'] = data = CommentedMap() @@ -44,7 +44,7 @@ def my_map(): return top -def test_align(my_map: CommentedMap): +def test_align(my_map: CommentedMap) -> None: buf = io.StringIO() write_aligned_yaml(my_map, buf, 1) file_contents = buf.getvalue() diff --git a/tests/yaml/test_model_to_yaml.py b/tests/yaml/test_model_to_yaml.py index db63265..06b6c17 100644 --- a/tests/yaml/test_model_to_yaml.py +++ b/tests/yaml/test_model_to_yaml.py @@ -1,6 +1,5 @@ from datetime import datetime from enum import Enum -from typing import List, Optional from pydantic import AnyHttpUrl, BaseModel, ByteSize, Field from tests.helper import dump_yaml @@ -9,7 +8,7 @@ from easyconfig.yaml.from_model import cmap_from_model -def test_simple_model(): +def test_simple_model() -> None: class SimpleModel(BaseModel): a: int = 5 b: int = 6 @@ -17,7 +16,7 @@ class SimpleModel(BaseModel): assert dump_yaml(cmap_from_model(SimpleModel())) == 'a: 5\nb: 6\n' -def test_base_overrides(): +def test_base_overrides() -> None: class SimpleModel(BaseModel): size: ByteSize url: AnyHttpUrl = 'http://test.de' @@ -25,22 +24,22 @@ class SimpleModel(BaseModel): assert dump_yaml(cmap_from_model(SimpleModel(size=ByteSize(1024)))) == 'size: 1024\nurl: http://test.de\n' -def test_simple_model_complex_types(): +def test_simple_model_complex_types() -> None: class SimpleModel(BaseModel): a: datetime = datetime(2000, 1, 1) assert dump_yaml(cmap_from_model(SimpleModel())) == "a: '2000-01-01T00:00:00'\n" -def test_simple_model_skip_none(): +def test_simple_model_skip_none() -> None: class SimpleModel(BaseModel): - a: Optional[int] = 5 + a: int | None = 5 b: int = 6 assert dump_yaml(cmap_from_model(SimpleModel(a=None))) == 'b: 6\n' -def test_simple_model_alias(): +def test_simple_model_alias() -> None: class SimpleModel(BaseModel): a: int = Field(5, alias='aaa') b: int = 6 @@ -48,7 +47,7 @@ class SimpleModel(BaseModel): assert dump_yaml(cmap_from_model(SimpleModel())) == 'aaa: 5\nb: 6\n' -def test_simple_model_description(): +def test_simple_model_description() -> None: class SimpleModel(BaseModel): a: int = Field(5, alias='aaa', description='Key A') b: int = Field(6, description='Key b') @@ -58,7 +57,7 @@ class SimpleModel(BaseModel): 'b: 6 # Key b\n' -def test_simple_model_skip_key(): +def test_simple_model_skip_key() -> None: class SimpleModel(BaseModel): a: int = Field(5, alias='aaa', description='Key A') b: int = Field(6, description='Key b', **{ARG_NAME_IN_FILE: False}) @@ -67,7 +66,7 @@ class SimpleModel(BaseModel): 'aaa: 5 # Key A\n' -def test_sub_model(): +def test_sub_model() -> None: class SubModel(BaseModel): aa: int = 5 ab: int = 6 @@ -83,7 +82,7 @@ class SimpleModel(BaseModel): ''' -def test_skip_sub_model(): +def test_skip_sub_model() -> None: class SubModel(BaseModel): aa: int = 5 ab: int = 6 @@ -95,7 +94,7 @@ class SimpleModel(BaseModel): assert dump_yaml(cmap_from_model(SimpleModel())) == 'b: 3\n' -def test_sub_model_alias_description(): +def test_sub_model_alias_description() -> None: class SubModel(BaseModel): aa: int = Field(5, alias='a', description='Key A') ab: int = 6 @@ -111,7 +110,7 @@ class SimpleModel(BaseModel): ''' -def test_multiline_comment(): +def test_multiline_comment() -> None: class SimpleModel(BaseModel): a: str = Field('value a', description='This is\nthe topmost\nvalue of the model') b: int = Field(3, description='\nThis is\nvalue b') @@ -125,7 +124,7 @@ class SimpleModel(BaseModel): '# value b\n' -def test_alias_not_in_file(): +def test_alias_not_in_file() -> None: class MyEnum(str, Enum): A = 'aa' @@ -135,7 +134,7 @@ class SimpleModel(BaseModel): c: MyEnum = MyEnum.A class EncapModel(BaseModel): - my_list: List[SimpleModel] = [] + my_list: list[SimpleModel] = [] assert dump_yaml(cmap_from_model(EncapModel(my_list=[SimpleModel(), SimpleModel(b=5), ]))) == \ 'my_list:\n' \ diff --git a/tests/yaml/test_yaml_value.py b/tests/yaml/test_yaml_value.py index d698850..4b2b238 100644 --- a/tests/yaml/test_yaml_value.py +++ b/tests/yaml/test_yaml_value.py @@ -1,7 +1,6 @@ # ruff: noqa: RUF012 import datetime -from typing import Dict, Set from pydantic import ( AnyHttpUrl, @@ -28,19 +27,19 @@ class BaseModel(_BaseModel): model_config = ConfigDict(validate_assignment=True, validate_default=True, extra='forbid') -def cmp_value(obj, target): +def cmp_value(obj, target) -> None: assert obj == target assert type(obj) is type(target) -def test_built_in_types(): +def test_built_in_types() -> None: class ConstrainedModel(BaseModel): is_bool: bool = True is_int: int = 10 is_str: str = 'asdf1!' - is_dict: Dict[str, int] = {'asdf': '10'} - is_set: Set[int] = {'10'} + is_dict: dict[str, int] = {'asdf': '10'} + is_set: set[int] = {'10'} m = ConstrainedModel() @@ -52,7 +51,7 @@ class ConstrainedModel(BaseModel): cmp_value(_get_yaml_value(m.is_set, m), CommentedSeq([10])) -def test_constrained_types(): +def test_constrained_types() -> None: class ConstrainedModel(BaseModel): negative_float: NegativeFloat = -5 negative_int: NegativeFloat = -3 @@ -81,7 +80,7 @@ class ConstrainedModel(BaseModel): cmp_value(_get_yaml_value(m.con_date, m, obj_name='con_date'), '2023-01-02') # yaml can't natively serialize dates -def test_strict_types(): +def test_strict_types() -> None: class StrictModel(BaseModel): strict_bool_true: StrictBool = True strict_bool_false: StrictBool = False @@ -104,7 +103,7 @@ class StrictModel(BaseModel): cmp_value(_get_yaml_value(m.strict_bytes_empty, m), b'') -def test_more_types(): +def test_more_types() -> None: class SimpleModel(BaseModel): size_raw: ByteSize = 100 size_str: ByteSize = '10kb' diff --git a/tox.ini b/tox.ini index 6646932..f1e0a27 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,19 @@ # content of: tox.ini , put in same dir as setup.py [tox] envlist = - py38 - py39 py310 py311 + py312 + py313 docs [gh-actions] python = - 3.8: py38 - 3.9: py39 - 3.10: py10, docs + 3.10: py10 3.11: py11 + 3.12: py12, docs + 3.13: py13 [testenv]