0.4.0 (#76)

.github/workflows/publish-pypi.yml

@@ -13,13 +13,13 @@ jobs:
       id-token: write
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: master
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     - name: Install setuptools
       run: |

.github/workflows/run_tox.yml

@@ -7,24 +7,24 @@ jobs:
     name: pre-commit
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-      - uses: pre-commit/[email protected].0
+          python-version: '3.12'
+      - uses: pre-commit/[email protected].1
 
   test:
     needs: pre-commit
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: check-ast
       - id: check-builtin-literals
@@ -13,17 +13,34 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.1.5
+    rev: v0.8.2
     hooks:
       - id: ruff
-      # - id: ruff-format
+        name: ruff unused imports
+        #  F401 [*] {name} imported but unused
+        args: [ "--select", "F401", "--extend-exclude", "__init__.py", "--fix"]
+
+      - id: ruff
+        #  I001 [*] Import block is un-sorted or un-formatted
+        # UP035 [*] Import from {target} instead: {names}
+        #  Q000 [*] Double quote found but single quotes preferred
+        #  Q001 [*] Double quote multiline found but single quotes preferred
+        args: [ "--select", "I001,UP035,Q000,Q001", "--fix"]
+
 
   - repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.10.0
     hooks:
       - id: rst-backticks
 
+  - repo: https://github.com/JelleZijlstra/autotyping
+    rev: 24.9.0
+    hooks:
+      - id: autotyping
+        types: [python]
+        args: [--safe]
+
+
   - repo: meta
     hooks:
       - id: check-hooks-apply

.ruff.toml

@@ -1,75 +1,106 @@
-
-line-length = 120
 indent-width = 4
+line-length  = 120
+
+target-version = "py310"
 
-target-version = "py38"
-
-# https://docs.astral.sh/ruff/settings/#ignore-init-module-imports
-ignore-init-module-imports = true
-src = ["src", "test"]
-extend-exclude = ["__init__.py"]
-
-select = [
-    "E", "W",   # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
-    "I",        # https://docs.astral.sh/ruff/rules/#isort-i
-    "UP",       # https://docs.astral.sh/ruff/rules/#pyupgrade-up
-
-    "A",        # https://docs.astral.sh/ruff/rules/#flake8-builtins-a
-    "ASYNC",    # https://docs.astral.sh/ruff/rules/#flake8-async-async
-    "C4",       # https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4
-    "EM",       # https://docs.astral.sh/ruff/rules/#flake8-errmsg-em
-    "FIX",      # https://docs.astral.sh/ruff/rules/#flake8-fixme-fix
-    "INP",      # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
-    "ISC",      # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
-    "PIE",      # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
-    "PT",       # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
-    "PTH",      # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
-    "RET",      # https://docs.astral.sh/ruff/rules/#flake8-return-ret
-    "SIM",      # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
-    "SLOT",     # https://docs.astral.sh/ruff/rules/#flake8-slots-slot
-    "T10",      # https://docs.astral.sh/ruff/rules/#flake8-debugger-t10
-    "TCH",      # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tch
-    "TD",       # https://docs.astral.sh/ruff/rules/#flake8-todos-td
-
-    "TRY",      # https://docs.astral.sh/ruff/rules/#tryceratops-try
-    "FLY",      # https://docs.astral.sh/ruff/rules/#flynt-fly
-    "PERF",     # https://docs.astral.sh/ruff/rules/#perflint-perf
-    "RUF",      # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
-
-    "PL",       # https://docs.astral.sh/ruff/rules/#pylint-pl
+src = [
+    "src",
+    "tests"
 ]
 
+
+[lint]
+select = ["ALL"]
+
 ignore = [
-    "A003",     # https://docs.astral.sh/ruff/rules/builtin-attribute-shadowing/
+    "D",        # https://docs.astral.sh/ruff/rules/#pydocstyle-d
+    "T20",      # https://docs.astral.sh/ruff/rules/#flake8-print-t20
+    "DTZ",      # https://docs.astral.sh/ruff/rules/#flake8-datetimez-dtz
+    "SLF",      # https://docs.astral.sh/ruff/rules/#flake8-self-slf
+
     "RET501",   # https://docs.astral.sh/ruff/rules/unnecessary-return-none/#unnecessary-return-none-ret501
     "TRY400",   # https://docs.astral.sh/ruff/rules/error-instead-of-exception/
-    "PLR1711",  # https://docs.astral.sh/ruff/rules/useless-return/
+
+    # https://docs.astral.sh/ruff/rules/#flake8-builtins-a
+    "A003",     # Python builtin is shadowed by class attribute {name} from {row}
+
+    # https://docs.astral.sh/ruff/rules/#pyflakes-f
+    "F401",     # {name} imported but unused; consider using importlib.util.find_spec to test for availability
+
+    # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
+    "S311",     # Standard pseudo-random generators are not suitable for cryptographic purposes
+
+    # https://docs.astral.sh/ruff/rules/#pyupgrade-up
+    "UP038",    # Use X | Y in {} call instead of (X, Y)
+
+    # https://docs.astral.sh/ruff/rules/#flake8-annotations-ann
+    "ANN101",   # Missing type annotation for {name} in method
+    "ANN102",   # Missing type annotation for {name} in classmethod
+    "ANN401",   # Dynamically typed expressions (typing.Any) are disallowed in {name}
+
+    # https://docs.astral.sh/ruff/rules/#flake8-blind-except-ble
+    "BLE001",   # Do not catch blind exception: {name}
+
+    # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
+    "RSE102",   # Unnecessary parentheses on raised exception
+
+    # https://docs.astral.sh/ruff/rules/#flake8-commas-com
+    "COM812",   # Trailing comma missing
+    "COM819",   # Trailing comma prohibited
+
+    # https://docs.astral.sh/ruff/rules/#warning-w_1
+    "PLW0603",  # Using the global statement to update {name} is discouraged
+
+    # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
+    "G004",     # Logging statement uses f-string
+
+    # https://docs.astral.sh/ruff/rules/#refactor-r
+    "PLR1711",  # Useless return statement at end of function
+
+    # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
+    "RUF005",  # Consider {expression} instead of concatenation
+
+    # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
+    "PT007",  # Wrong values type in @pytest.mark.parametrize expected {values} of {row}
 ]
 
 
 [format]
-# Use single quotes for non-triple-quoted strings.
 quote-style = "single"
 
 
-[lint.per-file-ignores]
-"docs/*" = [
-    "A001",     # A001 Variable `copyright` is shadowing a Python builtin
-    "E402",     # E402 Module level import not at top of file
-    "INP001",   # INP001 File `FILE_NAME` is part of an implicit namespace package. Add an `__init__.py`.
-]
+# https://docs.astral.sh/ruff/settings/#lintflake8-quotes
+[lint.flake8-quotes]
+inline-quotes    = "single"
+multiline-quotes = "single"
 
-"tests/*" = [
-    "INP001",   # INP001 File `FILE_NAME` is part of an implicit namespace package. Add an `__init__.py`.
-    "ISC002",   # ISC002 Implicitly concatenated string literals over multiple lines
-    "PLR2004",   # PLR2004 Magic value used in comparison, consider replacing 5 with a constant variable
+
+[lint.flake8-builtins]
+builtins-ignorelist = ["id", "input"]
+
+
+# https://docs.astral.sh/ruff/settings/#lintisort
+[lint.isort]
+lines-after-imports = 2     # https://docs.astral.sh/ruff/settings/#lint_isort_lines-after-imports
+
+
+[lint.per-file-ignores]
+"docs/conf.py" = [
+    "INP001",   # File `conf.py` is part of an implicit namespace package. Add an `__init__.py`.
+    "A001",     # Variable `copyright` is shadowing a Python builtin
+    "PTH118",   # `os.path.join()` should be replaced by `Path` with `/` operator
+    "PTH100",   # `os.path.abspath()` should be replaced by `Path.resolve()`
 ]
 
 "setup.py" = ["PTH123"]
-"src/easyconfig/yaml/from_model.py" = ["PLR0911"]   # PLR0911 Too many return statements (7 > 6)
 
+"tests/*" = [
+    "ANN",  # https://docs.astral.sh/ruff/rules/#flake8-annotations-ann
 
+    # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
+    "S101",     # Use of assert detected
 
-[lint.isort]
-# https://docs.astral.sh/ruff/settings/#isort-lines-after-imports
-lines-after-imports = 2
+    # https://docs.astral.sh/ruff/rules/#refactor-r
+    "PLR2004",  # Magic value used in comparison, consider replacing {value} with a constant variable
+    "PLR0913",  # Too many arguments in function definition ({c_args} > {max_args})
+]

docs/class_reference.rst

@@ -22,3 +22,10 @@ Subscription
 
 .. autoclass:: easyconfig.config_objs.ConfigObjSubscription
    :members:
+
+
+PreProcess
+======================================
+
+.. autoclass:: easyconfig.pre_process.PreProcess
+   :members:

docs/requirements.txt

@@ -1,5 +1,5 @@
 # Packages required to build the documentation
-sphinx                      == 7.2.6
-sphinx-autodoc-typehints    == 1.25.2
-sphinx_rtd_theme            == 2.0.0
-sphinx-exec-code            == 0.12
+sphinx                      == 8.1.3
+sphinx-autodoc-typehints    == 2.5.0
+sphinx_rtd_theme            == 3.0.2
+sphinx-exec-code            == 0.14

docs/usage.rst

@@ -77,11 +77,13 @@ Nested example with the convenience base classes from easyconfig.
     # ------------ hide: stop -------------
 
 
-Description and comments
+Default file generation
 --------------------------------------
 It's possible to specify a description through the pydantic ``Field``.
 The description will be created as a comment in the .yml file.
-Note that the comments will be aligned properly
+Note that the comments will be aligned properly.
+With the ``in_file`` argument it's possible to skip entries from appearing in the default file
+(e.g. for advanced settings). When added manually to the file these values will still be loaded as expected.
 
 .. exec_code::
     :language_output: yaml
@@ -94,6 +96,7 @@ Note that the comments will be aligned properly
     class MySimpleAppConfig(AppBaseModel):
         retries: int = Field(5, description='Amount of retries on error')
         url: str = Field('localhost', description='Url used for connection')
+        advanced: str = Field('something advanced', in_file=False)
         port: int = 443
 
 
@@ -125,7 +128,7 @@ yaml file
 
 .. exec_code::
     :language_output: yaml
-    :hide_code:
+    :hide:
 
     a = """
     env_var: "${MY_USER}"
@@ -145,7 +148,7 @@ yaml file
 
 .. exec_code::
     :language_output: yaml
-    :hide_code:
+    :hide:
     :caption_output: After expansion
 
 
@@ -222,3 +225,46 @@ This is especially useful feature if the application allows dynamic reloading of
     # This will trigger the callback
     CONFIG.load_config_file('/my/configuration/file.yml')
     # ------------ skip: stop -------------
+
+
+
+Preprocessing
+--------------------------------------
+With preprocessing it's possible to introduce changes in a non-breaking way
+
+
+.. exec_code::
+    :language_output: yaml
+
+    from pydantic import Field
+    from easyconfig import AppBaseModel, BaseModel, create_app_config
+
+
+    class HttpConfig(BaseModel):
+        url: str = 'localhost'
+        port: int = 443
+        retries: int = 3
+        timeout: int = 0
+
+
+    class MySimpleAppConfig(AppBaseModel):
+        http: HttpConfig = HttpConfig()
+
+
+    CONFIG = create_app_config(MySimpleAppConfig())
+
+    # Setup preprocessing, these are the migration steps from the old format
+    preprocess = CONFIG.load_preprocess
+    preprocess.rename_entry(['server'], 'http')
+    preprocess.move_entry(['wait time'], ['http', 'timeout'])
+    preprocess.set_log_func(print)  # This should normally be logger.info or logger.debug
+
+    # Load some old legacy format where http was still named server
+    CONFIG.load_config_dict({
+        'server': {     # this entry will be renamed to http
+            'retries': 5
+        },
+        'wait time': 10 # this entry will be moved to http.timeout
+    })
+
+    print(f'timeout: {CONFIG.http.timeout}')

readme.md

@@ -40,6 +40,13 @@ That way the users can have some guidance how to change the program behaviour.
 It's possible to use environment variable or files for expansion. Easyconfig will load all values
 
 # Changelog
+#### 0.4.0 (2024-01-10)
+- Minimum required python version is now 3.10
+- Added preprocessor to so it's possible to move and deprecate configuration entries
+- Added property to get the loaded configuration file
+- Many fixes
+- Updated CI and code linters
+
 #### 0.3.2 (2024-01-10)
 - Updated CI and code linters
 

requirements.txt

@@ -2,8 +2,9 @@
 -r requirements_setup.txt
 
 # testing dependencies
-pytest     == 7.4.4
-pre-commit == 3.5.0
+pre-commit  == 4.0.1
+pytest      == 8.3.4
 
 # linter
-ruff == 0.1.11
+ruff == 0.8.2
+pur  == 7.3.2