pyproject.toml

# Copyright The Lightning AI team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[metadata]
name = "litdata"
author = "Lightning-AI et al."
url = "https://github.com/Lightning-AI/lit-data"

[build-system]
requires = [
    "setuptools",
    "wheel",
]


[tool.black]
line-length = 120
exclude = '(_notebooks/.*)'

[tool.docformatter]
recursive = true
# this need to be shorter as some docstings are r"""...
wrap-summaries = 119
wrap-descriptions = 120
blank = true

[tool.codespell]
# Todo: enable also python files in a next step
skip = '*.py'
quiet-level = 3
# comma separated list of words; waiting for:
#  https://github.com/codespell-project/codespell/issues/2839#issuecomment-1731601603
# also adding links until they ignored by its: nature
#  https://github.com/codespell-project/codespell/issues/2243#issuecomment-1732019960
ignore-words-list = "te, compiletime"


[tool.ruff]
line-length = 120
# Enable Pyflakes `E` and `F` codes by default.
lint.select = [
    "E", "W",  # see: https://pypi.org/project/pycodestyle
    "F",  # see: https://pypi.org/project/pyflakes
    "S",  # see: https://pypi.org/project/flake8-bandit
    "RUF018",  # see: https://docs.astral.sh/ruff/rules/assignment-in-assert
]
lint.extend-select = [
    "I",  # see: isort
    "C4",  # see: https://pypi.org/project/flake8-comprehensions
    "SIM",  # see: https://pypi.org/project/flake8-simplify
    "RET",  # see: https://pypi.org/project/flake8-return
    "PT",  # see: https://pypi.org/project/flake8-pytest-style
]
lint.ignore = [
    "E731",  # Do not assign a lambda expression, use a def
    "S108",
    "E203", # conflicts with black
]
# Exclude a variety of commonly ignored directories.
exclude = [
    ".git",
    "docs",
    "_notebooks"
]
lint.ignore-init-module-imports = true

[tool.ruff.lint.per-file-ignores]
".actions/*" = ["S101", "S310"]
"setup.py" = ["S101", "SIM115"]
"examples/**" = [
    "S101",  # Use of `assert` detected
    "S113",  # todo: Probable use of requests call without
    "S104",  # Possible binding to all interface
    "F821",  # Undefined name `...`
    "S311",  # Standard pseudo-random generators are not suitable for cryptographic purposes
    "S501",  # Probable use of `requests` call with `verify=False` disabling SSL certificate checks
    "S108",  # Probable insecure usage of temporary file or directory: "/tmp/data/MNIST"
]
"litdata/**" = [
    "S101",  # todo: Use of `assert` detected
    "S105", "S106", "S107",  # todo: Possible hardcoded password: ...
    "S113",  # todo: Probable use of requests call without timeout
    "S301",  # todo: `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
    "S324",  # todo: Probable use of insecure hash functions in `hashlib`
    "S403",  # todo: `pickle`, `cPickle`, `dill`, and `shelve` modules are possibly insecure
    "S404",  # todo: `subprocess` module is possibly insecure
    "S602",  # todo: `subprocess` call with `shell=True` identified, security issue
    "S603",  # todo: `subprocess` call: check for execution of untrusted input
    "S605",  # todo: Starting a process with a shell: seems safe, but may be changed in the future; consider rewriting without `shell`
    "S607",  # todo: Starting a process with a partial executable path
    "RET504",  # todo:Unnecessary variable assignment before `return` statement
    "RET503",
    "S310",  # todo: Audit URL open for permitted schemes. Allowing use of `file:` or custom schemes is often unexpected.
]
"tests/**" = [
    "S101",  # Use of `assert` detected
    "S105", "S106",  # todo: Possible hardcoded password: ...
    "S301",  # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
    "S113",  # todo: Probable use of requests call without timeout
    "S311",  # todo: Standard pseudo-random generators are not suitable for cryptographic purposes
    "S108",  # todo: Probable insecure usage of temporary file or directory: "/tmp/sys-customizations-sync"
    "S202",  # Uses of `tarfile.extractall()`
    "S403",  # `pickle`, `cPickle`, `dill`, and `shelve` modules are possibly insecure
    "S404",  # `subprocess` module is possibly insecure
    "S602",  # todo: `subprocess` call with `shell=True` identified, security issue
    "S603",  # todo: `subprocess` call: check for execution of untrusted input
    "S605",  # todo: Starting a process with a shell: seems safe, but may be changed in the future; consider rewriting without `shell`
    "S607",  # todo: Starting a process with a partial executable path
    "RET504",  # todo:Unnecessary variable assignment before `return` statement
    "PT004",  # todo: Fixture `tmpdir_unittest_fixture` does not return anything, add leading underscore
    "PT011",  # todo: `pytest.raises(ValueError)` is too broad, set the `match` parameter or use a more specific exception
    "PT012",  # todo: `pytest.raises()` block should contain a single simple statement
    "PT019",  # todo: Fixture `_` without value is injected as parameter, use `@pytest.mark.usefixtures` instead
]

[tool.ruff.lint.mccabe]
# Unlike Flake8, default to a complexity level of 10.
max-complexity = 10


[tool.mypy]
files = [
    "litdata",
]
# This section is for folders with "-" as they are not valid python modules
exclude = [
]
install_types = "True"
non_interactive = "True"
disallow_untyped_defs = "True"
ignore_missing_imports = "True"
show_error_codes = "True"
warn_redundant_casts = "True"
warn_unused_configs = "True"
warn_unused_ignores = "True"
allow_redefinition = "True"
# disable this rule as the Trainer attributes are defined in the connectors, not in its __init__
disable_error_code = "attr-defined"
# style choices
warn_no_return = "False"

# Ignore mypy errors for these files
# TODO: the goal is for this to be empty
[[tool.mypy.overrides]]
# the list can be generated with:
# mypy --no-error-summary 2>&1 | tr ':' ' ' | awk '{print $1}' | sort | uniq | sed 's/\.py//g; s|src/||g;  s|\/|\.|g' | xargs -I {} echo '"{}",'
module = [
]
ignore_errors = "True"


[tool.coverage.report]
exclude_lines = [
    "pragma: no cover",
    "warnings",
    "pass",
    "rank_zero_warn",
    "raise NotImplementedError",
]


[tool.pytest.ini_options]
norecursedirs = [
    ".git",
    ".github",
    "dist",
    "build",
    "docs",
]
addopts = [
    "--strict-markers",
    "--doctest-modules",
    "--color=yes",
    "--disable-pytest-warnings",
    "--ignore=legacy/checkpoints",
]
markers = [
    "cloud: Run the cloud tests for example",
]
filterwarnings = [
    "error::FutureWarning",
]
xfail_strict = true
junit_duration_report = "call"