diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e009c6..c0d372e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,28 @@ repos: - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.8.0 hooks: - id: black language_version: python3 - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - repo: https://github.com/asottile/pyupgrade - rev: v3.7.0 + rev: v3.17.0 hooks: - id: pyupgrade args: - --py39-plus - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.11.2 hooks: - id: mypy # Override default --ignore-missing-imports @@ -37,6 +37,6 @@ repos: - pytest - types-setuptools - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 7.1.1 hooks: - id: flake8 diff --git a/dask_deltatable/utils.py b/dask_deltatable/utils.py index dabb6b4..eeb7b95 100644 --- a/dask_deltatable/utils.py +++ b/dask_deltatable/utils.py @@ -103,6 +103,7 @@ def get_partition_filters( if isinstance(filters[0][0], str): filters = cast(list[list[Filter]], [filters]) + filters = cast(list[list[Filter]], filters) allowed_ops = { "=": "=", diff --git a/dask_deltatable/write.py b/dask_deltatable/write.py index c71d71e..1c90a67 100644 --- a/dask_deltatable/write.py +++ b/dask_deltatable/write.py @@ -18,17 +18,22 @@ try: from deltalake.writer import MAX_SUPPORTED_PYARROW_WRITER_VERSION except ImportError: - from deltalake.writer import ( # type: ignore - MAX_SUPPORTED_WRITER_VERSION as MAX_SUPPORTED_PYARROW_WRITER_VERSION, - ) + # Black and mypy were arguing when doing this in one line, the type: ignore kept moving around + from deltalake.writer import MAX_SUPPORTED_WRITER_VERSION # type: ignore + + MAX_SUPPORTED_PYARROW_WRITER_VERSION = MAX_SUPPORTED_WRITER_VERSION + del MAX_SUPPORTED_WRITER_VERSION + +try: + from deltalake.writer import _enforce_append_only +except ImportError: + from deltalake.writer import __enforce_append_only as _enforce_append_only # type: ignore from deltalake.writer import ( - PYARROW_MAJOR_VERSION, AddAction, DeltaJSONEncoder, DeltaProtocolError, DeltaStorageHandler, - __enforce_append_only, get_file_stats_from_metadata, get_num_idx_cols_and_stats_columns, get_partitions_from_path, @@ -40,6 +45,8 @@ from . import utils from ._schema import pyarrow_to_deltalake, validate_compatible +PYARROW_MAJOR_VERSION = int(pa.__version__.split(".")[0]) + def to_deltalake( table_or_uri: str | Path | DeltaTable, @@ -138,7 +145,7 @@ def to_deltalake( if table: table.update_incremental() - __enforce_append_only(table=table, configuration=configuration, mode=mode) + _enforce_append_only(table=table, configuration=configuration, mode=mode) if filesystem is None: if table is not None: