From 490b5e816b804f338b0eb97f240ae874d4e15810 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 10 Dec 2024 08:35:51 +0100 Subject: [PATCH] Consistent way of checking Airflow version in providers (#44686) This PR introduces consistent way of checking version of Airflow by Airflow providers. So far there were about 6 different ways on how Providers checked for Airflow version - this PR aims to unify this approach for now and in the future - at least until minimum version of Airflow set to 2.11 where we are likely to introduce a simpler check via #44607. Until then all providers are going to have `version_references.py` module copied in their sources that they will be importing the constants from. This PR also adds pre-commit that checks if the ``version_compat.py`` module is imported from local package copy or maybe from another provider or test code - both causing unneeded dependencies from the provider - to another package or to test code respectively. --- .pre-commit-config.yaml | 7 + contributing-docs/08_static_code_checks.rst | 2 + .../doc/images/output_static-checks.svg | 148 +++++++++--------- .../doc/images/output_static-checks.txt | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 1 + .../MANAGING_PROVIDERS_LIFECYCLE.rst | 25 +++ .../aws/auth_manager/aws_auth_manager.py | 8 +- .../amazon/aws/hooks/redshift_sql.py | 8 +- .../amazon/aws/transfers/gcs_to_s3.py | 4 +- .../providers/amazon/aws/utils/__init__.py | 16 +- .../providers/amazon/version_compat.py | 37 +++++ .../cncf/kubernetes/cli/kubernetes_command.py | 6 +- .../cncf/kubernetes/version_compat.py | 37 +++++ .../common/compat/assets/__init__.py | 7 +- .../providers/common/compat/lineage/hook.py | 2 +- .../common/compat/standard/operators.py | 8 +- .../providers/common/compat/version_compat.py | 37 +++++ .../common/compat/version_references.py | 27 ---- .../providers/common/io/assets/file.py | 6 +- .../providers/common/io/version_compat.py | 37 +++++ .../providers/common/io/xcom/backend.py | 7 +- .../providers/edge/cli/edge_command.py | 4 +- .../edge/plugins/edge_executor_plugin.py | 6 +- .../airflow/providers/edge/version_compat.py | 37 +++++ .../edge/worker_api/routes/_v2_compat.py | 7 +- .../elasticsearch/log/es_task_handler.py | 6 +- .../providers/elasticsearch/version_compat.py | 37 +++++ .../airflow/providers/google/assets/gcs.py | 8 +- .../cloud/log/stackdriver_task_handler.py | 6 +- .../providers/google/version_compat.py | 37 +++++ .../providers/openlineage/extractors/base.py | 4 +- .../providers/openlineage/plugins/listener.py | 10 +- .../openlineage/plugins/openlineage.py | 4 +- .../providers/openlineage/utils/utils.py | 23 +-- .../providers/openlineage/version_compat.py | 37 +++++ .../opensearch/log/os_task_handler.py | 6 +- .../providers/opensearch/version_compat.py | 37 +++++ .../airflow/providers/presto/hooks/presto.py | 6 +- .../providers/presto/version_compat.py | 37 +++++ .../providers/standard/operators/python.py | 5 +- .../airflow/providers/standard/provider.yaml | 1 - .../providers/standard/sensors/date_time.py | 2 +- .../providers/standard/sensors/time.py | 2 +- .../providers/standard/sensors/time_delta.py | 2 +- .../standard/triggers/external_task.py | 2 +- .../providers/standard/triggers/temporal.py | 2 +- .../standard/utils/version_references.py | 26 --- .../providers/standard/version_compat.py | 37 +++++ .../airflow/providers/trino/hooks/trino.py | 6 +- .../airflow/providers/trino/version_compat.py | 37 +++++ .../amazon/aws/operators/test_redshift_sql.py | 4 +- .../tests/openlineage/extractors/test_base.py | 2 +- .../tests/standard/triggers/test_temporal.py | 3 +- .../pre_commit/check_imports_in_providers.py | 96 ++++++++++++ .../ci/pre_commit/common_precommit_utils.py | 21 +++ tests_common/test_utils/version_compat.py | 7 +- 56 files changed, 732 insertions(+), 267 deletions(-) create mode 100644 providers/src/airflow/providers/amazon/version_compat.py create mode 100644 providers/src/airflow/providers/cncf/kubernetes/version_compat.py create mode 100644 providers/src/airflow/providers/common/compat/version_compat.py delete mode 100644 providers/src/airflow/providers/common/compat/version_references.py create mode 100644 providers/src/airflow/providers/common/io/version_compat.py create mode 100644 providers/src/airflow/providers/edge/version_compat.py create mode 100644 providers/src/airflow/providers/elasticsearch/version_compat.py create mode 100644 providers/src/airflow/providers/google/version_compat.py create mode 100644 providers/src/airflow/providers/openlineage/version_compat.py create mode 100644 providers/src/airflow/providers/opensearch/version_compat.py create mode 100644 providers/src/airflow/providers/presto/version_compat.py delete mode 100644 providers/src/airflow/providers/standard/utils/version_references.py create mode 100644 providers/src/airflow/providers/standard/version_compat.py create mode 100644 providers/src/airflow/providers/trino/version_compat.py create mode 100755 scripts/ci/pre_commit/check_imports_in_providers.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49e74d453d09d..65e3a0895b2cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -174,6 +174,13 @@ repos: language: python additional_dependencies: ['rich>=12.4.4'] require_serial: true + - id: check-imports-in-providers + name: Check imports in providers + entry: ./scripts/ci/pre_commit/check_imports_in_providers.py + language: python + additional_dependencies: ['rich>=12.4.4', "ruff==0.8.1"] + files: ^providers/src/airflow/providers/.*\.py$ + require_serial: true - id: update-common-sql-api-stubs name: Check and update common.sql API stubs entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 01d8f9d303ee8..0775c83ef0621 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -182,6 +182,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-hooks-apply | Check if all hooks apply to the repository | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ +| check-imports-in-providers | Check imports in providers | | ++-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-incorrect-use-of-LoggingMixin | Make sure LoggingMixin is not used alone | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | check-init-decorator-arguments | Sync model __init__ and decorator arguments | | diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index 4a51fad162631..8d78c5924b4f1 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -1,4 +1,4 @@ - +