diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5adcc3ac5..4b214ce11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,10 +11,8 @@ concurrency: cancel-in-progress: true jobs: - Authorize: - environment: - ${{ github.event_name == 'pull_request_target' && + environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }} runs-on: ubuntu-latest @@ -30,8 +28,8 @@ jobs: - uses: actions/setup-python@v3 with: - python-version: '3.9' - architecture: 'x64' + python-version: "3.9" + architecture: "x64" - run: pip3 install hatch - run: hatch run tests.py3.9-2.7:type-check @@ -294,6 +292,55 @@ jobs: AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + Run-Performance-Tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + airflow-version: ["2.7"] + num-models: [1, 10, 50, 100] + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .nox + key: perf-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Run performance tests against against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + id: run-performance-tests + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance-setup + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance + + # read the performance results and set them as an env var for the next step + # format: NUM_MODELS={num_models}\nTIME={end - start}\n + cat /tmp/performance_results.txt > $GITHUB_STEP_SUMMARY + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + MODEL_COUNT: ${{ matrix.num-models }} + + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH Code-Coverage: if: github.event.action != 'labeled' @@ -309,7 +356,7 @@ jobs: - name: Set up Python 3.11 uses: actions/setup-python@v3 with: - python-version: '3.11' + python-version: "3.11" - name: Install coverage run: | pip3 install coverage diff --git a/dev/dags/dbt/perf/.gitignore b/dev/dags/dbt/perf/.gitignore new file mode 100644 index 000000000..49f147cb9 --- /dev/null +++ b/dev/dags/dbt/perf/.gitignore @@ -0,0 +1,4 @@ + +target/ +dbt_packages/ +logs/ diff --git a/dev/dags/dbt/perf/README.md b/dev/dags/dbt/perf/README.md new file mode 100644 index 000000000..31e67f483 --- /dev/null +++ b/dev/dags/dbt/perf/README.md @@ -0,0 +1,3 @@ +dbt project for running performance tests. + +The `models` directory gets populated by an integration test defined in `tests/perf`. diff --git a/dev/dags/dbt/perf/analyses/.gitkeep b/dev/dags/dbt/perf/analyses/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dev/dags/dbt/perf/dbt_project.yml b/dev/dags/dbt/perf/dbt_project.yml new file mode 100644 index 000000000..ebb027574 --- /dev/null +++ b/dev/dags/dbt/perf/dbt_project.yml @@ -0,0 +1,17 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "perf" +version: "1.0.0" +config-version: 2 + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" diff --git a/dev/dags/dbt/perf/macros/.gitkeep b/dev/dags/dbt/perf/macros/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dev/dags/dbt/perf/profiles.yml b/dev/dags/dbt/perf/profiles.yml new file mode 100644 index 000000000..5b3cf175d --- /dev/null +++ b/dev/dags/dbt/perf/profiles.yml @@ -0,0 +1,11 @@ +simple: + target: dev + outputs: + dev: + type: sqlite + threads: 1 + database: "database" + schema: "main" + schemas_and_paths: + main: "{{ env_var('DBT_SQLITE_PATH') }}/imdb.db" + schema_directory: "{{ env_var('DBT_SQLITE_PATH') }}" diff --git a/dev/dags/dbt/perf/seeds/.gitkeep b/dev/dags/dbt/perf/seeds/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dev/dags/dbt/perf/snapshots/.gitkeep b/dev/dags/dbt/perf/snapshots/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dev/dags/dbt/perf/tests/.gitkeep b/dev/dags/dbt/perf/tests/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/dev/dags/performance_dag.py b/dev/dags/performance_dag.py new file mode 100644 index 000000000..caf977817 --- /dev/null +++ b/dev/dags/performance_dag.py @@ -0,0 +1,36 @@ +""" +A DAG that uses Cosmos to render a dbt project for performance testing. +""" + +import airflow +from datetime import datetime +import os +from pathlib import Path + +from cosmos import DbtDag, ProjectConfig, ProfileConfig, RenderConfig + +DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" +DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) +DBT_SQLITE_PATH = str(DEFAULT_DBT_ROOT_PATH / "data") + +profile_config = ProfileConfig( + profile_name="simple", + target_name="dev", + profiles_yml_filepath=(DBT_ROOT_PATH / "simple/profiles.yml"), +) + +cosmos_perf_dag = DbtDag( + project_config=ProjectConfig( + DBT_ROOT_PATH / "perf", + env_vars={"DBT_SQLITE_PATH": DBT_SQLITE_PATH}, + ), + profile_config=profile_config, + render_config=RenderConfig( + dbt_deps=False, + ), + # normal dag parameters + schedule_interval=None, + start_date=datetime(2024, 1, 1), + catchup=False, + dag_id="performance_dag", +) diff --git a/pyproject.toml b/pyproject.toml index 7536e703a..522431da7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,16 +9,8 @@ description = "Orchestrate your dbt projects in Airflow" readme = "README.rst" license = "Apache-2.0" requires-python = ">=3.8" -authors = [ - { name = "Astronomer", email = "humans@astronomer.io" }, -] -keywords = [ - "airflow", - "apache-airflow", - "astronomer", - "dags", - "dbt", -] +authors = [{ name = "Astronomer", email = "humans@astronomer.io" }] +keywords = ["airflow", "apache-airflow", "astronomer", "dags", "dbt"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Web Environment", @@ -56,48 +48,23 @@ dbt-all = [ "dbt-spark", "dbt-vertica", ] -dbt-athena = [ - "dbt-athena-community", - "apache-airflow-providers-amazon>=8.0.0", -] -dbt-bigquery = [ - "dbt-bigquery", -] -dbt-databricks = [ - "dbt-databricks", -] -dbt-exasol = [ - "dbt-exasol", -] -dbt-postgres = [ - "dbt-postgres", -] -dbt-redshift = [ - "dbt-redshift", -] -dbt-snowflake = [ - "dbt-snowflake", -] -dbt-spark = [ - "dbt-spark", -] -dbt-vertica = [ - "dbt-vertica<=1.5.4", -] -openlineage = [ - "openlineage-integration-common", - "openlineage-airflow", -] -all = [ - "astronomer-cosmos[dbt-all]", - "astronomer-cosmos[openlineage]" -] -docs =[ +dbt-athena = ["dbt-athena-community", "apache-airflow-providers-amazon>=8.0.0"] +dbt-bigquery = ["dbt-bigquery"] +dbt-databricks = ["dbt-databricks"] +dbt-exasol = ["dbt-exasol"] +dbt-postgres = ["dbt-postgres"] +dbt-redshift = ["dbt-redshift"] +dbt-snowflake = ["dbt-snowflake"] +dbt-spark = ["dbt-spark"] +dbt-vertica = ["dbt-vertica<=1.5.4"] +openlineage = ["openlineage-integration-common", "openlineage-airflow"] +all = ["astronomer-cosmos[dbt-all]", "astronomer-cosmos[openlineage]"] +docs = [ "sphinx", "pydata-sphinx-theme", "sphinx-autobuild", "sphinx-autoapi", - "apache-airflow-providers-cncf-kubernetes>=5.1.1" + "apache-airflow-providers-cncf-kubernetes>=5.1.1", ] tests = [ "packaging", @@ -137,9 +104,7 @@ Documentation = "https://astronomer.github.io/astronomer-cosmos" path = "cosmos/__init__.py" [tool.hatch.build.targets.sdist] -include = [ - "/cosmos", -] +include = ["/cosmos"] [tool.hatch.build.targets.wheel] packages = ["cosmos"] @@ -175,51 +140,20 @@ matrix.airflow.dependencies = [ [tool.hatch.envs.tests.scripts] freeze = "pip freeze" type-check = "mypy cosmos" -test = 'pytest -vv --durations=0 . -m "not integration" --ignore=tests/test_example_dags.py --ignore=tests/test_example_dags_no_connections.py' -test-cov = """pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml --durations=0 -m "not integration" --ignore=tests/test_example_dags.py --ignore=tests/test_example_dags_no_connections.py""" -# we install using the following workaround to overcome installation conflicts, such as: -# apache-airflow 2.3.0 and dbt-core [0.13.0 - 1.5.2] and jinja2>=3.0.0 because these package versions have conflicting dependencies -test-integration-setup = """pip uninstall -y dbt-postgres dbt-databricks dbt-vertica; \ -rm -rf airflow.*; \ -airflow db init; \ -pip install 'dbt-core' 'dbt-databricks' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow'""" -test-integration = """rm -rf dbt/jaffle_shop/dbt_packages; -pytest -vv \ ---cov=cosmos \ ---cov-report=term-missing \ ---cov-report=xml \ ---durations=0 \ --m integration \ --k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)'""" -test-integration-expensive = """pytest -vv \ ---cov=cosmos \ ---cov-report=term-missing \ ---cov-report=xml \ ---durations=0 \ --m integration \ --k 'example_cosmos_python_models or example_virtualenv'""" -test-integration-sqlite-setup = """pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \ -rm -rf airflow.*; \ -airflow db init; \ -pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4' """ -test-integration-sqlite = """ -pytest -vv \ ---cov=cosmos \ ---cov-report=term-missing \ ---cov-report=xml \ ---durations=0 \ --m integration \ --k 'example_cosmos_sources or sqlite'""" +test = 'sh scripts/test/unit.sh' +test-cov = 'sh scripts/test/unit-cov.sh' +test-integration-setup = 'sh scripts/test/integration-setup.sh' +test-integration = 'sh scripts/test/integration.sh' +test-integration-expensive = 'sh scripts/test/integration-expensive.sh' +test-integration-sqlite-setup = 'sh scripts/test/integration-sqlite-setup.sh' +test-integration-sqlite = 'sh scripts/test/integration-sqlite.sh' +test-performance-setup = 'sh scripts/test/performance-setup.sh' +test-performance = 'sh scripts/test/performance.sh' [tool.pytest.ini_options] -filterwarnings = [ - "ignore::DeprecationWarning", -] +filterwarnings = ["ignore::DeprecationWarning"] minversion = "6.0" -markers = [ - "integration", - "sqlite" -] +markers = ["integration", "sqlite", "perf"] ###################################### # DOCS @@ -233,7 +167,7 @@ dependencies = [ "sphinx-autobuild", "sphinx-autoapi", "openlineage-airflow", - "apache-airflow-providers-cncf-kubernetes>=5.1.1" + "apache-airflow-providers-cncf-kubernetes>=5.1.1", ] [tool.hatch.envs.docs.scripts] diff --git a/scripts/test/integration-expensive.sh b/scripts/test/integration-expensive.sh new file mode 100644 index 000000000..24bace86d --- /dev/null +++ b/scripts/test/integration-expensive.sh @@ -0,0 +1,8 @@ +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'example_cosmos_python_models or example_virtualenv' diff --git a/scripts/test/integration-setup.sh b/scripts/test/integration-setup.sh new file mode 100644 index 000000000..eba4f1513 --- /dev/null +++ b/scripts/test/integration-setup.sh @@ -0,0 +1,6 @@ +# we install using the following workaround to overcome installation conflicts, such as: +# apache-airflow 2.3.0 and dbt-core [0.13.0 - 1.5.2] and jinja2>=3.0.0 because these package versions have conflicting dependencies +pip uninstall -y dbt-postgres dbt-databricks dbt-vertica; \ +rm -rf airflow.*; \ +airflow db init; \ +pip install 'dbt-core' 'dbt-databricks' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow' diff --git a/scripts/test/integration-sqlite-setup.sh b/scripts/test/integration-sqlite-setup.sh new file mode 100644 index 000000000..b8bce035c --- /dev/null +++ b/scripts/test/integration-sqlite-setup.sh @@ -0,0 +1,4 @@ +pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \ +rm -rf airflow.*; \ +airflow db init; \ +pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4' diff --git a/scripts/test/integration-sqlite.sh b/scripts/test/integration-sqlite.sh new file mode 100644 index 000000000..dc32324d4 --- /dev/null +++ b/scripts/test/integration-sqlite.sh @@ -0,0 +1,8 @@ +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'example_cosmos_sources or sqlite' diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh new file mode 100644 index 000000000..823f70a7e --- /dev/null +++ b/scripts/test/integration.sh @@ -0,0 +1,9 @@ +rm -rf dbt/jaffle_shop/dbt_packages; +pytest -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m integration \ + --ignore=tests/perf \ + -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)' diff --git a/scripts/test/performance-setup.sh b/scripts/test/performance-setup.sh new file mode 100644 index 000000000..b8bce035c --- /dev/null +++ b/scripts/test/performance-setup.sh @@ -0,0 +1,4 @@ +pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \ +rm -rf airflow.*; \ +airflow db init; \ +pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4' diff --git a/scripts/test/performance.sh b/scripts/test/performance.sh new file mode 100644 index 000000000..ea58c1960 --- /dev/null +++ b/scripts/test/performance.sh @@ -0,0 +1,5 @@ +pytest -vv \ + -s \ + -m 'perf' \ + --ignore=tests/test_example_dags.py \ + --ignore=tests/test_example_dags_no_connections.py diff --git a/scripts/test/unit-cov.sh b/scripts/test/unit-cov.sh new file mode 100644 index 000000000..89a6244ba --- /dev/null +++ b/scripts/test/unit-cov.sh @@ -0,0 +1,10 @@ +pytest \ + -vv \ + --cov=cosmos \ + --cov-report=term-missing \ + --cov-report=xml \ + --durations=0 \ + -m "not (integration or perf)" \ + --ignore=tests/perf \ + --ignore=tests/test_example_dags.py \ + --ignore=tests/test_example_dags_no_connections.py diff --git a/scripts/test/unit.sh b/scripts/test/unit.sh new file mode 100644 index 000000000..ecc1a049a --- /dev/null +++ b/scripts/test/unit.sh @@ -0,0 +1,7 @@ +pytest \ + -vv \ + --durations=0 \ + -m "not (integration or perf)" \ + --ignore=tests/perf \ + --ignore=tests/test_example_dags.py \ + --ignore=tests/test_example_dags_no_connections.py diff --git a/tests/perf/test_performance.py b/tests/perf/test_performance.py new file mode 100644 index 000000000..acf5d3544 --- /dev/null +++ b/tests/perf/test_performance.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import time +import os +from pathlib import Path +from contextlib import contextmanager +from typing import Generator + +try: + from functools import cache +except ImportError: + from functools import lru_cache as cache + +import pytest +from airflow.models.dagbag import DagBag +from dbt.version import get_installed_version as get_dbt_version +from packaging.version import Version + +EXAMPLE_DAGS_DIR = Path(__file__).parent.parent.parent / "dev/dags" +AIRFLOW_IGNORE_FILE = EXAMPLE_DAGS_DIR / ".airflowignore" +DBT_VERSION = Version(get_dbt_version().to_version_string()[1:]) + + +@cache +def get_dag_bag() -> DagBag: + """Create a DagBag by adding the files that are not supported to .airflowignore""" + # add everything to airflow ignore that isn't performance_dag.py + with open(AIRFLOW_IGNORE_FILE, "w+") as f: + for file in EXAMPLE_DAGS_DIR.iterdir(): + if file.is_file() and file.suffix == ".py": + if file.name != "performance_dag.py": + print(f"Adding {file.name} to .airflowignore") + f.write(f"{file.name}\n") + + print(AIRFLOW_IGNORE_FILE.read_text()) + + db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) + + assert db.dags + assert not db.import_errors + + return db + + +def generate_model_code(model_number: int) -> str: + """ + Generates code for a dbt model with a dependency on the previous model. Runs + a simple select statement on the previous model. + """ + if model_number == 0: + return f""" + {{{{ config(materialized='table') }}}} + + select 1 as id + """ + + return f""" + {{{{ config(materialized='table') }}}} + + select * from {{{{ ref('model_{model_number - 1}') }}}} + """ + + +@contextmanager +def generate_project( + project_path: Path, + num_models: int, +) -> Generator[Path, None, None]: + """ + Generate dbt models in the project directory. + """ + models_dir = project_path / "models" + + try: + # create the models directory + models_dir.mkdir(exist_ok=True) + + # create the models + for i in range(num_models): + model = models_dir / f"model_{i}.sql" + model.write_text(generate_model_code(i)) + + yield project_path + finally: + # clean up the models in the project_path / models directory + for model in models_dir.iterdir(): + model.unlink() + + +@pytest.mark.perf +def test_perf_dag(): + num_models = os.environ.get("MODEL_COUNT", 10) + + if type(num_models) is str: + num_models = int(num_models) + + print(f"Generating dbt project with {num_models} models") + + dbt_project_dir = EXAMPLE_DAGS_DIR / "dbt" / "perf" + + with generate_project(dbt_project_dir, num_models): + dag_bag = get_dag_bag() + + dag = dag_bag.get_dag("performance_dag") + + # verify the integrity of the dag + assert dag.task_count == num_models + + # measure the time before and after the dag is run + + start = time.time() + dag.test() + end = time.time() + + print(f"Ran {num_models} models in {end - start} seconds") + print(f"NUM_MODELS={num_models}\nTIME={end - start}") + + # write the results to a file + with open("/tmp/performance_results.txt", "w") as f: + f.write( + f"NUM_MODELS={num_models}\nTIME={end - start}\nMODELS_PER_SECOND={num_models / (end - start)}\nDBT_VERSION={DBT_VERSION}" + ) diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index 11655a31d..91fd1d6c2 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -26,6 +26,8 @@ "2.4": ["cosmos_seed_dag.py"], } +IGNORED_DAG_FILES = ["performance_dag.py"] + # Sort descending based on Versions and convert string to an actual version MIN_VER_DAG_FILE_VER: dict[Version, list[str]] = { Version(version): MIN_VER_DAG_FILE[version] for version in sorted(MIN_VER_DAG_FILE, key=Version, reverse=True) @@ -51,11 +53,17 @@ def get_dag_bag() -> DagBag: if Version(airflow.__version__) < min_version: print(f"Adding {files} to .airflowignore") file.writelines([f"{file}\n" for file in files]) - # The dbt sqlite adapter is only available until dbt 1.4 + + for dagfile in IGNORED_DAG_FILES: + print(f"Adding {dagfile} to .airflowignore") + file.writelines([f"{dagfile}\n"]) + + # The dbt sqlite adapter is only available until dbt 1.4 if DBT_VERSION >= Version("1.5.0"): file.writelines(["example_cosmos_sources.py\n"]) if DBT_VERSION < Version("1.6.0"): file.writelines(["example_model_version.py\n"]) + print(".airflowignore contents: ") print(AIRFLOW_IGNORE_FILE.read_text()) db = DagBag(EXAMPLE_DAGS_DIR, include_examples=False) diff --git a/tests/test_example_dags_no_connections.py b/tests/test_example_dags_no_connections.py index 5356c4ea6..ae7c354a1 100644 --- a/tests/test_example_dags_no_connections.py +++ b/tests/test_example_dags_no_connections.py @@ -22,6 +22,8 @@ "2.4": ["cosmos_seed_dag.py"], } +IGNORED_DAG_FILES = ["performance_dag.py"] + # Sort descending based on Versions and convert string to an actual version MIN_VER_DAG_FILE_VER: dict[Version, list[str]] = { Version(version): MIN_VER_DAG_FILE[version] for version in sorted(MIN_VER_DAG_FILE, key=Version, reverse=True) @@ -36,9 +38,11 @@ def get_dag_bag() -> DagBag: if Version(airflow.__version__) < min_version: print(f"Adding {files} to .airflowignore") file.writelines([f"{file_name}\n" for file_name in files]) - a = 1 + 2 - b = 3 + 4 - a + b + + for dagfile in IGNORED_DAG_FILES: + print(f"Adding {dagfile} to .airflowignore") + file.writelines([f"{dagfile}\n"]) + if DBT_VERSION >= Version("1.5.0"): file.writelines(["example_cosmos_sources.py\n"]) if DBT_VERSION < Version("1.6.0"):