From 55d4aeb26cb54d679e70ecaf10d21e12176c519f Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 5 Dec 2024 00:36:29 +0530 Subject: [PATCH] Update Sqlite to Postgres Replace sqlite to posgres --- .github/workflows/test.yml | 72 +----------------------- dev/dags/dbt/simple/profiles.yml | 15 ++--- pyproject.toml | 4 +- scripts/test/integration-sqlite-setup.sh | 4 -- scripts/test/integration-sqlite.sh | 9 --- scripts/test/integration.sh | 4 +- tests/dbt/test_graph.py | 4 -- 7 files changed, 12 insertions(+), 100 deletions(-) delete mode 100644 scripts/test/integration-sqlite-setup.sh delete mode 100644 scripts/test/integration-sqlite.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f9dbd2c4..2178b88e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main] + branches: [main, replace-sqlite-to-pg] pull_request_target: # Also run on pull requests originated from forks branches: [main] @@ -267,76 +267,6 @@ jobs: AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} -# TODO: https://github.com/astronomer/astronomer-cosmos/issues/1341 -# Run-Integration-Tests-Sqlite: -# needs: Authorize -# runs-on: ubuntu-latest -# strategy: -# matrix: -# python-version: ["3.11"] -# airflow-version: ["2.8"] -# -# steps: -# - uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha || github.ref }} -# - uses: actions/cache@v3 -# with: -# path: | -# ~/.cache/pip -# .local/share/hatch/ -# key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} -# -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# -# - name: Install packages and dependencies -# run: | -# python -m pip install uv -# uv pip install --system hatch -# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze -# -# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} -# run: | -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup -# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres -# AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} -# AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} -# AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} -# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH -# AIRFLOW__COSMOS__ENABLE_CACHE: 0 -# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} -# DATABRICKS_CLUSTER_ID: mock -# DATABRICKS_HOST: mock -# DATABRICKS_WAREHOUSE_ID: mock -# DATABRICKS_TOKEN: mock -# POSTGRES_HOST: localhost -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres -# POSTGRES_DB: postgres -# POSTGRES_SCHEMA: public -# POSTGRES_PORT: 5432 -# AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/" -# AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn -# -# - name: Upload coverage to Github -# uses: actions/upload-artifact@v4 -# with: -# name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} -# path: .coverage -# include-hidden-files: true -# -# env: -# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ -# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres -# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - Run-Integration-Tests-DBT-1-5-4: needs: Authorize runs-on: ubuntu-latest diff --git a/dev/dags/dbt/simple/profiles.yml b/dev/dags/dbt/simple/profiles.yml index 24841363e..7b838c70d 100644 --- a/dev/dags/dbt/simple/profiles.yml +++ b/dev/dags/dbt/simple/profiles.yml @@ -2,10 +2,11 @@ simple: target: dev outputs: dev: - type: sqlite - threads: 1 - database: 'database' - schema: 'main' - schemas_and_paths: - main: "{{ env_var('DBT_SQLITE_PATH') }}/imdb.db" - schema_directory: "{{ env_var('DBT_SQLITE_PATH') }}" + type: postgres + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + password: "{{ env_var('POSTGRES_PASSWORD') }}" + port: "{{ env_var('POSTGRES_PORT') | int }}" + dbname: "{{ env_var('POSTGRES_DB') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + threads: 4 diff --git a/pyproject.toml b/pyproject.toml index dd73c18da..5366e952d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,8 +168,6 @@ test-kubernetes-setup = "sh scripts/test/kubernetes-setup.sh" test-integration-dbt-1-5-4 = 'sh scripts/test/integration-dbt-1-5-4.sh' test-integration-expensive = 'sh scripts/test/integration-expensive.sh' test-integration-setup = 'sh scripts/test/integration-setup.sh' -test-integration-sqlite = 'sh scripts/test/integration-sqlite.sh' -test-integration-sqlite-setup = 'sh scripts/test/integration-sqlite-setup.sh' test-performance = 'sh scripts/test/performance.sh' test-performance-setup = 'sh scripts/test/performance-setup.sh' type-check = "pre-commit run mypy --files cosmos/**/*" @@ -177,7 +175,7 @@ type-check = "pre-commit run mypy --files cosmos/**/*" [tool.pytest.ini_options] filterwarnings = ["ignore::DeprecationWarning"] minversion = "6.0" -markers = ["integration", "sqlite", "perf"] +markers = ["integration", "perf"] ###################################### # DOCS diff --git a/scripts/test/integration-sqlite-setup.sh b/scripts/test/integration-sqlite-setup.sh deleted file mode 100644 index 9b0f5438d..000000000 --- a/scripts/test/integration-sqlite-setup.sh +++ /dev/null @@ -1,4 +0,0 @@ -pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \ -rm -rf airflow.*; \ -airflow db init; \ -pip install 'dbt-core==1.4' 'dbt-sqlite==1.4' 'dbt-databricks==1.4' 'dbt-postgres==1.4' #'databricks-sdk==0.16.0' diff --git a/scripts/test/integration-sqlite.sh b/scripts/test/integration-sqlite.sh deleted file mode 100644 index dab70efb4..000000000 --- a/scripts/test/integration-sqlite.sh +++ /dev/null @@ -1,9 +0,0 @@ -pytest -vv \ - --cov=cosmos \ - --cov-report=term-missing \ - --cov-report=xml \ - --durations=0 \ - -m integration \ - --ignore=tests/perf \ - --ignore=tests/test_example_k8s_dags.py \ - -k 'example_cosmos_sources or sqlite' diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh index db11bff6c..e8859836a 100644 --- a/scripts/test/integration.sh +++ b/scripts/test/integration.sh @@ -18,7 +18,7 @@ pytest -vv \ --cov-report=term-missing \ --cov-report=xml \ --durations=0 \ - -m integration \ + -m 'integration' \ --ignore=tests/perf \ --ignore=tests/test_example_k8s_dags.py \ - -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv or jaffle_shop_kubernetes)' + -k 'not ( example_cosmos_sources or example_cosmos_python_models or example_virtualenv or jaffle_shop_kubernetes)' diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index f5afa06c3..c63301f70 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -588,7 +588,6 @@ def test_load_via_dbt_ls_with_invalid_dbt_path(mock_which): assert err_info.value.args[0] == expected -@pytest.mark.sqlite @pytest.mark.parametrize("load_method", ["load_via_dbt_ls", "load_from_dbt_manifest"]) @pytest.mark.integration def test_load_via_dbt_ls_with_sources(load_method): @@ -601,7 +600,6 @@ def test_load_via_dbt_ls_with_sources(load_method): render_config=RenderConfig( dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, dbt_deps=False, - env_vars={"DBT_SQLITE_PATH": str(DBT_PROJECTS_ROOT_DIR / "data")}, source_rendering_behavior=SOURCE_RENDERING_BEHAVIOR, ), execution_config=ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name), @@ -1385,7 +1383,6 @@ def test_load_method_with_unsupported_render_config_selector_arg(load_method): dbt_graph.load(method=load_method) -@pytest.mark.sqlite @pytest.mark.integration def test_load_via_dbt_ls_with_project_config_vars(): """ @@ -1396,7 +1393,6 @@ def test_load_via_dbt_ls_with_project_config_vars(): dbt_graph = DbtGraph( project=ProjectConfig( dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name, - env_vars={"DBT_SQLITE_PATH": str(DBT_PROJECTS_ROOT_DIR / "data")}, dbt_vars={"animation_alias": "top_5_animated_movies"}, ), render_config=RenderConfig(