Update Sqlite to Postgres

Replace sqlite to posgres
astronomer · Dec 5, 2024 · 55d4aeb · 55d4aeb
1 parent cfb6bf2
commit 55d4aeb
Show file tree

Hide file tree

Showing 7 changed files with 12 additions and 100 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -2,7 +2,7 @@ name: test
 
 on:
   push: # Run on pushes to the default branch
-    branches: [main]
+    branches: [main, replace-sqlite-to-pg]
   pull_request_target: # Also run on pull requests originated from forks
     branches: [main]
 
@@ -267,76 +267,6 @@ jobs:
       AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
       DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
 
-# TODO: https://github.com/astronomer/astronomer-cosmos/issues/1341
-#  Run-Integration-Tests-Sqlite:
-#    needs: Authorize
-#    runs-on: ubuntu-latest
-#    strategy:
-#      matrix:
-#        python-version: ["3.11"]
-#        airflow-version: ["2.8"]
-#
-#    steps:
-#      - uses: actions/checkout@v3
-#        with:
-#          ref: ${{ github.event.pull_request.head.sha || github.ref }}
-#      - uses: actions/cache@v3
-#        with:
-#          path: |
-#            ~/.cache/pip
-#            .local/share/hatch/
-#          key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
-#
-#      - name: Set up Python ${{ matrix.python-version }}
-#        uses: actions/setup-python@v4
-#        with:
-#          python-version: ${{ matrix.python-version }}
-#
-#      - name: Install packages and dependencies
-#        run: |
-#          python -m pip install uv
-#          uv pip install --system hatch
-#          hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
-#
-#      - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
-#        run: |
-#          hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup
-#          hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite
-#        env:
-#          AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
-#          AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
-#          AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }}
-#          AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }}
-#          AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }}
-#          AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
-#          PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
-#          AIRFLOW__COSMOS__ENABLE_CACHE: 0
-#          COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
-#          DATABRICKS_CLUSTER_ID: mock
-#          DATABRICKS_HOST: mock
-#          DATABRICKS_WAREHOUSE_ID: mock
-#          DATABRICKS_TOKEN: mock
-#          POSTGRES_HOST: localhost
-#          POSTGRES_USER: postgres
-#          POSTGRES_PASSWORD: postgres
-#          POSTGRES_DB: postgres
-#          POSTGRES_SCHEMA: public
-#          POSTGRES_PORT: 5432
-#          AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/"
-#          AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn
-#
-#      - name: Upload coverage to Github
-#        uses: actions/upload-artifact@v4
-#        with:
-#          name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
-#          path: .coverage
-#          include-hidden-files: true
-#
-#    env:
-#      AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
-#      AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
-#      PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
-
   Run-Integration-Tests-DBT-1-5-4:
     needs: Authorize
     runs-on: ubuntu-latest

diff --git a/dev/dags/dbt/simple/profiles.yml b/dev/dags/dbt/simple/profiles.yml
@@ -2,10 +2,11 @@ simple:
   target: dev
   outputs:
     dev:
-      type: sqlite
-      threads: 1
-      database: 'database'
-      schema: 'main'
-      schemas_and_paths:
-        main: "{{ env_var('DBT_SQLITE_PATH') }}/imdb.db"
-      schema_directory: "{{ env_var('DBT_SQLITE_PATH') }}"
+      type: postgres
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      password: "{{ env_var('POSTGRES_PASSWORD') }}"
+      port: "{{ env_var('POSTGRES_PORT') | int }}"
+      dbname: "{{ env_var('POSTGRES_DB') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA') }}"
+      threads: 4
diff --git a/pyproject.toml b/pyproject.toml
@@ -168,16 +168,14 @@ test-kubernetes-setup = "sh scripts/test/kubernetes-setup.sh"
 test-integration-dbt-1-5-4 = 'sh scripts/test/integration-dbt-1-5-4.sh'
 test-integration-expensive = 'sh scripts/test/integration-expensive.sh'
 test-integration-setup = 'sh scripts/test/integration-setup.sh'
-test-integration-sqlite = 'sh scripts/test/integration-sqlite.sh'
-test-integration-sqlite-setup = 'sh scripts/test/integration-sqlite-setup.sh'
 test-performance = 'sh scripts/test/performance.sh'
 test-performance-setup = 'sh scripts/test/performance-setup.sh'
 type-check = "pre-commit run mypy --files cosmos/**/*"
 
 [tool.pytest.ini_options]
 filterwarnings = ["ignore::DeprecationWarning"]
 minversion = "6.0"
-markers = ["integration", "sqlite", "perf"]
+markers = ["integration", "perf"]
 
 ######################################
 # DOCS

diff --git a/scripts/test/integration-sqlite-setup.sh b/scripts/test/integration-sqlite-setup.sh
diff --git a/scripts/test/integration-sqlite.sh b/scripts/test/integration-sqlite.sh
diff --git a/scripts/test/integration.sh b/scripts/test/integration.sh
@@ -18,7 +18,7 @@ pytest -vv \
     --cov-report=term-missing \
     --cov-report=xml \
     --durations=0 \
-    -m integration  \
+    -m 'integration'  \
     --ignore=tests/perf \
     --ignore=tests/test_example_k8s_dags.py \
-    -k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv or jaffle_shop_kubernetes)'
+    -k 'not ( example_cosmos_sources or example_cosmos_python_models or example_virtualenv or jaffle_shop_kubernetes)'
diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py
@@ -588,7 +588,6 @@ def test_load_via_dbt_ls_with_invalid_dbt_path(mock_which):
     assert err_info.value.args[0] == expected
 
 
-@pytest.mark.sqlite
 @pytest.mark.parametrize("load_method", ["load_via_dbt_ls", "load_from_dbt_manifest"])
 @pytest.mark.integration
 def test_load_via_dbt_ls_with_sources(load_method):
@@ -601,7 +600,6 @@ def test_load_via_dbt_ls_with_sources(load_method):
         render_config=RenderConfig(
             dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name,
             dbt_deps=False,
-            env_vars={"DBT_SQLITE_PATH": str(DBT_PROJECTS_ROOT_DIR / "data")},
             source_rendering_behavior=SOURCE_RENDERING_BEHAVIOR,
         ),
         execution_config=ExecutionConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name),
@@ -1385,7 +1383,6 @@ def test_load_method_with_unsupported_render_config_selector_arg(load_method):
         dbt_graph.load(method=load_method)
 
 
-@pytest.mark.sqlite
 @pytest.mark.integration
 def test_load_via_dbt_ls_with_project_config_vars():
     """
@@ -1396,7 +1393,6 @@ def test_load_via_dbt_ls_with_project_config_vars():
     dbt_graph = DbtGraph(
         project=ProjectConfig(
             dbt_project_path=DBT_PROJECTS_ROOT_DIR / project_name,
-            env_vars={"DBT_SQLITE_PATH": str(DBT_PROJECTS_ROOT_DIR / "data")},
             dbt_vars={"animation_alias": "top_5_animated_movies"},
         ),
         render_config=RenderConfig(