diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..b16a22c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "Python 3", + "dockerComposeFile": "../docker-compose.yml", + "service": "devcontainer", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", + "postCreateCommand": "pip3 install --user -r requirements.txt", + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.defaultProfile.linux": "zsh" + }, + "extensions": [ + "GitHub.codespaces" + ] + } + } +} \ No newline at end of file diff --git a/.github/workflows/cd-pipeline.yml b/.github/workflows/cd-pipeline.yml new file mode 100644 index 0000000..bff2655 --- /dev/null +++ b/.github/workflows/cd-pipeline.yml @@ -0,0 +1,35 @@ +name: CD Pipeline + +on: + push: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + container: + image: ghcr.io/dbt-labs/dbt-postgres:1.6.3 + env: + POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ vars.POSTGRES_DB }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install dependencies + run: | + dbt deps + - name: Run unit tests + run: | + dbt test --target test --select tag:unit-test + - name: Run component tests + run: | + dbt test --target test --select tag:unit-test + deploy-test: + name: Deploy to test + needs: [test] + uses: ./.github/workflows/deploy.yml + with: + environment-name: test + secrets: inherit diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..b02dc38 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,66 @@ +name: Deploy + +on: + workflow_call: + inputs: + environment-name: + required: true + type: string + +env: + POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ vars.POSTGRES_DB }} + +jobs: + seed-source-tables: + name: Seed source tables + runs-on: ubuntu-latest + environment: ${{ inputs.environment-name }} + container: + image: ghcr.io/dbt-labs/dbt-postgres:1.6.3 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install dependencies + run: | + dbt deps + - name: Run seeds + run: | + dbt seed --target ${{ inputs.environment-name }} + source-contract-tests: + name: Run source contract tests + needs: [seed-source-tables] + runs-on: ubuntu-latest + environment: ${{ inputs.environment-name }} + container: + image: ghcr.io/dbt-labs/dbt-postgres:1.6.3 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install dependencies + run: | + dbt deps + - name: Run seeds + run: | + dbt seed --target ${{ inputs.environment-name }}\ + - name: Run source contract tests + run: | + dbt test --target ${{ inputs.environment-name }} --select tag:contract-test-source + deploy: + name: Deploy + needs: [source-contract-tests] + runs-on: ubuntu-latest + environment: ${{ inputs.environment-name }} + container: + image: ghcr.io/dbt-labs/dbt-postgres:1.6.3 + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install dependencies + run: | + dbt deps + - name: Run data transformations + run: | + dbt run --target ${{ inputs.environment-name }} diff --git a/.gitignore b/.gitignore index db37cdc..e079e98 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ target/ dbt_packages/ logs/ dbt -profiles.yml \ No newline at end of file +.env \ No newline at end of file diff --git a/README.md b/README.md index 6dd77a6..3451727 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,12 @@ Check that everything works dbt debug ``` +Seed the database + +``` +dbt seed +``` + ## Running the tests All tests @@ -88,6 +94,12 @@ Component tests dbt test --select tag:component ``` +Contract tests + +``` +dbt test --select tag:contract-test-source +``` + ## Running data quality tests ``` diff --git a/dbt_project.yml b/dbt_project.yml index 3c62417..7ab1444 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,3 @@ -# Name your project! Project names should contain only lowercase characters -# and underscores. A good package name should reflect your organization's -# name or the intended use of these models name: "dbt_testing_example" version: "1.0.0" config-version: 2 @@ -8,6 +5,8 @@ config-version: 2 # This setting configures which "profile" dbt uses for this project. profile: "dbt_testing_example" +require-dbt-version: ">=1.5.0" + # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! diff --git a/docker-compose.yml b/docker-compose.yml index ee6d86e..f79ba08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,20 @@ -version: '3.1' +version: '3.8' services: + devcontainer: + image: mcr.microsoft.com/devcontainers/python:0-3.10-bullseye + volumes: + - .:/workspaces:cached + network_mode: service:db + command: sleep infinity + db: image: postgres restart: always - ports: - - 5432:5432 environment: POSTGRES_PASSWORD: example adminer: image: adminer restart: always - ports: - - 8080:8080 \ No newline at end of file + network_mode: service:db \ No newline at end of file diff --git a/models/marts/_body_mass_indexes__models.yml b/models/marts/_body_mass_indexes__models.yml index f55b83e..3c79431 100644 --- a/models/marts/_body_mass_indexes__models.yml +++ b/models/marts/_body_mass_indexes__models.yml @@ -2,14 +2,30 @@ version: 2 models: - name: body_mass_indexes + config: + materialized: table + contract: + enforced: true columns: - name: user_id tags: ["data-quality"] + data_type: int + constraints: + - type: not_null + - type: check + expression: "user_id > 0" tests: - - not_null - relationships: to: source('gym_app', 'raw_height') field: user_id - relationships: to: source('gym_app', 'raw_weight') field: user_id + - name: created_date + data_type: date + - name: weight + data_type: float + - name: height + data_type: float + - name: bmi + data_type: decimal diff --git a/models/staging/gym_app/_gym_app__raw_height_schema.yml b/models/staging/gym_app/_gym_app__raw_height_schema.yml new file mode 100644 index 0000000..d960763 --- /dev/null +++ b/models/staging/gym_app/_gym_app__raw_height_schema.yml @@ -0,0 +1,28 @@ +version: 2 + +sources: + - name: gym_app + schema: dbt_testing_example + tables: + - name: raw_height + tags: ["data-quality", "contract-test-source"] + columns: + - name: height_unit + tests: + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: text + - accepted_values: + values: ["cm", "inches"] + - name: user_id + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: integer + - name: height + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: double precision + - dbt_utils.accepted_range: + min_value: 0 + inclusive: false diff --git a/models/staging/gym_app/_gym_app__raw_weight_schema.yml b/models/staging/gym_app/_gym_app__raw_weight_schema.yml new file mode 100644 index 0000000..3de1d5c --- /dev/null +++ b/models/staging/gym_app/_gym_app__raw_weight_schema.yml @@ -0,0 +1,28 @@ +version: 2 + +sources: + - name: gym_app + schema: dbt_testing_example + tables: + - name: raw_weight + tags: ["data-quality", "contract-test-source"] + columns: + - name: measurement_unit + tests: + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: text + - accepted_values: + values: ["kg", "pounds"] + - name: user_id + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: integer + - name: weight + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: double precision + - dbt_utils.accepted_range: + min_value: 0 + inclusive: false diff --git a/models/staging/gym_app/_gym_app__sources.yml b/models/staging/gym_app/_gym_app__sources.yml deleted file mode 100644 index 0e6587b..0000000 --- a/models/staging/gym_app/_gym_app__sources.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: 2 - -sources: - - name: gym_app - schema: dbt_testing_example - tables: - - name: raw_weight - tags: ["data-quality"] - columns: - - name: measurement_unit - tests: - - accepted_values: - values: ["kg"] - - name: user_id - tests: - - not_null - - name: weight - tests: - - not_null - - dbt_utils.accepted_range: - min_value: 0 - inclusive: false - - - name: raw_height - tags: ["data-quality"] - columns: - - name: height_unit - tests: - - accepted_values: - values: ["cm"] - - name: user_id - tests: - - not_null - - name: height - tests: - - not_null - - dbt_utils.accepted_range: - min_value: 0 - inclusive: false diff --git a/packages.yml b/packages.yml index 44a58b6..0efd7bf 100644 --- a/packages.yml +++ b/packages.yml @@ -1,6 +1,9 @@ packages: - package: dbt-labs/dbt_utils - version: 1.0.0 + version: 1.1.1 + + - package: calogica/dbt_expectations + version: 0.8.5 - git: "https://github.com/EqualExperts/dbt-unit-testing" - revision: v0.2.6 + revision: v0.2.7 diff --git a/profiles.example.yml b/profiles.example.yml deleted file mode 100644 index 561b709..0000000 --- a/profiles.example.yml +++ /dev/null @@ -1,12 +0,0 @@ -dbt_testing_example: - target: dev - outputs: - dev: - type: postgres - host: localhost - user: postgres - password: REPLACE - port: 5432 - dbname: postgres - schema: dbt_testing_example - threads: 4 diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..670cbe5 --- /dev/null +++ b/profiles.yml @@ -0,0 +1,21 @@ +dbt_testing_example: + target: dev + outputs: + dev: + type: postgres + host: localhost + user: postgres + password: "{{ env_var('POSTGRES_PASSWORD') }}" + port: 5432 + dbname: postgres + schema: dbt_testing_example + threads: 4 + test: + type: postgres + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + password: "{{ env_var('POSTGRES_PASSWORD') }}" + port: 5432 + dbname: "{{ env_var('POSTGRES_DB') }}" + schema: dbt_testing_example + threads: 4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f0e7ec5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +dbt-postgres \ No newline at end of file