ci_test_package.yml

name: CI test package

on:
  workflow_dispatch:
  # all PRs, important to note that `pull_request_target` workflows
  # will run in the context of the target branch of a PR
  pull_request_target:

env:
  # These are configured in GitHub secrets
  DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_test_project
  GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger
  DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
  DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}


jobs:
  integration-snowflake:
    runs-on: ubuntu-latest
    environment:
      name: Approve Integration Tests

    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR

      - name: Install tox
        run: python3 -m pip install tox

      - name: Run Snowflake Integration Tests
        run: tox -e integration_snowflake

      - name: Run Snowflake Tests
        run: tox -e snowflake

  sqlfluff-lint-models:
    name: Lint dbt models using SQLFluff
    runs-on: ubuntu-latest
    environment:
      name: Approve Integration Tests

    steps:
      - name: Checkout branch
        uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR

      - name: Install Python packages
        run: python -m pip install dbt-snowflake~=1.5.0 sqlfluff-templater-dbt

      - name: Test database connection
        run: dbt debug

      - name: Install dbt packages
        run: dbt deps

      - name: Get changed files
        id: get_file_changes
        uses: trilom/file-changes-action@v1.2.4
        with:
          output: ' '

      - name: Get new and changed .sql files in /models to lint
        id: get_files_to_lint
        shell: bash -l {0}
        run: |
          # Set the command in the $() brackets as an output to use in later steps
          echo "::set-output name=lintees::$(
          # Issue where grep regular expressions don't work as expected on the
          # Github Actions shell, check dbt/models/ folder
          echo \
          $(echo ${{ steps.get_file_changes.outputs.files_modified }} |
          tr -s ' ' '\n' |
          grep -E '^models.*[.]sql$' |
          tr -s '\n' ' ') \
          $(echo ${{ steps.get_file_changes.outputs.files_added }} |
          tr -s ' ' '\n' |
          grep -E '^models.*[.]sql$' |
          tr -s '\n' ' ')
          )"

      - name: Lint dbt models
        if: steps.get_files_to_lint.outputs.lintees != ''
        shell: bash -l {0}
        run: |
          sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
          sed -i '/^\[/!d' annotations.json # see https://github.com/sqlfluff/sqlfluff/issues/2244

      - name: Annotate
        uses: yuzutech/annotations-action@v0.3.0
        with:
          repo-token: "${{ secrets.GITHUB_TOKEN }}"
          title: "SQLFluff Lint"
          input: "./annotations.json"