From 23379d623a8a7ae9b4194aea830b3bccfb17948c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 15 Nov 2024 19:39:10 -0600 Subject: [PATCH] update the structure for the monorepo --- .github/workflows/_code-quality.yml | 23 +++++ .github/workflows/_integration-tests.yml | 81 +++++++++++++++ .github/workflows/_publish-internal.yml | 29 ++++++ .github/workflows/_publish-pypi.yml | 32 ++++++ .github/workflows/_unit-tests.yml | 38 +++++++ .github/workflows/code-quality.yml | 39 ------- .github/workflows/integration-tests.yml | 64 ------------ .github/workflows/publish-internal.yml | 102 ------------------ .github/workflows/publish-pypi.yml | 85 --------------- .github/workflows/publish.yml | 113 ++++++++++---------- .github/workflows/pull-request-checks.yml | 120 ++++++++++++++-------- .github/workflows/unit-tests.yml | 54 ---------- dbt-athena/.changes/header.tpl.md | 7 ++ dbt-athena/CHANGELOG.md | 0 dbt-athena/CONTRIBUTING.md | 0 dbt-athena/hatch.toml | 50 +++++++++ dbt-athena/pyproject.toml | 62 ++--------- dbt-athena/tests/functional/__init__.py | 0 18 files changed, 406 insertions(+), 493 deletions(-) create mode 100644 .github/workflows/_code-quality.yml create mode 100644 .github/workflows/_integration-tests.yml create mode 100644 .github/workflows/_publish-internal.yml create mode 100644 .github/workflows/_publish-pypi.yml create mode 100644 .github/workflows/_unit-tests.yml delete mode 100644 .github/workflows/code-quality.yml delete mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/publish-internal.yml delete mode 100644 .github/workflows/publish-pypi.yml delete mode 100644 .github/workflows/unit-tests.yml create mode 100644 dbt-athena/.changes/header.tpl.md create mode 100644 dbt-athena/CHANGELOG.md create mode 100644 dbt-athena/CONTRIBUTING.md create mode 100644 dbt-athena/hatch.toml create mode 100644 dbt-athena/tests/functional/__init__.py diff --git a/.github/workflows/_code-quality.yml b/.github/workflows/_code-quality.yml new file mode 100644 index 00000000..dda5fc58 --- /dev/null +++ b/.github/workflows/_code-quality.yml @@ -0,0 +1,23 @@ +name: "Code quality" + +on: + workflow_dispatch: + inputs: + branch: + description: "Choose the branch to check" + type: string + default: "main" + repository: + description: "Choose the repository to check, when using a fork" + type: string + default: "dbt-labs/dbt-athena" + +permissions: + contents: read + +jobs: + code-quality: + uses: dbt-labs/dbt-adapters/.workflows/_code-quality.yml@monorepo-prep + with: + branch: ${{ inputs.branch }} + repository: ${{ inputs.repository }} diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml new file mode 100644 index 00000000..943fa7d8 --- /dev/null +++ b/.github/workflows/_integration-tests.yml @@ -0,0 +1,81 @@ +name: "Integration tests" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to test" + type: string + default: "dbt-athena" + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-athena" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: string + default: ${{ vars.DEFAULT_PYTHON_VERSION }} + workflow_dispatch: + inputs: + package: + description: "Choose the package to test" + type: choice + options: ["dbt-athena", "dbt-athena-community"] + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-athena" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: choice + options: ["3.9", "3.10", "3.11", "3.12"] + +permissions: + id-token: write + contents: read + +env: + DBT_TEST_ATHENA_S3_STAGING_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/staging/ + DBT_TEST_ATHENA_S3_TMP_TABLE_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/tmp_tables/ + DBT_TEST_ATHENA_REGION_NAME: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} + DBT_TEST_ATHENA_DATABASE: awsdatacatalog + DBT_TEST_ATHENA_SCHEMA: dbt-tests + DBT_TEST_ATHENA_WORK_GROUP: athena-dbt-tests + DBT_TEST_ATHENA_THREADS: 16 + DBT_TEST_ATHENA_POLL_INTERVAL: 0.5 + DBT_TEST_ATHENA_NUM_RETRIES: 3 + +jobs: + integration-tests: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: pypa/hatch@install + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }} + aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} + - run: hatch run integration-tests + working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml new file mode 100644 index 00000000..ace5d115 --- /dev/null +++ b/.github/workflows/_publish-internal.yml @@ -0,0 +1,29 @@ +name: "Publish Internally" + +on: + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: string + default: "dbt-athena" + deploy-to: + description: "Choose whether to publish to test or prod" + type: string + default: "test" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + +defaults: + run: + shell: bash + +jobs: + publish-internal: + uses: dbt-labs/dbt-adapters/.github/workflows/_publish-internal.yml + with: + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml new file mode 100644 index 00000000..84de1ce5 --- /dev/null +++ b/.github/workflows/_publish-pypi.yml @@ -0,0 +1,32 @@ +name: "Publish to PyPI" + +on: + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: string + default: "dbt-athena" + deploy-to: + description: "Choose whether to publish to test or prod" + type: string + default: "test" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + publish-pypi: + uses: dbt-labs/dbt-adapters/.workflows/_publish-pypi.yml@monorepo-prep + with: + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml new file mode 100644 index 00000000..a1ebec58 --- /dev/null +++ b/.github/workflows/_unit-tests.yml @@ -0,0 +1,38 @@ +name: "Unit tests" + +on: + workflow_dispatch: + inputs: + package: + description: "Choose the package to test" + type: choice + options: ["dbt-athena", "dbt-athena-community"] + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-athena" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: choice + options: ["3.9", "3.10", "3.11", "3.12"] + +permissions: + contents: read + +jobs: + unit-tests: + uses: dbt-labs/dbt-adapters/.github/workflows/_unit-tests.yml@monorepo-prep + with: + package: ${{ inputs.package }} + branch: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + os: ${{ inputs.os }} + python-version: ${{ inputs.python-version }} diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 425940da..00000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: "Code quality" - -on: - workflow_call: - inputs: - branch: - description: "Choose the branch to check" - type: string - default: "main" - repository: - description: "Choose the repository to check, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - workflow_dispatch: - inputs: - branch: - description: "Choose the branch to check" - type: string - default: "main" - repository: - description: "Choose the repository to check, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - -permissions: - contents: read - -jobs: - code-quality: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - repository: ${{ inputs.repository }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 33ab6073..00000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Integration tests" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to test" - type: string - default: "dbt-athena" - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - workflow_dispatch: - inputs: - package: - description: "Choose the package to test" - type: choice - options: ["dbt-athena", "dbt-athena-community"] - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - -permissions: - id-token: write - contents: read - -jobs: - integration-tests: - runs-on: ubuntu-latest - env: - DBT_TEST_ATHENA_S3_STAGING_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/staging/ - DBT_TEST_ATHENA_S3_TMP_TABLE_DIR: ${{ vars.DBT_TEST_ATHENA_S3_BUCKET }}/tmp_tables/ - DBT_TEST_ATHENA_REGION_NAME: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} - DBT_TEST_ATHENA_DATABASE: awsdatacatalog - DBT_TEST_ATHENA_SCHEMA: dbt-tests - DBT_TEST_ATHENA_WORK_GROUP: athena-dbt-tests - DBT_TEST_ATHENA_THREADS: 16 - DBT_TEST_ATHENA_POLL_INTERVAL: 0.5 - DBT_TEST_ATHENA_NUM_RETRIES: 3 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - repository: ${{ inputs.repository }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pypa/hatch@install - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }} - aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }} - - run: hatch run integration-tests - working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/publish-internal.yml b/.github/workflows/publish-internal.yml deleted file mode 100644 index c7170ba8..00000000 --- a/.github/workflows/publish-internal.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: "Publish Internally" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "prod" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - workflow_dispatch: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "test" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - -defaults: - run: - shell: bash - -jobs: - publish: - runs-on: ubuntu-latest - environment: - name: ${{ inputs.deploy-to }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pypa/hatch@install - - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ vars.AWS_REGION }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - id: package - run: | - # strip the pre-release off to find all iterations of this patch - hatch version release - echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} - - id: published - run: | - versions_published="$(aws codeartifact list-package-versions \ - --domain ${{ vars.AWS_DOMAIN }} \ - --repository ${{ vars.AWS_REPOSITORY }} \ - --format pypi \ - --package ${{ inputs.package }} \ - --output json \ - --query 'versions[*].version' | jq -r '.[]' | grep "^${{ steps.package.outputs.version }}" || true )" # suppress pipefail only here - echo "versions=$(echo "${versions_published[*]}"| tr '\n' ',')" >> $GITHUB_OUTPUT - - id: next - uses: dbt-labs/dbt-release/.github/actions/next-cloud-release-version@main - with: - version_number: ${{ steps.package.outputs.version }} - versions_published: ${{ steps.published.outputs.versions }} - - name: "Update version to internal PyPI format" - run: | - VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) - tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/athena/__version__.py - working-directory: ./${{ inputs.package }} - - name: "Remove dbt-core from build requirements" - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml - working-directory: ./${{ inputs.package }} - - run: | - export HATCH_INDEX_USER=${{ secrets.AWS_USER }} - - export HATCH_INDEX_AUTH=$(aws codeartifact get-authorization-token \ - --domain ${{ vars.AWS_DOMAIN }} \ - --output text \ - --query authorizationToken) - - export HATCH_INDEX_REPO=$(aws codeartifact get-repository-endpoint \ - --domain ${{ vars.AWS_DOMAIN }} \ - --repository ${{ vars.AWS_REPOSITORY }} \ - --format pypi \ - --output text \ - --query repositoryEndpoint) - - hatch build --clean - hatch run build:check-all - hatch publish - working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml deleted file mode 100644 index 379d8b90..00000000 --- a/.github/workflows/publish-pypi.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: "Publish to PyPI" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "prod" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - workflow_dispatch: - inputs: - package: - description: "Choose the package to publish" - type: string - default: "dbt-athena" - deploy-to: - description: "Choose whether to publish to test or prod" - type: string - default: "test" - branch: - description: "Choose the branch to publish" - type: string - default: "main" - -permissions: - contents: read - -defaults: - run: - shell: bash - -jobs: - publish: - runs-on: ubuntu-latest - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }} - permissions: - # this permission is required for trusted publishing - # see https://github.com/marketplace/actions/pypi-publish - id-token: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} - - uses: pypa/hatch@install - # hatch will build using test PyPI first and fall back to prod PyPI when deploying to test - # this is done via environment variables in the test environment in GitHub - - run: hatch build && hatch run build:check-all - working-directory: ./${{ inputs.package }} - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - packages-dir: ${{ inputs.package }}/dist/ - - verify: - runs-on: ubuntu-latest - needs: publish - # check the correct index - environment: - name: ${{ inputs.deploy-to }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - id: version - run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} - - uses: nick-fields/retry@v3 - with: - timeout_seconds: 10 - retry_wait_seconds: 10 - max_attempts: 15 # 5 minutes: (10s timeout + 10s delay) * 15 attempts - command: wget ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }}/${{ steps.version.outputs.version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5fa44b21..699440e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,67 +1,70 @@ name: "Publish" on: - workflow_dispatch: - inputs: - deploy-to: - description: "Choose whether to deploy to test or prod" - type: environment - default: "prod" - branch: - description: "Choose the branch to release from" - type: string - default: "main" - pypi-internal: - description: "Publish Internally" - type: boolean - default: true - pypi-public: - description: "Publish to PyPI" - type: boolean - default: false + workflow_dispatch: + inputs: + deploy-to: + description: "Choose whether to deploy to test or prod" + type: environment + default: "prod" + branch: + description: "Choose the branch to release from" + type: string + default: "main" + pypi-internal: + description: "Publish Internally" + type: boolean + default: true + pypi-public: + description: "Publish to PyPI" + type: boolean + default: false # don't attempt to release the same target in parallel concurrency: - group: ${{ github.workflow }}-${{ inputs.deploy-to }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ inputs.deploy-to }} + cancel-in-progress: true jobs: - unit-tests: - uses: ./.github/workflows/unit-tests.yml - with: - branch: ${{ inputs.branch }} + unit-tests: + uses: dbt-labs/dbt-adapters/.github/workflows/_unit-tests.yml@monorepo-prep + with: + package: "dbt-athena" + branch: ${{ inputs.branch }} - integration-tests: - uses: ./.github/workflows/integration-tests.yml - with: - branch: ${{ inputs.branch }} - repository: ${{ github.repository }} - secrets: inherit + integration-tests: + uses: ./.github/workflows/integration-tests.yml + with: + branch: ${{ inputs.branch }} + repository: ${{ github.repository }} + secrets: inherit - publish-internal: - if: ${{ inputs.pypi-internal == true }} - needs: [unit-tests, integration-tests] - uses: ./.github/workflows/publish-internal.yml - with: - deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} - secrets: inherit + publish-internal: + if: ${{ inputs.pypi-internal == true }} + needs: [unit-tests, integration-tests] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-internal.yml@monorepo-prep + with: + package: "dbt-athena" + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} + secrets: inherit - publish-pypi: - if: ${{ inputs.pypi-public == true }} - needs: [unit-tests, integration-tests] - uses: ./.github/workflows/publish-pypi.yml - with: - deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + publish-pypi: + if: ${{ inputs.pypi-public == true }} + needs: [unit-tests, integration-tests] + uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@monorepo-prep + with: + package: "dbt-athena" + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} - publish-pypi-dbt-athena-community: - if: ${{ inputs.pypi-public == true }} - # dbt-athena-community is hard pinned to dbt-athena to ensure they are the same - # this means we need to finish publishing dbt-athena before starting to build dbt-athena-community - needs: publish-pypi - uses: ./.github/workflows/publish-pypi.yml - with: - package: "dbt-athena-community" - deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + publish-pypi-dbt-athena-community: + if: ${{ inputs.pypi-public == true }} + # dbt-athena-community is hard pinned to dbt-athena to ensure they are the same + # this means we need to finish publishing dbt-athena before starting to build dbt-athena-community + needs: publish-pypi + uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@monorepo-prep + with: + package: "dbt-athena-community" + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index f977dfc1..9e9db193 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,50 +1,86 @@ name: "Pull request checks" on: - pull_request_target: - types: [opened, reopened, synchronize] + pull_request_target: + types: [opened, reopened, synchronize] # only run this once per PR at a time concurrency: - group: ${{ github.workflow }}-${{ github.event.number }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.number }} + cancel-in-progress: true jobs: - code-quality: - uses: ./.github/workflows/code-quality.yml - with: - branch: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - - unit-tests: - uses: ./.github/workflows/unit-tests.yml - with: - package: ${{ matrix.package }} - branch: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - strategy: - matrix: - package: ["dbt-athena", "dbt-athena-community"] - - integration-tests: - uses: ./.github/workflows/integration-tests.yml - with: - # integration test runs can't run in parallel for now, so only run dbt-athena - # both will run post merge and unit tests run for both, so this is sufficient - branch: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - secrets: inherit - - # This job does nothing and is only used for branch protection - results: - name: "Pull request checks" - if: always() - needs: - - code-quality - - unit-tests - - integration-tests - runs-on: ubuntu-latest - steps: - - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} + code-quality: + uses: dbt-labs/dbt-adapters/.github/workflows/_code-quality.yml@monorepo-prep + strategy: + matrix: + package: ["dbt-athena", "dbt-athena-community"] + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' + skip_label: 'Skip Changelog' + secrets: inherit + + verify-builds: + uses: dbt-labs/dbt-adapters/.github/workflows/_verify-build.yml + strategy: + matrix: + package: ["dbt-athena", "dbt-athena-community"] + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit-tests: + uses: dbt-labs/dbt-adapters/.github/workflows/_unit-tests.yml@monorepo-prep + strategy: + matrix: + package: ["dbt-athena", "dbt-athena-community"] + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + integration-tests: + uses: ./.github/workflows/_integration-tests.yml + strategy: + matrix: + package: ["dbt-athena", "dbt-athena-community"] + os: [ubuntu-22.04] + python-version: ["3.9", "3.10", "3.11", "3.12"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + secrets: inherit + + # This job does nothing and is only used for branch protection + results: + name: "Pull request checks" + if: always() + needs: [code-quality, changelog, verify-builds, unit-tests, integration-tests] + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 71e5559d..00000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: "Unit tests" - -on: - workflow_call: - inputs: - package: - description: "Choose the package to test" - type: string - default: "dbt-athena" - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - workflow_dispatch: - inputs: - package: - description: "Choose the package to test" - type: choice - options: ["dbt-athena", "dbt-athena-community"] - branch: - description: "Choose the branch to test" - type: string - default: "main" - repository: - description: "Choose the repository to test, when using a fork" - type: string - default: "dbt-labs/dbt-athena" - -permissions: - contents: read - -jobs: - unit-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - repository: ${{ inputs.repository }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - uses: pypa/hatch@install - - run: hatch run unit-tests - shell: bash - working-directory: ./${{ inputs.package }} diff --git a/dbt-athena/.changes/header.tpl.md b/dbt-athena/.changes/header.tpl.md new file mode 100644 index 00000000..44118343 --- /dev/null +++ b/dbt-athena/.changes/header.tpl.md @@ -0,0 +1,7 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), +and is generated by [Changie](https://github.com/miniscruff/changie). diff --git a/dbt-athena/CHANGELOG.md b/dbt-athena/CHANGELOG.md new file mode 100644 index 00000000..e69de29b diff --git a/dbt-athena/CONTRIBUTING.md b/dbt-athena/CONTRIBUTING.md new file mode 100644 index 00000000..e69de29b diff --git a/dbt-athena/hatch.toml b/dbt-athena/hatch.toml new file mode 100644 index 00000000..40106b58 --- /dev/null +++ b/dbt-athena/hatch.toml @@ -0,0 +1,50 @@ +[envs.default] +dependencies = [ + "moto~=5.0.13", + "pre-commit~=3.5", + "pyparsing~=3.1.4", +] +[envs.default.scripts] +setup = [ + "pre-commit install", + "cp -n test.env.example test.env", +] +code-quality = "pre-commit run --all-files" +unit-tests = "pytest --cov=dbt --cov-report=html:htmlcov {args:tests/unit}" +integration-tests = "python -m pytest -n auto {args:tests/functional}" +all-tests = ["unit-tests", "integration-tests"] + +[envs.hatch-test] +randomize = true +parallel = true +retries = 2 +extra-dependencies = [ + "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-tests-adapter~=1.9.2", + "pytest-cov~=5.0", + "pytest-dotenv~=0.5", + "pytest-xdist~=3.6", +] + +[envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] +[envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt_athena-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-athena", +] +check-sdist = [ + "twine check dist/*", + "find ./dist/dbt_athena-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-athena", +] diff --git a/dbt-athena/pyproject.toml b/dbt-athena/pyproject.toml index 27b84498..c3fb7fc3 100644 --- a/dbt-athena/pyproject.toml +++ b/dbt-athena/pyproject.toml @@ -39,69 +39,27 @@ Homepage = "https://github.com/dbt-labs/dbt-athena/dbt-athena" Documentation = "https://docs.getdbt.com" Repository = "https://github.com/dbt-labs/dbt-athena.git#subdirectory=dbt-athena" Issues = "https://github.com/dbt-labs/dbt-athena/issues" +Changelog = "https://github.com/dbt-labs/dbt-athena/blob/main/dbt-athena/CHANGELOG.md" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.build.targets.sdist] -include = ["src/dbt"] - -[tool.hatch.build.targets.wheel] -packages = ["src/dbt"] - [tool.hatch.version] path = "src/dbt/adapters/athena/__version__.py" -[tool.hatch.envs.default] -dependencies = [ - "dbt-tests-adapter~=1.9.2", - "moto~=5.0.13", - "pre-commit~=3.5", - "pyparsing~=3.1.4", - "pytest~=8.3", - "pytest-cov~=5.0", - "pytest-dotenv~=0.5", - "pytest-xdist~=3.6", -] -[tool.hatch.envs.default.scripts] -setup = [ - "pre-commit install", - "cp -n test.env.example test.env", -] -code-quality = "pre-commit run --all-files" -unit-tests = "pytest --cov=dbt --cov-report=html:htmlcov {args:tests/unit}" -integration-tests = "python -m pytest -n auto {args:tests/functional}" -all-tests = ["unit-tests", "integration-tests"] +[tool.hatch.build.targets.sdist] +packages = ["src/dbt/adapters", "src/dbt/include"] +sources = ["src"] -[tool.hatch.envs.build] -detached = true -dependencies = [ - "wheel", - "twine", - "check-wheel-contents", -] -[tool.hatch.envs.build.scripts] -check-all = [ - "- check-wheel", - "- check-sdist", -] -check-wheel = [ - "check-wheel-contents dist/*.whl --ignore W007,W008", - "find ./dist/dbt_athena-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", - "pip freeze | grep dbt-athena", -] -check-sdist = [ - "twine check dist/*", - "find ./dist/dbt_athena-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", - "pip freeze | grep dbt-athena", -] +[tool.hatch.build.targets.wheel] +packages = ["src/dbt/adapters", "src/dbt/include"] +sources = ["src"] [tool.pytest] -testpaths = [ - "tests/unit", - "tests/functional", -] +testpaths = ["tests/unit", "tests/functional"] +color = true +csv = "results.csv" filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", diff --git a/dbt-athena/tests/functional/__init__.py b/dbt-athena/tests/functional/__init__.py new file mode 100644 index 00000000..e69de29b