diff --git a/.github/workflows/e2e/get_bundle_info/action.yaml b/.github/workflows/e2e/get_bundle_info/action.yaml new file mode 100644 index 0000000..5c2e3ff --- /dev/null +++ b/.github/workflows/e2e/get_bundle_info/action.yaml @@ -0,0 +1,57 @@ +name: Get Deployment Bundle Info +description: Get deployment bundle info from Astronomer API +inputs: + deployment_id: + description: The deployment ID + required: true + organization_id: + description: The organization ID + required: true + astro_api_token: + description: The Astronomer API token + required: true + astronomer_host: + description: The Astronomer host + required: true + expected_status_code: + description: The expected status code + default: 200 + +outputs: + desired_bundle_version: + description: The desired bundle version + value: ${{ steps.get-deployment-info.outputs.desired_bundle_version }} + bundle_type: + description: The bundle type + value: ${{ steps.get-deployment-info.outputs.bundle_type }} + updated_at: + description: The timestamp at which the bundle was last updated + value: ${{ steps.get-deployment-info.outputs.updated_at }} + +runs: + using: "composite" + steps: + - name: Get Deployment Info + id: get-deployment-info + shell: bash + run: | + STATUS_CODE=$(curl -s -w "%{http_code}" -o response.json -H "Authorization: Bearer ${{ inputs.astro_api_token }}" "https://api.${{ inputs.astronomer_host }}/v1alpha1/organizations/${{ inputs.organization_id }}/deployments/${{ inputs.deployment_id }}/deploys") + if [[ $STATUS_CODE -ne ${{ inputs.expected_status_code }} ]]; then + echo "Failed to get expected status code from GET Deployment API. Status code: $STATUS_CODE" + exit 1 + fi + if [[ $(cat response.json | jq -r '.deploys | length') -eq 0 ]]; then + echo "No deploys found for the deployment: ${{ inputs.deployment_id }}" + exit 1 + fi + + # sort by updatedAt to fetch the latest deploy object + cat response.json | jq '.deploys | sort_by(.updatedAt)' > response_sorted.json + + desired_bundle_version=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .bundles[0].desiredVersion' | head -n 1) + bundle_type=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .bundles[0].bundleType' | head -n 1) + updated_at=$(cat response_sorted.json | jq -r '.[] | select(.type == "BUNDLE" and .bundles[0].bundleType == "dbt") | .updatedAt' | head -n 1) + + echo "desired_bundle_version=$desired_bundle_version" >> $GITHUB_OUTPUT + echo "bundle_type=$bundle_type" >> $GITHUB_OUTPUT + echo "updated_at=$updated_at" >> $GITHUB_OUTPUT diff --git a/.github/workflows/e2e/validate_deployment/action.yaml b/.github/workflows/e2e/validate_deployment/action.yaml index 650c622..890a64b 100644 --- a/.github/workflows/e2e/validate_deployment/action.yaml +++ b/.github/workflows/e2e/validate_deployment/action.yaml @@ -5,6 +5,9 @@ inputs: is_dag_only_deploy: description: Whether the deploy operation was DAG-only default: false + is_no_deploy: + description: If the deploy test was a no-op + default: false dag_tarball_version_before: description: The desired DAG tarball version before the test required: true @@ -26,6 +29,15 @@ runs: shell: bash run: | + if [[ "${{ inputs.is_no_deploy }}" == "true" ]]; then + if [[ "${{ inputs.dag_tarball_version_before }}" == "${{ inputs.dag_tarball_version_after }}" && "${{ inputs.image_version_before }}" == "${{ inputs.image_version_after }}" ]]; then + echo "Deploy Action validation succeeded: no deploy operation" + exit 0 + fi + echo "Deploy Action validation failed: deploy operation was not a no-op" + exit 1 + fi + # If it's dag only deploy then validate that only desiredDagTarballVersion was updated if [[ "${{ inputs.is_dag_only_deploy }}" == "true" ]]; then if [[ "${{ inputs.dag_tarball_version_before }}" != "${{ inputs.dag_tarball_version_after }}" && "${{ inputs.image_version_before }}" == "${{ inputs.image_version_after }}" ]]; then diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 10ba0f0..25b51cd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -4,7 +4,6 @@ on: push: branches: - main - - e2e-tests workflow_dispatch: inputs: workspace_id: @@ -137,7 +136,7 @@ jobs: - name: Mock git commands run: | - mv e2e-setup/mocks/git.sh /usr/local/bin/git + mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git chmod +x /usr/local/bin/git - name: Install dependencies @@ -224,6 +223,13 @@ jobs: - name: Set CLI context run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + - name: Mock git commands for DAG Deploy + run: | + + # set to dag deploy git mock, with deploy-type set to image-and-dags to test that it will do a image deploy + mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + - name: Get Deployment Info Before Test id: get-deployment-before uses: ./.github/workflows/e2e/get_deployment_info @@ -239,7 +245,8 @@ jobs: deployment-id: ${{ matrix.deployment_id }} workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} pytest: true - deploy-image: true + action: deploy + deploy-type: image-and-dags root-folder: e2e-setup/astro-project - name: Get Deployment Info After Test @@ -259,10 +266,201 @@ jobs: dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} + infer-deploy: + name: Infer Deploy Test + runs-on: ubuntu-latest + needs: [pytests-test, create-test-deployments] + strategy: + max-parallel: 1 + matrix: + deployment_id: + [ + "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", + "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", + ] + deploy_type: [dags, image-and-dags] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Mock git commands + run: | + + # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock + if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then + mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git + else + mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git + fi + chmod +x /usr/local/bin/git + + - name: Install dependencies + run: | + + sudo apt-get install jq + + # we need to pre-install the CLI to set the context + curl -sSL https://install.astronomer.io | sudo bash -s + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Get Deployment Info Before Test + id: get-deployment-before + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Run Deploy Action + uses: ./ + with: + deployment-id: ${{ matrix.deployment_id }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + parse: true + action: deploy + root-folder: e2e-setup/astro-project + + - name: Get Deployment Info After Test + id: get-deployment-after + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Validate Deploy Action + uses: ./.github/workflows/e2e/validate_deployment + with: + is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }} + dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} + image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} + dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} + image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} + + no-deploy-test: + name: No Deploy Test + runs-on: ubuntu-latest + needs: [infer-image-deploy, create-test-deployments] + strategy: + matrix: + deployment_id: + [ + "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", + "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", + ] + deploy_type: [infer, dbt, dags, image-and-dags] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Setup dependencies + run: | + + sudo apt-get install jq + + # we need to pre-install the CLI to set the context + curl -sSL https://install.astronomer.io | sudo bash -s + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Get Deployment Info Before Test + id: get-deployment-before + if: ${{ matrix.deploy_type != 'dbt' }} + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Run Deploy Action + uses: ./ + with: + deployment-id: ${{ matrix.deployment_id }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + action: deploy + deploy-type: ${{ matrix.deploy_type }} + root-folder: ${{ matrix.deploy_type == 'dbt' && 'e2e-setup/dbt' || 'e2e-setup/astro-project' }} + + - name: Get Deployment Info After Test + id: get-deployment-after + if: ${{ matrix.deploy_type != 'dbt' }} + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Validate no image or dag deploy happened + if: ${{ matrix.deploy_type != 'dbt' }} + uses: ./.github/workflows/e2e/validate_deployment + with: + is_no_deploy: true + dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} + image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} + dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} + image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} + + - name: Get DBT bundle info + id: get-dbt-bundle-info + if: ${{ matrix.deploy_type == 'dbt' }} + uses: ./.github/workflows/e2e/get_bundle_info + with: + deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Validate no DBT deploy happened + id: validate-dbt-deploy-action + if: ${{ matrix.deploy_type == 'dbt' }} + shell: bash + run: | + + if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" != "" ]]; then + echo "DBT Deploy Action validation failed: DBT bundle found, but there should be no deploy" + exit 1 + fi + + echo "DBT Deploy Action validation succeeded: DBT bundle not found" + exit 0 + custom-docker-image-test: name: Custom Docker Image Test runs-on: ubuntu-latest - needs: [pytests-test, create-test-deployments] + needs: [no-deploy-test, create-test-deployments] strategy: matrix: deployment_id: @@ -316,7 +514,8 @@ jobs: deployment-id: ${{ matrix.deployment_id }} workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} image-name: custom-image:latest - deploy-image: true + action: deploy + deploy-type: image-and-dags root-folder: e2e-setup/astro-project - name: Get Deployment Info After Test @@ -336,6 +535,92 @@ jobs: dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desiredDagTarballVersion }} image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} + # DBT Deploy test would test the DAG only deploy functionality in deploy action + dbt-deploy-test: + name: DBT Deploy Test + runs-on: ubuntu-latest + needs: [custom-docker-image-test, create-test-deployments] + strategy: + matrix: + deployment_id: + [ + "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", + "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", + ] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Mock git commands + run: | + mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + + - name: Install dependencies + id: setup + run: | + + sudo apt-get install jq + # we need to pre-install the CLI to set the context + curl -sSL https://install.astronomer.io | sudo bash -s + + test_start_time=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ") + echo "test_start_time=$test_start_time" >> $GITHUB_OUTPUT + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: DBT Deploy to Astro + uses: ./ + with: + deployment-id: ${{ matrix.deployment_id }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + parse: true + root-folder: e2e-setup/dbt + deploy-type: dbt + description: "test-dbt-deploy-action" + + - name: Get DBT bundle info + id: get-dbt-bundle-info + uses: ./.github/workflows/e2e/get_bundle_info + with: + deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Validate DBT Deploy Action + id: validate-dbt-deploy-action + shell: bash + run: | + if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then + echo "DBT Deploy Action validation failed: DBT bundle not found" + exit 1 + fi + + updated_at=$(date -u -d "${{ steps.get-dbt-bundle-info.outputs.updated_at }}" +"%s.%N") + test_start_time=$(date -u -d "${{ steps.setup.outputs.test_start_time }}" +"%s.%N") + if [[ $(echo "$updated_at > $test_start_time" | bc -l) ]]; then + echo "DBT Deploy Action validation succeeded: DBT bundle found" + exit 0 + fi + + echo "DBT Deploy Action validation failed: DBT bundle was not updated" + exit 1 + # Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests deployment-preview-test: name: Deployment Preview Test @@ -366,7 +651,7 @@ jobs: secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - name: Install dependencies - id: install-pre-reqs + id: setup run: | sudo apt-get install jq @@ -374,6 +659,9 @@ jobs: # we need to pre-install the CLI to set the context curl -sSL https://install.astronomer.io | sudo bash -s + test_start_time=$(date -u +"%Y-%m-%dT%H:%M:%S.%NZ") + echo "test_start_time=$test_start_time" >> $GITHUB_OUTPUT + - name: Set CLI context run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} @@ -409,6 +697,11 @@ jobs: dag_tarball_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_dag_tarball_version }} image_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_image_version }} + - name: Mock git commands + run: | + mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + - name: Get Deployment Info Before Test id: get-deployment-before-deploy uses: ./.github/workflows/e2e/get_deployment_info @@ -423,10 +716,10 @@ jobs: uses: ./ with: deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} - workspace: ${{ secrets.WORKSPACE_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} action: deploy-deployment-preview + deploy-type: image-and-dags root-folder: e2e-setup/astro-project - deploy-image: true preview-name: test-preview-${{ matrix.deployment_id }} - name: Get Deployment Info After Test @@ -449,7 +742,7 @@ jobs: - name: Mock git commands for DAG Deploy run: | - mv e2e-setup/mocks/git.sh /usr/local/bin/git + mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git chmod +x /usr/local/bin/git - name: Get Deployment Info Before DAG Deploy Test @@ -466,8 +759,10 @@ jobs: uses: ./ with: deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} - workspace: ${{ secrets.WORKSPACE_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + action: deploy-deployment-preview parse: true + deploy-type: dags-only root-folder: e2e-setup/astro-project preview-name: test-preview-${{ matrix.deployment_id }} @@ -490,6 +785,51 @@ jobs: dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }} image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }} + - name: Mock git commands for DBT Deploy + run: | + mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + + - name: DBT Deploy to Astro + id: deployment-preview-dbt-deploy + uses: ./ + with: + deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + action: deploy-deployment-preview + deploy-type: dbt + description: "test-dbt-deploy-action" + root-folder: e2e-setup/dbt + preview-name: test-preview-${{ matrix.deployment_id }} + + - name: Get DBT bundle info + id: get-dbt-bundle-info + uses: ./.github/workflows/e2e/get_bundle_info + with: + deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Validate DBT Deploy Action + id: validate-dbt-deploy-action + shell: bash + run: | + if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then + echo "DBT Deploy Action validation failed: DBT bundle not found" + exit 1 + fi + + updated_at=$(date -u -d "${{ steps.get-dbt-bundle-info.outputs.updated_at }}" +"%s.%N") + test_start_time=$(date -u -d "${{ steps.setup.outputs.test_start_time }}" +"%s.%N") + if [[ $(echo "$updated_at > $test_start_time" | bc -l) ]]; then + echo "DBT Deploy Action validation succeeded: DBT bundle found" + exit 0 + fi + + echo "DBT Deploy Action validation failed: DBT bundle was not updated" + exit 1 + - name: Delete Deployment Preview id: delete-deployment-preview # ensure that we always try delete the deployment preview even if the previous steps fail @@ -497,7 +837,7 @@ jobs: uses: ./ with: deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} - workspace: ${{ secrets.WORKSPACE_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} action: delete-deployment-preview preview-name: test-preview-${{ matrix.deployment_id }} @@ -522,6 +862,7 @@ jobs: dag-deploy-test, pytests-test, custom-docker-image-test, + dbt-deploy-test, deployment-preview-test, ] # ensure that we always try delete the deployment even if any of the tests fail diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..288ff44 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @astronomer/dev-ex diff --git a/README.md b/README.md index e840e71..694fc45 100644 --- a/README.md +++ b/README.md @@ -48,21 +48,23 @@ The following table lists the configuration options for the Deploy to Astro acti | Name | Default | Description | | ---|---|--- | -| `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. Don't sepcify anything if you are deploying to a regular deployment. | +| `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `deploy`, `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. If using `deploy` or `deploy-deployment-preview` one should also specify `deploy-type`. | +| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. | | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. | | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` | | `description` | | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. | -| `root-folder` | `.` | Specifies the path to the Astro project directory that contains the `dags` folder. | +| `root-folder` | `.` | Path to the Astro project, or dbt project for dbt deploys. | | `parse` | `false` | When set to `true`, DAGs are parsed for errors before deploying to Astro. Note that when an image deploy is performed (i.e. `astro deploy`), parsing is also executed by default. Parsing is _not_ performed automatically for DAG-only deploys (i.e. `astro deploy --dags`). | | `pytest` | `false` | When set to `true`, all pytests in the `tests` directory of your Astro project are run before deploying to Astro. See [Run tests with pytest](https://docs.astronomer.io/astro/cli/test-your-astro-project-locally#run-tests-with-pytest) | | `pytest-file` | (all tests run) | Specifies a custom pytest file to run with the pytest command. For example, you could specify `/tests/test-tags.py`.| | `force` | `false` | When set to `true`, your code is deployed and skips any pytest or parsing errors. | -| `image-name` | | Specifies a custom, locally built image to deploy. | +| `image-name` | | Specifies a custom, locally built image to deploy. To be used with `deploy-type` set to `image-and-dags` or `infer` | | `workspace` | | Workspace id to select. Only required when `ASTRO_API_TOKEN` is given an organization token. | | `preview-name` | `false` | Specifies custom preview name. By default this is branch name “_” deployment name. | | `checkout` | `true` | Whether to checkout the repo as the first step. Set this to false if you want to modify repo contents before invoking the action. Your custom checkout step needs to have `fetch-depth` of `0` and `ref` equal to `${{ github.event.after }}` so all the commits in the PR are checked out. Look at the checkout step that runs within this action for reference. | -| `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. | +| `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. NOTE: This option is deprecated and will be removed in a future release. Use `deploy-type: image-and-dags` instead. | | `build-secrets` | `` | Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'. | +| `mount-path` | `` | Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name} | | `checkout-submodules` | `false` | Whether to checkout submodules when cloning the repository: `false` to disable (default), `true` to checkout submodules or `recursive` to recursively checkout submodules. Works only when `checkout` is set to `true`. Works only when `checkout` is set to `true`. | @@ -97,7 +99,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Deploy to Astro - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: deployment-id: parse: true @@ -112,7 +114,7 @@ In the following example, the folder `/example-dags/` is specified as the root f ```yaml steps: - name: Deploy to Astro - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: deployment-id: root-folder: /example-dags/ @@ -125,7 +127,7 @@ In the following example, the pytest located at `/tests/test-tags.py` runs befor ```yaml steps: - name: Deploy to Astro - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: deployment-id: pytest: true @@ -139,7 +141,7 @@ In the following example, `force` is enabled and both the DAG parse and pytest p ```yaml steps: - name: Deploy to Astro - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: deployment-id: force: true @@ -182,13 +184,48 @@ jobs: build-args: | - name: Deploy to Astro - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: deployment-id: + deploy-type: image-and-dags image-name: ${{ steps.image_tag.outputs.image_tag }} ``` +### Deploy a DBT project + +In the following example we would be deploying the dbt project located at `dbt` folder in the Github repo + +```yaml +steps: +- name: DBT Deploy to Astro + uses: astronomer/deploy-action@v0.6 + with: + deployment-id: + deploy-type: dbt + root-folder: dbt +``` + +### Deploy DAGs anad DBT project from same repo + +In the following example we would setup a workflow to deploy dags/images located at `astro-project` and dbt deploy from dbt project located at `dbt` folder in the same Github repo + +```yaml +steps: +- name: DBT Deploy to Astro + uses: astronomer/deploy-action@v0.6 + with: + deployment-id: + deploy-type: dbt + root-folder: dbt +- name: DAGs/Image Deploy to Astro + uses: astronomer/deploy-action@v0.6 + with: + deployment-id: + root-folder: astro-project/ + parse: true +``` + ## Deployment Preview Templates This section contains four workflow files that you will need in your repository to have a full Deployment Preview Cycle running for your Deployment. A Deployment Preview is an Astro Deployment that mirrors the configuration of your original Deployment. This Deployment Preview can be used to test your new pipelines changes before pushing them to your original Deployment. The scripts below will take your pipeline changes through the following flow: @@ -217,7 +254,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Create Deployment Preview - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: action: create-deployment-preview deployment-id: @@ -242,12 +279,39 @@ jobs: runs-on: ubuntu-latest steps: - name: Deploy to Deployment Preview - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: action: deploy-deployment-preview deployment-id: ``` +## DBT Deploy to Deployment Preview + +```yaml +name: Astronomer - DBT Deploy code to Preview + +on: + pull_request: + branches: + - main + +env: + ## Sets Deployment API key credentials as environment variables + ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Deploy to Deployment Preview + uses: astronomer/deploy-action@v0.6 + with: + action: deploy-deployment-preview + deploy-type: dbt + deployment-id: + root-folder: dbt +``` + ## Delete Deployment Preview ```yaml @@ -266,8 +330,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Create Deployment Preview - uses: astronomer/deploy-action@v0.4 + - name: Delete Deployment Preview + uses: astronomer/deploy-action@v0.6 with: action: delete-deployment-preview deployment-id: @@ -292,7 +356,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Deploy to Astro - uses: astronomer/deploy-action@v0.4 + uses: astronomer/deploy-action@v0.6 with: deployment-id: ``` diff --git a/action.yaml b/action.yaml index 9c2284b..1bcad97 100644 --- a/action.yaml +++ b/action.yaml @@ -7,8 +7,8 @@ branding: inputs: root-folder: required: false - default: ./ - description: "Path to the Astro project folder that contains the 'dags' folder." + default: "" + description: "Path to the Astro project, or dbt project for dbt deploys." parse: required: false default: false @@ -28,11 +28,11 @@ inputs: image-name: required: false default: no-custom-image - description: "Specify a custom built image to deploy to an Asto Deployment." + description: "Specify a custom built image to deploy to an Asto Deployment. To be used with 'deploy-type' set to 'image-and-dags' or 'infer'" action: required: false default: deploy - description: "Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either 'create-deployment-preview', 'delete-deployment-preview' or 'deploy-deployment-preview'. Don't specify anything if you are deploying to a regular Deployment." + description: "Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either 'deploy', 'create-deployment-preview', 'delete-deployment-preview', or 'deploy-deployment-preview'. If using 'deploy' or 'deploy-deployment-preview' one should also specify 'deploy-type'." deployment-name: required: false description: "The name of the Deployment you want to make preview from or are deploying to." @@ -78,10 +78,18 @@ inputs: deploy-image: required: false default: false - description: "If true image and DAGs will deploy" + description: "If true image and DAGs will deploy. NOTE: This option is deprecated and will be removed in a future release. Use `deploy-type: image-and-dags` instead." + deploy-type: + required: false + default: "infer" + description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image or DBT project. Specify either 'infer', 'dags-only', 'image-and-dags', or 'dbt'. 'infer' option would infer between DAG only deploy and image and DAG deploy based on updated files." build-secrets: required: false description: "Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'" + mount-path: + required: false + default: "" + description: "Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name}" checkout-submodules: required: false default: false @@ -102,15 +110,63 @@ runs: ref: ${{ github.event.after }} clean: false submodules: ${{ inputs.checkout-submodules }} + - name: Warn about deprecated deploy-image option + if: inputs.deploy-image == true + shell: bash + run: | + echo "The 'deploy-image' option is deprecated and will be removed in 1.0 release. Use 'deploy-type: image-and-dags' instead." - name: Install Astro CLI run: | - curl -sSL https://install.astronomer.io | sudo bash -s ${{ inputs.cli-version }} + + echo ::group::Install Astro CLI + CLI_VERSION=${{ inputs.cli-version }} + # if Astro CLI is pre-installed, fetch it's version + if command -v astro &> /dev/null; then + # "astro version" commands returns the CLI version, its output is of the form: "Astro CLI Version: 1.29.0" + CLI_VERSION=$(astro version | awk '{print $4}') + fi + + # Check if the Astro CLI version is less than 1.28.1 for dbt-deploy + if [[ "${{ inputs.deploy-type }}" == "dbt" && $CLI_VERSION != "" ]]; then + #install semver to compare versions + npm install -g semver + + REQUIRED_VERSION="1.28.1" + CURRENT_VERSION=$(astro version | awk '{print $4}') + if ! semver -r ">${REQUIRED_VERSION}" "${CURRENT_VERSION}"; then + echo "DBT Deploy requires Astro CLI version $REQUIRED_VERSION or higher" + exit 1 + fi + fi + + # skip Astro CLI installation if already installed + if command -v astro &> /dev/null; then + echo "Astro CLI is already installed" + exit 0 + fi + + # check if CLI_VERSION does not starts with "v", then add "v" to it + if [[ $CLI_VERSION != "" && $CLI_VERSION != v* ]]; then + CLI_VERSION=v$CLI_VERSION + fi + + if [[ $CLI_VERSION == "" ]]; then + curl -sSL https://install.astronomer.io | sudo bash -s + else + curl -sSL https://install.astronomer.io | sudo bash -s -- $CLI_VERSION + fi + echo ::endgroup:: shell: bash - - name: Deployment Preview action + - name: Determine Deploy Deployment run: | - # set action - ACTION=nothing + echo ::group::Determine Deploy Deployment + + # validate action input + if [[ ${{ inputs.action }} != create-deployment-preview && ${{ inputs.action }} != delete-deployment-preview && ${{ inputs.action }} != deploy-deployment-preview && ${{ inputs.action }} != deploy ]]; then + echo ERROR: you specified an improper action input. Action must be deploy, deploy-deployment-preview, create-deployment-preview, or delete-deployment-preview. + exit 1 # terminate and indicate error + fi # Select workspace if specified if [[ "${{ inputs.workspace }}" != "" ]]; then @@ -170,15 +226,6 @@ runs: # Get original Deployment ID echo "ORIGINAL_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT - - # don't skip deploy - echo "SKIP_DEPLOY=false" >> $GITHUB_OUTPUT - - # image deploy only - echo "IMAGE_DEPLOY_ONLY=true" >> $GITHUB_OUTPUT - - # set action - ACTION=create fi # delete deployment preview and skip deploy if action is delete-deployment-preview @@ -199,15 +246,6 @@ runs: fi # delete branch deployment astro deployment delete -n $BRANCH_DEPLOYMENT_NAME -f - - # skip deploy - echo "SKIP_DEPLOY=true" >> $GITHUB_OUTPUT - - # not image deploy only - echo "IMAGE_DEPLOY_ONLY=false" >> $GITHUB_OUTPUT - - # set action - ACTION=delete fi # # deploy to deployment preview if action is deploy-deployment-preview @@ -235,84 +273,130 @@ runs: # Get Deployment ID echo "FINAL_DEPLOYMENT_ID=$(astro deployment inspect --clean-output -n $BRANCH_DEPLOYMENT_NAME --key metadata.deployment_id)" >> $GITHUB_OUTPUT - - # don't skip deploy - echo "SKIP_DEPLOY=false" >> $GITHUB_OUTPUT - - # not image deploy only - echo "IMAGE_DEPLOY_ONLY=false" >> $GITHUB_OUTPUT - - # set action - ACTION=deploy-preview fi - # if action is deploy set final Deployment id to Deployment id + # if action is deploy or dbt deploy simply set final Deployment id to Deployment id to streamline the deployment id reference in rest of the workflow if [[ ${{ inputs.action }} == deploy ]]; then echo "FINAL_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT - - # don't skip deploy - echo "SKIP_DEPLOY=false" >> $GITHUB_OUTPUT - - # not image deploy only - echo "IMAGE_DEPLOY_ONLY=false" >> $GITHUB_OUTPUT - - # set action - ACTION=deploy fi - if [[ $ACTION == nothing ]]; then - echo ERROR: you specified an improper action input. Action must be deploy, deploy-deployment-preview, create-deployment-preview, or delete-deployment-preview. - exit 1 # terminate and indicate error - fi + echo ::endgroup:: shell: bash id: deployment-preview - name: Determine if DAG Deploy is enabled run: | - if [[ ${{steps.deployment-preview.outputs.SKIP_DEPLOY}} == false && ${{steps.deployment-preview.outputs.IMAGE_DEPLOY_ONLY}} == false ]]; then + + echo ::group::Determine if DAG Deploy is enabled + if [[ ${{ inputs.action }} != create-deployment-preview && ${{ inputs.action }} != delete-deployment-preview ]]; then echo "DAG_DEPLOY_ENABLED=$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key configuration.dag_deploy_enabled)" >> $GITHUB_OUTPUT else echo "DAG_DEPLOY_ENABLED=false" >> $GITHUB_OUTPUT fi + echo ::endgroup:: shell: bash id: dag-deploy-enabled - - name: Get Deployment Type + - name: Get DBT Deploy Options + if: ${{ inputs.deploy-type == 'dbt' }} run: | + + echo ::group::Get DBT Deploy Options cd ${{ inputs.root-folder }} branch=$(echo "${GITHUB_REF#refs/heads/}") echo "Branch pushed to: $branch" git fetch origin $branch files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }}) - echo "files changed: $files" - dags_only=1 + echo "Files changed: $files" + DBT_DEPLOY=false for file in $files; do - if [[ $file != *"dags/"* ]]; then - echo $file is not a DAG, triggering a full image build - dags_only=0 - break + if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then + echo $file is part of configured root folder, so would be triggering a dbt deploy + DBT_DEPLOY=true fi done - if [[ ${{steps.dag-deploy-enabled.outputs.DAG_DEPLOY_ENABLED}} == false ]]; then - dags_only=0 + options="" + if [[ $dbt_deploy == 1 ]]; then + # Add mount path option + if [[ "${{ inputs.mount-path }}" != "" ]]; then + options="$options --mount-path=${{ inputs.mount-path }}" + fi + + # Add description option + if [[ "${{ inputs.description }}" != "" ]]; then + options="$options --description '${{ inputs.description }}'" + fi fi - if [[ ${{steps.deployment-preview.outputs.SKIP_DEPLOY}} == true ]]; then + echo "DBT_DEPLOY=$DBT_DEPLOY" >> $GITHUB_OUTPUT + echo "DBT_OPTIONS=$options" >> $GITHUB_OUTPUT + echo ::endgroup:: + shell: bash + id: dbt-deploy-options + - name: Get Deploy Type + if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'image-and-dags' }} + run: | + + # infer based on files changed to deploy only dags or image and dags + echo ::group::Get Deploy Type + cd ${{ inputs.root-folder }} + branch=$(echo "${GITHUB_REF#refs/heads/}") + echo "Branch pushed to: $branch" + git fetch origin $branch + files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }}) + echo "files changed: $files" + DAGS_ONLY_DEPLOY=false + + SKIP_IMAGE_OR_DAGS_DEPLOY=true + + # This for loop checks for following cases: + # 1. If no file is part of the input root folder, then it skips deploy + # 2. If any file is not part of the dags folder, then it triggers a full image build + # 3. If all files are part of the dags folder and input root folder, then it triggers a DAG-only deploy + for file in $files; do + if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then + echo $file is part of the input root folder + SKIP_IMAGE_OR_DAGS_DEPLOY=false + if [[ $file == *"dags/"* && ${{ inputs.deploy-type }} != 'image-and-dags' ]]; then + echo $file is part of dags folder, triggering a DAG-only deploy + DAGS_ONLY_DEPLOY=true + break + fi + fi + done + + # Note: the order of these following checks is important to ensure that we skip/trigger deploy correctly in following cases: + # 1. When there is no change in the input root folder we should skip deploy, but not when it's deployment preview create action + # 2. When user has passed a custom image then we would need to deploy the image + # 3. When the action is deployment preview delete, then we should skip any form of deploy + if [[ $SKIP_IMAGE_OR_DAGS_DEPLOY == true ]]; then # skip all deploy steps - dags_only=2 + DAGS_ONLY_DEPLOY=false fi - if [[ ${{ inputs.deploy-image }} == true ]]; then - # make sure image and DAGs deploys because deploy-image is true - dags_only=0 + # check if user has passed a custom image or the action has created a new deployment preview, then we would need to deploy the image + if [[ ${{ inputs.image-name }} != "no-custom-image" || ${{ steps.dag-deploy-enabled.outputs.DAG_DEPLOY_ENABLED }} == false || ${{ inputs.action }} == create-deployment-preview ]]; then + SKIP_IMAGE_OR_DAGS_DEPLOY=false + DAGS_ONLY_DEPLOY=false fi - echo "DAGS_ONLY=$dags_only" >> $GITHUB_OUTPUT + if [[ ${{ inputs.action }} == delete-deployment-preview ]]; then + # skip all deploy steps + SKIP_IMAGE_OR_DAGS_DEPLOY=true + DAGS_ONLY_DEPLOY=false + fi + + echo "DAGS_ONLY_DEPLOY=$DAGS_ONLY_DEPLOY" >> $GITHUB_OUTPUT + echo "SKIP_IMAGE_OR_DAGS_DEPLOY=$SKIP_IMAGE_OR_DAGS_DEPLOY" >> $GITHUB_OUTPUT + echo ::endgroup:: shell: bash - id: deployment-type + id: deploy-type # If only DAGs changed and dag deploys is enabled, do a DAG-only deploy - - name: setup deploy options + - name: Setup image or dags deploy options + if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'image-and-dags' }} run: | + + echo ::group::Setup image or dags deploy options options="" # add parse option @@ -326,7 +410,7 @@ runs: fi # add custom image option - if [[ ${{ inputs.image-name }} != no-custom-image ]]; then + if [[ ${{ inputs.image-name }} != no-custom-image && inputs.deploy-type != "dags-only" ]]; then options="$options --image-name ${{ inputs.image-name }}" fi @@ -346,15 +430,19 @@ runs: fi echo "OPTIONS=$options" >> $GITHUB_OUTPUT + echo ::endgroup:: shell: bash id: deploy-options - name: Determine if Development Mode is enabled run: | - if [[ ${{steps.deployment-preview.outputs.SKIP_DEPLOY}} == false ]]; then + + echo ::group::Determine if Development Mode is enabled + if [[ ${{ inputs.action }} != delete-deployment-preview ]]; then DEPLOYMENT_TYPE=$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key configuration.deployment_type) # only inspect development mode if deployment is not hybrid if [[ $DEPLOYMENT_TYPE != "HYBRID" ]]; then DEV_MODE=$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key configuration.is_development_mode) + echo "Deployment development mode: $DEV_MODE" if [[ $DEV_MODE == "" ]]; then echo "DEVELOPMENT_MODE=false" >> $GITHUB_OUTPUT else @@ -366,46 +454,68 @@ runs: else echo "DEVELOPMENT_MODE=false" >> $GITHUB_OUTPUT fi + echo ::endgroup:: shell: bash id: development-mode - name: Override to wake up the Deployment + if: ${{ inputs.action != 'delete-deployment-preview' && steps.development-mode.outputs.DEVELOPMENT_MODE == 'true' }} run: | - if [[ ${{steps.deployment-preview.outputs.SKIP_DEPLOY}} == false && ${{steps.development-mode.outputs.DEVELOPMENT_MODE}} == true ]]; then - astro deployment wake-up ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --force - # Give it some time to wake up - sleep 60 - fi + echo ::group::Override to wake up the Deployment + astro deployment wake-up ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --force + # Give it some time to wake up + sleep 60 + echo ::endgroup:: shell: bash - name: DAG Deploy to Astro - if: steps.deployment-type.outputs.DAGS_ONLY == 1 + if: ${{ (inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'infer') && steps.deploy-type.outputs.DAGS_ONLY_DEPLOY == 'true' }} run: | + + echo ::group::DAG Deploy to Astro + # Deploy only dags cd ${{ inputs.root-folder }} astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --dags ${{steps.deploy-options.outputs.OPTIONS}} + echo ::endgroup:: shell: bash # If any other files changed or dag deploys is disabled, deploy the entire Astro project - name: Image and DAG Deploy to Astro - if: steps.deployment-type.outputs.DAGS_ONLY == 0 + if: ${{ inputs.deploy-image == true || ( (inputs.deploy-type == 'image-and-dags' || inputs.deploy-type == 'infer') && steps.deploy-type.outputs.DAGS_ONLY_DEPLOY == 'false' && steps.deploy-type.outputs.SKIP_IMAGE_OR_DAGS_DEPLOY == 'false' ) }} run: | + echo ::group::Image and DAG Deploy to Astro + # Deploy image and DAGs cd ${{ inputs.root-folder }} astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.deploy-options.outputs.OPTIONS}} + echo ::endgroup:: + shell: bash + - name: DBT Deploy to Astro + if: ${{ inputs.deploy-type == 'dbt' && steps.dbt-deploy-options.outputs.DBT_DEPLOY == 'true' }} + run: | + + echo ::group::DBT Deploy to Astro + cd ${{ inputs.root-folder }} + astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}} + echo ::endgroup:: shell: bash - name: Copy Airflow Connections, Variables, and Pools + if: ${{ inputs.action == 'create-deployment-preview' }} run: | - if [[ ${{ inputs.action }} == create-deployment-preview ]]; then - if [[ ${{ inputs.copy-connections }} == true ]]; then - astro deployment connection copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} - fi - if [[ ${{ inputs.copy-airflow-variables }} == true ]]; then - astro deployment airflow-variable copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} - fi - if [[ ${{ inputs.copy-pools }} == true ]]; then - astro deployment pool copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} - fi + + echo ::group::Copy Airflow Connections, Variables, and Pools + if [[ ${{ inputs.copy-connections }} == true ]]; then + astro deployment connection copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} + fi + if [[ ${{ inputs.copy-airflow-variables }} == true ]]; then + astro deployment airflow-variable copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} + fi + if [[ ${{ inputs.copy-pools }} == true ]]; then + astro deployment pool copy --source-id ${{steps.deployment-preview.outputs.ORIGINAL_DEPLOYMENT_ID}} --target-id ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} fi + echo ::endgroup:: shell: bash - name: Remove override on Deployment to resume schedule + if: ${{ steps.development-mode.outputs.DEVELOPMENT_MODE == 'true' }} run: | - if [[ ${{steps.deployment-preview.outputs.SKIP_DEPLOY}} == false && ${{steps.development-mode.outputs.DEVELOPMENT_MODE}} == true ]]; then - astro deployment wake-up ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --remove-override --force - fi + + echo ::group::Remove override on Deployment to resume schedule + astro deployment wake-up ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --remove-override --force + echo ::endgroup:: shell: bash diff --git a/e2e-setup/README.md b/e2e-setup/README.md index 28dac4c..070bef0 100644 --- a/e2e-setup/README.md +++ b/e2e-setup/README.md @@ -6,18 +6,27 @@ The E2E tests could be triggered automatically when a new commit is pushed to ma ## Folder Structure +``` e2e-setup/ ├── astro-project/ ├── deployment-templates/ -│ ├── deployment-hibernate.yaml -│ └── deployment.yaml +│ ├── deployment-hibernate.yaml +│ └── deployment.yaml ├── mocks/ -│ └── git.sh +│ ├── dag-deploy-git.sh +│ └── dbt-deploy-git.sh +├── dbt/ +│ └── dbt_project.yml +``` ### astro-project Astro project folder contains a basic airflow project initialized via Astro CLI, which will deployed as part of tests via deploy action +### dbt + +The dbt folder contains a basic sample dbt_project.yml file, which is used to define the configuration of a dbt project. This file includes settings such as the project name, version, and other configurations necessary for running dbt commands. + ### deployment-templates Deployment templates contains the basic templates used by e2e tests to create required deployments against which tests would be executed diff --git a/e2e-setup/dbt/dbt_project.yml b/e2e-setup/dbt/dbt_project.yml new file mode 100644 index 0000000..377ac5f --- /dev/null +++ b/e2e-setup/dbt/dbt_project.yml @@ -0,0 +1,26 @@ +name: "jaffle_shop" + +config-version: 2 +version: "0.1" + +profile: "jaffle_shop" + +model-paths: ["models"] +seed-paths: ["seeds"] +test-paths: ["tests"] +analysis-paths: ["analysis"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_modules" + - "logs" + +require-dbt-version: [">=1.0.0", "<2.0.0"] + +models: + jaffle_shop: + materialized: table + staging: + materialized: view diff --git a/e2e-setup/mocks/git.sh b/e2e-setup/mocks/dag-deploy-git.sh similarity index 99% rename from e2e-setup/mocks/git.sh rename to e2e-setup/mocks/dag-deploy-git.sh index 8365d3e..4229a91 100755 --- a/e2e-setup/mocks/git.sh +++ b/e2e-setup/mocks/dag-deploy-git.sh @@ -10,4 +10,4 @@ elif [[ "$1" == "fetch" ]]; then else echo "Error: git mock script isn't configured to handle $1" >&2 exit 1 -fi \ No newline at end of file +fi diff --git a/e2e-setup/mocks/dbt-deploy-git.sh b/e2e-setup/mocks/dbt-deploy-git.sh new file mode 100644 index 0000000..a415600 --- /dev/null +++ b/e2e-setup/mocks/dbt-deploy-git.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# hack to mock git commands as part of action.yaml so that we could simulate dbt deploy scenario without making any additional commits + +# Check if the script was invoked with "git diff" +if [[ "$1" == "diff" ]]; then + echo "e2e-setup/dbt/dbt_project.yml" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi diff --git a/e2e-setup/mocks/image-deploy-git.sh b/e2e-setup/mocks/image-deploy-git.sh new file mode 100644 index 0000000..792ca88 --- /dev/null +++ b/e2e-setup/mocks/image-deploy-git.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# hack to mock git commands as part of action.yaml so that we could simulate image deploy scenario without making any additional commits + +# Check if the script was invoked with "git diff" +if [[ "$1" == "diff" ]]; then + echo "e2e-setup/astro-project/Dockerfile" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi