add temporary astro project at root level #104
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests for Astro Deploy Action | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - dbt-deploy | |
| workflow_dispatch: | |
| inputs: | |
| workspace_id: | |
| description: "Workspace ID" | |
| required: false | |
| default: "" | |
| org_id: | |
| description: "Organization ID" | |
| required: false | |
| default: "" | |
| astronomer_host: | |
| description: "Astronomer Host" | |
| required: false | |
| default: "" | |
| token: | |
| description: "API Token" | |
| required: false | |
| default: "" | |
| env: | |
| ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} | |
| jobs: | |
| redact-inputs: | |
| name: Redact Inputs | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Redact Inputs | |
| run: | | |
| # add-mask will redact the value of the input, but will still log the value, | |
| # hence using a known workaround: https://github.com/actions/runner/issues/643 | |
| WORKSPACE_ID=$(jq -r '.inputs.workspace_id' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$WORKSPACE_ID | |
| ORG_ID=$(jq -r '.inputs.org_id' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$ORG_ID | |
| ASTRONOMER_HOST=$(jq -r '.inputs.astronomer_host' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$ASTRONOMER_HOST | |
| TOKEN=$(jq -r '.inputs.token' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$TOKEN | |
| # Create test deployments would use the template files and generate deployments with unique names | |
| create-test-deployments: | |
| name: Create Deployment | |
| needs: redact-inputs | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| deployment: [deployment-hibernate.yaml, deployment.yaml] | |
| outputs: | |
| DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.DEPLOYMENT_ID }} | |
| HIBERNATE_DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.HIBERNATE_DEPLOYMENT_ID }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| sudo add-apt-repository ppa:rmescandon/yq | |
| sudo apt update | |
| sudo apt install yq -y | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| # Generate a random test_uid and set it as an environment variable, so we can append it to the deployment name | |
| test_uid=$(uuidgen) | |
| echo "test_uid=$test_uid" >> $GITHUB_ENV | |
| - name: Create Deployment | |
| id: create-deployment | |
| run: | | |
| # Update the deployment name to maintain uniqueness | |
| sed -i "s| name:.*| name: deploy-action-e2e-${{ env.test_uid }}|g" e2e-setup/deployment-templates/${{ matrix.deployment }} | |
| DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -) | |
| if [[ "${{ matrix.deployment }}" == "deployment.yaml" ]]; then | |
| echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT | |
| else | |
| echo "HIBERNATE_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT | |
| fi | |
| # DAG Deploy test would test the DAG only deploy functionality in deploy action | |
| dag-deploy-test: | |
| name: DAG Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Mock git commands | |
| run: | | |
| mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| parse: true | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: true | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| pytests-test: | |
| name: Pytest Test | |
| runs-on: ubuntu-latest | |
| needs: [dag-deploy-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Run Pytest | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| pytest: true | |
| action: deploy | |
| deploy-type: image-and-dags | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| infer-image-deploy: | |
| name: Infer Image Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [pytests-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Run Deploy Action | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| pytest: true | |
| action: deploy | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| custom-docker-image-test: | |
| name: Custom Docker Image Test | |
| runs-on: ubuntu-latest | |
| needs: [infer-image-deploy, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Build Docker image | |
| run: cd e2e-setup/astro-project && docker build -t custom-image:latest . | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| image-name: custom-image:latest | |
| action: deploy | |
| deploy-type: image-and-dags | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desiredDagTarballVersion }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| # DBT Deploy test would test the DAG only deploy functionality in deploy action | |
| dbt-deploy-test: | |
| name: DBT Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [custom-docker-image-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Mock git commands | |
| run: | | |
| mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: DBT Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| parse: true | |
| root-folder: e2e-setup/dbt | |
| deploy-type: dbt | |
| description: "test-dbt-deploy-action" | |
| - name: Get DBT bundle info | |
| id: get-dbt-bundle-info | |
| uses: ./.github/workflows/e2e/get_bundle_info | |
| with: | |
| deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| search_term: "test-dbt-deploy-action" # We are passing description in the dbt deploy action and using that as search term over here because that's the only search term we have | |
| - name: Validate DBT Deploy Action | |
| id: validate-dbt-deploy-action | |
| shell: bash | |
| run: | | |
| if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then | |
| echo "DBT Deploy Action validation failed: DBT bundle not found" | |
| exit 1 | |
| fi | |
| if [[ "${{ steps.get-dbt-bundle-info.outputs.bundle_type }}" != "dbt" ]]; then | |
| echo "DBT Deploy Action validation failed: DBT bundle not found" | |
| exit 1 | |
| fi | |
| echo "DBT Deploy Action validation succeeded: DBT bundle found" | |
| exit 0 | |
| # Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests | |
| deployment-preview-test: | |
| name: Deployment Preview Test | |
| runs-on: ubuntu-latest | |
| needs: [create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install dependencies | |
| id: install-pre-reqs | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Create Deployment Preview | |
| id: create-deployment-preview | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: create-deployment-preview | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| root-folder: e2e-setup/astro-project | |
| # TODO: allow copying airflow resources, once we add wait in the deploy action for deployment preview to be healthy | |
| copy-connections: false | |
| copy-airflow-variables: false | |
| copy-pools: false | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after-create-preview | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| id: validate-deployment-preview-create | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: "" | |
| image_version_before: "12.1.0" | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_image_version }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Deploy to Deployment Preview | |
| id: deployment-preview-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: deploy-deployment-preview | |
| deploy-type: image-and-dags | |
| root-folder: e2e-setup/astro-project | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after-deploy-preview | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-deploy.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| id: validate-deployment-preview-deploy | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_image_version }} | |
| - name: Mock git commands for DAG Deploy | |
| run: | | |
| mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Get Deployment Info Before DAG Deploy Test | |
| id: get-deployment-before-dag-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: DAG Deploy to Astro | |
| id: deployment-preview-dag-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: deploy-deployment-preview | |
| parse: true | |
| deploy-type: dags-only | |
| root-folder: e2e-setup/astro-project | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get Deployment Info After DAG Deploy Test | |
| id: get-deployment-after-dag-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| - name: Validate DAG Deploy Action | |
| id: validate-dag-deploy-action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: true | |
| dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }} | |
| - name: Mock git commands for DBT Deploy | |
| run: | | |
| mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: DBT Deploy to Astro | |
| id: deployment-preview-dbt-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: deploy-deployment-preview | |
| deploy-type: dbt | |
| description: "test-dbt-deploy-action" | |
| root-folder: e2e-setup/dbt | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get DBT bundle info | |
| id: get-dbt-bundle-info | |
| uses: ./.github/workflows/e2e/get_bundle_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| search_term: "test-dbt-deploy-action" # We are passing description in the dbt deploy action and using that as search term over here because that's the only search term we have | |
| - name: Validate DBT Deploy Action | |
| id: validate-dbt-deploy-action | |
| shell: bash | |
| run: | | |
| if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then | |
| echo "DBT Deploy Action validation failed: DBT bundle not found" | |
| exit 1 | |
| fi | |
| if [[ "${{ steps.get-dbt-bundle-info.outputs.bundle_type }}" != "dbt" ]]; then | |
| echo "DBT Deploy Action validation failed: DBT bundle not found" | |
| exit 1 | |
| fi | |
| echo "DBT Deploy Action validation succeeded: DBT bundle found" | |
| exit 0 | |
| - name: Delete Deployment Preview | |
| id: delete-deployment-preview | |
| # ensure that we always try delete the deployment preview even if the previous steps fail | |
| if: always() | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| action: delete-deployment-preview | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Validate Deployment Preview Deletion | |
| id: validate-deployment-preview-deletion | |
| # ensure that we always try delete the deployment preview even if the previous steps fail | |
| if: always() | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| expected_status_code: 404 | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} | |
| astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} | |
| astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| delete-test-deployments: | |
| name: Delete Deployment | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| create-test-deployments, | |
| dag-deploy-test, | |
| pytests-test, | |
| custom-docker-image-test, | |
| dbt-deploy-test, | |
| deployment-preview-test, | |
| ] | |
| # ensure that we always try delete the deployment even if any of the tests fail | |
| if: always() | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get Astro Environment Info | |
| id: get-astro-env-info | |
| uses: ./.github/workflows/e2e/get_astro_env_info | |
| with: | |
| input_workspace_id: ${{ github.event.inputs.workspace_id }} | |
| input_organization_id: ${{ github.event.inputs.org_id }} | |
| input_astronomer_host: ${{ github.event.inputs.astronomer_host }} | |
| input_astro_api_token: ${{ github.event.inputs.token }} | |
| secret_workspace_id: ${{ secrets.WORKSPACE_ID }} | |
| secret_organization_id: ${{ secrets.ORGANIZATION_ID }} | |
| secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
| secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
| - name: Install Astro CLI and set context | |
| run: | | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} | |
| astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} | |
| - name: Delete Deployment | |
| run: | | |
| astro deployment delete -f ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} | |
| astro deployment delete -f ${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} |