code review changes: allow inputs for worflow_dispatch #75
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests for Astro Deploy Action | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - e2e-tests | |
| workflow_dispatch: | |
| inputs: | |
| workspace_id: | |
| description: "Workspace ID" | |
| required: false | |
| default: "" | |
| org_id: | |
| description: "Organization ID" | |
| required: false | |
| default: "" | |
| astronomer_host: | |
| description: "Astronomer Host" | |
| required: false | |
| default: "" | |
| token: | |
| description: "API Token" | |
| required: false | |
| default: "" | |
| env: | |
| ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} | |
| jobs: | |
| set-env-vars: | |
| name: Set Environment Variables | |
| runs-on: ubuntu-latest | |
| outputs: | |
| workspace_id: ${{ steps.set-env.outputs.workspace_id }} | |
| org_id: ${{ steps.set-env.outputs.org_id }} | |
| token: ${{ steps.set-env.outputs.token }} | |
| astronomer_host: ${{ steps.set-env.outputs.astronomer_host }} | |
| steps: | |
| - name: Set environment variables from inputs | |
| id: set-env | |
| run: | | |
| echo "Setting environment variables..." | |
| if [ "${{ github.event.inputs.workspace_id }}" != "" ]; then | |
| echo "Using provided workspace_id" | |
| echo "workspace_id=${{ github.event.inputs.workspace_id }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "workspace_id=${{ secrets.WORKSPACE_ID }}" >> $GITHUB_OUTPUT | |
| fi | |
| if [ "${{ github.event.inputs.org_id }}" != "" ]; then | |
| echo "Using provided org_id" | |
| echo "org_id=${{ github.event.inputs.org_id }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "org_id=${{ secrets.ORGANIZATION_ID }}" >> $GITHUB_OUTPUT | |
| fi | |
| if [ "${{ github.event.inputs.astronomer_host }}" != "" ]; then | |
| echo "Using provided astronomer_host" | |
| echo "astronomer_host=${{ github.event.inputs.astronomer_host }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "astronomer_host=${{ secrets.ASTRONOMER_HOST }}" >> $GITHUB_OUTPUT | |
| fi | |
| if [ "${{ github.event.inputs.token }}" != "" ]; then | |
| echo "Using provided token" | |
| echo "ASTRO_API_TOKEN=${{ github.event.inputs.token }}" >> $GITHUB_ENV | |
| echo "token=${{ github.event.inputs.token }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "ASTRO_API_TOKEN=${{ secrets.ASTRO_API_TOKEN }}" >> $GITHUB_ENV | |
| echo "token=${{ secrets.ASTRO_API_TOKEN }}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Redact sensitive information | |
| run: | | |
| echo "::add-mask::${{ steps.set-env.outputs.workspace_id }}" | |
| echo "::add-mask::${{ steps.set-env.outputs.org_id }}" | |
| echo "::add-mask::${{ steps.set-env.outputs.astronomer_host }}" | |
| echo "::add-mask::${{ steps.set-env.outputs.token }}" | |
| # Create test deployments would use the template files and generate deployments with unique names | |
| create-test-deployments: | |
| name: Create Deployment | |
| runs-on: ubuntu-latest | |
| needs: set-env-vars | |
| strategy: | |
| matrix: | |
| deployment: [deployment-hibernate.yaml, deployment.yaml] | |
| outputs: | |
| DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.DEPLOYMENT_ID }} | |
| HIBERNATE_DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.HIBERNATE_DEPLOYMENT_ID }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| sudo add-apt-repository ppa:rmescandon/yq | |
| sudo apt update | |
| sudo apt install yq -y | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| astro context switch ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| astro workspace switch ${{ needs.set-env-vars.outputs.workspace_id }} | |
| # Generate a random test_uid and set it as an environment variable, so we can append it to the deployment name | |
| test_uid=$(uuidgen) | |
| echo "test_uid=$test_uid" >> $GITHUB_ENV | |
| - name: Create Deployment | |
| id: create-deployment | |
| run: | | |
| # Update the deployment name to maintain uniqueness | |
| sed -i "s| name:.*| name: deploy-action-e2e-${{ env.test_uid }}|g" e2e-setup/deployment-templates/${{ matrix.deployment }} | |
| DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -) | |
| if [[ "${{ matrix.deployment }}" == "deployment.yaml" ]]; then | |
| echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT | |
| else | |
| echo "HIBERNATE_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT | |
| fi | |
| # DAG Deploy test would test the DAG only deploy functionality in deploy action | |
| dag-deploy-test: | |
| name: DAG Deploy Test | |
| runs-on: ubuntu-latest | |
| needs: [set-env-vars, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Mock git commands | |
| run: | | |
| mv e2e-setup/mocks/git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ needs.set-env-vars.outputs.workspace_id }} | |
| parse: true | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: true | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| pytests-test: | |
| name: Pytest Test | |
| runs-on: ubuntu-latest | |
| needs: [set-env-vars, dag-deploy-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Run Pytest | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ needs.set-env-vars.outputs.workspace_id }} | |
| pytest: true | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| custom-docker-image-test: | |
| name: Custom Docker Image Test | |
| runs-on: ubuntu-latest | |
| needs: [set-env-vars, pytests-test, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Build Docker image | |
| run: cd e2e-setup/astro-project && docker build -t custom-image:latest . | |
| - name: Set CLI context | |
| run: astro context switch ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Deploy to Astro | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ needs.set-env-vars.outputs.workspace_id }} | |
| image-name: custom-image:latest | |
| root-folder: e2e-setup/astro-project | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ matrix.deployment_id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desiredDagTarballVersion }} | |
| image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
| # Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests | |
| deployment-preview-test: | |
| name: Deployment Preview Test | |
| runs-on: ubuntu-latest | |
| needs: [set-env-vars, create-test-deployments] | |
| strategy: | |
| matrix: | |
| deployment_id: | |
| [ | |
| "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
| "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| id: install-pre-reqs | |
| run: | | |
| sudo apt-get install jq | |
| # we need to pre-install the CLI to set the context | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| - name: Set CLI context | |
| run: astro context switch ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Create Deployment Preview | |
| id: create-deployment-preview | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ matrix.deployment_id }} | |
| workspace: ${{ needs.set-env-vars.outputs.workspace_id }} | |
| action: create-deployment-preview | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| root-folder: e2e-setup/astro-project | |
| # TODO: allow copying airflow resources, once we add wait in the deploy action for deployment preview to be healthy | |
| copy-connections: false | |
| copy-airflow-variables: false | |
| copy-pools: false | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after-create-preview | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| id: validate-deployment-preview-create | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: "" | |
| image_version_before: "12.1.0" | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_image_version }} | |
| - name: Get Deployment Info Before Test | |
| id: get-deployment-before-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Deploy to Deployment Preview | |
| id: deployment-preview-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ secrets.WORKSPACE_ID }} | |
| action: deploy-deployment-preview | |
| root-folder: e2e-setup/astro-project | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get Deployment Info After Test | |
| id: get-deployment-after-deploy-preview | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-deploy.outputs.preview-id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Validate Deploy Action | |
| id: validate-deployment-preview-deploy | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| dag_tarball_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_image_version }} | |
| - name: Mock git commands for DAG Deploy | |
| run: | | |
| mv e2e-setup/mocks/git.sh /usr/local/bin/git | |
| chmod +x /usr/local/bin/git | |
| - name: Get Deployment Info Before DAG Deploy Test | |
| id: get-deployment-before-dag-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: DAG Deploy to Astro | |
| id: deployment-preview-dag-deploy | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ secrets.WORKSPACE_ID }} | |
| parse: true | |
| root-folder: e2e-setup/astro-project | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Get Deployment Info After DAG Deploy Test | |
| id: get-deployment-after-dag-deploy | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| - name: Validate DAG Deploy Action | |
| id: validate-dag-deploy-action | |
| uses: ./.github/workflows/e2e/validate_deployment | |
| with: | |
| is_dag_only_deploy: true | |
| dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }} | |
| dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }} | |
| image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }} | |
| - name: Delete Deployment Preview | |
| id: delete-deployment-preview | |
| # ensure that we always try delete the deployment preview even if the previous steps fail | |
| if: always() | |
| uses: ./ | |
| with: | |
| deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| workspace: ${{ secrets.WORKSPACE_ID }} | |
| action: delete-deployment-preview | |
| preview-name: test-preview-${{ matrix.deployment_id }} | |
| - name: Validate Deployment Preview Deletion | |
| id: validate-deployment-preview-deletion | |
| # ensure that we always try delete the deployment preview even if the previous steps fail | |
| if: always() | |
| uses: ./.github/workflows/e2e/get_deployment_info | |
| with: | |
| expected_status_code: 404 | |
| deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
| organization_id: ${{ needs.set-env-vars.outputs.org_id }} | |
| astro_api_token: ${{ needs.set-env-vars.outputs.token }} | |
| astronomer_host: ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| delete-test-deployments: | |
| name: Delete Deployment | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| set-env-vars, | |
| create-test-deployments, | |
| dag-deploy-test, | |
| pytests-test, | |
| custom-docker-image-test, | |
| deployment-preview-test, | |
| ] | |
| # ensure that we always try delete the deployment even if any of the tests fail | |
| if: always() | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Astro CLI and set context | |
| run: | | |
| curl -sSL https://install.astronomer.io | sudo bash -s | |
| astro context switch ${{ needs.set-env-vars.outputs.astronomer_host }} | |
| astro workspace switch ${{ needs.set-env-vars.outputs.workspace_id }} | |
| - name: Delete Deployment | |
| run: | | |
| astro deployment delete -f ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} | |
| astro deployment delete -f ${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} |