fix create deployment preview image deploy #64
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: E2E Tests for Astro Deploy Action | |
on: | |
push: | |
branches: | |
- main | |
- e2e-tests | |
- dbt-deploy | |
workflow_dispatch: | |
env: | |
ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} | |
jobs: | |
# Create test deployments would use the template files and generate deployments with unique names | |
create-test-deployments: | |
name: Create Deployment | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
deployment: [deployment-hibernate.yaml, deployment.yaml] | |
outputs: | |
DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.DEPLOYMENT_ID }} | |
HIBERNATE_DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.HIBERNATE_DEPLOYMENT_ID }} | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@v4 | |
- name: Install dependencies | |
run: | | |
sudo add-apt-repository ppa:rmescandon/yq | |
sudo apt update | |
sudo apt install yq -y | |
curl -sSL https://install.astronomer.io | sudo bash -s | |
astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
astro workspace switch ${{ secrets.WORKSPACE_ID }} | |
# Generate a random test_uid and set it as an environment variable, so we can append it to the deployment name | |
test_uid=$(uuidgen) | |
echo "test_uid=$test_uid" >> $GITHUB_ENV | |
- name: Create Deployment | |
id: create-deployment | |
run: | | |
# Update the deployment name to maintain uniqueness | |
sed -i "s| name:.*| name: deploy-action-e2e-${{ env.test_uid }}|g" e2e-setup/deployment-templates/${{ matrix.deployment }} | |
DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -) | |
if [[ "${{ matrix.deployment }}" == "deployment.yaml" ]]; then | |
echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT | |
else | |
echo "HIBERNATE_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT | |
fi | |
# DAG Deploy test would test the DAG only deploy functionality in deploy action | |
dag-deploy-test: | |
name: DAG Deploy Test | |
runs-on: ubuntu-latest | |
needs: create-test-deployments | |
strategy: | |
matrix: | |
deployment_id: | |
[ | |
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
] | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@v4 | |
- name: Mock git commands | |
run: | | |
mv e2e-setup/mocks/git.sh /usr/local/bin/git | |
chmod +x /usr/local/bin/git | |
- name: Install dependencies | |
run: | | |
sudo apt-get install jq | |
# we need to pre-install the CLI to set the context | |
curl -sSL https://install.astronomer.io | sudo bash -s | |
- name: Set CLI context | |
run: astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
- name: Get Deployment Info Before Test | |
id: get-deployment-before | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Deploy to Astro | |
uses: ./ | |
with: | |
deployment-id: ${{ matrix.deployment_id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
parse: true | |
root-folder: e2e-setup/astro-project | |
- name: Get Deployment Info After Test | |
id: get-deployment-after | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate Deploy Action | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
is_dag_only_deploy: true | |
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
pytests-test: | |
name: Pytest Test | |
runs-on: ubuntu-latest | |
needs: [dag-deploy-test, create-test-deployments] | |
strategy: | |
matrix: | |
deployment_id: | |
[ | |
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
] | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@v4 | |
- name: Install dependencies | |
run: | | |
sudo apt-get install jq | |
# we need to pre-install the CLI to set the context | |
curl -sSL https://install.astronomer.io | sudo bash -s | |
- name: Set CLI context | |
run: astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
- name: Get Deployment Info Before Test | |
id: get-deployment-before | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Run Pytest | |
uses: ./ | |
with: | |
deployment-id: ${{ matrix.deployment_id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
pytest: true | |
action: deploy | |
deploy-type: image-and-dags | |
root-folder: e2e-setup/astro-project | |
- name: Get Deployment Info After Test | |
id: get-deployment-after | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate Deploy Action | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
custom-docker-image-test: | |
name: Custom Docker Image Test | |
runs-on: ubuntu-latest | |
needs: [pytests-test, create-test-deployments] | |
strategy: | |
matrix: | |
deployment_id: | |
[ | |
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
] | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@v4 | |
- name: Install dependencies | |
run: | | |
sudo apt-get install jq | |
# we need to pre-install the CLI to set the context | |
curl -sSL https://install.astronomer.io | sudo bash -s | |
- name: Build Docker image | |
run: cd e2e-setup/astro-project && docker build -t custom-image:latest . | |
- name: Set CLI context | |
run: astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
- name: Get Deployment Info Before Test | |
id: get-deployment-before | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Deploy to Astro | |
uses: ./ | |
with: | |
deployment-id: ${{ matrix.deployment_id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
action: deploy | |
deploy-type: image-and-dags | |
image-name: custom-image:latest | |
root-folder: e2e-setup/astro-project | |
- name: Get Deployment Info After Test | |
id: get-deployment-after | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate Deploy Action | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desiredDagTarballVersion }} | |
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
# DBT Deploy test would test the DAG only deploy functionality in deploy action | |
dbt-deploy-test: | |
name: DBT Deploy Test | |
runs-on: ubuntu-latest | |
needs: [custom-docker-image-test, create-test-deployments] | |
strategy: | |
matrix: | |
deployment_id: | |
[ | |
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
] | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@v4 | |
- name: Mock git commands | |
run: | | |
mv e2e-setup/mocks/git.sh /usr/local/bin/git | |
chmod +x /usr/local/bin/git | |
- name: Install dependencies | |
run: | | |
sudo apt-get install jq | |
# we need to pre-install the CLI to set the context | |
curl -sSL https://install.astronomer.io | sudo bash -s | |
- name: Set CLI context | |
run: astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
- name: Get Deployment Info Before Test | |
id: get-deployment-before | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: DBT Deploy to Astro | |
uses: ./ | |
with: | |
deployment-id: ${{ matrix.deployment_id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
parse: true | |
root-folder: e2e-setup/dbt | |
deploy-type: dbt | |
- name: Get Deployment Info After Test | |
id: get-deployment-after | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ matrix.deployment_id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate Deploy Action | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
is_dag_only_deploy: true | |
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} | |
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} | |
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} | |
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} | |
# Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests | |
deployment-preview-test: | |
name: Deployment Preview Test | |
runs-on: ubuntu-latest | |
needs: create-test-deployments | |
strategy: | |
matrix: | |
deployment_id: | |
[ | |
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}", | |
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}", | |
] | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@v4 | |
- name: Install dependencies | |
id: install-pre-reqs | |
run: | | |
sudo apt-get install jq | |
# we need to pre-install the CLI to set the context | |
curl -sSL https://install.astronomer.io | sudo bash -s | |
- name: Set CLI context | |
run: astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
- name: Create Deployment Preview | |
id: create-deployment-preview | |
uses: ./ | |
with: | |
deployment-id: ${{ matrix.deployment_id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
action: create-deployment-preview | |
preview-name: test-preview-${{ matrix.deployment_id }} | |
root-folder: e2e-setup/astro-project | |
# TODO: allow copying airflow resources, once we add wait in the deploy action for deployment preview to be healthy | |
copy-connections: false | |
copy-airflow-variables: false | |
copy-pools: false | |
- name: Get Deployment Info After Test | |
id: get-deployment-after-create-preview | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate Deploy Action | |
id: validate-deployment-preview-create | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
dag_tarball_version_before: "" | |
image_version_before: "12.1.0" | |
dag_tarball_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_dag_tarball_version }} | |
image_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_image_version }} | |
- name: Get Deployment Info Before Test | |
id: get-deployment-before-deploy | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Deploy to Deployment Preview | |
id: deployment-preview-deploy | |
uses: ./ | |
with: | |
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
action: deploy-deployment-preview | |
deploy-type: image-and-dags | |
root-folder: e2e-setup/astro-project | |
preview-name: test-preview-${{ matrix.deployment_id }} | |
- name: Get Deployment Info After Test | |
id: get-deployment-after-deploy-preview | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ steps.deployment-preview-deploy.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate Deploy Action | |
id: validate-deployment-preview-deploy | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
dag_tarball_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_dag_tarball_version }} | |
image_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_image_version }} | |
dag_tarball_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_dag_tarball_version }} | |
image_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_image_version }} | |
- name: Mock git commands for DAG Deploy | |
run: | | |
cp /usr/local/bin/git /usr/local/bin/git-original | |
mv e2e-setup/mocks/git.sh /usr/local/bin/git | |
chmod +x /usr/local/bin/git | |
- name: Get Deployment Info Before DAG Deploy Test | |
id: get-deployment-before-dag-deploy | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: DAG Deploy to Astro | |
id: deployment-preview-dag-deploy | |
uses: ./ | |
with: | |
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
action: deploy-deployment-preview | |
parse: true | |
deploy-type: dags-only | |
root-folder: e2e-setup/astro-project | |
preview-name: test-preview-${{ matrix.deployment_id }} | |
- name: Revert mock git commands | |
run: | | |
mv /usr/local/bin/git-original /usr/local/bin/git | |
chmod +x /usr/local/bin/git | |
- name: Get Deployment Info After DAG Deploy Test | |
id: get-deployment-after-dag-deploy | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate DAG Deploy Action | |
id: validate-dag-deploy-action | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
is_dag_only_deploy: true | |
dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }} | |
image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }} | |
dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }} | |
image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }} | |
- name: Get Deployment Info Before DBT Deploy Test | |
id: get-deployment-before-dbt-deploy | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: DBT Deploy to Astro | |
id: deployment-preview-dbt-deploy | |
uses: ./ | |
with: | |
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
action: deploy-deployment-preview | |
deploy-type: dbt | |
root-folder: e2e-setup/astro-project | |
preview-name: test-preview-${{ matrix.deployment_id }} | |
- name: Get Deployment Info After DBT Deploy Test | |
id: get-deployment-after-dbt-deploy | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
- name: Validate DBT Deploy Action | |
id: validate-dbt-deploy-action | |
uses: ./.github/workflows/e2e/validate_deployment | |
with: | |
is_dag_only_deploy: true | |
dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }} | |
image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }} | |
dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }} | |
image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }} | |
- name: Delete Deployment Preview | |
id: delete-deployment-preview | |
# ensure that we always try delete the deployment preview even if the previous steps fail | |
if: always() | |
uses: ./ | |
with: | |
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
workspace: ${{ secrets.WORKSPACE_ID }} | |
action: delete-deployment-preview | |
preview-name: test-preview-${{ matrix.deployment_id }} | |
- name: Validate Deployment Preview Deletion | |
id: validate-deployment-preview-deletion | |
# ensure that we always try delete the deployment preview even if the previous steps fail | |
if: always() | |
uses: ./.github/workflows/e2e/get_deployment_info | |
with: | |
expected_status_code: 404 | |
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} | |
organization_id: ${{ secrets.ORGANIZATION_ID }} | |
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} | |
astronomer_host: ${{ secrets.ASTRONOMER_HOST }} | |
delete-test-deployments: | |
name: Delete Deployment | |
runs-on: ubuntu-latest | |
needs: | |
[ | |
create-test-deployments, | |
dag-deploy-test, | |
pytests-test, | |
custom-docker-image-test, | |
dbt-deploy-test, | |
deployment-preview-test, | |
] | |
# ensure that we always try delete the deployment even if any of the tests fail | |
if: always() | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@v4 | |
- name: Install Astro CLI and set context | |
run: | | |
curl -sSL https://install.astronomer.io | sudo bash -s | |
astro context switch ${{ secrets.ASTRONOMER_HOST }} | |
astro workspace switch ${{ secrets.WORKSPACE_ID }} | |
- name: Delete Deployment | |
run: | | |
astro deployment delete -f ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }} | |
astro deployment delete -f ${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }} |