Skip to content

fix create deployment preview image deploy #67

fix create deployment preview image deploy

fix create deployment preview image deploy #67

Workflow file for this run

name: E2E Tests for Astro Deploy Action
on:
push:
branches:
- main
- e2e-tests
- dbt-deploy
workflow_dispatch:
env:
ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }}
jobs:
# Create test deployments would use the template files and generate deployments with unique names
create-test-deployments:
name: Create Deployment
runs-on: ubuntu-latest
strategy:
matrix:
deployment: [deployment-hibernate.yaml, deployment.yaml]
outputs:
DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.DEPLOYMENT_ID }}
HIBERNATE_DEPLOYMENT_ID: ${{ steps.create-deployment.outputs.HIBERNATE_DEPLOYMENT_ID }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
run: |
sudo add-apt-repository ppa:rmescandon/yq
sudo apt update
sudo apt install yq -y
curl -sSL https://install.astronomer.io | sudo bash -s
astro context switch ${{ secrets.ASTRONOMER_HOST }}
astro workspace switch ${{ secrets.WORKSPACE_ID }}
# Generate a random test_uid and set it as an environment variable, so we can append it to the deployment name
test_uid=$(uuidgen)
echo "test_uid=$test_uid" >> $GITHUB_ENV
- name: Create Deployment
id: create-deployment
run: |
# Update the deployment name to maintain uniqueness
sed -i "s| name:.*| name: deploy-action-e2e-${{ env.test_uid }}|g" e2e-setup/deployment-templates/${{ matrix.deployment }}
DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -)
if [[ "${{ matrix.deployment }}" == "deployment.yaml" ]]; then
echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT
else
echo "HIBERNATE_DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT
fi
# DAG Deploy test would test the DAG only deploy functionality in deploy action
dag-deploy-test:
name: DAG Deploy Test
runs-on: ubuntu-latest
needs: create-test-deployments
strategy:
matrix:
deployment_id:
[
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}",
]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Mock git commands
run: |
mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: Install dependencies
run: |
sudo apt-get install jq
# we need to pre-install the CLI to set the context
curl -sSL https://install.astronomer.io | sudo bash -s
- name: Set CLI context
run: astro context switch ${{ secrets.ASTRONOMER_HOST }}
- name: Get Deployment Info Before Test
id: get-deployment-before
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Deploy to Astro
uses: ./
with:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ secrets.WORKSPACE_ID }}
parse: true
root-folder: e2e-setup/astro-project
- name: Get Deployment Info After Test
id: get-deployment-after
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate Deploy Action
uses: ./.github/workflows/e2e/validate_deployment
with:
is_dag_only_deploy: true
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
pytests-test:
name: Pytest Test
runs-on: ubuntu-latest
needs: [dag-deploy-test, create-test-deployments]
strategy:
matrix:
deployment_id:
[
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}",
]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
run: |
sudo apt-get install jq
# we need to pre-install the CLI to set the context
curl -sSL https://install.astronomer.io | sudo bash -s
- name: Set CLI context
run: astro context switch ${{ secrets.ASTRONOMER_HOST }}
- name: Get Deployment Info Before Test
id: get-deployment-before
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Run Pytest
uses: ./
with:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ secrets.WORKSPACE_ID }}
pytest: true
action: deploy
deploy-type: image-and-dags
root-folder: e2e-setup/astro-project
- name: Get Deployment Info After Test
id: get-deployment-after
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate Deploy Action
uses: ./.github/workflows/e2e/validate_deployment
with:
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
custom-docker-image-test:
name: Custom Docker Image Test
runs-on: ubuntu-latest
needs: [pytests-test, create-test-deployments]
strategy:
matrix:
deployment_id:
[
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}",
]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
run: |
sudo apt-get install jq
# we need to pre-install the CLI to set the context
curl -sSL https://install.astronomer.io | sudo bash -s
- name: Build Docker image
run: cd e2e-setup/astro-project && docker build -t custom-image:latest .
- name: Set CLI context
run: astro context switch ${{ secrets.ASTRONOMER_HOST }}
- name: Get Deployment Info Before Test
id: get-deployment-before
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Deploy to Astro
uses: ./
with:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy
deploy-type: image-and-dags
image-name: custom-image:latest
root-folder: e2e-setup/astro-project
- name: Get Deployment Info After Test
id: get-deployment-after
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate Deploy Action
uses: ./.github/workflows/e2e/validate_deployment
with:
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desiredDagTarballVersion }}
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
# DBT Deploy test would test the DAG only deploy functionality in deploy action
dbt-deploy-test:
name: DBT Deploy Test
runs-on: ubuntu-latest
needs: [custom-docker-image-test, create-test-deployments]
strategy:
matrix:
deployment_id:
[
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}",
]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Mock git commands
run: |
mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: Install dependencies
run: |
sudo apt-get install jq
# we need to pre-install the CLI to set the context
curl -sSL https://install.astronomer.io | sudo bash -s
- name: Set CLI context
run: astro context switch ${{ secrets.ASTRONOMER_HOST }}
- name: Get Deployment Info Before Test
id: get-deployment-before
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: DBT Deploy to Astro
uses: ./
with:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ secrets.WORKSPACE_ID }}
parse: true
root-folder: e2e-setup/dbt
deploy-type: dbt
- name: Get Deployment Info After Test
id: get-deployment-after
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate Deploy Action
uses: ./.github/workflows/e2e/validate_deployment
with:
is_dag_only_deploy: true
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
# Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests
deployment-preview-test:
name: Deployment Preview Test
runs-on: ubuntu-latest
needs: create-test-deployments
strategy:
matrix:
deployment_id:
[
"${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
"${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}",
]
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install dependencies
id: install-pre-reqs
run: |
sudo apt-get install jq
# we need to pre-install the CLI to set the context
curl -sSL https://install.astronomer.io | sudo bash -s
- name: Set CLI context
run: astro context switch ${{ secrets.ASTRONOMER_HOST }}
- name: Create Deployment Preview
id: create-deployment-preview
uses: ./
with:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: create-deployment-preview
preview-name: test-preview-${{ matrix.deployment_id }}
root-folder: e2e-setup/astro-project
# TODO: allow copying airflow resources, once we add wait in the deploy action for deployment preview to be healthy
copy-connections: false
copy-airflow-variables: false
copy-pools: false
- name: Get Deployment Info After Test
id: get-deployment-after-create-preview
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate Deploy Action
id: validate-deployment-preview-create
uses: ./.github/workflows/e2e/validate_deployment
with:
dag_tarball_version_before: ""
image_version_before: "12.1.0"
dag_tarball_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after-create-preview.outputs.desired_image_version }}
- name: Get Deployment Info Before Test
id: get-deployment-before-deploy
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Deploy to Deployment Preview
id: deployment-preview-deploy
uses: ./
with:
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy-deployment-preview
deploy-type: image-and-dags
root-folder: e2e-setup/astro-project
preview-name: test-preview-${{ matrix.deployment_id }}
- name: Get Deployment Info After Test
id: get-deployment-after-deploy-preview
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.deployment-preview-deploy.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate Deploy Action
id: validate-deployment-preview-deploy
uses: ./.github/workflows/e2e/validate_deployment
with:
dag_tarball_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before-deploy.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after-deploy-preview.outputs.desired_image_version }}
- name: Mock git commands for DAG Deploy
run: |
mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: Get Deployment Info Before DAG Deploy Test
id: get-deployment-before-dag-deploy
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: DAG Deploy to Astro
id: deployment-preview-dag-deploy
uses: ./
with:
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy-deployment-preview
parse: true
deploy-type: dags-only
root-folder: e2e-setup/astro-project
preview-name: test-preview-${{ matrix.deployment_id }}
- name: Get Deployment Info After DAG Deploy Test
id: get-deployment-after-dag-deploy
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate DAG Deploy Action
id: validate-dag-deploy-action
uses: ./.github/workflows/e2e/validate_deployment
with:
is_dag_only_deploy: true
dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }}
- name: Get Deployment Info Before DBT Deploy Test
id: get-deployment-before-dbt-deploy
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Mock git commands for DAG Deploy
run: |
mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: DBT Deploy to Astro
id: deployment-preview-dbt-deploy
uses: ./
with:
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy-deployment-preview
deploy-type: dbt
root-folder: e2e-setup/astro-project
preview-name: test-preview-${{ matrix.deployment_id }}
- name: Get Deployment Info After DBT Deploy Test
id: get-deployment-after-dbt-deploy
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Validate DBT Deploy Action
id: validate-dbt-deploy-action
uses: ./.github/workflows/e2e/validate_deployment
with:
is_dag_only_deploy: true
dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }}
- name: Delete Deployment Preview
id: delete-deployment-preview
# ensure that we always try delete the deployment preview even if the previous steps fail
if: always()
uses: ./
with:
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: delete-deployment-preview
preview-name: test-preview-${{ matrix.deployment_id }}
- name: Validate Deployment Preview Deletion
id: validate-deployment-preview-deletion
# ensure that we always try delete the deployment preview even if the previous steps fail
if: always()
uses: ./.github/workflows/e2e/get_deployment_info
with:
expected_status_code: 404
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
delete-test-deployments:
name: Delete Deployment
runs-on: ubuntu-latest
needs:
[
create-test-deployments,
dag-deploy-test,
pytests-test,
custom-docker-image-test,
dbt-deploy-test,
deployment-preview-test,
]
# ensure that we always try delete the deployment even if any of the tests fail
if: always()
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Astro CLI and set context
run: |
curl -sSL https://install.astronomer.io | sudo bash -s
astro context switch ${{ secrets.ASTRONOMER_HOST }}
astro workspace switch ${{ secrets.WORKSPACE_ID }}
- name: Delete Deployment
run: |
astro deployment delete -f ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}
astro deployment delete -f ${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}