diff --git a/.github/workflows/ci-astro-deploy.yml b/.github/workflows/ci-astro-deploy.yml index 764a08e00..e57c4cf1b 100644 --- a/.github/workflows/ci-astro-deploy.yml +++ b/.github/workflows/ci-astro-deploy.yml @@ -9,34 +9,86 @@ on: required: false default: '' + environment_to_deploy: + description: 'astro cloud deployment to deploy to' + required: true + type: choice + options: + - both + - astro-sdk-integration-tests + - astro-sdk-integration-tests-on-KE + dags_to_trigger_after_deployment: + description: | + Comma separated list of dag_ids to trigger after deployment + (e.g. "example_mssql_transform, example_load_file") + required: false + type: string + default: '' + + jobs: - deploy: - env: - ASTRO_DOCKER_REGISTRY: ${{ secrets.ASTRO_DOCKER_REGISTRY }} - ASTRO_ORGANIZATION_ID: ${{ secrets.ASTRO_ORGANIZATION_ID }} - ASTRO_DEPLOYMENT_ID: ${{ secrets.ASTRO_DEPLOYMENT_ID }} - ASTRO_KEY_ID: ${{ secrets.ASTRO_KEY_ID }} - ASTRO_KEY_SECRET: ${{ secrets.ASTRO_KEY_SECRET }} - ASTRO_DEPLOYMENT_ID_SINGLE_WORKER: ${{ secrets.ASTRO_DEPLOYMENT_ID_SINGLE_WORKER }} - ASTRO_KEY_ID_SINGLE_WORKER: ${{ secrets.ASTRO_KEY_ID_SINGLE_WORKER }} - ASTRO_KEY_SECRET_SINGLE_WORKER: ${{ secrets.ASTRO_KEY_SECRET_SINGLE_WORKER }} - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.git_rev }} + deploy-to-astro-sdk-integration-tests: + if: | + contains(fromJSON('["both", "astro-sdk-integration-tests"]'), inputs.environment_to_deploy) || + github.event_name == 'schedule' + uses: ./.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml + with: + git_rev: ${{ inputs.git_rev }} + environment_to_deploy: 'astro-sdk-integration-tests' + secrets: + docker_registry: ${{ secrets.ASTRO_DOCKER_REGISTRY }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID }} + astronomer_key_id: ${{ secrets.ASTRO_KEY_ID }} + astronomer_key_secret: ${{ secrets.ASTRO_KEY_SECRET }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + wait-for-deployment-to-be-ready-and-trigger-dags-for-astro-sdk-integration-tests: + if: | + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.dags_to_trigger_after_deployment != '') + needs: deploy-to-astro-sdk-integration-tests + uses: ./.github/workflows/reuse-wf-trigger-dag.yaml + with: + git_rev: ${{ inputs.git_rev }} + dags_to_trigger_after_deployment: ${{ inputs.dags_to_trigger_after_deployment }} + secrets: + astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }} + deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID }} + astronomer_key_id: ${{ secrets.ASTRO_KEY_ID }} + astronomer_key_secret: ${{ secrets.ASTRO_KEY_SECRET }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + bearer_token: ${{ secrets.BEARER_TOKEN }} + + deploy-to-astro-sdk-integration-tests-on-KE: + if: | + contains(fromJSON('["both", "astro-sdk-integration-tests-on-KE"]'), inputs.environment_to_deploy) || + github.event_name == 'schedule' + uses: ./.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml + with: + git_rev: ${{ inputs.git_rev }} + environment_to_deploy: 'astro-sdk-integration-tests-on-KE' + secrets: + docker_registry: ${{ secrets.ASTRO_DOCKER_REGISTRY }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID_KE }} + astronomer_key_id: ${{ secrets.ASTRO_KEY_ID_KE }} + astronomer_key_secret: ${{ secrets.ASTRO_KEY_SECRET_KE }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - - name: deploy - working-directory: python-sdk/tests_integration/astro_deploy - run: | - echo "deploying ${{ inputs.git_rev }}" - bash deploy.sh \ - $ASTRO_DOCKER_REGISTRY \ - $ASTRO_ORGANIZATION_ID \ - $ASTRO_DEPLOYMENT_ID \ - $ASTRO_KEY_ID \ - $ASTRO_KEY_SECRET \ - $ASTRO_DEPLOYMENT_ID_SINGLE_WORKER \ - $ASTRO_KEY_ID_SINGLE_WORKER \ - $ASTRO_KEY_SECRET_SINGLE_WORKER + wait-for-deployment-to-be-ready-and-trigger-dags-for-astro-sdk-integration-tests-on-KE: + if: | + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && inputs.dags_to_trigger_after_deployment != '') + needs: deploy-to-astro-sdk-integration-tests-on-KE + uses: ./.github/workflows/reuse-wf-trigger-dag.yaml + with: + git_rev: ${{ inputs.git_rev }} + dags_to_trigger_after_deployment: ${{ inputs.dags_to_trigger_after_deployment }} + secrets: + astro_subdomain: ${{ secrets.ASTRO_SUBDOMAIN }} + deployment_id: ${{ secrets.ASTRO_DEPLOYMENT_ID_KE }} + astronomer_key_id: ${{ secrets.ASTRO_KEY_ID_KE }} + astronomer_key_secret: ${{ secrets.ASTRO_KEY_SECRET_KE }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + bearer_token: ${{ secrets.BEARER_TOKEN }} diff --git a/.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml b/.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml new file mode 100644 index 000000000..a28c6ef9d --- /dev/null +++ b/.github/workflows/reuse-wf-deploy-to-astro-cloud.yaml @@ -0,0 +1,199 @@ +--- +name: (Reusable workflows) Deploy to astro cloud + +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + git_rev: + description: 'The git revision to deploy' + required: false + type: string + default: '' + environment_to_deploy: + description: 'astro cloud deployment to deploy to' + required: true + type: string + secrets: + docker_registry: + description: 'astro cloud docker registry' + required: true + organization_id: + description: 'astro cloud organization_id' + required: true + deployment_id: + description: 'astro cloud deployment_id' + required: true + astronomer_key_id: + description: 'astro cloud astronomer_key_id' + required: true + astronomer_key_secret: + description: 'astro cloud astronomer_key_secret' + required: true + SLACK_WEBHOOK_URL: + description: 'slack webhook url for sending notification' + required: true + +jobs: + deploy-to-astro-cloud: + runs-on: 'ubuntu-20.04' + steps: + - name: checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.git_rev }} + + - name: get git revision + id: get_git_revision + run: echo "git_rev=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: deploy + working-directory: python-sdk/tests_integration/astro_deploy + run: | + echo "deploying ${{ inputs.git_rev }} to ${{ inputs.environment_to_deploy }}" + bash deploy.sh ${{ secrets.docker_registry }}\ + ${{ secrets.organization_id }} \ + ${{ secrets.deployment_id }} \ + ${{ secrets.astronomer_key_id }} \ + ${{ secrets.astronomer_key_secret }} + + - name: send succeeded notification to Slack + if: success() && github.event_name == 'workflow_dispatch' + uses: slackapi/slack-github-action@v1.23.0 + with: + # yamllint disable rule:line-length + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "✅ Deploy succeeded" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Environment to deploy:*\n ${{ inputs.environment_to_deploy }}" + }, + { + "type": "mrkdwn", + "text": "*Deployed git revision*:\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ steps.get_git_revision.outputs.git_rev }}|${{ steps.get_git_revision.outputs.git_rev }}>" + }, + { + "type": "mrkdwn", + "text": "*Link to workflow run:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}|link>" + }, + { + "type": "mrkdwn", + "text": "*Triggered by:*\n<${{ github.server_url }}/${{ github.triggering_actor }}|${{ github.triggering_actor }}>" + } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*---Workflow Detail---*" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Workflow name:*\n<${{ github.server_url }}/${{ github.repository }}/actions/workflows/${{ github.workflow }}|${{ github.workflow }}>" + }, + { + "type": "mrkdwn", + "text": "*Event name:*\n${{ github.event_name }}" + }, + { + "type": "mrkdwn", + "text": "*Workflow Ref:*\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref }}|${{ github.ref_name }}>" + }, + { + "type": "mrkdwn", + "text": "*Workflow Sha:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" + } + ] + } + ] + } + # yamllint enable rule:line-length + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + + - name: send failure notification to Slack + if: failure() + uses: slackapi/slack-github-action@v1.23.0 + with: + # yamllint disable rule:line-length + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "❌ Deploy failed" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Environment to deploy:*\n ${{ inputs.environment_to_deploy }}" + }, + { + "type": "mrkdwn", + "text": "*Deployed git revision*:\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ steps.get_git_revision.outputs.git_rev }}|${{ steps.get_git_revision.outputs.git_rev }}>" + }, + { + "type": "mrkdwn", + "text": "*Link to workflow run:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}|link>" + }, + { + "type": "mrkdwn", + "text": "*Triggered by:*\n<${{ github.server_url }}/${{ github.triggering_actor }}|${{ github.triggering_actor }}>" + } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*---Workflow Detail---*" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Workflow name:*\n<${{ github.server_url }}/${{ github.repository }}/actions/workflows/${{ github.workflow }}|${{ github.workflow }}>" + }, + { + "type": "mrkdwn", + "text": "*Event name:*\n${{ github.event_name }}" + }, + { + "type": "mrkdwn", + "text": "*Workflow Ref:*\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref }}|${{ github.ref_name }}>" + }, + { + "type": "mrkdwn", + "text": "*Workflow Sha:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" + } + ] + } + ] + } + # yamllint enable rule:line-length + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/reuse-wf-trigger-dag.yaml b/.github/workflows/reuse-wf-trigger-dag.yaml new file mode 100644 index 000000000..ddb0e4241 --- /dev/null +++ b/.github/workflows/reuse-wf-trigger-dag.yaml @@ -0,0 +1,82 @@ +--- +name: (Reusable workflows) Wait for deployment and trigger dags + +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + git_rev: + description: 'The git revision to deploy' + required: false + type: string + default: '' + dags_to_trigger_after_deployment: + description: | + Comma separated list of dag_ids to trigger after deployment + (e.g. "example_mssql_transform, example_load_file") + required: false + type: string + default: '' + secrets: + astro_subdomain: + description: 'astro cloud subdomain' + required: true + deployment_id: + description: 'astro cloud deployment_id' + required: true + astronomer_key_id: + description: 'astro cloud astronomer_key_id' + required: true + astronomer_key_secret: + description: 'astro cloud astronomer_key_secret' + required: true + organization_id: + description: 'astro cloud organization_id' + required: true + bearer_token: + description: 'workspace bearer token' + required: true + +jobs: + wait-for-deployment-to-be-ready-and-trigger-dag: + runs-on: 'ubuntu-20.04' + steps: + + - name: Wait for deployment to be healthy + run: | + astro_core_api="https://api.astronomer.io/v1alpha1/organizations/${{secrets.organization_id }}/\ + deployments" + tries=15 + health_flag=false + + while [[ $tries -gt 0 && $health_flag == false ]]; do + sleep 120 + response=$(curl -s -H "Authorization: Bearer ${{ secrets.bearer_token }}" -X GET \ + "$astro_core_api?deploymentIds=${{ secrets.deployment_id }}") + echo "response is $response" + deployment_status=$(echo "$response" | jq -r '.deployments[0].status') + echo "Deployment status is: $deployment_status" + echo "Waiting for deployment to be in ready state!!!" + if [[ $deployment_status == "HEALTHY" ]]; then + health_flag=true + fi + tries=$((tries - 1)) + done + if [[ $health_flag == false ]]; then + echo "Timed out waiting for deployment ${{ secrets.deployment_id }} to be HEALTHY" + exit 1 + fi + echo "${{ secrets.deployment_id }} is in HEALTHY state now" + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.git_rev }} + + - name: Trigger DAG(s) + run: | + python3 python-sdk/dev/integration_test_scripts/trigger_dag.py \ + "org-airflow-team-org" \ + ${{ secrets.deployment_id }} \ + ${{ secrets.astronomer_key_id }} \ + ${{ secrets.astronomer_key_secret }} \ + --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}" diff --git a/python-sdk/dev/integration_test_scripts/trigger_dag.py b/python-sdk/dev/integration_test_scripts/trigger_dag.py new file mode 100644 index 000000000..55bca8bea --- /dev/null +++ b/python-sdk/dev/integration_test_scripts/trigger_dag.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import argparse +import logging +import sys + +import requests + +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + +def get_access_token(api_key_id: str, api_key_secret: str) -> str: + """ + Gets bearer access token for the Astro Cloud deployment needed for REST API authentication. + + :param api_key_id: API key ID of the Astro Cloud deployment + :param api_key_secret: API key secret of the Astro Cloud deployment + """ + request_json = { + "client_id": api_key_id, + "client_secret": api_key_secret, + "audience": "astronomer-ee", + "grant_type": "client_credentials", + } + response = requests.post("https://auth.astronomer.io/oauth/token", json=request_json) + response_json = response.json() + return response_json["access_token"] + + +def trigger_dag_runs( + *, dag_ids_name: list[str], astro_subdomain: str, deployment_id: str, bearer_token: str +) -> None: + """ + Triggers the DAG using Airflow REST API. + + :param dag_ids_name: list of dag_id to trigger + :param astro_subdomain: subdomain of the Astro Cloud (e.g., https://.astronomer.run/) + :param deployment_id: ID of the Astro Cloud deployment. Using this, we generate the short deployment ID needed + for the construction of Airflow endpoint to hit + :param bearer_token: bearer token to be used for authentication with the Airflow REST API + """ + short_deployment_id = f"d{deployment_id[-7:]}" + integration_tests_deployment_url = f"https://{astro_subdomain}.astronomer.run/{short_deployment_id}" + headers = { + "Content-Type": "application/json", + "Cache-Control": "no-cache", + "Authorization": f"Bearer {bearer_token}", + } + failed_dag_ids: list[str] = [] + for dag_id in dag_ids_name: + dag_trigger_url = f"{integration_tests_deployment_url}/api/v1/dags/{dag_id}/dagRuns" + response = requests.post(dag_trigger_url, headers=headers, json={}) + + if response.status_code != 200: + failed_dag_ids.append(dag_id) + + logging.info(f"Response for {dag_id} DAG trigger is %s", response.json()) + + if failed_dag_ids: + for dag_id in failed_dag_ids: + logging.error("Failed to run DAG %s", {dag_id}) + sys.exit(1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("astro_subdomain", help="subdomain of the Astro Cloud", type=str) + parser.add_argument("deployment_id", help="ID of the deployment in Astro Cloud", type=str) + parser.add_argument("astronomer_key_id", help="Key ID of the Astro Cloud deployment", type=str) + parser.add_argument("astronomer_key_secret", help="Key secret of the Astro Cloud deployment", type=str) + parser.add_argument( + "--dag-ids", + help=( + "Comma separated list of dag_ids_name to trigger" + " e.g. 'example_async_adf_run_pipeline, example_async_batch'" + ), + default="example_master_dag", + # fallback to "example_master_dag" if empty string "" is provided + type=lambda dag_ids: dag_ids if dag_ids else "example_master_dag", + ) + + args = parser.parse_args() + token = get_access_token(args.astronomer_key_id.strip(), args.astronomer_key_secret.strip()) + + input_dag_ids = args.dag_ids + dag_ids = [dag_id.strip() for dag_id in input_dag_ids.split(",")] + + trigger_dag_runs( + dag_ids_name=dag_ids, + astro_subdomain=args.astro_subdomain, + deployment_id=args.deployment_id, + bearer_token=token, + ) diff --git a/python-sdk/tests_integration/astro_deploy/Dockerfile b/python-sdk/tests_integration/astro_deploy/Dockerfile index 478d97562..aa111076b 100644 --- a/python-sdk/tests_integration/astro_deploy/Dockerfile +++ b/python-sdk/tests_integration/astro_deploy/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/astronomer/astro-runtime:8.8.0-base +FROM quay.io/astronomer/astro-runtime:9.1.0-base USER root RUN apt-get update -y && apt-get install -y git @@ -18,6 +18,7 @@ ENV SETUPTOOLS_USE_DISTUTILS=stdlib COPY python-sdk /tmp/python-sdk RUN pip install /tmp/python-sdk[all] RUN pip install "apache-airflow-providers-slack>=8.0.0" +RUN pip install astronomer-starship-provider RUN mkdir -p ${AIRFLOW_HOME}/dags RUN mkdir -p ${AIRFLOW_HOME}/tests @@ -25,6 +26,7 @@ RUN mkdir -p ${AIRFLOW_HOME}/tests COPY example_dags/ ${AIRFLOW_HOME}/dags/ COPY master_dag.py/ ${AIRFLOW_HOME}/dags/ COPY example_snowflake_cleanup.py/ ${AIRFLOW_HOME}/dags/ +COPY astronomer_migration_dag.py/ ${AIRFLOW_HOME}/dags/ COPY tests/ ${AIRFLOW_HOME}/tests/ RUN ls ${AIRFLOW_HOME}/dags/ diff --git a/python-sdk/tests_integration/astro_deploy/astronomer_migration_dag.py b/python-sdk/tests_integration/astro_deploy/astronomer_migration_dag.py new file mode 100644 index 000000000..359b67db3 --- /dev/null +++ b/python-sdk/tests_integration/astro_deploy/astronomer_migration_dag.py @@ -0,0 +1,16 @@ +"""Astronomer migration DAG to transform metadata from source deployment to target Astro Cloud deployment.""" +from datetime import datetime + +from airflow import DAG +from astronomer.starship.operators import AstroMigrationOperator + +with DAG( + dag_id="astronomer_migration_dag", + start_date=datetime(2020, 8, 15), + schedule_interval=None, +) as dag: + AstroMigrationOperator( # nosec B106 + task_id="export_meta", + deployment_url='{{ dag_run.conf["deployment_url"] }}', + token='{{ dag_run.conf["astro_token"] }}', + ) diff --git a/python-sdk/tests_integration/astro_deploy/deploy.sh b/python-sdk/tests_integration/astro_deploy/deploy.sh index 4b03bd77f..9dad9ea65 100644 --- a/python-sdk/tests_integration/astro_deploy/deploy.sh +++ b/python-sdk/tests_integration/astro_deploy/deploy.sh @@ -25,10 +25,7 @@ function echo_help() { echo "ASTRO_DEPLOYMENT_ID Astro cloud Deployment id" echo "ASTRO_KEY_ID Astro cloud service account API key id" echo "ASTRO_KEY_SECRET Astro cloud service account API key secret" - echo "ASTRO_DEPLOYMENT_ID_SINGLE_WORKER Astro cloud Deployment id" - echo "ASTRO_KEY_ID_SINGLE_WORKER Astro cloud service account for Single Worker API key id" - echo "ASTRO_KEY_SECRET_SINGLE_WORKER Astro cloud service account for Single Worker API key secret" - echo "bash deploy.sh " + echo "bash deploy.sh " } if [ "$1" == "-h" ]; then @@ -48,11 +45,8 @@ ASTRO_ORGANIZATION_ID=$2 ASTRO_DEPLOYMENT_ID=$3 ASTRO_KEY_ID=$4 ASTRO_KEY_SECRET=$5 -ASTRO_DEPLOYMENT_ID_SINGLE_WORKER=$6 -ASTRO_KEY_ID_SINGLE_WORKER=$7 -ASTRO_KEY_SECRET_SINGLE_WORKER=$8 MASTER_DAG_DOCKERFILE="Dockerfile" -MASTER_DAG_MUTLI_WORKER_DOCKERFILE="Dockerfile.single_worker" + clean @@ -128,6 +122,4 @@ cp -r "${PROJECT_PATH}"/tests/data "${SCRIPT_PATH}"/tests/data deploy $ASTRO_DOCKER_REGISTRY $ASTRO_ORGANIZATION_ID $ASTRO_DEPLOYMENT_ID $ASTRO_KEY_ID $ASTRO_KEY_SECRET $MASTER_DAG_DOCKERFILE -deploy $ASTRO_DOCKER_REGISTRY $ASTRO_ORGANIZATION_ID $ASTRO_DEPLOYMENT_ID_SINGLE_WORKER $ASTRO_KEY_ID_SINGLE_WORKER $ASTRO_KEY_SECRET_SINGLE_WORKER $MASTER_DAG_MUTLI_WORKER_DOCKERFILE - clean