-
Notifications
You must be signed in to change notification settings - Fork 45
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable GH actions job for daily deploy and Master DAG Trigger (#2034)
Currently we are not running Astro SDK Example DAG daily, This PR enable Actions job to deploy everyday and trigger example Master Dag for celery and Kubernetes deployment --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
- Loading branch information
1 parent
e5cf304
commit 54795d4
Showing
7 changed files
with
476 additions
and
40 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,199 @@ | ||
| --- | ||
| name: (Reusable workflows) Deploy to astro cloud | ||
|
|
||
| on: # yamllint disable-line rule:truthy | ||
| workflow_call: | ||
| inputs: | ||
| git_rev: | ||
| description: 'The git revision to deploy' | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| environment_to_deploy: | ||
| description: 'astro cloud deployment to deploy to' | ||
| required: true | ||
| type: string | ||
| secrets: | ||
| docker_registry: | ||
| description: 'astro cloud docker registry' | ||
| required: true | ||
| organization_id: | ||
| description: 'astro cloud organization_id' | ||
| required: true | ||
| deployment_id: | ||
| description: 'astro cloud deployment_id' | ||
| required: true | ||
| astronomer_key_id: | ||
| description: 'astro cloud astronomer_key_id' | ||
| required: true | ||
| astronomer_key_secret: | ||
| description: 'astro cloud astronomer_key_secret' | ||
| required: true | ||
| SLACK_WEBHOOK_URL: | ||
| description: 'slack webhook url for sending notification' | ||
| required: true | ||
|
|
||
| jobs: | ||
| deploy-to-astro-cloud: | ||
| runs-on: 'ubuntu-20.04' | ||
| steps: | ||
| - name: checkout | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| ref: ${{ inputs.git_rev }} | ||
|
|
||
| - name: get git revision | ||
| id: get_git_revision | ||
| run: echo "git_rev=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT | ||
|
|
||
| - name: deploy | ||
| working-directory: python-sdk/tests_integration/astro_deploy | ||
| run: | | ||
| echo "deploying ${{ inputs.git_rev }} to ${{ inputs.environment_to_deploy }}" | ||
| bash deploy.sh ${{ secrets.docker_registry }}\ | ||
| ${{ secrets.organization_id }} \ | ||
| ${{ secrets.deployment_id }} \ | ||
| ${{ secrets.astronomer_key_id }} \ | ||
| ${{ secrets.astronomer_key_secret }} | ||
| - name: send succeeded notification to Slack | ||
| if: success() && github.event_name == 'workflow_dispatch' | ||
| uses: slackapi/[email protected] | ||
| with: | ||
| # yamllint disable rule:line-length | ||
| payload: | | ||
| { | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "✅ Deploy succeeded" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "fields": [ | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Environment to deploy:*\n ${{ inputs.environment_to_deploy }}" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Deployed git revision*:\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ steps.get_git_revision.outputs.git_rev }}|${{ steps.get_git_revision.outputs.git_rev }}>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Link to workflow run:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}|link>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Triggered by:*\n<${{ github.server_url }}/${{ github.triggering_actor }}|${{ github.triggering_actor }}>" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "*---Workflow Detail---*" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "fields": [ | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Workflow name:*\n<${{ github.server_url }}/${{ github.repository }}/actions/workflows/${{ github.workflow }}|${{ github.workflow }}>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Event name:*\n${{ github.event_name }}" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Workflow Ref:*\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref }}|${{ github.ref_name }}>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Workflow Sha:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| # yamllint enable rule:line-length | ||
| env: | ||
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | ||
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | ||
|
|
||
| - name: send failure notification to Slack | ||
| if: failure() | ||
| uses: slackapi/[email protected] | ||
| with: | ||
| # yamllint disable rule:line-length | ||
| payload: | | ||
| { | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "❌ Deploy failed" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "fields": [ | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Environment to deploy:*\n ${{ inputs.environment_to_deploy }}" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Deployed git revision*:\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ steps.get_git_revision.outputs.git_rev }}|${{ steps.get_git_revision.outputs.git_rev }}>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Link to workflow run:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}|link>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Triggered by:*\n<${{ github.server_url }}/${{ github.triggering_actor }}|${{ github.triggering_actor }}>" | ||
| } | ||
| ] | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "*---Workflow Detail---*" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "fields": [ | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Workflow name:*\n<${{ github.server_url }}/${{ github.repository }}/actions/workflows/${{ github.workflow }}|${{ github.workflow }}>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Event name:*\n${{ github.event_name }}" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Workflow Ref:*\n<${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref }}|${{ github.ref_name }}>" | ||
| }, | ||
| { | ||
| "type": "mrkdwn", | ||
| "text": "*Workflow Sha:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| # yamllint enable rule:line-length | ||
| env: | ||
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | ||
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| --- | ||
| name: (Reusable workflows) Wait for deployment and trigger dags | ||
|
|
||
| on: # yamllint disable-line rule:truthy | ||
| workflow_call: | ||
| inputs: | ||
| git_rev: | ||
| description: 'The git revision to deploy' | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| dags_to_trigger_after_deployment: | ||
| description: | | ||
| Comma separated list of dag_ids to trigger after deployment | ||
| (e.g. "example_mssql_transform, example_load_file") | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| secrets: | ||
| astro_subdomain: | ||
| description: 'astro cloud subdomain' | ||
| required: true | ||
| deployment_id: | ||
| description: 'astro cloud deployment_id' | ||
| required: true | ||
| astronomer_key_id: | ||
| description: 'astro cloud astronomer_key_id' | ||
| required: true | ||
| astronomer_key_secret: | ||
| description: 'astro cloud astronomer_key_secret' | ||
| required: true | ||
| organization_id: | ||
| description: 'astro cloud organization_id' | ||
| required: true | ||
| bearer_token: | ||
| description: 'workspace bearer token' | ||
| required: true | ||
|
|
||
| jobs: | ||
| wait-for-deployment-to-be-ready-and-trigger-dag: | ||
| runs-on: 'ubuntu-20.04' | ||
| steps: | ||
|
|
||
| - name: Wait for deployment to be healthy | ||
| run: | | ||
| astro_core_api="https://api.astronomer.io/v1alpha1/organizations/${{secrets.organization_id }}/\ | ||
| deployments" | ||
| tries=15 | ||
| health_flag=false | ||
| while [[ $tries -gt 0 && $health_flag == false ]]; do | ||
| sleep 120 | ||
| response=$(curl -s -H "Authorization: Bearer ${{ secrets.bearer_token }}" -X GET \ | ||
| "$astro_core_api?deploymentIds=${{ secrets.deployment_id }}") | ||
| echo "response is $response" | ||
| deployment_status=$(echo "$response" | jq -r '.deployments[0].status') | ||
| echo "Deployment status is: $deployment_status" | ||
| echo "Waiting for deployment to be in ready state!!!" | ||
| if [[ $deployment_status == "HEALTHY" ]]; then | ||
| health_flag=true | ||
| fi | ||
| tries=$((tries - 1)) | ||
| done | ||
| if [[ $health_flag == false ]]; then | ||
| echo "Timed out waiting for deployment ${{ secrets.deployment_id }} to be HEALTHY" | ||
| exit 1 | ||
| fi | ||
| echo "${{ secrets.deployment_id }} is in HEALTHY state now" | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| ref: ${{ inputs.git_rev }} | ||
|
|
||
| - name: Trigger DAG(s) | ||
| run: | | ||
| python3 python-sdk/dev/integration_test_scripts/trigger_dag.py \ | ||
| "org-airflow-team-org" \ | ||
| ${{ secrets.deployment_id }} \ | ||
| ${{ secrets.astronomer_key_id }} \ | ||
| ${{ secrets.astronomer_key_secret }} \ | ||
| --dag-ids "${{ inputs.dags_to_trigger_after_deployment }}" |
Oops, something went wrong.