update README

astronomer · Sep 18, 2024 · 002d95e · 002d95e
1 parent 81b3c21
commit 002d95e
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 30 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - main
-      - e2e-tests
+      - dbt-deploy
   workflow_dispatch:
     inputs:
       workspace_id:
@@ -260,10 +260,83 @@ jobs:
           dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
           image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
 
+  infer-image-deploy:
+    name: Pytest Test
+    runs-on: ubuntu-latest
+    needs: [pytests-test, create-test-deployments]
+    strategy:
+      matrix:
+        deployment_id:
+          [
+            "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
+            "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}",
+          ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Astro Environment Info
+        id: get-astro-env-info
+        uses: ./.github/workflows/e2e/get_astro_env_info
+        with:
+          input_workspace_id: ${{ github.event.inputs.workspace_id }}
+          input_organization_id: ${{ github.event.inputs.org_id }}
+          input_astronomer_host: ${{ github.event.inputs.astronomer_host }}
+          input_astro_api_token: ${{ github.event.inputs.token }}
+          secret_workspace_id: ${{ secrets.WORKSPACE_ID }}
+          secret_organization_id: ${{ secrets.ORGANIZATION_ID }}
+          secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
+          secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+
+          sudo apt-get install jq
+
+          # we need to pre-install the CLI to set the context
+          curl -sSL https://install.astronomer.io | sudo bash -s
+
+      - name: Set CLI context
+        run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Get Deployment Info Before Test
+        id: get-deployment-before
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Run Deploy Action
+        uses: ./
+        with:
+          deployment-id: ${{ matrix.deployment_id }}
+          workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }}
+          pytest: true
+          action: deploy
+
+      - name: Get Deployment Info After Test
+        id: get-deployment-after
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Validate Deploy Action
+        uses: ./.github/workflows/e2e/validate_deployment
+        with:
+          dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
+          image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
+          dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
+          image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
+
   custom-docker-image-test:
     name: Custom Docker Image Test
     runs-on: ubuntu-latest
-    needs: [pytests-test, create-test-deployments]
+    needs: [infer-image-deploy, create-test-deployments]
     strategy:
       matrix:
         deployment_id:

diff --git a/README.md b/README.md
@@ -48,7 +48,8 @@ The following table lists the configuration options for the Deploy to Astro acti
 
 | Name | Default | Description |
 | ---|---|--- |
-| `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `create-deployment-preview`, `delete-deployment-preview`, `deploy-deployment-preview` or `dbt-deploy`. |
+| `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `deploy`, `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. If using `deploy` or `deploy-deployment-preview` one should also specify `deploy-type`. |
+| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. |
 | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. |
 | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` |
 | `description` |  | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. |
@@ -57,11 +58,11 @@ The following table lists the configuration options for the Deploy to Astro acti
 | `pytest` | `false` | When set to `true`, all pytests in the `tests` directory of your Astro project are run before deploying to Astro. See [Run tests with pytest](https://docs.astronomer.io/astro/cli/test-your-astro-project-locally#run-tests-with-pytest) |
 | `pytest-file` | (all tests run) | Specifies a custom pytest file to run with the pytest command. For example, you could specify `/tests/test-tags.py`.|
 | `force` | `false` | When set to `true`, your code is deployed and skips any pytest or parsing errors. |
-| `image-name` | | Specifies a custom, locally built image to deploy. |
+| `image-name` | | Specifies a custom, locally built image to deploy. To be used with `deploy-type` set to `image-and-dags` or `infer` |
 | `workspace` | | Workspace id to select. Only required when `ASTRO_API_TOKEN` is given an organization token. |
 | `preview-name` | `false` | Specifies custom preview name. By default this is branch name “_” deployment name. |
 | `checkout` | `true` | Whether to checkout the repo as the first step. Set this to false if you want to modify repo contents before invoking the action. Your custom checkout step needs to have `fetch-depth` of `0` and `ref` equal to `${{ github.event.after }}` so all the commits in the PR are checked out. Look at the checkout step that runs within this action for reference. |
-| `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. |
+| `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. NOTE: This option is deprecated and will be removed in a future release. Use `deploy-type: image-and-dags` instead. |
 | `build-secrets` | `` | Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'. |
 | `mount-path` | `` | Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name} |
 | `checkout-submodules` | `false` | Whether to checkout submodules when cloning the repository: `false` to disable (default), `true` to checkout submodules or `recursive` to recursively checkout submodules. Works only when `checkout` is set to `true`. Works only when `checkout` is set to `true`. |
@@ -186,6 +187,7 @@ jobs:
       uses: astronomer/[email protected]
       with:
         deployment-id: <deployment id>
+        deploy-type: image-and-dags
         image-name: ${{ steps.image_tag.outputs.image_tag }}
 
 ```
@@ -200,7 +202,7 @@ steps:
   uses: astronomer/[email protected]
   with:
     deployment-id: <deployment id>
-    action: dbt-deploy
+    deploy-type: dbt
     root-folder: dbt
 ```
 
@@ -214,12 +216,12 @@ steps:
   uses: astronomer/[email protected]
   with:
     deployment-id: <deployment id>
+    deploy-type: dbt
     root-folder: dbt
 - name: DAGs/Image Deploy to Astro
   uses: astronomer/[email protected]
   with:
     deployment-id: <deployment id>
-    action: deploy
     root-folder: astro-project/
     parse: true
 ```
@@ -304,7 +306,8 @@ jobs:
     - name: Deploy to Deployment Preview
       uses: astronomer/[email protected]
       with:
-        action: dbt-deploy-deployment-preview
+        action: deploy-deployment-preview
+        deploy-type: dbt
         deployment-id: <orginal deployment id>
         root-folder: dbt
 ```

diff --git a/action.yaml b/action.yaml
@@ -28,7 +28,7 @@ inputs:
   image-name:
     required: false
     default: no-custom-image
-    description: "Specify a custom built image to deploy to an Asto Deployment."
+    description: "Specify a custom built image to deploy to an Asto Deployment.  To be used with 'deploy-type' set to 'image-and-dags' or 'infer'"
   action:
     required: false
     default: deploy
@@ -82,7 +82,7 @@ inputs:
   deploy-type:
     required: false
     default: "infer"
-    description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image or DBT project. Specify either 'infer', 'dags-only', 'image-and-dags', or 'dbt'."
+    description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image or DBT project. Specify either 'infer', 'dags-only', 'image-and-dags', or 'dbt'. 'infer' option would infer between DAG only deploy and image and DAG deploy based on updated files."
   build-secrets:
     required: false
     description: "Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'"
@@ -129,7 +129,7 @@ runs:
         # Check if the Astro CLI version is less than 1.28.1 for dbt-deploy
         if [[ "${{ inputs.deploy-type }}" == "dbt" && $CLI_VERSION != "" ]]; then
           REQUIRED_VERSION="1.28.1"
-          CURRENT_VERSION=$(astro --version | awk '{print $3}')
+          CURRENT_VERSION=$(astro version | awk '{print $4}')
           if [[ $CURRENT_VERSION =~ ([0-9]+)\.([0-9]+)\.([0-9]+) ]]; then
             CURRENT_MAJOR=${BASH_REMATCH[1]}
             CURRENT_MINOR=${BASH_REMATCH[2]}
@@ -374,7 +374,7 @@ runs:
       shell: bash
       id: dbt-deploy-options
     - name: Get Deploy Type
-      if: ${{ inputs.deploy-type == 'infer' }}
+      if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' }}
       run: |
 
         # infer based on files changed to deploy only dags or image and dags
@@ -447,7 +447,7 @@ runs:
         fi
 
         # add custom image option
-        if [[ ${{ inputs.image-name }} != no-custom-image ]]; then
+        if [[ ${{ inputs.image-name }} != no-custom-image && inputs.deploy-type != "dags-only" ]]; then
           options="$options --image-name ${{ inputs.image-name }}"
         fi
 
@@ -504,7 +504,7 @@ runs:
         echo ::endgroup::
       shell: bash
     - name: DAG Deploy to Astro
-      if: ${{ inputs.deploy-type == 'dags-only' || (inputs.deploy-type == 'infer' && steps.deployment-type.outputs.DAGS_ONLY == '1') }}
+      if: ${{ (inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'infer') && steps.deployment-type.outputs.DAGS_ONLY == '1' }}
       run: |
 
         echo ::group::DAG Deploy to Astro

diff --git a/e2e-setup/README.md b/e2e-setup/README.md
@@ -6,18 +6,27 @@ The E2E tests could be triggered automatically when a new commit is pushed to ma
 
 ## Folder Structure
 
+```
 e2e-setup/
 ├── astro-project/
 ├── deployment-templates/
-│ ├── deployment-hibernate.yaml
-│ └── deployment.yaml
+│   ├── deployment-hibernate.yaml
+│   └── deployment.yaml
 ├── mocks/
-│ └── git.sh
+│   ├── dag-deploy-git.sh
+│   └── dbt-deploy-git.sh
+├── dbt/
+│   └── dbt_project.yml
+```
 
 ### astro-project
 
 Astro project folder contains a basic airflow project initialized via Astro CLI, which will deployed as part of tests via deploy action
 
+### dbt
+
+The dbt folder contains a basic sample dbt_project.yml file, which is used to define the configuration of a dbt project. This file includes settings such as the project name, version, and other configurations necessary for running dbt commands.
+
 ### deployment-templates
 
 Deployment templates contains the basic templates used by e2e tests to create required deployments against which tests would be executed

diff --git a/e2e-setup/mocks/git.sh b/e2e-setup/mocks/git.sh