From 822be311c7037511a2eb077ae42fd478e2262d5e Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 31 Oct 2024 12:47:02 -0700
Subject: [PATCH 01/22] fix: comment out veda-config related steps

---
 .github/workflows/pr.yml | 191 +++++++++++++++++++--------------------
 1 file changed, 94 insertions(+), 97 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 0951ccf..edf61d9 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -1,15 +1,12 @@
 # This GitHub Actions workflow automates the process of
 # publishing dataset collections to a staging environment
-# and creating a pull request (PR) in the veda-config repository
-# with the dataset configuration.
 # It is triggered by a pull request to the main branch
 # that modifies any files within the ingestion-data/dataset-config/ directory
 # The workflow includes steps to
 #   - publish the datasets,
-#   - create a PR in veda-config repository,
 #   - constantly updates the status of the workflow in the PR comment
 
-name: Publish collection to staging and create dataset config PR
+name: Publish collection to staging
 
 on:
   pull_request:
@@ -168,18 +165,18 @@ jobs:
           $status_message"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
-      # Update PR comment
-      - name: Update PR comment for PR creation
-        if: success()
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-        run: |
-          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-          UPDATED_BODY="$CURRENT_BODY
+      # # Update PR comment
+      # - name: Update PR comment for PR creation
+      #   if: success()
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      #   run: |
+      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+      #     UPDATED_BODY="$CURRENT_BODY
 
-          **Creating a PR in veda-config...**"
-          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+      #     **Creating a PR in veda-config...**"
+      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -198,85 +195,85 @@ jobs:
             python3 scripts/mdx.py "$file"
           done
 
-      - name: Set up Git
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
-      - name: Clone `veda-config`
-        env:
-          VEDA_CONFIG_GH_TOKEN: ${{ secrets.VEDA_CONFIG_GH_TOKEN }}
-        run: git clone https://${{ env.VEDA_CONFIG_GH_TOKEN }}@github.com/${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}.git
-
-      # Creates a PR in veda-config with the following changes:
-      # 1. the mdx files for all published collections
-      # 2. updates the stac/raster urls in .env file
-      # This step needs a GH_TOKEN that has permissions to create a PR in veda-config
-      - name: Create PR with changes
-        id: create-pr
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VEDA_CONFIG_GH_TOKEN: ${{ secrets.VEDA_CONFIG_GH_TOKEN }}
-          COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
-          PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
-        run: |
-          files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
-          hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
-          NEW_BRANCH="add-dataset-$hash"
-          cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
-          git fetch origin
-          if git ls-remote --exit-code --heads origin $NEW_BRANCH; then
-            git push origin --delete $NEW_BRANCH
-          fi
-          git checkout -b $NEW_BRANCH
-
-          # Update the env vars to staging based on env vars
-          sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
-          cp -r ../datasets/* datasets/
-          git add .
-          git commit -m "Add dataset(s)"
-          git push origin $NEW_BRANCH
-          PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_GH_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
-
-          echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
-          echo "PR creation succeeded"
-
-      # Updates the comment with a link to the above PR
-      - name: Update PR comment with PR creation result
-        if: success()
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-        run: |
-          PR_URL=${{ steps.create-pr.outputs.PR_URL }}
-          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-          UPDATED_BODY="$CURRENT_BODY
-
-          **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
-          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      - name: Update PR comment on PR creation failure
-        if: failure() && steps.create-pr.outcome == 'failure'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-        run: |
-          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-          UPDATED_BODY="$CURRENT_BODY
-
-          **Failed ❌ to create a PR with the dataset configuration. 😔 **"
-          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      # If the workflow fails at any point, the PR comment will be updated
-      - name: Update PR comment on overall workflow failure
-        if: failure() && steps.create-pr.outcome != 'failure'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-        run: |
-          WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-          UPDATED_BODY="$CURRENT_BODY
-
-          ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
-          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+      # - name: Set up Git
+      #   run: |
+      #     git config --global user.name "github-actions[bot]"
+      #     git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      # - name: Clone `veda-config`
+      #   env:
+      #     VEDA_CONFIG_GH_TOKEN: ${{ secrets.VEDA_CONFIG_GH_TOKEN }}
+      #   run: git clone https://${{ env.VEDA_CONFIG_GH_TOKEN }}@github.com/${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}.git
+
+      # # Creates a PR in veda-config with the following changes:
+      # # 1. the mdx files for all published collections
+      # # 2. updates the stac/raster urls in .env file
+      # # This step needs a GH_TOKEN that has permissions to create a PR in veda-config
+      # - name: Create PR with changes
+      #   id: create-pr
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     VEDA_CONFIG_GH_TOKEN: ${{ secrets.VEDA_CONFIG_GH_TOKEN }}
+      #     COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
+      #     PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
+      #   run: |
+      #     files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
+      #     hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
+      #     NEW_BRANCH="add-dataset-$hash"
+      #     cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
+      #     git fetch origin
+      #     if git ls-remote --exit-code --heads origin $NEW_BRANCH; then
+      #       git push origin --delete $NEW_BRANCH
+      #     fi
+      #     git checkout -b $NEW_BRANCH
+
+      #     # Update the env vars to staging based on env vars
+      #     sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
+      #     cp -r ../datasets/* datasets/
+      #     git add .
+      #     git commit -m "Add dataset(s)"
+      #     git push origin $NEW_BRANCH
+      #     PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_GH_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
+
+      #     echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
+      #     echo "PR creation succeeded"
+
+      # # Updates the comment with a link to the above PR
+      # - name: Update PR comment with PR creation result
+      #   if: success()
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      #   run: |
+      #     PR_URL=${{ steps.create-pr.outputs.PR_URL }}
+      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+      #     UPDATED_BODY="$CURRENT_BODY
+
+      #     **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
+      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      # - name: Update PR comment on PR creation failure
+      #   if: failure() && steps.create-pr.outcome == 'failure'
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      #   run: |
+      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+      #     UPDATED_BODY="$CURRENT_BODY
+
+      #     **Failed ❌ to create a PR with the dataset configuration. 😔 **"
+      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+      # # If the workflow fails at any point, the PR comment will be updated
+      # - name: Update PR comment on overall workflow failure
+      #   if: failure() && steps.create-pr.outcome != 'failure'
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      #   run: |
+      #     WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+      #     UPDATED_BODY="$CURRENT_BODY
+
+      #     ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
+      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"

From 79cfb96f37bf5ef8fe0c41bbd4d7ddecb7b9d59d Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 31 Oct 2024 15:26:24 -0700
Subject: [PATCH 02/22] fix: update paths

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index edf61d9..b9659b0 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -14,7 +14,7 @@ on:
       - main
     paths:
       # Run the workflow only if files inside this path are updated
-      - ingestion-data/dataset-config/*
+      - ingestion-data/staging/dataset-config/*
 
 jobs:
   dataset-publication-and-configuration:

From 29e5868e327b6faeaf410ffda7fb7c41d8526791 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Mon, 4 Nov 2024 13:28:59 -0800
Subject: [PATCH 03/22] fix: update paths for testing

---
 .github/workflows/pr.yml                      |  3 +-
 .../testing/dataset-config/test.json          | 40 +++++++++++++++++++
 2 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 ingestion-data/testing/dataset-config/test.json

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index b9659b0..8d88328 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -14,7 +14,8 @@ on:
       - main
     paths:
       # Run the workflow only if files inside this path are updated
-      - ingestion-data/staging/dataset-config/*
+      # - ingestion-data/staging/dataset-config/*
+      - ingestion-data/testing/dataset-config/*
 
 jobs:
   dataset-publication-and-configuration:
diff --git a/ingestion-data/testing/dataset-config/test.json b/ingestion-data/testing/dataset-config/test.json
new file mode 100644
index 0000000..20667ca
--- /dev/null
+++ b/ingestion-data/testing/dataset-config/test.json
@@ -0,0 +1,40 @@
+{
+    "collection": "bangladesh-landcover-2001-2020-test",
+    "title": "Annual land cover maps for 2001 and 2020",
+    "description": "The annual land cover maps of 2001 and 2021 were captured using combined Moderate Resolution Imaging Spectroradiometer (MODIS) Annual Land Cover Type dataset (MCD12Q1 V6, dataset link: https://lpdaac.usgs.gov/products/mcd12q1v006/). The actual data product provides global land cover types at yearly intervals (2001-2020) at 500 meters with six different types of land cover classification. Among six different schemes, The International Geosphere–Biosphere Programme (IGBP) land cover classification selected and further simplified to dominant land cover classes (water, urban, cropland, native vegetation) for two different years to illustrate the changes in land use and land cover of the country.",
+    "type": "cog",
+    "spatial_extent": {
+        "xmin": 88.02591469087191,
+        "ymin": 20.742099910319755,
+        "xmax": 92.68367943903164,
+        "ymax": 26.63504817414382
+    },
+    "temporal_extent": {
+        "startdate": "2001-01-01T00:00:00Z",
+        "enddate": "2020-12-31T23:59:59Z"
+    },
+    "license": "CC0-1.0",
+    "is_periodic": false,
+    "time_density": "year",
+    "stac_version": "1.0.0",
+    "discovery_items": [
+        {
+            "prefix": "EIS/COG/coastal-flooding-and-slr/",
+            "bucket": "veda-data-store-staging",
+            "filename_regex": "^(.*)MODIS_LC_2001_BD_v2.cog.tif$",
+            "discovery": "s3",
+            "start_datetime": "2001-01-01T00:00:00Z",
+            "end_datetime": "2001-12-31T23:59:59Z",
+            "upload": false
+        },
+        {
+            "prefix": "EIS/COG/coastal-flooding-and-slr/",
+            "bucket": "veda-data-store-staging",
+            "filename_regex": "^(.*)MODIS_LC_2020_BD.cog.tif$",
+            "discovery": "s3",
+            "start_datetime": "2020-01-01T00:00:00Z",
+            "end_datetime": "2020-12-31T23:59:59Z",
+            "upload": false
+        }
+    ]
+}
\ No newline at end of file

From db787de47978d8dd1957346be667aa4ac10a0360 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Mon, 4 Nov 2024 15:54:35 -0800
Subject: [PATCH 04/22] fix: add trigger to test pr workflow

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 8d88328..d22298f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -12,6 +12,7 @@ on:
   pull_request:
     branches:
       - main
+      - feat/remove-veda-config-pr-creation
     paths:
       # Run the workflow only if files inside this path are updated
       # - ingestion-data/staging/dataset-config/*

From 9e943a9646cccd57e3ec63a16d1f22a53d4fc9a9 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Mon, 4 Nov 2024 16:02:48 -0800
Subject: [PATCH 05/22] fix: change branches to list, see if action is
 triggered

---
 .github/workflows/pr.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d22298f..c464b51 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -10,9 +10,7 @@ name: Publish collection to staging
 
 on:
   pull_request:
-    branches:
-      - main
-      - feat/remove-veda-config-pr-creation
+    branches: ['main', 'feat/remove-veda-config-pr-creation']
     paths:
       # Run the workflow only if files inside this path are updated
       # - ingestion-data/staging/dataset-config/*

From 954ddaf86a4b68a4a2e83e5c7b698de1eafd4169 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Mon, 4 Nov 2024 16:14:41 -0800
Subject: [PATCH 06/22] fix: update branches again to see if action will
 trigger

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index c464b51..7e826db 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -10,7 +10,7 @@ name: Publish collection to staging
 
 on:
   pull_request:
-    branches: ['main', 'feat/remove-veda-config-pr-creation']
+    branches: ['main', 'feat/automated-staging-publish']
     paths:
       # Run the workflow only if files inside this path are updated
       # - ingestion-data/staging/dataset-config/*

From 951b41e0a9d9ce7579008bf9470a21731e5ce679 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Mon, 4 Nov 2024 16:36:18 -0800
Subject: [PATCH 07/22] fix: remove commented out sections, change workflow
 fail condition

---
 .github/workflows/pr.yml | 106 +++++----------------------------------
 1 file changed, 12 insertions(+), 94 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 7e826db..d41f925 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -165,19 +165,6 @@ jobs:
           $status_message"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
-      # # Update PR comment
-      # - name: Update PR comment for PR creation
-      #   if: success()
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-      #   run: |
-      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-      #     UPDATED_BODY="$CURRENT_BODY
-
-      #     **Creating a PR in veda-config...**"
-      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -195,85 +182,16 @@ jobs:
             python3 scripts/mdx.py "$file"
           done
 
-      # - name: Set up Git
-      #   run: |
-      #     git config --global user.name "github-actions[bot]"
-      #     git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
-      # - name: Clone `veda-config`
-      #   env:
-      #     VEDA_CONFIG_GH_TOKEN: ${{ secrets.VEDA_CONFIG_GH_TOKEN }}
-      #   run: git clone https://${{ env.VEDA_CONFIG_GH_TOKEN }}@github.com/${{ vars.VEDA_CONFIG_REPO_ORG }}/${{ vars.VEDA_CONFIG_REPO_NAME }}.git
-
-      # # Creates a PR in veda-config with the following changes:
-      # # 1. the mdx files for all published collections
-      # # 2. updates the stac/raster urls in .env file
-      # # This step needs a GH_TOKEN that has permissions to create a PR in veda-config
-      # - name: Create PR with changes
-      #   id: create-pr
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     VEDA_CONFIG_GH_TOKEN: ${{ secrets.VEDA_CONFIG_GH_TOKEN }}
-      #     COMMENT_ID: ${{ steps.publish-collections.outputs.COMMENT_ID }}
-      #     PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
-      #   run: |
-      #     files_string=$(IFS=$'\n'; echo "${PUBLISHED_COLLECTION_FILES[*]}")
-      #     hash=$(echo -n "$files_string" | md5sum | cut -d ' ' -f 1)
-      #     NEW_BRANCH="add-dataset-$hash"
-      #     cd ${{ vars.VEDA_CONFIG_REPO_NAME }}
-      #     git fetch origin
-      #     if git ls-remote --exit-code --heads origin $NEW_BRANCH; then
-      #       git push origin --delete $NEW_BRANCH
-      #     fi
-      #     git checkout -b $NEW_BRANCH
-
-      #     # Update the env vars to staging based on env vars
-      #     sed -i "s|${{ vars.ENV_FROM }}|${{ vars.ENV_TO }}|g" .env
-      #     cp -r ../datasets/* datasets/
-      #     git add .
-      #     git commit -m "Add dataset(s)"
-      #     git push origin $NEW_BRANCH
-      #     PR_URL=$(GITHUB_TOKEN=$VEDA_CONFIG_GH_TOKEN gh pr create -H $NEW_BRANCH -B develop --title 'Add dataset [Automated workflow]' --body-file <(echo "Add datasets (Automatically created by Github action)"))
-
-      #     echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT
-      #     echo "PR creation succeeded"
-
-      # # Updates the comment with a link to the above PR
-      # - name: Update PR comment with PR creation result
-      #   if: success()
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-      #   run: |
-      #     PR_URL=${{ steps.create-pr.outputs.PR_URL }}
-      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-      #     UPDATED_BODY="$CURRENT_BODY
-
-      #     **A PR has been created with the dataset configuration: 🗺️ [PR link]($PR_URL)**"
-      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      # - name: Update PR comment on PR creation failure
-      #   if: failure() && steps.create-pr.outcome == 'failure'
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-      #   run: |
-      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-      #     UPDATED_BODY="$CURRENT_BODY
-
-      #     **Failed ❌ to create a PR with the dataset configuration. 😔 **"
-      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
-
-      # # If the workflow fails at any point, the PR comment will be updated
-      # - name: Update PR comment on overall workflow failure
-      #   if: failure() && steps.create-pr.outcome != 'failure'
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #     COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
-      #   run: |
-      #     WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-      #     CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
-      #     UPDATED_BODY="$CURRENT_BODY
+      # If the workflow fails at any point, the PR comment will be updated
+      - name: Update PR comment on overall workflow failure
+        if: failure()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
 
-      #     ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
-      #     gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+          ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"

From 0846b3aaeac35a34de2cb4a85d995bde378ec501 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Mon, 4 Nov 2024 16:51:40 -0800
Subject: [PATCH 08/22] fix: attempt to fix workflow status link

---
 .github/workflows/pr.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d41f925..27b73b3 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -35,9 +35,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-          body='### Workflow Status
-          **Starting workflow...** [View action run]($WORKFLOW_URL)
-          '
+          body="### Workflow Status
+          **Starting workflow...** [View action run]($WORKFLOW_URL)"
+          echo "Body content: $body"
 
           # Get the PR number
           PR_NUMBER=${{ github.event.pull_request.number }}

From e26b970f08cdb5cbbb327a765e68c26494cf0677 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 5 Nov 2024 10:48:07 -0800
Subject: [PATCH 09/22] fix: update environment to point to staging

---
 .github/workflows/pr.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 27b73b3..dab5782 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -22,6 +22,7 @@ jobs:
       pull-requests: write
       contents: read
     runs-on: ubuntu-latest
+    environment: staging
 
     steps:
       - uses: actions/checkout@v4
@@ -73,12 +74,13 @@ jobs:
       - name: Get auth token
         id: get-token
         run: |
+          echo "Vars: $vars"
           response=$(curl -X POST \
             ${{ vars.STAGING_COGNITO_DOMAIN }}/oauth2/token \
             -H "Content-Type: application/x-www-form-urlencoded" \
             -d "grant_type=client_credentials" \
             -d "client_id=${{ vars.STAGING_CLIENT_ID }}" \
-            -d "client_secret=${{ secrets.STAGING_CLIENT_SECRET }}"
+            -d "client_secret=${{ vars.STAGING_CLIENT_SECRET }}"
           )
 
           access_token=$(echo "$response" | jq -r '.access_token')

From 8aeaefab90bc89dde33f50e59689ca15e72f86a7 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:03:32 -0800
Subject: [PATCH 10/22] fix: add debugging

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index dab5782..cdad2a3 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -132,6 +132,7 @@ jobs:
               )
 
               status_code=$(tail -n1 <<< "$response")
+              echo "Status code is: $status_code"
 
               # Update status message based on response code
               if [ "$status_code" -eq 200 ] || [ "$status_code" -eq 201 ]; then

From e1a59cc86fd00e35b1fa31e24bb8c3101f5591f3 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:38:26 -0800
Subject: [PATCH 11/22] fix: change test collection to use valid one

---
 .../testing/dataset-config/test.json          | 45 ++++++++-----------
 1 file changed, 18 insertions(+), 27 deletions(-)

diff --git a/ingestion-data/testing/dataset-config/test.json b/ingestion-data/testing/dataset-config/test.json
index 20667ca..753d634 100644
--- a/ingestion-data/testing/dataset-config/test.json
+++ b/ingestion-data/testing/dataset-config/test.json
@@ -1,40 +1,31 @@
 {
-    "collection": "bangladesh-landcover-2001-2020-test",
-    "title": "Annual land cover maps for 2001 and 2020",
-    "description": "The annual land cover maps of 2001 and 2021 were captured using combined Moderate Resolution Imaging Spectroradiometer (MODIS) Annual Land Cover Type dataset (MCD12Q1 V6, dataset link: https://lpdaac.usgs.gov/products/mcd12q1v006/). The actual data product provides global land cover types at yearly intervals (2001-2020) at 500 meters with six different types of land cover classification. Among six different schemes, The International Geosphere–Biosphere Programme (IGBP) land cover classification selected and further simplified to dominant land cover classes (water, urban, cropland, native vegetation) for two different years to illustrate the changes in land use and land cover of the country.",
-    "type": "cog",
+    "collection": "hls-swir-falsecolor-composite-TEST",
+    "title": "HLS SWIR FalseColor Composite",
     "spatial_extent": {
-        "xmin": 88.02591469087191,
-        "ymin": 20.742099910319755,
-        "xmax": 92.68367943903164,
-        "ymax": 26.63504817414382
+        "xmin": -156.75,
+        "ymin": 20.80,
+        "xmax": -156.55,
+        "ymax": 20.94
     },
     "temporal_extent": {
-        "startdate": "2001-01-01T00:00:00Z",
-        "enddate": "2020-12-31T23:59:59Z"
+        "startdate": "2023-08-08T00:00:00Z",
+        "enddate": "2023-08-08T23:59:59Z"
     },
+    "data_type": "cog",
     "license": "CC0-1.0",
+    "description": "HLS falsecolor composite imagery using Bands 12, 8A, and 4.",
     "is_periodic": false,
-    "time_density": "year",
-    "stac_version": "1.0.0",
+    "time_density": "day",
+    "sample_files": [
+        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-08_SWIR_falsecolor_cog.tif",
+        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-13_SWIR_falsecolor_cog.tif"
+    ],
     "discovery_items": [
         {
-            "prefix": "EIS/COG/coastal-flooding-and-slr/",
-            "bucket": "veda-data-store-staging",
-            "filename_regex": "^(.*)MODIS_LC_2001_BD_v2.cog.tif$",
             "discovery": "s3",
-            "start_datetime": "2001-01-01T00:00:00Z",
-            "end_datetime": "2001-12-31T23:59:59Z",
-            "upload": false
-        },
-        {
-            "prefix": "EIS/COG/coastal-flooding-and-slr/",
+            "prefix": "maui-fire/",
             "bucket": "veda-data-store-staging",
-            "filename_regex": "^(.*)MODIS_LC_2020_BD.cog.tif$",
-            "discovery": "s3",
-            "start_datetime": "2020-01-01T00:00:00Z",
-            "end_datetime": "2020-12-31T23:59:59Z",
-            "upload": false
+            "filename_regex": "(.*)SWIR_falsecolor(.*).tif$"
         }
     ]
-}
\ No newline at end of file
+}

From d41d629ba22b2efecaf8886f4edf7bb856d98857 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:41:31 -0800
Subject: [PATCH 12/22] fix: remove unnecessary echoes

---
 .github/workflows/pr.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index cdad2a3..5db0e1a 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -38,7 +38,6 @@ jobs:
           WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           body="### Workflow Status
           **Starting workflow...** [View action run]($WORKFLOW_URL)"
-          echo "Body content: $body"
 
           # Get the PR number
           PR_NUMBER=${{ github.event.pull_request.number }}
@@ -132,7 +131,6 @@ jobs:
               )
 
               status_code=$(tail -n1 <<< "$response")
-              echo "Status code is: $status_code"
 
               # Update status message based on response code
               if [ "$status_code" -eq 200 ] || [ "$status_code" -eq 201 ]; then
@@ -143,7 +141,7 @@ jobs:
                 all_failed=false
               else
                 echo "$collection_id failed to publish ❌"
-                status_message+="- **$collection_id**: Failed to publish ❌
+                status_message+="- **$collection_id**: Failed to publish. Error code $status_code. ❌
                 "
               fi
             else

From e40df511961a99a3d58714c7c96bb01dd2a9d7bd Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 6 Nov 2024 14:29:02 -0800
Subject: [PATCH 13/22] fix: change staging client secret to retrieve from
 secrets

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 5db0e1a..0e27b08 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -79,7 +79,7 @@ jobs:
             -H "Content-Type: application/x-www-form-urlencoded" \
             -d "grant_type=client_credentials" \
             -d "client_id=${{ vars.STAGING_CLIENT_ID }}" \
-            -d "client_secret=${{ vars.STAGING_CLIENT_SECRET }}"
+            -d "client_secret=${{ secrets.STAGING_CLIENT_SECRET }}"
           )
 
           access_token=$(echo "$response" | jq -r '.access_token')

From 9345bfff40498781704eb0e6a3cfe8b9befdc17a Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 13 Nov 2024 09:12:50 -0800
Subject: [PATCH 14/22] fix: update to only look at added files

---
 .github/workflows/pr.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 0e27b08..8a3bd7f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -58,12 +58,12 @@ jobs:
 
           echo "COMMENT_ID=$COMMENT_ID" >> $GITHUB_OUTPUT
 
-      # Find only the updated files (file that differ from base)
+      # Find only the newly added files
       # Only .json files
       # The files are outputted to GITHUB_OUTPUT, which can be used in subsequent steps
       - name: Get updated files
         id: changed-files
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v45
         with:
           files: |
             **.json
@@ -93,7 +93,7 @@ jobs:
       - name: Publish all updated collections
         id: publish-collections
         env:
-          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+          ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
           WORKFLOWS_URL: ${{ vars.STAGING_WORKFLOWS_URL }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AUTH_TOKEN: ${{ steps.get-token.outputs.ACCESS_TOKEN }}
@@ -118,7 +118,7 @@ jobs:
           status_message='### Collection Publication Status
           '
 
-          for file in "${ALL_CHANGED_FILES[@]}"; do
+          for file in "${ADDED_FILES[@]}"; do
             echo $file
             if [ -f "$file" ]; then
               dataset_config=$(jq '.' "$file")

From a80142f2704d5266b920054228fab8fb3d76dbff Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 13 Nov 2024 09:14:10 -0800
Subject: [PATCH 15/22] fix: rename step

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 8a3bd7f..b47b50e 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -61,7 +61,7 @@ jobs:
       # Find only the newly added files
       # Only .json files
       # The files are outputted to GITHUB_OUTPUT, which can be used in subsequent steps
-      - name: Get updated files
+      - name: Get newly added files
         id: changed-files
         uses: tj-actions/changed-files@v45
         with:

From f10e8547ce5632e2c6cac4f6625986ccbaf2e8be Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 13 Nov 2024 10:08:28 -0800
Subject: [PATCH 16/22] fix: add second test json file

---
 .../testing/dataset-config/test-2.json        | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 ingestion-data/testing/dataset-config/test-2.json

diff --git a/ingestion-data/testing/dataset-config/test-2.json b/ingestion-data/testing/dataset-config/test-2.json
new file mode 100644
index 0000000..498b4f6
--- /dev/null
+++ b/ingestion-data/testing/dataset-config/test-2.json
@@ -0,0 +1,31 @@
+{
+    "collection": "hls-swir-falsecolor-composite-TEST-2",
+    "title": "HLS SWIR FalseColor Composite",
+    "spatial_extent": {
+        "xmin": -156.75,
+        "ymin": 20.80,
+        "xmax": -156.55,
+        "ymax": 20.94
+    },
+    "temporal_extent": {
+        "startdate": "2023-08-08T00:00:00Z",
+        "enddate": "2023-08-08T23:59:59Z"
+    },
+    "data_type": "cog",
+    "license": "CC0-1.0",
+    "description": "HLS falsecolor composite imagery using Bands 12, 8A, and 4.",
+    "is_periodic": false,
+    "time_density": "day",
+    "sample_files": [
+        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-08_SWIR_falsecolor_cog.tif",
+        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-13_SWIR_falsecolor_cog.tif"
+    ],
+    "discovery_items": [
+        {
+            "discovery": "s3",
+            "prefix": "maui-fire/",
+            "bucket": "veda-data-store-staging",
+            "filename_regex": "(.*)SWIR_falsecolor(.*).tif$"
+        }
+    ]
+}

From 70ab8aec7daf99fb2589cf8569d451b8a2bb1ed4 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 13 Nov 2024 10:26:14 -0800
Subject: [PATCH 17/22] fix: add step to list newly added files

---
 .github/workflows/pr.yml                      |  8 +++++
 .../testing/dataset-config/test-2.json        | 31 -------------------
 2 files changed, 8 insertions(+), 31 deletions(-)
 delete mode 100644 ingestion-data/testing/dataset-config/test-2.json

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index b47b50e..b05888f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -68,6 +68,14 @@ jobs:
           files: |
             **.json
 
+      - name: List all newly added files
+        env:
+          ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
+        run: |
+          for file in ${ADDED_FILES}; do
+            echo "$file was added"
+          done
+
       # Uses service client creds to get token
       # No username/password needed
       - name: Get auth token
diff --git a/ingestion-data/testing/dataset-config/test-2.json b/ingestion-data/testing/dataset-config/test-2.json
deleted file mode 100644
index 498b4f6..0000000
--- a/ingestion-data/testing/dataset-config/test-2.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-    "collection": "hls-swir-falsecolor-composite-TEST-2",
-    "title": "HLS SWIR FalseColor Composite",
-    "spatial_extent": {
-        "xmin": -156.75,
-        "ymin": 20.80,
-        "xmax": -156.55,
-        "ymax": 20.94
-    },
-    "temporal_extent": {
-        "startdate": "2023-08-08T00:00:00Z",
-        "enddate": "2023-08-08T23:59:59Z"
-    },
-    "data_type": "cog",
-    "license": "CC0-1.0",
-    "description": "HLS falsecolor composite imagery using Bands 12, 8A, and 4.",
-    "is_periodic": false,
-    "time_density": "day",
-    "sample_files": [
-        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-08_SWIR_falsecolor_cog.tif",
-        "s3://veda-data-store-staging/maui-fire/Lahaina_HLS_2023-08-13_SWIR_falsecolor_cog.tif"
-    ],
-    "discovery_items": [
-        {
-            "discovery": "s3",
-            "prefix": "maui-fire/",
-            "bucket": "veda-data-store-staging",
-            "filename_regex": "(.*)SWIR_falsecolor(.*).tif$"
-        }
-    ]
-}

From 6210ae3851cbffac8eb91b0457b015057287eb9b Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Wed, 13 Nov 2024 10:36:45 -0800
Subject: [PATCH 18/22] fix: update step name

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index b05888f..618e99e 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -98,7 +98,7 @@ jobs:
       # Used by other steps
       # If none of the requests are successful, workflow fails
       # Updates the PR comment with status of collection publication
-      - name: Publish all updated collections
+      - name: Publish all newly added collections
         id: publish-collections
         env:
           ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}

From eb835b34208fa2d55b8b5a0e4e6c2e8b9f41bebd Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 14 Nov 2024 10:42:08 -0800
Subject: [PATCH 19/22] fix: update based on feedback to separate into
 different jobs

---
 .github/workflows/pr.yml | 59 +++++++++++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 618e99e..a5c7662 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -16,14 +16,21 @@ on:
       # - ingestion-data/staging/dataset-config/*
       - ingestion-data/testing/dataset-config/*
 
+  push:
+    branches:
+      - main
+
+permissions:
+  pull-requests: write
+  contents: read
+
 jobs:
   dataset-publication-and-configuration:
-    permissions:
-      pull-requests: write
-      contents: read
+    if: ${{ github.event_name == 'pull_request' && (github.event.action == 'synchronize' || github.event.action == 'opened') }}
     runs-on: ubuntu-latest
     environment: staging
-
+    outputs:
+      publishedCollections: ${{ steps.publish-collections.outputs.success_collections }}
     steps:
       - uses: actions/checkout@v4
 
@@ -98,7 +105,7 @@ jobs:
       # Used by other steps
       # If none of the requests are successful, workflow fails
       # Updates the PR comment with status of collection publication
-      - name: Publish all newly added collections
+      - name: Publish all newly added collections to staging
         id: publish-collections
         env:
           ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
@@ -180,17 +187,6 @@ jobs:
           python-version: '3.9'
           cache: 'pip'
 
-      # Creates a slim dataset mdx file for each collection based on the dataset config json
-      - name: Create dataset mdx for given collections
-        env:
-          PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
-        run: |
-          pip install -r scripts/requirements.txt
-          for file in "${PUBLISHED_COLLECTION_FILES[@]}"
-          do
-            python3 scripts/mdx.py "$file"
-          done
-
       # If the workflow fails at any point, the PR comment will be updated
       - name: Update PR comment on overall workflow failure
         if: failure()
@@ -204,3 +200,34 @@ jobs:
 
           ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
+  create-mdx-files:
+    runs-on: ubuntu-latest
+    needs: dataset-publication-and-configuration
+    steps:
+      - name: Use output from dataset-publication-and-configuration
+
+        run: |
+          echo "The output from the previous step is: ${{ needs.dataset-publication-and-configuration.outputs.publishedCollections }}"
+
+      # Creates a slim dataset mdx file for each collection based on the dataset config json
+      - name: Create dataset mdx for given collections
+        env:
+          PUBLISHED_COLLECTION_FILES: ${{ needs.dataset-publication-and-configuration.outputs.publishedCollections }}
+        run: echo "NO-OP step"
+        # run: |
+        #   pip install -r scripts/requirements.txt
+        #   for file in "${PUBLISHED_COLLECTION_FILES[@]}"
+        #   do
+        #     python3 scripts/mdx.py "$file"
+        #   done
+
+  publish-to-prod-on-pr-merge:
+    if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Publish to production on PR merge
+        run: echo "NO-OP. This step runs when a PR is merged."

From 8fabafa69d7483ec9a8adc76c8df4410d52b64ca Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 14 Nov 2024 11:00:01 -0800
Subject: [PATCH 20/22] fix: add notification step

---
 .github/workflows/pr.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index a5c7662..3d1fd59 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -222,6 +222,16 @@ jobs:
         #     python3 scripts/mdx.py "$file"
         #   done
 
+  open-veda-config-pr-notification:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Notify user to open a new PR
+        run: |
+          echo "The action is complete. Please open a new Pull Request in veda-config for the changes."
+
   publish-to-prod-on-pr-merge:
     if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true }}
     runs-on: ubuntu-latest

From 66626d1019317128725de862351e6bd2c7c27bb2 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 14 Nov 2024 11:01:54 -0800
Subject: [PATCH 21/22] fix: clean up step

---
 .github/workflows/pr.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 3d1fd59..d5f1339 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -225,9 +225,6 @@ jobs:
   open-veda-config-pr-notification:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
       - name: Notify user to open a new PR
         run: |
           echo "The action is complete. Please open a new Pull Request in veda-config for the changes."

From 57f1005c2c9fe728bcdfe922c0a540c6c7648473 Mon Sep 17 00:00:00 2001
From: Jennifer Tran <12633533+botanical@users.noreply.github.com>
Date: Thu, 14 Nov 2024 11:02:28 -0800
Subject: [PATCH 22/22] fix: add dependency

---
 .github/workflows/pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index d5f1339..9084db7 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -224,6 +224,7 @@ jobs:
 
   open-veda-config-pr-notification:
     runs-on: ubuntu-latest
+    needs: create-mdx-files
     steps:
       - name: Notify user to open a new PR
         run: |