From c96c9bb22cd7d8c77a2b118f672e6699f3706693 Mon Sep 17 00:00:00 2001
From: wrongkindofdoctor <20195932+wrongkindofdoctor@users.noreply.github.com>
Date: Wed, 2 Oct 2024 09:47:56 -0400
Subject: [PATCH] add wkdir cleanup between each test run step and separate obs
 data fetching for set 1 tests in ci config file

---
 .github/workflows/mdtf_tests.yml | 41 +++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/mdtf_tests.yml b/.github/workflows/mdtf_tests.yml
index 76614495a..b6e00913d 100644
--- a/.github/workflows/mdtf_tests.yml
+++ b/.github/workflows/mdtf_tests.yml
@@ -97,7 +97,7 @@ jobs:
         mkdir wkdir
         ## make input data directories
         mkdir -p inputdata/obs_data
-    - name: Get Observational Data for Set 1
+    - name: Get Observational Data for Set 1a
       run: |
         echo "${PWD}"
         cd ../
@@ -106,18 +106,12 @@ jobs:
         # attempt FTP data fetch
         # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual
         # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos.
-        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
-        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/EOF_500hPa_obs_data.tar --output EOF_500hPa_obs_data.tar
+        # curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/EOF_500hPa_obs_data.tar --output EOF_500hPa_obs_data.tar
         curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/Wheeler_Kiladis_obs_data.tar --output Wheeler_Kiladis_obs_data.tar
-        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_teleconnection_obs_data.tar --output MJO_teleconnection_obs_data.tar
-        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_suite_obs_data.tar --output MJO_suite_obs_data.tar
         curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/precip_diurnal_cycle_obs_data.tar --output precip_diurnal_cycle_obs_data.tar
-        echo "Untarring set 1 NCAR/CESM standard test files"
-        tar -xvf convective_transition_diag_obs_data.tar
-        tar -xvf EOF_500hPa_obs_data.tar
+        echo "Untarring set 1a NCAR/CESM standard test files"
+        # tar -xvf EOF_500hPa_obs_data.tar
         tar -xvf precip_diurnal_cycle_obs_data.tar
-        tar -xvf MJO_teleconnection_obs_data.tar
-        tar -xvf MJO_suite_obs_data.tar
         tar -xvf Wheeler_Kiladis_obs_data.tar
         # clean up tarballs
         rm -f *.tar
@@ -133,6 +127,23 @@ jobs:
         ./mdtf -f ${{matrix.json-file-1a}}
         # Debug POD log(s)
         # cat ${POD_OUTPUT}/MDTF_NCAR.Synthetic_1975_1981/Wheeler_Kiladis/Wheeler_Kiladis.log
+    - name: Get observational data for set 1b
+      run: |
+        # clean up data from previous runs
+        echo "deleting data from set 1a"
+        cd ../wkdir
+        rm -rf *
+        cd ../inputdata/obs_data
+        rm -rf *
+        cd ../../
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_teleconnection_obs_data.tar --output MJO_teleconnection_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_suite_obs_data.tar --output MJO_suite_obs_data.tar
+        tar -xvf MJO_teleconnection_obs_data.tar
+        tar -xvf MJO_suite_obs_data.tar
+        tar -xvf convective_transition_diag_obs_data.tar
+        # clean up tarballs
+        rm -f *.tar
     - name: Run diagnostic tests set 1b
       run: |
         ./mdtf -f ${{matrix.json-file-1b}}
@@ -141,8 +152,10 @@ jobs:
         echo "${PWD}"
         # remove data from previous run
         # Actions moves you to the root repo directory in every step, so need to cd again
+        echo "deleting data from set 1b"
+        cd ../wkdir
+        rm -rf *
         cd ../inputdata/obs_data
-        echo "deleting obs data from set 1"
         rm -rf *
         cd ../../
         echo "Available Space"
@@ -157,8 +170,6 @@ jobs:
         rm -f *.tar
     - name: Run diagnostic tests set 2
       run: |
-        micromamba activate _MDTF_base
-        # run the test PODs
         ./mdtf -f ${{matrix.json-file-2}}
         # Uncomment the following line for debugging
         #cat ../wkdir/MDTF_GFDL.Synthetic_1_10/MJO_prop_amp/MJO_prop_amp.log
@@ -167,8 +178,10 @@ jobs:
         echo "${PWD}"
         # remove data from previous run
         # Actions moves you to the root repo directory in every step, so need to cd again
+        echo "deleting data from set 2"
+        cd ../wkdir
+        rm -rf *
         cd ../inputdata/obs_data
-        echo "deleting obs data from set 2"
         rm -rf *
         cd ../../
         echo "Available Space"