NOAA-EMC · TerrenceMcGuinness-NOAA · Aug 28, 2023 · Aug 28, 2023 · Aug 30, 2023 · Sep 12, 2023
@@ -9,7 +9,7 @@ arguments:
   resens: 192
   comrot: ${RUNTESTS}/COMROT
   expdir: ${RUNTESTS}/EXPDIR
-  icsdir: /scratch1/NCEPDEV/stmp2/Rahul.Mahajan/ICSDIR/IC_C384
+  icsdir: ${ICSDIR_ROOT}/C384C192
   idate: 2023040118
   edate: 2023040200
   nens: 2

@@ -5,10 +5,11 @@ experiment:
 arguments:
   pslot: ${pslot}
   app: ATM
-  resdet: C384
+  resdet: 384
+  resens: 192
   comrot: ${RUNTESTS}/COMROT
   expdir: ${RUNTESTS}/EXPDIR
-  icsdir: /scratch1/NCEPDEV/stmp2/Rahul.Mahajan/ICSDIR/IC_C384
+  icsdir: ${ICSDIR_ROOT}/C384C192
   idate: 2023040118
   edate: 2023040200
   nens: 0

@@ -41,22 +41,24 @@ done
 
 cd "${repodir}" || exit 1
 # clone copy of repo
-if [[ -d global-workflow ]]; then
-  rm -Rf global-workflow
+if [[ -d HOMEgfs ]]; then
+  rm -Rf HOMEgfs
 fi
 
-git clone "${REPO_URL}"
-cd global-workflow || exit 1
-
-pr_state=$("${GH}" pr view "${PR}" --json state --jq '.state')
-if [[ "${pr_state}" != "OPEN" ]]; then
-  title=$("${GH}" pr view "${PR}" --json title --jq '.title')
-  echo "PR ${title} is no longer open, state is ${pr_state} ... quitting"
-  exit 1
-fi
+git clone "${REPO_URL}" HOMEgfs
+cd HOMEgfs || exit 1
 
+if [[ "${PR}" != "develop" ]]; then
+  pr_state=$("${GH}" pr view "${PR}" --json state --jq '.state')
+  if [[ "${pr_state}" != "OPEN" ]]; then
+    title=$("${GH}" pr view "${PR}" --json title --jq '.title')
+    echo "PR ${title} is no longer open, state is ${pr_state} ... quitting"
+    exit 1
+  fi
 # checkout pull request
 "${GH}" pr checkout "${PR}" --repo "${REPO_URL}"
+fi
+
 HOMEgfs="${PWD}"
 source "${HOMEgfs}/ush/detect_machine.sh"
 

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eux
 
-#####################################################################################
+##############################################################################################
 #
 # Script description: Top level driver script for checking PR
 #                     ready for CI regression testing
@@ -11,9 +11,10 @@ set -eux
 # This script uses GitHub CLI to check for Pull Requests with CI-Ready-${machine} tags on the
 # development branch for the global-workflow repo.  It then stages tests directories per
 # PR number and calls clone-build_ci.sh to perform a clone and full build from $(HOMEgfs)/sorc
-# of the PR. It then is ready to run a suite of regression tests with various
-# configurations with run_tests.py.
-#######################################################################################
+# of the PR. It then creates an expiment directory in ${GFS_CI_ROOT}/PR/$[pr}/RUNTESTS
+# for each yaml case file in ${HOMEgfs}/ci/cases/pr.  At his point the run-ci.sh script
+# will advance the experiment using Rocoto while check_ci.sh script will tests for complition
+#######################################################################################i######
 
 #################################################################
 # TODO using static build for GitHub CLI until fixed in HPC-Stack
@@ -145,7 +146,7 @@ for pr in ${pr_list}; do
     # loop over every yaml file in the PR's ci/cases
     # and create an run directory for each one for this PR loop
     #############################################################
-    HOMEgfs_PR="${pr_dir}/global-workflow"
+    HOMEgfs_PR="${pr_dir}/HOMEgfs"
     export HOMEgfs_PR
     cd "${HOMEgfs_PR}"
     pr_sha=$(git rev-parse --short HEAD)

@@ -0,0 +1,142 @@
+#!/bin/bash
+set -eux
+
+##############################################################################################
+#
+# Script description: Top level driver script for checking PR
+#                     ready for CI regression testing
+#
+# Abstract:
+#
+# This script runs the CI Expirments foreach case files found in $HOMEgfs/ci/cases/weekly
+# from the develop branch for the global-workflow repo that is intended to run on a weekly basis
+# from a cron job. When ran it will clone and build from the EMC's global-workflow and
+# create each expiment directory in ${GFS_CI_ROOT}/develop/RUNTESTS
+# for each yaml case file in ${HOMEgfs}/ci/cases/weekly.  As each EXPDIR is created it will
+# run run-check_ci.sh agaist it to advance the experiment using Rocoto.  After each
+# weekly case file is finished it will email from the EMAIL_LIST the results.
+#######################################################################################i######
+
+#################################################################
+# TODO using static build for GitHub CLI until fixed in HPC-Stack
+#################################################################
+export GH=${HOME}/bin/gh
+export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"}
+
+#EMAIL_LIST="[email protected], [email protected], [email protected]"
+EMAIL_LIST="[email protected]"
+
+################################################################
+# Setup the reletive paths to scripts and PS4 for better logging 
+################################################################
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+scriptname=$(basename "${BASH_SOURCE[0]}")
+echo "Begin ${scriptname} at $(date -u)" || true
+export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
+
+#########################################################################
+#  Set up runtime environment varibles for accounts on supproted machines
+#########################################################################
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+case ${MACHINE_ID} in
+  hera | orion)
+    echo "Running Automated Testing on ${MACHINE_ID}"
+    source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh"
+    ;;
+  *)
+    echo "Unsupported platform. Exiting with error."
+    exit 1
+    ;;
+esac
+
+######################################################
+# setup runtime env for correct python install and git
+######################################################
+set +x
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/modulefiles"
+module load "module_gwsetup.${MACHINE_ID}"
+set -x
+
+#############################################################
+# Clone, checkout, build, develop branch and creat set of
+# cases from $HOMEgfs/ci/cases/weekly 
+#############################################################
+
+  echo "Building development branch on ${MACHINE_ID}"
+  develop_dir="${GFS_CI_ROOT}/develop"
+  RUNTESTS="${develop_dir}/RUNTESTS"
+  export RUNTESTS
+  mkdir -p "${RUNTESTS}"
+  date_stamp=$(date | awk '{print $1"-"$2"-"$3"-"$6}') || true
+  export CI_LOG="${develop_dir}/ci_${date_stamp}.log"
+  rm -f "${CI_LOG}"
+  # call clone-build_ci to clone and build the develop branch
+  set +e
+  rm -Rf "${develop_dir:?}/HOMEgfs"
+  "${HOMEgfs}/ci/scripts/clone-build_ci.sh" -p develop -d "${develop_dir}" -o "${CI_LOG}"
+  #echo "SKIPPING: ${HOMEgfs}/ci/scripts/clone-build_ci.sh -p develop -d ${develop_dir} -o ${CI_LOG}"
+  ci_status=$?
+  set -e
+  if [[ ${ci_status} -eq 0 ]]; then
+
+    #############################################################
+    # loop over every yaml file in ci/cases/weekly dir
+    # and create an EXPDIR directory for each one
+    #############################################################
+    HOMEgfs_PR="${develop_dir}/HOMEgfs"
+    export HOMEgfs_PR
+    cd "${HOMEgfs_PR}"
+    pr_sha=$(git rev-parse --short HEAD)
+
+    for yaml_config in "${HOMEgfs_PR}/ci/cases/weekly/"*.yaml; do
+      case=$(basename "${yaml_config}" .yaml) || true
+      pslot="${case}_${pr_sha}"
+      export pslot
+      # Do not quit on errors so we can email results
+      set +e
+      "${HOMEgfs_PR}/ci/scripts/create_experiment.py" --yaml "${yaml_config}" --dir "${HOMEgfs_PR}"
+      ci_status=$?
+      if [[ ${ci_status} -eq 0 ]]; then
+        {
+          echo "Created experiment:            *SUCCESS*"
+          echo "Case setup: Completed at $(date) for experiment ${pslot}" || true
+        } >> "${CI_LOG}"
+        rm -f "${develop_dir}/RUNTESTS/ci.log"
+        "${HOMEgfs}/ci/scripts/run-check_ci.sh" "${develop_dir}" "${pslot}" 2>> "${develop_dir}/output_${case}.stderr" > "${develop_dir}/output_${case}.stdout"
+        ci_status=$?
+        stdout_errors=$(cat "${develop_dir}/output_${case}.stdout")
+        if [[ "FATAL ERROR" == *"${stdout_errors}"* ]]; then
+          ci_status=1
+        fi
+        if [[ ${ci_status} -eq 0 ]]; then
+          {
+            echo -e "\n**** CASE ${case} SUCCEDED at $(date) ****" || true
+            cat "${develop_dir}/RUNTESTS/ci.log"
+          } >> "${CI_LOG}"
+        else
+         {
+          echo -e "\n**** CASE ${case} FAILED ****"
+          cat "${develop_dir}/output_${case}.stderr"
+          cat "${develop_dir}/RUNTESTS/ci.log"
+         } >> "${CI_LOG}"
+        fi
+      else 
+        {
+          echo "Failed to create experiment:  *FAIL* ${pslot}"
+          echo "Experiment setup: failed at $(date) for experiment ${pslot}" || true
+          echo ""
+          cat "${HOMEgfs_PR}/ci/scripts/"setup_*.std*
+        } >> "${CI_LOG}"
+      fi
+    done
+
+   else 
+   {
+      echo "Failed on cloning and building global-workflow"
+      echo "CI on ${MACHINE_ID^} failed to build on $(date) for repo ${REPO_URL}" || true
+   } >> "${CI_LOG}"
+  fi
+  mail -s "Weekly CI Tests Results for ${date_stamp}" "${EMAIL_LIST}" < "${CI_LOG}"
+
@@ -25,8 +25,8 @@ HOMEgfs="${TEST_DIR}/HOMEgfs"
 RUNTESTS="${TEST_DIR}/RUNTESTS"
 
 # Source modules and setup logging
-echo "Source modules."
-source "${HOMEgfs}/workflow/gw_setup.sh"
+echo "NOT Source modules."
+#source "${HOMEgfs}/workflow/gw_setup.sh"
 
 # cd into the experiment directory
 echo "cd ${RUNTESTS}/EXPDIR/${pslot}"
@@ -83,7 +83,7 @@ while true; do
      echo "${error_logs}"
     } >> "${RUNTESTS}/ci.log"
     sed -i "s/\`\`\`//2g" "${RUNTESTS}/ci.log"
-    sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep "${pr}\/RUNTESTS" |  awk '{print $1}' | xargs scancel || true
+    sacct --format=jobid,jobname%35,WorkDir%100,stat | grep "${pslot}" | grep RUNTESTS |  awk '{print $1}' | xargs scancel || true
     rc=1
     break
   fi

@@ -0,0 +1,56 @@
+#! /usr/bin/env python3
+
+import time
+from pathlib import Path
+from github import Github
+from datetime import datetime
+
+now = datetime.now()
+date = now.strftime("%m %d %Y")
+
+# Provide your personal access token or use a GitHub token if available
+with open(Path.home().joinpath('.gittokens','access_token.txt'), 'r') as file:
+    access_token = file.read().strip()
+
+# Create a PyGitHub instance using the access token
+github = Github(access_token)
+
+# Specify the repository where you want to open the Pull Request and modify the label
+repo_owner = 'TerrenceMcGuinness-NOAA'
+repo_name = 'global-workflow'
+pr_title = 'Weekly Tests '+date
+pr_body = 'Weekly High Resolution Forecast Tests'
+label_names = ['CI-Hera-Ready','CI-Orion-Ready']
+
+# Get the repository object
+repo = github.get_repo(f'{repo_owner}/{repo_name}')
+sb = repo.get_branch('develop')
+
+# Create a new branch for the Weekly test Pull Request
+branch_name = 'weekly_tests'
+repo.create_git_ref(ref=f'refs/heads/{branch_name}', sha=sb.commit.sha)
+time.sleep(5)
+branch = repo.get_branch(branch_name)
+
+pr_cases = repo.get_contents('/ci/cases/pr', ref=branch.name)
+for file in pr_cases:
+    if file.name.endswith('.yaml'):
+        repo.delete_file(path=file.path, message='deleting file for weekly tests', sha=file.sha, branch=branch_name)
+
+weekly_directory = repo.get_contents('/ci/cases/weekly',branch.name)
+cases = [file for file in weekly_directory if file.name.endswith('.yaml')]
+for case in cases:
+    file_name = case.name
+    file_content = case.decoded_content.decode('utf-8')
+    repo.create_file(path=file_name, message='adding a new file for weekly tests', content=file_content, branch=branch_name)
+
+new_commit = repo.get_commits()[0]
+
+# Create a new Pull Request
+pr = repo.create_pull(title=pr_title, body=pr_body, head='weekly_tests', base='develop')
+
+# Add labels to the Pull Request
+for label_name in label_names:
+    pr.add_to_labels(label_name)
+
+print(f'Pull Request {pr_title} successfully created')