Merge pull request #183 from nusbaume/ParamGen_namelist

Add ParamGen and multi-namelist definition functionality

.github/scripts/branch_pr_issue_closer.py

@@ -53,9 +53,9 @@ def  project_card_move(oa_token, column_id, card_id):
     """
 
     #create required argument strings from inputs:
-    github_oa_header = ''' "Authorization: token {0}" '''.format(oa_token)
-    github_url_str = '''https://api.github.com/projects/columns/cards/{0}/moves'''.format(card_id)
-    json_post_inputs = ''' '{{"position":"top", "column_id":{}}}' '''.format(column_id)
+    github_oa_header = f''' "Authorization: token {oa_token}" '''
+    github_url_str = f'''https://api.github.com/projects/columns/cards/{card_id}/moves'''
+    json_post_inputs = f''' '{{"position":"top", "column_id":{column_id}}}' '''
 
     #Create curl command line string:
     curl_cmdline = '''curl -H '''+github_oa_header+''' -H "Accept: application/vnd.github.inertia-preview+json" -X POST -d '''+\
@@ -101,7 +101,7 @@ def end_script(msg):
     """
     Prints message to screen, and then exits script.
     """
-    print("\n{}\n".format(msg))
+    print(f"\n{msg}\n")
     print("Issue closing check has completed successfully.")
     sys.exit(0)
 
@@ -173,7 +173,7 @@ def _main_prog():
         first_word = post_msg_word_list[0]
 
         #Print merged pr number to screen:
-        print("Merged PR: {}".format(first_word))
+        print(f"Merged PR: {first_word}")
 
         try:
             #Try assuming the word is just a number:
@@ -267,10 +267,10 @@ def _main_prog():
     issue_pattern = re.compile(r'#[0-9]+(\s|,|$)|.')
 
     #Create new "close" issues list:
-    close_issues = list()
+    close_issues = []
 
     #Create new "closed" PR list:
-    close_pulls = list()
+    close_pulls = []
 
     #Search text right after keywords for possible issue numbers:
     for match in word_matches:
@@ -364,20 +364,20 @@ def _main_prog():
                         end_script(endmsg)
 
     #Print project name associated with merged PR:
-    print("merged PR project name: {}".format(proj_mod_name))
+    print(f"merged PR project name: {proj_mod_name}")
 
     #++++++++++++++++++++++++++++++++++++++++
     #Extract repo project "To do" card issues
     #++++++++++++++++++++++++++++++++++++++++
 
     #Initalize issue counting dictionary:
-    proj_issues_count = dict()
+    proj_issues_count = {}
 
     #Initalize issue id to project card id dictionary:
-    proj_issue_card_ids = dict()
+    proj_issue_card_ids = {}
 
     #Initialize list for issues that have already been closed:
-    already_closed_issues = list()
+    already_closed_issues = []
 
     #Loop over all repo projects:
     for project in projects:
@@ -467,7 +467,7 @@ def _main_prog():
             cam_issue = cam_repo.get_issue(number=issue_num)
             #Close issue:
             cam_issue.edit(state='closed')
-            print("Issue #{} has been closed.".format(issue_num))
+            print(f"Issue #{issue_num} has been closed.")
         else:
             #Extract card id from id dictionary:
             if issue_num in proj_issue_card_ids:
@@ -498,7 +498,7 @@ def _main_prog():
 
         #Close Pull Request:
         cam_pull.edit(state='closed')
-        print("Pull Request #{} has been closed.".format(pull_num))
+        print(f"Pull Request #{pull_num} has been closed.")
 
     #++++++++++
     #End script

.github/scripts/pr_mod_file_tests.py

@@ -16,8 +16,6 @@
 
 import sys
 import os
-import subprocess
-import shlex
 import argparse
 
 from stat import S_ISREG
@@ -63,7 +61,7 @@ def _file_is_python(filename):
             #If no ".py" extension exists, then
             #open the file and look for a shabang
             #that contains the word "python".
-            with open(filename, "r") as mod_file:
+            with open(filename, "r", encoding='utf-8') as mod_file:
                 #Loop over lines in file:
                 for line in mod_file:
 
@@ -170,7 +168,7 @@ def _main_prog():
     #++++++++++++++++++++++++++++++
 
     #Create empty list to store python files:
-    pyfiles = list()
+    pyfiles = []
 
     #Extract Github file objects:
     file_obj_list = pull_req.get_files()
@@ -220,14 +218,14 @@ def _main_prog():
             print("+++++++++++++++++++++++++++++++++++++++++++++++")
 
             #Raise test failure exception:
-            fail_msg = "One or more files are below allowed pylint " \
-                       "score of {}.\nPlease see pylint message(s) " \
-                       "above for possible fixes."
+            fail_msg = "One or more files are below allowed pylint "
+            fail_msg += f"score of {pylev}.\nPlease see pylint message(s) "
+            fail_msg += "above for possible fixes."
             raise PrModTestFail(fail_msg)
-        else:
-            #All tests have passed, so exit normally:
-            print("All pylint tests passed!")
-            sys.exit(0)
+
+        #All tests have passed, so exit normally:
+        print("All pylint tests passed!")
+        sys.exit(0)
 
     #If no python files exist in PR, then exit script:
     else:

.github/workflows/branch_push_workflow.yml

@@ -16,25 +16,23 @@ jobs:
   #This job is designed to close any issues or pull requests specified
   #in the body of a pull request merged into a non-default branch.
   issue_closer:
+    if: github.repository == 'NCAR/CAMDEN' # Only run on main repo
     runs-on: ubuntu-latest
     steps:
     # Acquire github action routines
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     # Acquire specific version of python
-    - name: Set up Python 3.6
-      if: github.repository == 'NCAR/CAMDEN' # Only run on main repo
+    - name: Set up Python 3.10
       uses: actions/setup-python@v1
       with:
-        python-version: '3.6' # Semantic version range syntax or exact version of a Python version
+        python-version: '3.10' # Semantic version range syntax or exact version of a Python version
     # Install required python packages
     - name: Install dependencies
-      if: github.repository == 'NCAR/CAMDEN' # Only run on main repo
       run: |
         python -m pip install --upgrade pip  # Install latest version of PIP
         pip install PyGithub                 # Install PyGithub python package
     # Run CAM issue-closing script
     - name: Run python Github issue-closing script
-      if: github.repository == 'NCAR/CAMDEN' # Only run on main repo
       env:
         ACCESS_TOKEN: ${{ secrets.WRITE_ACCESS_TOKEN }}
       run: .github/scripts/branch_pr_issue_closer.py --access_token $ACCESS_TOKEN --trigger_sha $GITHUB_SHA

.github/workflows/pr_open_sync_workflow.yml

@@ -13,11 +13,11 @@ jobs:
     strategy:
       matrix:
         #All of these python versions will be used to run tests:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
       fail-fast: false
     steps:
     # Acquire github action routines:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     # Acquire specific version of python:
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
@@ -35,7 +35,9 @@ jobs:
         pip install pytest
     # Run "checkout_externals":
     - name: Checkout externals
-      run: manage_externals/checkout_externals --externals Externals_CAM.cfg ccpp-framework
+      run: |
+        manage_externals/checkout_externals --externals Externals_CAM.cfg ccpp-framework
+        manage_externals/checkout_externals --externals Externals.cfg cime
     # Run python unit and doctests:
     - name: python unit tests
       run: |
@@ -47,12 +49,20 @@ jobs:
         python cime_config/cam_autogen.py
         # CAM build cache doctests:
         python cime_config/cam_build_cache.py
+        # Namelist reader generator doctests:
+        python cime_config/create_readnl_files.py
         # Registry generator doctests:
         pytest src/data/generate_registry_data.py --doctest-modules
+        # ParamGen atm_in namelist writer doctests:
+        pytest cime_config/atm_in_paramgen.py --doctest-modules
         # Registry generator unit tests:
         python test/unit/test_registry.py
+        # Namelist reader autogeneration unit tests
+        python test/unit/create_readnl_files_tests.py
         # Physics variable init (phys_init) generator unit tests:
         python test/unit/write_init_unit_tests.py
+        # ParamGen atm_in namelist writer unit tests:
+        python test/unit/test_atm_in_paramgen.py
 
 #####
 
@@ -65,12 +75,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     # acquire github action routines
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     # acquire specific version of python
-    - name: Set up Python 3.8
+    - name: Set up Python 3.10
       uses: actions/setup-python@v1
       with:
-        python-version: '3.8' # Use python 3.8
+        python-version: "3.10" # Use python 3.8
     # install required python packages
     - name: Install dependencies
       run: |

.gitignore

@@ -1,7 +1,10 @@
 # Ignore externals
 ccpp_framework
+ccs_config
 chem_proc
 cime
+share
+libraries
 components
 manage_externals.log
 src/physics/carma/base

Externals.cfg

@@ -1,42 +1,49 @@
-[cice]
-tag = cice5_20210802
+[ccs_config]
+tag = ccs_config_cesm0.0.28
+protocol = git
+repo_url = https://github.com/ESMCI/ccs_config_cesm
+local_path = ccs_config
+required = True
+
+[cice5]
+tag = cice5_20220204
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CICE5
-local_path = components/cice
+local_path = components/cice5
 required = True
 
 [cice6]
-tag = cesm_cice6_2_0_5
+tag = cesm_cice6_2_0_21
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CICE
-local_path = components/cice6
+local_path = components/cice
 externals = Externals.cfg
-required = False
+required = True
 
 [cmeps]
-tag = cmeps0.13.40
+tag = cmeps0.13.62
 protocol = git
 repo_url = https://github.com/ESCOMP/CMEPS.git
 local_path = components/cmeps
 required = True
 
 [cdeps]
-tag = cdeps0.12.32
+tag = cdeps0.12.46
 protocol = git
 repo_url = https://github.com/ESCOMP/CDEPS.git
 local_path = components/cdeps
 externals =  Externals_CDEPS.cfg
 required = True
 
 [cpl7]
-tag = cpl7.0.5
+tag = cpl7.0.12
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps
 local_path = components/cpl7
 required = True
 
 [share]
-tag = share1.0.8
+tag = share1.0.11
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_share
 local_path = share
@@ -50,29 +57,29 @@ local_path = libraries/mct
 required = True
 
 [parallelio]
-tag = pio2_5_4
+tag = pio2_5_6
 protocol = git
 repo_url = https://github.com/NCAR/ParallelIO
 local_path = libraries/parallelio
 required = True
 
 [cime]
-branch = CAMDEN_ParamGen
+tag = cime6.0.38
 protocol = git
-repo_url = https://github.com/gold2718/cime
+repo_url = https://github.com/ESMCI/cime
 local_path = cime
 required = True
 
 [cism]
-tag = cismwrap_2_1_93
+tag = cismwrap_2_1_95
 protocol = git
 repo_url = https://github.com/ESCOMP/CISM-wrapper
 local_path = components/cism
 externals = Externals_CISM.cfg
 required = True
 
 [clm]
-tag = ctsm5.1.dev064
+tag = ctsm5.1.dev082
 protocol = git
 repo_url = https://github.com/ESCOMP/CTSM
 local_path = components/clm
@@ -88,7 +95,7 @@ externals = Externals_FMS.cfg
 required = True
 
 [mosart]
-tag = mosart1_0_44
+tag = mosart1_0_45
 protocol = git
 repo_url = https://github.com/ESCOMP/MOSART
 local_path = components/mosart

Externals_CAM.cfg

@@ -2,7 +2,7 @@
 local_path = ccpp_framework
 protocol = git
 repo_url = https://github.com/gold2718/ccpp-framework
-tag = CPF_0.2.030
+tag = CPF_0.2.032
 required = True
 
 [cosp2]
@@ -23,7 +23,7 @@ required = False
 local_path = src/physics/ncar_ccpp
 protocol = git
 repo_url = https://github.com/NCAR/atmospheric_physics
-tag = atmos_phys0_00_014
+tag = atmos_phys0_00_015
 required = True
 
 [silhs]