Merge branch 'master' into python_package_reorg

.git-blame-ignore-revs

@@ -0,0 +1,3 @@
+# Migrated code style to Black
+eabf877cbb86b281fdd37a3fa3cc0edf9b8eb874
+321463922724b225988e517da54a18bad90bc316

.github/PULL_REQUEST_TEMPLATE

@@ -4,13 +4,13 @@ Lines should be wrapped at about 72 characters. Please also update
 the CIME documentation, if necessary, in doc/source/rst and indicate
 below if you need to have the gh-pages html regenerated.]
 
-Test suite: 
-Test baseline: 
-Test namelist changes: 
+Test suite:
+Test baseline:
+Test namelist changes:
 Test status: [bit for bit, roundoff, climate changing]
 
 Fixes [CIME Github issue #]
 
-User interface changes?: 
+User interface changes?:
 
 Update gh-pages html (Y/N)?:

.github/workflows/srt.yml

@@ -12,6 +12,27 @@ on:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      # Offical action is deprecated in favor of pre-commit.ci
+      # Should evaulate switching or just running manually.
+      # - name: Runs pre-commit action
+      #   # Do not run if using act tooling (https://github.com/nektos/act)
+      #   if: ${{ !env.ACT }}
+      #   uses: pre-commit/action@v2.0.3
+      - name: Runs pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit run -a
+
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
@@ -41,6 +62,12 @@ jobs:
 
       - uses: actions/checkout@v2
 
+      - name: ccs_config checkout
+        uses: actions/checkout@v2
+        with:
+          repository: ESMCI/ccs_config_cesm
+          path: ccs_config
+
       - name: share checkout
         uses: actions/checkout@v2
         with:
@@ -142,11 +169,12 @@ jobs:
           pwd
           mkdir -p $HOME/cesm/scratch
           mkdir -p $HOME/cesm/inputdata
-          cd $HOME/work/cime/cime/scripts/tests
+          cd $HOME/work/cime/cime
           export NETCDF=$HOME/netcdf-fortran
           export PATH=$NETCDF/bin:$PATH
           export LD_LIBRARY_PATH=$NETCDF/lib:$HOME/pnetcdf/lib:$LD_LIBRARY_PATH
-          ./scripts_regression_tests.py --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
+          python -m pip install pytest pytest-cov
+          pytest -vvv --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
 
 #     the following can be used by developers to login to the github server in case of errors
 #     see https://github.com/marketplace/actions/debugging-with-tmate for further details

.github/workflows/srt_nuopc.yml

@@ -32,7 +32,7 @@ jobs:
       NETCDF_C_PATH: /usr
       NETCDF_FORTRAN_PATH: ${HOME}/netcdf-fortran
       PNETCDF_PATH: ${HOME}/pnetcdf
-      ESMF_VERSION: ESMF_8_2_0_beta_snapshot_14
+      ESMF_VERSION: ESMF_8_3_0_beta_snapshot_05
       CIME_MODEL: cesm
 
     # Steps represent a sequence of tasks that will be executed as part of the job
@@ -41,6 +41,12 @@ jobs:
 
       - uses: actions/checkout@v2
 
+      - name: ccs_config checkout
+        uses: actions/checkout@v2
+        with:
+          repository: ESMCI/ccs_config_cesm
+          path: ccs_config
+
       - name: share checkout
         uses: actions/checkout@v2
         with:
@@ -180,12 +186,13 @@ jobs:
         run: |
           mkdir -p $HOME/cesm/scratch
           mkdir -p $HOME/cesm/inputdata
-          cd $HOME/work/cime/cime/scripts/tests
+          cd $HOME/work/cime/cime
           export NETCDF=$HOME/netcdf-fortran
           export PATH=$NETCDF/bin:$PATH
           export LD_LIBRARY_PATH=$NETCDF/lib:$HOME/pnetcdf/lib:$LD_LIBRARY_PATH
           export ESMFMKFILE=$HOME/ESMF/lib/libg/Linux.gfortran.64.openmpi.default/esmf.mk
-          ./scripts_regression_tests.py --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
+          python -m pip install pytest pytest-cov
+          pytest -vvv --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
 
 
 #     the following can be used by developers to login to the github server in case of errors

.gitignore

@@ -25,4 +25,3 @@ scripts/Tools/JENKINS*
 components
 libraries
 share
-

.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+exclude: ^utils/.*$
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: check-xml
+        files: config/
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 21.10b0
+    hooks:
+      - id: black
+        files: scripts/lib/CIME
+  - repo: https://github.com/PyCQA/pylint
+    rev: v2.11.1
+    hooks:
+      - id: pylint
+        args:
+          - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error
+        files: scripts/lib/CIME
+        exclude: scripts/lib/CIME/tests

CIME/BuildTools/cmakemacroswriter.py

@@ -12,6 +12,7 @@
 
 from CIME.BuildTools.macrowriterbase import MacroWriterBase
 from CIME.XML.standard_module_setup import *
+
 logger = logging.getLogger(__name__)
 
 
@@ -64,9 +65,13 @@ def shell_command_strings(self, command):
         # counter so that we get a different value next time.
         var_name = "CIME_TEMP_SHELL" + str(self._var_num)
         self._var_num += 1
-        set_up = "execute_process(COMMAND " + command + \
-                 " OUTPUT_VARIABLE " + var_name + \
-                 " OUTPUT_STRIP_TRAILING_WHITESPACE)"
+        set_up = (
+            "execute_process(COMMAND "
+            + command
+            + " OUTPUT_VARIABLE "
+            + var_name
+            + " OUTPUT_STRIP_TRAILING_WHITESPACE)"
+        )
         tear_down = "unset(" + var_name + ")"
         return (set_up, "${" + var_name + "}", tear_down)
 

CIME/BuildTools/configure.py

@@ -19,12 +19,27 @@
 from CIME.utils import expect, safe_copy
 from CIME.XML.compilers import Compilers
 from CIME.XML.env_mach_specific import EnvMachSpecific
+from CIME.XML.files import Files
+
+import shutil
 
 logger = logging.getLogger(__name__)
 
-def configure(machobj, output_dir, macros_format, compiler, mpilib, debug,
-              comp_interface, sysos, unit_testing=False, noenv=False, threaded=False,
-              extra_machines_dir=None):
+
+def configure(
+    machobj,
+    output_dir,
+    macros_format,
+    compiler,
+    mpilib,
+    debug,
+    comp_interface,
+    sysos,
+    unit_testing=False,
+    noenv=False,
+    threaded=False,
+    extra_machines_dir=None,
+):
     """Add Macros, Depends, and env_mach_specific files to a directory.
 
     Arguments:
@@ -41,16 +56,60 @@ def configure(machobj, output_dir, macros_format, compiler, mpilib, debug,
                          searched for a config_compilers.xml file.
     """
     # Macros generation.
-    suffixes = {'Makefile': 'make', 'CMake': 'cmake'}
-    macro_maker = Compilers(machobj, compiler=compiler, mpilib=mpilib,
-                            extra_machines_dir=extra_machines_dir)
+    suffixes = {"Makefile": "make", "CMake": "cmake"}
+
+    new_cmake_macros_dir = Files(comp_interface=comp_interface).get_value(
+        "CMAKE_MACROS_DIR"
+    )
+    macro_maker = None
     for form in macros_format:
-        out_file_name = os.path.join(output_dir,"Macros."+suffixes[form])
-        macro_maker.write_macros_file(macros_file=out_file_name, output_format=suffixes[form])
 
-    copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler)
-    generate_env_mach_specific(output_dir, machobj, compiler, mpilib,
-                               debug, comp_interface, sysos, unit_testing, threaded, noenv=noenv)
+        if (
+            form == "CMake"
+            and new_cmake_macros_dir is not None
+            and os.path.exists(new_cmake_macros_dir)
+            and not "CIME_NO_CMAKE_MACRO" in os.environ
+        ):
+            if not os.path.isfile(os.path.join(output_dir, "Macros.cmake")):
+                safe_copy(
+                    os.path.join(new_cmake_macros_dir, "Macros.cmake"), output_dir
+                )
+            if not os.path.exists(os.path.join(output_dir, "cmake_macros")):
+                shutil.copytree(
+                    new_cmake_macros_dir, os.path.join(output_dir, "cmake_macros")
+                )
+
+        else:
+            logger.warning("Using deprecated CIME makefile generators")
+            if macro_maker is None:
+                macro_maker = Compilers(
+                    machobj,
+                    compiler=compiler,
+                    mpilib=mpilib,
+                    extra_machines_dir=extra_machines_dir,
+                )
+
+            out_file_name = os.path.join(output_dir, "Macros." + suffixes[form])
+            macro_maker.write_macros_file(
+                macros_file=out_file_name, output_format=suffixes[form]
+            )
+
+    copy_depends_files(
+        machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler
+    )
+    generate_env_mach_specific(
+        output_dir,
+        machobj,
+        compiler,
+        mpilib,
+        debug,
+        comp_interface,
+        sysos,
+        unit_testing,
+        threaded,
+        noenv=noenv,
+    )
+
 
 def copy_depends_files(machine_name, machines_dir, output_dir, compiler):
     """
@@ -74,21 +133,40 @@ def copy_depends_files(machine_name, machines_dir, output_dir, compiler):
                 if not os.path.exists(outputdfile):
                     safe_copy(dfile, outputdfile)
 
-class FakeCase(object):
 
+class FakeCase(object):
     def __init__(self, compiler, mpilib, debug, comp_interface):
         # PIO_VERSION is needed to parse config_machines.xml but isn't otherwise used
         # by FakeCase
-        self._vals = {"COMPILER":compiler, "MPILIB":mpilib, "DEBUG":debug,
-                      "COMP_INTERFACE":comp_interface, "PIO_VERSION":2,
-                      "SMP_PRESENT":False}
+        self._vals = {
+            "COMPILER": compiler,
+            "MPILIB": mpilib,
+            "DEBUG": debug,
+            "COMP_INTERFACE": comp_interface,
+            "PIO_VERSION": 2,
+            "SMP_PRESENT": False,
+        }
 
     def get_value(self, attrib):
-        expect(attrib in self._vals, "FakeCase does not support getting value of '%s'" % attrib)
+        expect(
+            attrib in self._vals,
+            "FakeCase does not support getting value of '%s'" % attrib,
+        )
         return self._vals[attrib]
 
-def generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
-                               comp_interface, sysos, unit_testing, threaded, noenv=False):
+
+def generate_env_mach_specific(
+    output_dir,
+    machobj,
+    compiler,
+    mpilib,
+    debug,
+    comp_interface,
+    sysos,
+    unit_testing,
+    threaded,
+    noenv=False,
+):
     """
     env_mach_specific generation.
     """
@@ -97,20 +175,25 @@ def generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
         logger.warning("{} already exists, delete to replace".format(ems_path))
         return
 
-    ems_file = EnvMachSpecific(output_dir, unit_testing=unit_testing, standalone_configure=True)
-    ems_file.populate(machobj,attributes={"mpilib":mpilib,"compiler":compiler,"threaded":threaded})
+    ems_file = EnvMachSpecific(
+        output_dir, unit_testing=unit_testing, standalone_configure=True
+    )
+    ems_file.populate(
+        machobj,
+        attributes={"mpilib": mpilib, "compiler": compiler, "threaded": threaded},
+    )
     ems_file.write()
 
     if noenv:
         return
 
     fake_case = FakeCase(compiler, mpilib, debug, comp_interface)
     ems_file.load_env(fake_case)
-    for shell in ('sh', 'csh'):
+    for shell in ("sh", "csh"):
         ems_file.make_env_mach_specific_file(shell, fake_case, output_dir=output_dir)
         shell_path = os.path.join(output_dir, ".env_mach_specific." + shell)
-        with open(shell_path, 'a') as shell_file:
-            if shell == 'sh':
+        with open(shell_path, "a") as shell_file:
+            if shell == "sh":
                 shell_file.write("\nexport COMPILER={}\n".format(compiler))
                 shell_file.write("export MPILIB={}\n".format(mpilib))
                 shell_file.write("export DEBUG={}\n".format(repr(debug).upper()))