Updates pre-commit config and adds related fixes

.github/workflows/srt.yml

@@ -24,7 +24,7 @@ jobs:
           python-version: 3.9
       - name: Install and run pre-commit
         uses: pre-commit/action@v2.0.3
-  
+
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: 'utils/'
+exclude: ^utils/.*$
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -16,12 +16,8 @@ repos:
           - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error
         files: 'scripts/lib/CIME'
         exclude: 'scripts/lib/CIME/tests'
-  - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
-      - id: flake8
   - repo: https://github.com/psf/black
     rev: 21.10b0
     hooks:
       - id: black
-        exclude: "config"
+        files: 'scripts/lib/CIME'

config/cesm/machines/config_machines.xml

@@ -543,7 +543,7 @@ This allows using a different mpirun command to launch unit tests
 <!--  Due to a bug in romio these are not working -->
         <command name="load">cray-netcdf-hdf5parallel</command>
         <command name="load">cray-hdf5-parallel</command>
-        <command name="load">cray-parallel-netcdf</command> 
+        <command name="load">cray-parallel-netcdf</command>
 <!--
         <command name="load">cray-hdf5</command>
         <command name="load">cray-netcdf</command>

conftest.py

@@ -20,16 +20,13 @@ def pytest_addoption(parser):
     scripts_regression_tests.setup_arguments(parser)
 
     # verbose and debug flags already exist
-    parser.addoption("--silent", action="store_true",
-                     help="Disable all logging")
+    parser.addoption("--silent", action="store_true", help="Disable all logging")
 
 
 def pytest_configure(config):
     kwargs = vars(config.option)
 
-    utils.configure_logging(kwargs['verbose'],
-                            kwargs['debug'],
-                            kwargs['silent'])
+    utils.configure_logging(kwargs["verbose"], kwargs["debug"], kwargs["silent"])
 
     scripts_regression_tests.configure_tests(**kwargs)
 

doc/source/users_guide/testing.rst

@@ -486,13 +486,13 @@ Install dependencies::
 
 Run full suite::
 
-  pytest -vvv 
+  pytest -vvv
 
 Run just unit tests::
 
   pytest -vvv scripts/lib/CIME/tests/test_unit*
 
-Run a test class:: 
+Run a test class::
 
   pytest -vvv scripts/lib/CIME/tests/test_unit_case.py
 

scripts/lib/CIME/SystemTests/ers2.py

@@ -36,7 +36,7 @@ def _ers2_second_phase(self):
         stop_n = self._case.get_value("STOP_N")
         stop_option = self._case.get_value("STOP_OPTION")
 
-        rest_n = int(stop_n/2 + 1)
+        rest_n = int(stop_n / 2 + 1)
         stop_new = rest_n
 
         self._case.set_value("REST_OPTION", stop_option)

scripts/lib/CIME/SystemTests/pgn.py

@@ -269,7 +269,9 @@ def run_phase(self):
         logger.debug("PGN_INFO: Case name is:{}".format(casename))
 
         for icond in range(NUMBER_INITIAL_CONDITIONS):
-            for iprt, (prt_name, prt_value) in enumerate(PERTURBATIONS.items()):  # pylint: disable=unused-variable
+            for iprt, (prt_name, prt_value) in enumerate(
+                PERTURBATIONS.items()
+            ):  # pylint: disable=unused-variable
                 iinst = pg._sub2instance(icond, iprt, len(PERTURBATIONS))
                 fname = os.path.join(
                     rundir,

scripts/lib/CIME/SystemTests/test_utils/user_nl_utils.py

@@ -5,6 +5,7 @@
 import os
 import glob
 
+
 def append_to_user_nl_files(caseroot, component, contents):
     """
     Append the string given by 'contents' to the end of each user_nl file for
@@ -30,11 +31,12 @@ def append_to_user_nl_files(caseroot, component, contents):
     files = _get_list_of_user_nl_files(caseroot, component)
 
     if len(files) == 0:
-        raise RuntimeError('No user_nl files found for component ' + component)
+        raise RuntimeError("No user_nl files found for component " + component)
 
     for one_file in files:
-        with open(one_file, 'a') as user_nl_file:
-            user_nl_file.write('\n' + contents + '\n')
+        with open(one_file, "a") as user_nl_file:
+            user_nl_file.write("\n" + contents + "\n")
+
 
 def _get_list_of_user_nl_files(path, component):
     """Get a list of all user_nl files in the current path for the component
@@ -45,7 +47,7 @@ def _get_list_of_user_nl_files(path, component):
     The list of returned files gives their full path.
     """
 
-    file_pattern = 'user_nl_' + component + '*'
+    file_pattern = "user_nl_" + component + "*"
     file_list = glob.glob(os.path.join(path, file_pattern))
 
     return file_list

scripts/lib/CIME/XML/machines.py

@@ -194,12 +194,19 @@ def _probe_machine_name_one_guess(self, nametomatch):
                 logger.debug("machine regex string is " + regex_str)
                 # an environment variable can be used
                 if regex_str.startswith("$ENV"):
-                    machine_value = self.get_resolved_value(regex_str, allow_unresolved_envvars=True)
+                    machine_value = self.get_resolved_value(
+                        regex_str, allow_unresolved_envvars=True
+                    )
                     if not machine_value.startswith("$ENV"):
                         try:
                             match, this_machine = machine_value.split(":")
                         except ValueError:
-                            expect(False,"Bad formation of NODENAME_REGEX.  Expected envvar:value, found {}".format(regex_str))
+                            expect(
+                                False,
+                                "Bad formation of NODENAME_REGEX.  Expected envvar:value, found {}".format(
+                                    regex_str
+                                ),
+                            )
                         if match == this_machine:
                             machine = machtocheck
                             break

scripts/lib/CIME/XML/test_reporter.py

@@ -7,9 +7,10 @@
 import urllib.request
 from CIME.XML.standard_module_setup import *
 from CIME.XML.generic_xml import GenericXML
-from CIME.utils import expect,get_model
+from CIME.utils import expect, get_model
 import ssl
-#pylint: disable=protected-access
+
+# pylint: disable=protected-access
 ssl._create_default_https_context = ssl._create_unverified_context
 
 

scripts/lib/CIME/provenance.py

@@ -470,6 +470,7 @@ def _save_prerun_timing_e3sm(case, lid):
             ) as fd:
                 fd.write("{}\n".format(syslog_jobid))
 
+
 def _cleanup_spio_stats(case):
     rundir = case.get_value("RUNDIR")
     for item in glob.glob(os.path.join(rundir, "io_perf_summary*")):
@@ -482,7 +483,12 @@ def _cleanup_spio_stats(case):
     try:
         os.makedirs(spio_stats_dir)
     except OSError:
-        logger.warning("{} could not be created. Scorpio I/O statistics will be stored in the run directory.".format(spio_stats_dir))
+        logger.warning(
+            "{} could not be created. Scorpio I/O statistics will be stored in the run directory.".format(
+                spio_stats_dir
+            )
+        )
+
 
 def _save_prerun_provenance_e3sm(case, lid):
     _cleanup_spio_stats(case)

scripts/lib/CIME/tests/scripts_regression_tests.py

@@ -76,41 +76,74 @@ def cleanup(test_root):
 
 
 def setup_arguments(parser):
-    parser.add_argument("--fast", action="store_true",
-                        help="Skip full system tests, which saves a lot of time")
-
-    parser.add_argument("--no-batch", action="store_true",
-                        help="Do not submit jobs to batch system, run locally."
-                        " If false, will default to machine setting.")
-
-    parser.add_argument("--no-fortran-run", action="store_true",
-                        help="Do not run any fortran jobs. Implies --fast"
-                        " Used for github actions")
-
-    parser.add_argument("--no-cmake", action="store_true",
-                        help="Do not run cmake tests")
-
-    parser.add_argument("--no-teardown", action="store_true",
-                        help="Do not delete directories left behind by testing")
-
-    parser.add_argument("--machine",
-                        help="Select a specific machine setting for cime", default=None)
-
-    parser.add_argument("--compiler",
-                        help="Select a specific compiler setting for cime", default=None)
-
-    parser.add_argument( "--mpilib",
-                        help="Select a specific compiler setting for cime", default=None)
-
-    parser.add_argument( "--test-root",
-                        help="Select a specific test root for all cases created by the testing", default=None)
-
-    parser.add_argument("--timeout", type=int,
-                        help="Select a specific timeout for all tests", default=None)
-
-
-def configure_tests(timeout, no_fortran_run, fast, no_batch, no_cmake,
-                    no_teardown, machine, compiler, mpilib, test_root, **kwargs):
+    parser.add_argument(
+        "--fast",
+        action="store_true",
+        help="Skip full system tests, which saves a lot of time",
+    )
+
+    parser.add_argument(
+        "--no-batch",
+        action="store_true",
+        help="Do not submit jobs to batch system, run locally."
+        " If false, will default to machine setting.",
+    )
+
+    parser.add_argument(
+        "--no-fortran-run",
+        action="store_true",
+        help="Do not run any fortran jobs. Implies --fast" " Used for github actions",
+    )
+
+    parser.add_argument(
+        "--no-cmake", action="store_true", help="Do not run cmake tests"
+    )
+
+    parser.add_argument(
+        "--no-teardown",
+        action="store_true",
+        help="Do not delete directories left behind by testing",
+    )
+
+    parser.add_argument(
+        "--machine", help="Select a specific machine setting for cime", default=None
+    )
+
+    parser.add_argument(
+        "--compiler", help="Select a specific compiler setting for cime", default=None
+    )
+
+    parser.add_argument(
+        "--mpilib", help="Select a specific compiler setting for cime", default=None
+    )
+
+    parser.add_argument(
+        "--test-root",
+        help="Select a specific test root for all cases created by the testing",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        help="Select a specific timeout for all tests",
+        default=None,
+    )
+
+
+def configure_tests(
+    timeout,
+    no_fortran_run,
+    fast,
+    no_batch,
+    no_cmake,
+    no_teardown,
+    machine,
+    compiler,
+    mpilib,
+    test_root,
+    **kwargs
+):
     config = CIME.utils.get_cime_config()
 
     if timeout:
@@ -177,8 +210,7 @@ def configure_tests(timeout, no_fortran_run, fast, no_batch, no_cmake,
 
 
 def _main_func(description):
-    help_str = \
-"""
+    help_str = """
 {0} [TEST] [TEST]
 OR
 {0} --help
@@ -192,25 +224,27 @@ def _main_func(description):
 
     \033[1;32m# Run test test_wait_for_test_all_pass from class M_TestWaitForTests \033[0m
     > {0} M_TestWaitForTests.test_wait_for_test_all_pass
-""".format(os.path.basename(sys.argv[0]))
+""".format(
+        os.path.basename(sys.argv[0])
+    )
 
-    parser = argparse.ArgumentParser(usage=help_str,
-                                     description=description,
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser = argparse.ArgumentParser(
+        usage=help_str,
+        description=description,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
 
     setup_arguments(parser)
 
-    parser.add_argument("--verbose", action="store_true",
-                        help="Enable verbose logging")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
 
-    parser.add_argument("--debug", action="store_true",
-                        help="Enable debug logging")
+    parser.add_argument("--debug", action="store_true", help="Enable debug logging")
 
-    parser.add_argument("--silent", action="store_true",
-                        help="Disable all logging")
+    parser.add_argument("--silent", action="store_true", help="Disable all logging")
 
-    parser.add_argument("tests", nargs="*",
-                        help="Specific tests to run e.g. test_unit*")
+    parser.add_argument(
+        "tests", nargs="*", help="Specific tests to run e.g. test_unit*"
+    )
 
     ns, args = parser.parse_known_args()