Merge remote-tracking branch 'origin' into precommit

fischer-ncar · Dec 15, 2021 · 074b194 · 074b194
2 parents 3214639 + baf320d
commit 074b194
Show file tree

Hide file tree

Showing 64 changed files with 400 additions and 344 deletions.
diff --git a/.github/workflows/srt.yml b/.github/workflows/srt.yml
@@ -12,6 +12,19 @@ on:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install and run pre-commit
+        uses: pre-commit/action@v2.0.3
+
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
@@ -142,11 +155,12 @@ jobs:
           pwd
           mkdir -p $HOME/cesm/scratch
           mkdir -p $HOME/cesm/inputdata
-          cd $HOME/work/cime/cime/scripts/tests
+          cd $HOME/work/cime/cime
           export NETCDF=$HOME/netcdf-fortran
           export PATH=$NETCDF/bin:$PATH
           export LD_LIBRARY_PATH=$NETCDF/lib:$HOME/pnetcdf/lib:$LD_LIBRARY_PATH
-          ./scripts_regression_tests.py --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
+          python -m pip install pytest pytest-cov
+          pytest -vvv --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
 
 #     the following can be used by developers to login to the github server in case of errors
 #     see https://github.com/marketplace/actions/debugging-with-tmate for further details

diff --git a/.github/workflows/srt_nuopc.yml b/.github/workflows/srt_nuopc.yml
@@ -180,12 +180,13 @@ jobs:
         run: |
           mkdir -p $HOME/cesm/scratch
           mkdir -p $HOME/cesm/inputdata
-          cd $HOME/work/cime/cime/scripts/tests
+          cd $HOME/work/cime/cime
           export NETCDF=$HOME/netcdf-fortran
           export PATH=$NETCDF/bin:$PATH
           export LD_LIBRARY_PATH=$NETCDF/lib:$HOME/pnetcdf/lib:$LD_LIBRARY_PATH
           export ESMFMKFILE=$HOME/ESMF/lib/libg/Linux.gfortran.64.openmpi.default/esmf.mk
-          ./scripts_regression_tests.py --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
+          python -m pip install pytest pytest-cov
+          pytest -vvv --no-fortran-run --compiler gnu --mpilib openmpi --machine ubuntu-latest
 
 
 #     the following can be used by developers to login to the github server in case of errors

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,9 @@ repos:
     hooks:
       - id: pylint
         args:
-          - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed
+          - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error
+        files: 'scripts/lib/CIME'
+        exclude: 'scripts/lib/CIME/tests'
   - repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
     hooks:

diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
@@ -486,7 +486,7 @@ This allows using a different mpirun command to launch unit tests
     <DESC>This is using the cray env emulator on cheyenne </DESC>
     <NODENAME_REGEX>$ENV{CRAY_CPU_TARGET}:broadwell</NODENAME_REGEX>
     <OS>CNL</OS>
-    <COMPILERS>gnu,cray</COMPILERS>
+    <COMPILERS>cray,gnu</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
 <!--    <PROJECT></PROJECT> -->
     <CIME_OUTPUT_ROOT>/glade/scratch/$USER/CPE</CIME_OUTPUT_ROOT>
@@ -505,54 +505,49 @@ This allows using a different mpirun command to launch unit tests
       <executable>srun</executable>
       <arguments>
         <arg name="num_tasks">--ntasks={{ total_tasks }}</arg>
-        <arg name="cpu_bind">--cpu_bind=sockets --cpu_bind=verbose</arg>
+        <arg name="labelstdout">--label</arg>
         <arg name="kill-on-bad-exit">--kill-on-bad-exit</arg>
       </arguments>
     </mpirun>
     <module_system type="module">
-      <init_path lang="perl">/usr/local/Modules/init/perl.pm</init_path>
-      <init_path lang="python">/usr/local/Modules/init/python.py</init_path>
-      <init_path lang="sh">/usr/local/Modules/init/sh</init_path>
-      <init_path lang="csh">/usr/local/Modules/init/csh</init_path>
-      <cmd_path lang="perl">/usr/local/Modules/bin/modulecmd perl</cmd_path>
-      <cmd_path lang="python">/usr/local/Modules/bin/modulecmd python</cmd_path>
+      <init_path lang="perl">/opt/cray/pe/modules/default/init/perl.pm</init_path>
+      <init_path lang="python">/opt/cray/pe/modules/default/init/python.py</init_path>
+      <init_path lang="sh">/opt/cray/pe/modules/default/init/sh</init_path>
+      <init_path lang="csh">/opt/cray/pe/modules/default/init/csh</init_path>
+      <cmd_path lang="perl">/opt/cray/pe/modules/default/bin/modulecmd perl</cmd_path>
+      <cmd_path lang="python">/opt/cray/pe/modules/default/bin/modulecmd python</cmd_path>
       <cmd_path lang="sh">module</cmd_path>
       <cmd_path lang="csh">module</cmd_path>
-      <modules>
-        <command name="unload">cpe-cray</command>
-        <command name="unload">cpe-gnu</command>
-        <command name="unload">cray-netcdf-hdf5para</command>
-        <command name="unload">cray-netcdf</command>
-      </modules>
       <modules compiler="cray">
-        <command name="load">cpe-cray</command>
-        <command name="switch">cce cce/11.0.2</command>
+        <command name="load">PrgEnv-cray/8.1.0</command>
+        <command name="switch">cce cce/12.0.3</command>
       </modules>
       <modules compiler="gnu">
-        <command name="load">cpe-gnu</command>
+        <command name="load">PrgEnv-gnu/8.1.0</command>
         <command name="switch">gcc gcc/9.3.0</command>
       </modules>
       <modules>
         <command name="load">perftools-base</command>
         <command name="swap">craype craype/2.7.9</command>
       </modules>
       <modules compiler="!intel">
-        <command name="switch">cray-libsci/20.12.1.2</command>
+        <command name="switch">cray-libsci/21.08.1.2</command>
       </modules>
       <modules>
         <command name="unload">cray-mpich</command>
-        <command name="load">cray-mpich/8.1.6.55</command>
+        <command name="load">cray-mpich/8.1.9</command>
       </modules>
       <modules mpilib="mpich">
         <command name="unload">cray-netcdf</command>
         <command name="unload">cray-hdf5</command>
-<!--  Due to a bug in romio these are not working
+<!--  Due to a bug in romio these are not working -->
         <command name="load">cray-netcdf-hdf5parallel</command>
         <command name="load">cray-hdf5-parallel</command>
-        <command name="load">cray-parallel-netcdf</command> -->
+        <command name="load">cray-parallel-netcdf</command> 
+<!--
         <command name="load">cray-hdf5</command>
         <command name="load">cray-netcdf</command>
-
+-->
       </modules>
       <modules mpilib="mpi-serial">
         <command name="unload">cray-netcdf-hdf5parallel</command>
@@ -2064,6 +2059,7 @@ This allows using a different mpirun command to launch unit tests
       <cmd_path lang="sh">module</cmd_path>
       <modules>
         <command name="purge"></command>
+        <command name="load">lang/python/3.7.0</command>
       </modules>
       <modules compiler="gnu">
         <command name="load">compiler/gnu/9.3.0</command>
@@ -2098,68 +2094,68 @@ This allows using a different mpirun command to launch unit tests
       </modules>
       <modules compiler="gnu" mpilib="mvapich2" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/gfortran/9.3.0</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mvapich2-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-O</command>
       </modules>
       <modules compiler="gnu" mpilib="mvapich2" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/gfortran/9.3.0</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mvapich2-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-g</command>
       </modules>
       <modules compiler="gnu" mpilib="mpi-serial" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/gfortran/9.3.0</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mpiuni-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-O</command>
       </modules>
       <modules compiler="gnu" mpilib="mpi-serial" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/gfortran/9.3.0</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mpiuni-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-g</command>
       </modules>
 
       <modules compiler="nag" mpilib="mvapich2" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/nag/6.2</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mvapich2-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-O</command>
       </modules>
       <modules compiler="nag" mpilib="mvapich2" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/nag/6.2</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mvapich2-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-g</command>
       </modules>
       <modules compiler="nag" mpilib="mpi-serial" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/nag/6.2</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mpiuni-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-g</command>
       </modules>
       <modules compiler="nag" mpilib="mpi-serial" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/nag/6.2</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mpiuni-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-O</command>
       </modules>
       <modules compiler="intel" mpilib="mpi-serial" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/intel/20.0.1</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mpiuni-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-g</command>
       </modules>
       <modules compiler="intel" mpilib="mpi-serial" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/intel/20.0.1</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mpiuni-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-O</command>
       </modules>
       <modules compiler="intel" mpilib="mvapich2" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/intel/20.0.1</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mvapich2-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-g</command>
       </modules>
       <modules compiler="intel" mpilib="mvapich2" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/intel/20.0.1</command>
-        <command name="load">esmf-8.2.0b13-ncdfio-mvapich2-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-O</command>
       </modules>
       <modules compiler="pgi" mpilib="mpi-serial" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/pgi/20.1</command>
-        <command name="load">esmf-8.2.0b15-ncdfio-mpiuni-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-g</command>
       </modules>
       <modules compiler="pgi" mpilib="mpi-serial" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/pgi/20.1</command>
-        <command name="load">esmf-8.2.0b15-ncdfio-mpiuni-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mpiuni-O</command>
       </modules>
       <modules compiler="pgi" mpilib="mvapich2" DEBUG="TRUE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/pgi/20.1</command>
-        <command name="load">esmf-8.2.0b15-ncdfio-mvapich2-g</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-g</command>
       </modules>
       <modules compiler="pgi" mpilib="mvapich2" DEBUG="FALSE">
         <command name="use">/fs/cgd/data0/modules/modulefiles/esmfpkgs/pgi/20.1</command>
-        <command name="load">esmf-8.2.0b15-ncdfio-mvapich2-O</command>
+        <command name="load">esmf-8.2.0-ncdfio-mvapich2-O</command>
       </modules>
     </module_system>
     <environment_variables>

diff --git a/conftest.py b/conftest.py
@@ -0,0 +1,39 @@
+import os
+import sys
+
+CIMEROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "scripts", "lib"))
+sys.path.insert(0, CIMEROOT)
+
+import pytest
+
+from CIME import utils
+from CIME.tests import scripts_regression_tests
+
+os.environ["CIME_GLOBAL_WALLTIME"] = "0:05:00"
+
+
+def pytest_addoption(parser):
+    # set addoption as add_argument to use common argument setup
+    # pytest's addoption has same signature as add_argument
+    setattr(parser, "add_argument", parser.addoption)
+
+    scripts_regression_tests.setup_arguments(parser)
+
+    # verbose and debug flags already exist
+    parser.addoption("--silent", action="store_true",
+                     help="Disable all logging")
+
+
+def pytest_configure(config):
+    kwargs = vars(config.option)
+
+    utils.configure_logging(kwargs['verbose'],
+                            kwargs['debug'],
+                            kwargs['silent'])
+
+    scripts_regression_tests.configure_tests(**kwargs)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup(pytestconfig):
+    os.chdir(CIMEROOT)
diff --git a/doc/source/users_guide/cime-change-namelist.rst b/doc/source/users_guide/cime-change-namelist.rst
@@ -298,7 +298,7 @@ CIME calls **$SRCROOT/components/cam/cime_config/buildnml** to generate the CAM'
 
 CAM-specific CIME xml variables are set in **$SRCROOT/components/cam/cime_config/config_component.xml** and are used by CAM's **buildnml** script to generate the namelist.
 
-For complete documentation of namelist settings, see `CAM namelist variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_.
+For complete documentation of namelist settings, see `CAM namelist variables <https://www.cesm.ucar.edu/models/cesm2/settings/current/cam_nml.html>`_.
 
 To modify CAM namelist settings, add the appropriate keyword/value pair at the end of the **$CASEROOT/user_nl_cam** file. (See the documentation for each file at the top of that file.)
 
@@ -316,7 +316,7 @@ CIME calls **$SRCROOT/components/clm/cime_config/buildnml** to generate the CLM
 
 CLM-specific CIME xml variables are set in **$SRCROOT/components/clm/cime_config/config_component.xml** and are used by CLM's **buildnml** script to generate the namelist.
 
-For complete documentation of namelist settings, see `CLM namelist variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_.
+For complete documentation of namelist settings, see `CLM namelist variables <https://www.cesm.ucar.edu/models/cesm2/settings/current/clm5_0_nml.html>`_.
 
 To modify CLM namelist settings, add the appropriate keyword/value pair at the end of the **$CASEROOT/user_nl_clm** file.
 
@@ -336,34 +336,32 @@ CICE
 
 CIME calls **$SRCROOT/components/cice/cime_config/buildnml** to generate the CICE namelist variables.
 
-For complete documentation of namelist settings, see `CICE namelist variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_.
+For complete documentation of namelist settings, see `CICE namelist variables <https://www.cesm.ucar.edu/models/cesm2/settings/current/cice_nml.html>`_.
 
 To modify CICE namelist settings, add the appropriate keyword/value pair at the end of the **$CASEROOT/user_nl_cice** file.
 (See the documentation for each file at the top of that file.)
 To see the result of your change, call `preview_namelists <../Tools_user/preview_namelists.html>`_ and verify that the changes appear correctly in **CaseDocs/ice_in**.
 
-In addition, `case.setup <../Tools_user/case.setup.html>`_  creates CICE's compile time `block decomposition variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_ in **env_build.xml** as follows:
+In addition, `case.setup <../Tools_user/case.setup.html>`_  creates CICE's compile time `block decomposition variables <https://www.cesm.ucar.edu/models/cesm2/settings/current/cice_input.html>`_ in **env_build.xml**.
 
 POP2
 ~~~~
 
 CIME calls **$SRCROOT/components/pop2/cime_config/buildnml** to generate the POP2 namelist variables.
 
-For complete documentation of namelist settings, see `POP2 namelist variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_.
+For complete documentation of namelist settings, see `POP2 namelist variables <https://www.cesm.ucar.edu/models/cesm2/settings/current/pop2_nml.html>`_.
 
 To modify POP2 namelist settings, add the appropriate keyword/value pair at the end of the **$CASEROOT/user_nl_pop2** file.
 (See the documentation for each file at the top of that file.)
 To see the result of your change, call `preview_namelists <../Tools_user/preview_namelists.html>`_ and verify that the changes appear correctly in **CaseDocs/ocn_in**.
 
-In addition, `case.setup <../Tools_user/case.setup.html>`_ generates POP2's compile-time `block decomposition variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_ in **env_build.xml** as shown here:
-
 CISM
 ~~~~
 
-See `CISM namelist variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_ for a complete description of the CISM runtime namelist variables. This includes variables that appear both in **cism_in** and in **cism.config**.
+See `CISM namelist variables <https://www.cesm.ucar.edu/models/cesm2/settings/current/cism_nml.html>`_ for a complete description of the CISM runtime namelist variables. This includes variables that appear both in **cism_in** and in **cism.config**.
 
 To modify any of these settings, add the appropriate keyword/value pair at the end of the **user_nl_cism** file. (See the documentation for each file at the top of that file.)
 Note that there is no distinction between variables that will appear in **cism_in** and those that will appear in **cism.config**: simply add a new variable setting in **user_nl_cism**, and it will be added to the appropriate place in **cism_in** or **cism.config**.
 To see the result of your change, call `preview_namelists <../Tools_user/preview_namelists.html>`_ and verify that the changes appear correctly in **CaseDocs/cism_in** and **CaseDocs/cism.config**.
 
-Some CISM runtime settings are sets via **env_run.xml**, as documented in `CISM runtime variables <http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_.
+Some CISM runtime settings are sets via **env_run.xml**, as documented in `CISM runtime variables <https://www.cesm.ucar.edu/models/cesm2/settings/current/cism_input.html>`_.
diff --git a/doc/source/users_guide/grids.rst b/doc/source/users_guide/grids.rst
@@ -94,7 +94,7 @@ The steps for adding a new component grid to the model system follow. This proce
    If you are introducing just one new grid, you can leverage SCRIP grid files that are already in place for the other components.
    There is no supported functionality for creating the SCRIP format file.
 
-2. Build the **check_map** utility by following the instructions in **$CCSMROOT/mapping/check_maps/INSTALL**. Also confirm that the `ESMF <http://www.cesm.ucar.edu/models2.0/external-link-here>`_ toolkit is installed on your machine.
+2. Build the **check_map** utility by following the instructions in **$CCSMROOT/mapping/check_maps/INSTALL**. Also confirm that the ESMF toolkit is installed on your machine.
 
    When you add new user-defined grid files, you also need to generate a set of mapping files so the coupler can send data from a component on one grid to a component on another grid.
    There is an ESMF tool that tests the mapping file by comparing a mapping of a smooth function to its true value on the destination grid.

diff --git a/doc/source/users_guide/running-a-case.rst b/doc/source/users_guide/running-a-case.rst
@@ -484,8 +484,7 @@ for a component set using full active model components.
 If short-term archiving is turned on, the model archives the
 component restart data sets and pointer files into
 **$DOUT_S_ROOT/rest/yyyy-mm-dd-sssss**, where yyyy-mm-dd-sssss is the
-model date at the time of the restart. (See `below for more details
-<http://www.cesm.ucar.edu/models/cesm2.0/external-link-here>`_.)
+model date at the time of the restart. (See below for more details.)
 
 ---------------------------------
 Backing up to a previous restart