Skip to content

Commit

Permalink
Merge remote-tracking branch 'E3SM-Project/E3SM/master' into ocean/no…
Browse files Browse the repository at this point in the history 
…nhydro
  • Loading branch information
@scalandr
scalandr committed May 19, 2023
2 parents 707cc46 + aff425b commit a882fd9
Show file tree
Hide file tree
Showing 199 changed files with 12,648 additions and 3,066 deletions.
2 changes: 1 addition & 1 deletion cime
Submodule cime updated 34 files
+17 −0 .github/workflows/stale.yml
+3 −1 .github/workflows/testing.yml
+10 −6 CIME/Tools/Makefile
+2 −1 CIME/XML/env_batch.py
+3 −2 CIME/XML/generic_xml.py
+7 −0 CIME/build.py
+16 −2 CIME/case/case.py
+4 −1 CIME/case/case_clone.py
+1 −4 CIME/non_py/src/CMake/CIME_utils.cmake
+0 −52 CIME/non_py/src/CMake/FindpFUnit.cmake
+0 −4 CIME/non_py/src/CMake/README.md
+0 −227 CIME/non_py/src/CMake/pFUnit_utils.cmake
+8 −8 CIME/non_py/src/components/xcpl_comps_nuopc/xatm/src/atm_comp_nuopc.F90
+1 −1 CIME/non_py/src/components/xcpl_comps_nuopc/xglc/src/glc_comp_nuopc.F90
+8 −8 CIME/non_py/src/components/xcpl_comps_nuopc/xice/src/ice_comp_nuopc.F90
+8 −8 CIME/non_py/src/components/xcpl_comps_nuopc/xlnd/src/lnd_comp_nuopc.F90
+8 −8 CIME/non_py/src/components/xcpl_comps_nuopc/xocn/src/ocn_comp_nuopc.F90
+8 −8 CIME/non_py/src/components/xcpl_comps_nuopc/xrof/src/rof_comp_nuopc.F90
+2 −2 CIME/non_py/src/components/xcpl_comps_nuopc/xshare/dead_methods_mod.F90
+8 −8 CIME/non_py/src/components/xcpl_comps_nuopc/xwav/src/wav_comp_nuopc.F90
+7 −7 CIME/non_py/src/timing/perf_mod.F90
+20 −20 CIME/non_py/src/timing/perf_utils.F90
+3 −1 CIME/test_utils.py
+9 −4 CIME/tests/test_sys_cime_case.py
+4 −0 CIME/tests/test_sys_unittest.py
+1 −1 CIME/tests/test_unit_compare_test_results.py
+3 −3 Externals.cfg
+30 −43 doc/source/users_guide/unit_testing.rst
+3 −1 scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt
+1 −1 scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf
+4 −1 scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt
+1 −1 scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf
+1 −50 scripts/fortran_unit_testing/README
+7 −58 scripts/fortran_unit_testing/run_tests.py
25 changes: 25 additions & 0 deletions cime_config/allactive/config_compsets.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@
<lname>1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>WCYCL1850_chemUCI-Linozv3</alias>
<lname>1850SOI_EAM%CHEMUCI-LINOZV3_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>WCYCL1850-1pctCO2</alias>
<lname>1850SOI_EAM%CMIP6-1pctCO2_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV</lname>
Expand All @@ -70,6 +75,11 @@
<lname>20TRSOI_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>WCYCL20TR_chemUCI-Linozv3</alias>
<lname>20TRSOI_EAM%CHEMUCI-LINOZV3_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>WCYCLSSP585</alias>
<lname>SSP585SOI_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV</lname>
Expand Down Expand Up @@ -302,6 +312,21 @@
<lname>1950SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBISMF_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>CRYO20TR</alias>
<lname>20TRSOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBISMF_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>CRYOSSP585</alias>
<lname>SSP585SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBISMF_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>CRYOSSP370</alias>
<lname>SSP370SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBISMF_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>BGCEXP_BDRD_CNPECACNT_SSP585_CMIP6</alias>
<lname>SSP585_EAM%CMIP6_ELM%CNPECACNTBC_MPASSI%BGC_MPASO%OIECO_MOSART_SGLC_SWAV_BGC%BDRD</lname>
Expand Down
123 changes: 73 additions & 50 deletions cime_config/allactive/config_pesall.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@
</nthrds>
</pes>
</mach>
<mach name="theta|pm-gpu|cori-knl|cori-haswell|jlse">
<mach name="theta|pm-cpu|alvarez|pm-gpu|cori-knl|cori-haswell|jlse">
<pes compset="any" pesize="any">
<comment>allactive: default, 1 node x MAX_MPITASKS_PER_NODE mpi x 1 omp @ root 0</comment>
<ntasks>
Expand All @@ -122,7 +122,7 @@
</ntasks>
</pes>
</mach>
<mach name="pm-cpu|alvarez|chicoma-cpu">
<mach name="chicoma-cpu">
<pes compset="any" pesize="any">
<comment>allactive: default, 1 node x MAX_MPITASKS_PER_NODE mpi x 2 omp @ root 0</comment>
<ntasks>
Expand Down Expand Up @@ -152,8 +152,8 @@
<ntasks_atm>56</ntasks_atm>
<ntasks_lnd>56</ntasks_lnd>
<ntasks_rof>56</ntasks_rof>
<ntasks_ice>36</ntasks_ice>
<ntasks_ocn>36</ntasks_ocn>
<ntasks_ice>56</ntasks_ice>
<ntasks_ocn>56</ntasks_ocn>
<ntasks_glc>16</ntasks_glc>
<ntasks_wav>16</ntasks_wav>
<ntasks_cpl>56</ntasks_cpl>
Expand Down Expand Up @@ -290,7 +290,7 @@
<grid name="a%ne30np4">
<mach name="pm-cpu|alvarez">
<pes compset="JRA_ELM.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="any">
<comment>"pm-cpu 4 nodes, 256 partition"</comment>
<comment>"pm-cpu 4 nodes, 256 partition, 128x1, c8"</comment>
<ntasks>
<ntasks_atm>-4</ntasks_atm>
<ntasks_lnd>-4</ntasks_lnd>
Expand All @@ -299,8 +299,11 @@
<ntasks_ocn>-4</ntasks_ocn>
<ntasks_glc>-1</ntasks_glc>
<ntasks_wav>-1</ntasks_wav>
<ntasks_cpl>-4</ntasks_cpl>
<ntasks_cpl>64</ntasks_cpl>
</ntasks>
<pstrid>
<pstrid_cpl>8</pstrid_cpl>
</pstrid>
</pes>
</mach>
<mach name="cori-haswell">
Expand Down Expand Up @@ -348,8 +351,8 @@
<ntasks_atm>280</ntasks_atm>
<ntasks_lnd>280</ntasks_lnd>
<ntasks_rof>280</ntasks_rof>
<ntasks_ice>240</ntasks_ice>
<ntasks_ocn>240</ntasks_ocn>
<ntasks_ice>256</ntasks_ice>
<ntasks_ocn>256</ntasks_ocn>
<ntasks_glc>-1</ntasks_glc>
<ntasks_wav>-1</ntasks_wav>
<ntasks_cpl>280</ntasks_cpl>
Expand Down Expand Up @@ -823,7 +826,6 @@
<pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+SWAV.*" pesize="any">
<comment>ne120-wcycl on 42 nodes 128x1c8 ~0.7 sypd</comment>
<MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
<MAX_TASKS_PER_NODE>256</MAX_TASKS_PER_NODE>
<ntasks>
<ntasks_atm>3072</ntasks_atm>
<ntasks_cpl>384</ntasks_cpl>
Expand Down Expand Up @@ -1611,8 +1613,8 @@
<ntasks_atm>280</ntasks_atm>
<ntasks_lnd>280</ntasks_lnd>
<ntasks_rof>280</ntasks_rof>
<ntasks_ice>240</ntasks_ice>
<ntasks_ocn>84</ntasks_ocn>
<ntasks_ice>256</ntasks_ice>
<ntasks_ocn>112</ntasks_ocn>
<ntasks_cpl>280</ntasks_cpl>
</ntasks>
<nthrds>
Expand Down Expand Up @@ -1664,32 +1666,35 @@
</mach>
<mach name="pm-cpu|alvarez">
<pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="any">
<comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes </comment>
<MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
<comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 7 nodes, 128x1 c8 </comment>
<MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
<ntasks>
<ntasks_atm>320</ntasks_atm>
<ntasks_lnd>320</ntasks_lnd>
<ntasks_rof>320</ntasks_rof>
<ntasks_ice>320</ntasks_ice>
<ntasks_ocn>128</ntasks_ocn>
<ntasks_cpl>320</ntasks_cpl>
<ntasks_atm>640</ntasks_atm>
<ntasks_lnd>640</ntasks_lnd>
<ntasks_rof>640</ntasks_rof>
<ntasks_ice>640</ntasks_ice>
<ntasks_ocn>256</ntasks_ocn>
<ntasks_cpl>80</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>2</nthrds_atm>
<nthrds_lnd>2</nthrds_lnd>
<nthrds_rof>2</nthrds_rof>
<nthrds_ice>2</nthrds_ice>
<nthrds_ocn>2</nthrds_ocn>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>320</rootpe_ocn>
<rootpe_ocn>640</rootpe_ocn>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
<pstrid>
<pstrid_cpl>8</pstrid_cpl>
</pstrid>
</pes>
<pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="L">
<comment> -compset A_WCYCL* -res ne30pg2_oECv3 with MPASO on 58 nodes, ~20 sypd</comment>
Expand Down Expand Up @@ -2334,7 +2339,7 @@
</mach>
<mach name="pm-cpu|alvarez">
<pes compset="any" pesize="any">
<comment>"pm-cpu ne30np4 and ne30np4.pg2 2 nodes 2 threads"</comment>
<comment>"pm-cpu ne30np4 and ne30np4.pg2 2 nodes 1 thread, 128x1 c8"</comment>
<ntasks>
<ntasks_atm>-2</ntasks_atm>
<ntasks_lnd>-2</ntasks_lnd>
Expand All @@ -2343,16 +2348,11 @@
<ntasks_ocn>-2</ntasks_ocn>
<ntasks_glc>-2</ntasks_glc>
<ntasks_wav>-2</ntasks_wav>
<ntasks_cpl>-2</ntasks_cpl>
<ntasks_cpl>32</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>2</nthrds_atm>
<nthrds_lnd>2</nthrds_lnd>
<nthrds_rof>2</nthrds_rof>
<nthrds_ice>2</nthrds_ice>
<nthrds_ocn>2</nthrds_ocn>
<nthrds_cpl>2</nthrds_cpl>
</nthrds>
<pstrid>
<pstrid_cpl>8</pstrid_cpl>
</pstrid>
</pes>
</mach>
<mach name="cori-knl">
Expand Down Expand Up @@ -2766,33 +2766,34 @@
</mach>
<mach name="pm-cpu|alvarez">
<pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="any">
<comment> 8 nodes, 64x2 </comment>
<MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
<MAX_TASKS_PER_NODE>256</MAX_TASKS_PER_NODE>
<comment> 8 nodes, 128x1 c8</comment>
<ntasks>
<ntasks_atm>320</ntasks_atm>
<ntasks_lnd>320</ntasks_lnd>
<ntasks_rof>320</ntasks_rof>
<ntasks_ice>360</ntasks_ice>
<ntasks_ocn>192</ntasks_ocn>
<ntasks_cpl>320</ntasks_cpl>
<ntasks_atm>640</ntasks_atm>
<ntasks_lnd>640</ntasks_lnd>
<ntasks_rof>640</ntasks_rof>
<ntasks_ice>640</ntasks_ice>
<ntasks_ocn>384</ntasks_ocn>
<ntasks_cpl>80</ntasks_cpl>
</ntasks>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>320</rootpe_ocn>
<rootpe_ocn>640</rootpe_ocn>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
<nthrds>
<nthrds_atm>2</nthrds_atm>
<nthrds_lnd>2</nthrds_lnd>
<nthrds_rof>2</nthrds_rof>
<nthrds_ice>2</nthrds_ice>
<nthrds_ocn>2</nthrds_ocn>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<pstrid>
<pstrid_cpl>1</pstrid_cpl>
</pstrid>
</pes>
</mach>
</grid>
Expand Down Expand Up @@ -3023,6 +3024,28 @@
</nthrds>
</pes>
</mach>
<mach name="ruby">
<pes compset="any" pesize="any">
<ntasks>
<ntasks_atm>-4</ntasks_atm>
<ntasks_lnd>-4</ntasks_lnd>
<ntasks_rof>-4</ntasks_rof>
<ntasks_ice>-4</ntasks_ice>
<ntasks_ocn>-4</ntasks_ocn>
<ntasks_glc>-4</ntasks_glc>
<ntasks_wav>-4</ntasks_wav>
<ntasks_cpl>-4</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
</pes>
</mach>
<mach name="quartz">
<pes compset="any" pesize="any">
<ntasks>
Expand Down
1 change: 1 addition & 0 deletions cime_config/config_files.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@
<value component="mosart" >$COMP_ROOT_DIR_ROF/cime_config/testdefs/testmods_dirs</value>
<value component="scream" >$COMP_ROOT_DIR_ATM/cime_config/testdefs/testmods_dirs</value>
<value component="mpaso" >$COMP_ROOT_DIR_OCN/cime_config/testdefs/testmods_dirs</value>
<value component="mpassi" >$COMP_ROOT_DIR_ICE/cime_config/testdefs/testmods_dirs</value>
</values>
<group>case_last</group>
<file>env_case.xml</file>
Expand Down
2 changes: 1 addition & 1 deletion cime_config/config_grids.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2692,7 +2692,7 @@
<domain name="SOwISC12to60E2r4">
<nx>569915</nx>
<ny>1</ny>
<file grid="ice|ocn">$DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to60E2r4-nomask.210119.nc</file>
<file grid="ice|ocn">$DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to60E2r4.210119.nc</file>
<desc>SOwISC12to60E2r4 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid:</desc>
</domain>

Expand Down
4 changes: 4 additions & 0 deletions cime_config/config_inputdata.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
<protocol>wget</protocol>
<address>https://web.lcrc.anl.gov/public/e3sm/inputdata/</address>
</server>
<server>
<protocol>wget</protocol>
<address>https://portal.nersc.gov/project/e3sm/inputdata</address>
</server>
<server>
<protocol>svn</protocol>
<address>https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata</address>
Expand Down
30 changes: 27 additions & 3 deletions cime_config/customize/provenance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,10 @@ def _record_timing(case, lid):
with open(os.path.join(full_timing_dir, "GIT_DESCRIBE.{}".format(lid)), "w") as fd:
fd.write(desc)

# What this block does is mysterious to me (JGF)
# Collect syslog if enabled on machine. (Sarat)
# Ref: https://github.com/ESMCI/cime/blob/655de638182ba9381a5d6607cdbade3b0a70a040/CIME/case/case.py#L1696
# If machines_dir has a syslog.<machine_name> script, it is copied as caseroot/Tools/mach_syslog and run.
# Otherwise, syslog.noop is used and no output is produced.
if job_id is not None:
_record_syslog(case, lid, job_id, caseroot, rundir, full_timing_dir)

Expand All @@ -422,10 +425,13 @@ def _record_queue_info(mach, job_id, lid, full_timing_dir):
_record_nersc_queue(job_id, lid, full_timing_dir)
elif mach in ["anvil", "chrysalis", "compy"]:
_record_anl_queue(job_id, lid, full_timing_dir)
#TODO: Add Perlmutter
elif mach in ["frontier", "crusher"]:
_record_slurm_queue(job_id, lid, full_timing_dir)
elif mach == "summit":
_record_olcf_queue(job_id, lid, full_timing_dir)


# TODO: Switch to generic Slurm routine
def _record_nersc_queue(job_id, lid, full_timing_dir):
for cmd, filename in [
("sinfo -a -l", "sinfol"),
Expand All @@ -441,7 +447,24 @@ def _record_nersc_queue(job_id, lid, full_timing_dir):
utils.run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir)
utils.gzip_existing_file(os.path.join(full_timing_dir, filename))

# Generic Slurm queue info (Frontier)
# TODO: Consolidate _record routines based on batch system if generalization is adequate
def _record_slurm_queue(job_id, lid, full_timing_dir):
for cmd, filename in [
("sinfo -a -l", "sinfol"),
("scontrol show jobid %s" % job_id, "scontrol_jobid"),
(
"squeue -o '%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l %.20S %.20V'",
"squeuef",
),
("squeue -t R -o '%.10i %R'", "squeues"),
]:
filename = "%s.%s" % (filename, lid)
utils.run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir)
utils.gzip_existing_file(os.path.join(full_timing_dir, filename))


# TODO: Switch to generic Slurm routine
def _record_anl_queue(job_id, lid, full_timing_dir):
for cmd, filename in [
("sinfo -l", "sinfol"),
Expand Down Expand Up @@ -763,7 +786,8 @@ def _get_batch_job_id_for_syslog(case):
"""
mach = case.get_value("MACH")
try:
if mach in ["anvil", "chrysalis", "compy", "cori-haswell", "cori-knl", "pm-cpu", "pm-gpu", "alvarez"]:
if mach in ["anvil", "chrysalis", "compy", "cori-haswell", "cori-knl", "pm-cpu", "pm-gpu", "alvarez","frontier","crusher"]:
# Note: Besides, SLURM_JOB_ID, equivalent SLURM_JOBID is also present on some systems (Frontier).
return os.environ["SLURM_JOB_ID"]
elif mach in ["theta"]:
return os.environ["COBALT_JOBID"]
Expand Down
14 changes: 14 additions & 0 deletions cime_config/machines/Depends.anlgce.gnu.cmake
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# For this file, see internal compiler error on anlgce with -O2
set(NOOPT
eam/src/physics/cam/zm_conv.F90)

if (NOT DEBUG)
foreach(ITEM IN LISTS NOOPT)
e3sm_remove_flags("${ITEM}" "-O2")
e3sm_remove_flags("${ITEM}" "-O")
endforeach()
endif()




Loading

0 comments on commit a882fd9

Please sign in to comment.