Merge pull request #3139 from ESMCI/jgfouca/branch-for-acme-split-201…

…9-06-10

Jgfouca/branch for acme split 2019 06 10

Changes:
* Fix bug in PET timestamp (need to use build as a library, not a subprocess) (E3SM-Project/E3SM#2949)
* Fix bug in generating baselines (Need to use the base case id when parsing test opts.) (E3SM-Project/E3SM#2950)
* Change FFLAGS_NOPT to be an append to the normal set of flags. This reduces duplication with FFLAGS (E3SM-Project/E3SM#2922)
* Add ability to ask for CMake args (get_standard_cmake_args) (E3SM-Project/E3SM#2922)
* CMake macros should NOT be looking at environment, just CMake vars (E3SM-Project/E3SM#2922)
* case_setup will make both macro types (Make and CMake) (E3SM-Project/E3SM#2922)
* Import a FindPetsc CMake module (E3SM-Project/E3SM#2922)
* Implement a more-advanced way of sharing builds within test suites (Implements a concept of build groups within test_scheduler.) (E3SM-Project/E3SM#2961)

Test suite: scripts_regression_tests
Test baseline:
Test namelist changes:
Test status: [bit for bit, roundoff, climate changing]

Fixes [CIME Github issue #]

User interface changes?:

Update gh-pages html (Y/N)?:

Code review:

config/cesm/machines/Depends.babbageKnc

@@ -1,6 +1,6 @@
 
 shr_ncread_mod.o: shr_ncread_mod.F90
-	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $<
+	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS) $(FFLAGS_NOOPT) $<
 
 quadrature_mod.o: quadrature_mod.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -DIS_ACCELERATOR $<

config/cesm/machines/Depends.cray

@@ -1,6 +1,6 @@
 NOOPTOBJS= ice_boundary.o dyn_comp.o unicon.o
 
 $(NOOPTOBJS): %.o: %.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 
 

config/cesm/machines/Depends.intelmic

@@ -3,4 +3,4 @@
 #	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< 
 
 shr_ncread_mod.o: shr_ncread_mod.F90
-	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $<
+	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS) $(FFLAGS_NOOPT) $<

config/cesm/machines/Depends.intelmic14

@@ -3,4 +3,4 @@
 #	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< 
 
 shr_ncread_mod.o: shr_ncread_mod.F90
-	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $<
+	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS) $(FFLAGS_NOOPT) $<

config/cesm/machines/Depends.nag

@@ -1,4 +1,4 @@
 wrap_mpi.o: wrap_mpi.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 fft99.o: fft99.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<

config/cesm/machines/config_compilers.xml

@@ -243,7 +243,6 @@ using a fortran linker.
   </FFLAGS>
   <FFLAGS_NOOPT>
     <base> -O0 </base>
-    <append compile_threaded="TRUE"> -qopenmp </append>
   </FFLAGS_NOOPT>
   <FIXEDFLAGS>
     <base> -fixed  </base>
@@ -301,13 +300,7 @@ using a fortran linker.
     <append MODEL="cism"> -mismatch_all </append>
   </FFLAGS>
   <FFLAGS_NOOPT>
-    <base>-Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/config/cesm/machines/nag_mpi_argument.txt</base>
-    <append DEBUG="FALSE"> -ieee=full     </append>
-    <!-- Hack! If DEBUG="TRUE", put runtime checks in FFLAGS, but not into     -->
-    <!-- FFLAGS_NOOPT, allowing strict checks to be removed from files by      -->
-    <!-- having them use FFLAGS_NOOPT in Depends.nag                           -->
-    <append DEBUG="TRUE"> -g -time -f2003 -ieee=stop </append>
-    <append DEBUG="TRUE" compile_threaded="FALSE"> -gline        </append>
+    <base> -O0 </base>
   </FFLAGS_NOOPT>
   <FIXEDFLAGS>
     <base> -fixed </base>
@@ -379,8 +372,7 @@ using a fortran linker.
     <append MODEL="docn"> -Mnovect </append>
   </FFLAGS>
   <FFLAGS_NOOPT>
-    <base> -O0 -g -Ktrap=fp -Mbounds -Kieee  </base>
-    <append compile_threaded="TRUE"> -mp </append>
+    <base> -O0 </base>
   </FFLAGS_NOOPT>
   <FIXEDFLAGS>
     <base> -Mfixed </base>

config/e3sm/machines/Depends.cray

@@ -1,5 +1,5 @@
 NOOPTOBJS= ice_boundary.o dyn_comp.o
 
 $(NOOPTOBJS): %.o: %.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 

config/e3sm/machines/Depends.cray.cmake

@@ -0,0 +1 @@
+list(NOOPT_FILES APPEND cice/src/mpi/ice_boundary.F90 cice/src/serial/ice_boundary.F90)

config/e3sm/machines/Depends.gnu.cmake

@@ -0,0 +1 @@
+set_property(SOURCE cam/src/dynamics/fv/geopk.F90 APPEND_STRING PROPERTY COMPILE_FLAGS " -fcray-pointer ")

config/e3sm/machines/Depends.cetus → config/e3sm/machines/Depends.ibm

@@ -41,23 +41,6 @@ endif
 buffer.o: buffer.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $<
 
-## lnd files taking more than a minute to compile
-# this takes 4 mins to compile with -O3 -qsmp=omp
-BiogeophysRestMod.o: BiogeophysRestMod.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 -qsmp=noopt $<
-
-# this takes 17 mins to compile with -O3 -qsmp=omp
-CNrestMod.o: CNrestMod.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 -qsmp=noopt $<
-
-# this takes 4 mins to compile with -qsmp=omp
-clmtypeInitMod.o: clmtypeInitMod.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qsmp=noopt $<
-
-# this takes 2 mins to compile with -qsmp=omp
-clmtype.o: clmtype.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qsmp=noopt $<
-
 # disable inlining (some issues with pure functions therein)
 advance_xm_wpxp_module.o: advance_xm_wpxp_module.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Q! $<

config/e3sm/machines/Depends.ibm.cmake

@@ -0,0 +1,56 @@
+# These routines have problems with stacksize when omp is invoked add -qsmallstack to resolve
+set(SSOBJS
+  cam/src/chemistry/mozart/mo_sethet.F90
+  cam/src/chemistry/mozart/mo_drydep.F90)
+
+foreach(ITEM IN LISTS SSOBJS)
+  set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -qsmallstack ")
+endforeach()
+
+if (compile_threaded)
+  set_property(SOURCE share/util/shr_reprosum_mod.F90 APPEND_STRING PROPERTY COMPILE_FLAGS " -qsmp=noauto:noomp ")
+endif()
+
+# These routines benefit from -qnostrict without violating the bfb test
+set(PERFOBJS
+  homme/src/share/prim_advection_base.F90
+  homme/src/share/vertremap_base.F90
+  homme/src/share/edge_mod_base.F90
+  homme/src/share/derivative_mod_base.F90
+  homme/src/share/bndry_mod_base.F90
+  homme/src/theta-l/prim_advance_mod.F90
+  homme/src/pese/prim_advance_mod.F90
+  homme/src/theta/prim_advance_mod.F90
+  homme/src/preqx/share/prim_advance_mod.F90
+  cam/src/physics/cam/uwshcu.F90
+  cam/src/chemistry/aerosol/wetdep.F90)
+
+#Model crashes if these files are compiled with O3(default) optimizations
+set(REDUCEOPT
+  cam/src/chemistry/bulk_aero/seasalt_model.F90
+  cam/src/chemistry/modal_aero/seasalt_model.F90
+  cam/src/chemistry/mozart/linoz_data.F90)
+
+set(NOINLINE
+  cam/src/physics/clubb/advance_xm_wpxp_module.F90
+  cam/src/physics/clubb/advance_wp2_wp3_module.F90)
+
+if (NOT DEBUG)
+  foreach(ITEM IN LISTS PERFOBJS)
+    set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -qnostrict ")
+  endforeach()
+
+  foreach(ITEM IN LISTS REDUCEOPT)
+    set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -O2 ")
+  endforeach()
+
+endif()
+
+# These files take long time to compile with default optimization flags.
+# Reducing optimization gives <1min build-times and little impact on model run time.
+# begin
+list(APPEND NOOPT_FILES cam/src/utils/buffer.F90)
+
+foreach(ITEN IN LISTS NOINLINE)
+  set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -Q! ")
+endforeach()

config/e3sm/machines/Depends.intel.cmake

@@ -0,0 +1,50 @@
+set(PERFOBJS
+  homme/src/share/prim_advection_base.F90
+  homme/src/share/vertremap_base.F90
+  homme/src/share/edge_mod_base.F90
+  homme/src/share/derivative_mod_base.F90
+  homme/src/share/bndry_mod_base.F90
+  homme/src/theta-l/prim_advance_mod.F90
+  homme/src/pese/prim_advance_mod.F90
+  homme/src/theta/prim_advance_mod.F90
+  homme/src/preqx/share/prim_advance_mod.F90
+  homme/src/preqx/share/viscosity_preqx_base.F90
+  homme/src/share/viscosity_base.F90
+  homme/src/theta-l/viscosity_theta.F90
+  homme/src/theta/viscosity_theta.F90
+  homme/src/theta-l/eos.F90
+  homme/src/theta/eos.F90
+  homme/src/theta/hevi_mod.F90
+  cam/src/physics/cam/uwshcu.F90)
+
+set(REDUCED_PRECISION_OBJS share/util/shr_wv_sat_mod.F90)
+
+set(SHR_RANDNUM_FORT_OBJS
+  share/RandNum/src/kissvec/kissvec_mod.F90
+  share/RandNum/src/mt19937/mersennetwister_mod.F90
+  share/RandNum/src/dsfmt_f03/dSFMT_interface.F90
+  share/RandNum/src/shr_RandNum_mod.F90)
+
+set(SHR_RANDNUM_C_OBJS
+  share/RandNum/src/dsfmt_f03/dSFMT.c
+  share/RandNum/src/dsfmt_f03/dSFMT_utils.c
+  share/RandNum/src/kissvec/kissvec.c)
+
+if (NOT DEBUG)
+  foreach(ITEM IN LISTS PERFOBJS)
+    set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -O3 -fp-model fast -no-prec-div ")
+  endforeach()
+
+  foreach(ITEM IN LISTS REDUCED_PRECISION_OBJS)
+    set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -fimf-precision=low -fp-model fast ")
+  endforeach()
+
+  foreach(ITEM IN LISTS SHR_RANDNUM_FORT_OBJS)
+    set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -O3 -fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits ")
+  endforeach()
+
+  foreach(ITEM IN LISTS SHR_RANDNUM_C_OBJS)
+    set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -O3 -fp-model fast ")
+  endforeach()
+
+endif()

config/e3sm/machines/Depends.nag

@@ -1,4 +1,4 @@
 wrap_mpi.o: wrap_mpi.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 fft99.o: fft99.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<

config/e3sm/machines/Depends.nag.cmake

@@ -0,0 +1 @@
+list(NOOPT_FILES APPEND cam/src/control/wrap_mpi.F90 cam/src/utils/fft99.F90)

config/e3sm/machines/Depends.summit.cmake

@@ -0,0 +1,22 @@
+list(APPEND NOOPT_FILES
+  cam/src/dynamics/eul/dyn_comp.F90
+  cam/src/dynamics/fv/dyn_comp.F90
+  cam/src/dynamics/se/dyn_comp.F90
+  cam/src/dynamics/sld/dyn_comp.F90
+  cam/src/physics/cam/microp_aero.F90)
+
+set(FILES_NEED_CUDA_FLAGS
+  homme/src/preqx_acc/bndry_mod.F90
+  homme/src/preqx_acc/derivative_mod.F90
+  homme/src/preqx_acc/edge_mod.F90
+  homme/src/share/element_mod.F90
+  homme/src/preqx_acc/element_state.F90
+  homme/src/preqx_acc/openacc_utils_mod.F90
+  homme/src/preqx_acc/prim_advection_mod.F90
+  homme/src/share/prim_si_mod.F90
+  homme/src/preqx_acc/model_init_mod.F90
+  homme/src/preqx_acc/viscosity_mod.F90
+  homme/src/preqx_acc/prim_driver_mod.F90
+  homme/src/share/prim_driver_base.F90
+  homme/src/share/physics_mod.F90
+  cam/src/control/physconst.F90)

config/e3sm/machines/Depends.summit.pgiacc

@@ -1,8 +1,8 @@
 dyn_comp.o: dyn_comp.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 
 microp_aero.o: microp_aero.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 
 
 

config/e3sm/machines/Depends.summit.pgiacc.cmake

@@ -0,0 +1,3 @@
+foreach(ITEM IN LISTS FILES_NEED_CUDA_FLAGS)
+  set_property(SOURCE ${ITEM} APPEND_STRING PROPERTY COMPILE_FLAGS " -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel ")
+endforeach()