Merge remote-tracking branch 'origin/mrnorman/hommme/create-preqx-acc…

…-target' (PR #730) This pull request is making a new target-based structure for HOMME to specify new solvers. Instead of having src/share/openacc, I have src/preqx and src/preqx_acc, both of which overload the same module files. See https://acme-climate.atlassian.net/wiki/display/PERF/HOMME+Cleanup+and+Restructuring+Discussion for more documentation of how this works. [BFB]
ESMCI · Mar 22, 2016 · 8b5cda8 · 8b5cda8
2 parents fac6ad2 + f5e6c9a
commit 8b5cda8
Show file tree

Hide file tree

Showing 91 changed files with 1,893 additions and 1,611 deletions.
diff --git a/cime/machines-acme/Depends.intel b/cime/machines-acme/Depends.intel
@@ -1,9 +1,10 @@
 #
 PERFOBJS=\
-prim_advection_mod.o \
-edge_mod.o \
-derivative_mod.o \
-bndry_mod.o \
+prim_advection_mod_base.o \
+vertremap_mod_base.o \
+edge_mod_base.o \
+derivative_mod_base.o \
+bndry_mod_base.o \
 prim_advance_mod.o \
 uwshcu.o
 

diff --git a/cime/machines-acme/Depends.intel14 b/cime/machines-acme/Depends.intel14
@@ -6,23 +6,23 @@
 #ecosys_mod.o: ecosys_mod.F90
 #	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -CU $<
 
-prim_advection_mod.o: prim_advection_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3  -no-prec-div  $<
 
-edge_mod.o: edge_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
 
-derivative_mod.o: derivative_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
+PERFOBJS=\
+prim_advection_mod_base.o \
+vertremap_mod_base.o \
+edge_mod_base.o \
+derivative_mod_base.o \
+bndry_mod_base.o \
+prim_advance_mod.o \
+uwshcu.o \
+wetdep.o
 
-bndry_mod.o: bndry_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
-
-prim_advance_mod.o: prim_advance_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
-
-uwshcu.o: uwshcu.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
-
-wetdep.o: wetdep.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
+ifeq ($(DEBUG),FALSE)
+  $(PERFOBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div $<
+  $(REDUCED_OPT_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O1 $<
+  $(REDUCED_PRECISION_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $<
+endif
diff --git a/cime/machines-acme/Depends.intelmic b/cime/machines-acme/Depends.intelmic
@@ -1,6 +1,6 @@
 
-derivative_mod.o: derivative_mod.F90
-	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< 
+#derivative_mod_base.o: derivative_mod_base.F90
+#	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< 
 
 shr_ncread_mod.o: shr_ncread_mod.F90
 	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $<
diff --git a/cime/machines-acme/Depends.intelmic14 b/cime/machines-acme/Depends.intelmic14
@@ -1,6 +1,6 @@
 
-derivative_mod.o: derivative_mod.F90
-	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< 
+#derivative_mod_base.o: derivative_mod_base.F90
+#	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $< 
 
 shr_ncread_mod.o: shr_ncread_mod.F90
 	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $<
diff --git a/cime/machines-acme/Depends.mira b/cime/machines-acme/Depends.mira
@@ -13,13 +13,16 @@ shr_reprosum_mod.o: shr_reprosum_mod.F90
 
 # These routines benefit from -qnostrict without violating the bfb test
 PERFOBJS=\
-prim_advection_mod.o \
-edge_mod.o \
-derivative_mod.o \
-bndry_mod.o \
+prim_advection_mod_base.o \
+vertremap_mod_base.o \
+edge_mod_base.o \
+derivative_mod_base.o \
+bndry_mod_base.o \
 prim_advance_mod.o \
 uwshcu.o \
-wetdep.o 
+wetdep.o
+
+
 ifeq ($(DEBUG),FALSE)
   $(PERFOBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qnostrict  $<

diff --git a/cime/scripts/Tools/config_definition.xml b/cime/scripts/Tools/config_definition.xml
@@ -1363,6 +1363,14 @@
 	 sdesc="CAM dynamical core"
 	 ></entry>
 
+  <entry id="CAM_TARGET" 
+	 type="char"
+	 valid_values="preqx,preqx_acc" 
+	 value="preqx" 
+	 group="build_component_cam"
+	 sdesc="CAM-SE cmake target (only used with CAM_DYCORE == 'se')"
+	 ></entry>
+
   <entry id="CAM_NML_USE_CASE" 
 	 type="char"
 	 valid_values=""

diff --git a/cime/scripts/create_newcase b/cime/scripts/create_newcase
@@ -96,6 +96,9 @@ OPTIONS
                                     or read from ~/.cesm_proj or ~/.ccsm_proj
   -compiler <name>         Specify a compiler for the target machine (optional)
                            default: default compiler for the target machine
+  -camse_target <name>     Spedify a target for the cam-se dycore (ignored for other atmospheric dycores)
+                           default: preqx
+                           options: preqx, preqx-acc
   -mpilib <name>           Specify a mpi library for the target machine (optional)
                            default: default mpi library for the target machine
                            allowed: openmpi, mpich, ibm, mpi-serial, etc
@@ -166,6 +169,7 @@ GetOptions(
     "confopts=s"                => \$opts{'confopts'},
     "project=s"                 => \$opts{'project'},  
     "compiler=s"                => \$opts{'compiler'},  
+    "camse_target=s"            => \$opts{'camse_target'},  
     "mpilib=s"                  => \$opts{'mpilib'},  
     "res=s"                     => \$opts{'res'},
     "h|help"                    => \$opts{'help'},
@@ -211,6 +215,7 @@ my $pecount;
 my $xmlmode;
 my $project;
 my $compiler;
+my $camse_target;
 my $mpilib;
 if (!$opts{'list'} && !$opts{'testlist'}) {
     # Check for manditory case input
@@ -260,6 +265,13 @@ if (!$opts{'list'} && !$opts{'testlist'}) {
 	$compiler = $opts{'compiler'};
     }
 
+    # Check if machine compiler option is given
+    if ($opts{'camse_target'}) {
+      $camse_target = $opts{'camse_target'};
+    } else {
+      $camse_target = "preqx";
+    }
+
     # Check if machine mpilib option is given
     $mpilib = 'unset';
     if ($opts{'mpilib'}) {
@@ -502,36 +514,51 @@ if ($mach =~ /(.*)_(.*)/){
 }
 $cfg_ref->set('CCSM_MACHDIR', "$machdir");
 
+# Handle the camse_target option
+# Do this before handling the compiler so that default compiler can be overwritten
+if ($camse_target eq "preqx") {
+    $cfg_ref->set('CAM_TARGET', "preqx");
+} elsif ($camse_target eq "preqx_acc") {
+    $cfg_ref->set('CAM_TARGET', "preqx_acc");
+} else {
+    die "ERROR: option -camse_target must be preqx (default) or preqx_acc"
+}
+
 # Check that compiler request for target machine matches a supported value
 # Or set default compiler - if not provided compiler request
 
 my $compilers;
 if ($mach =~ /userdefined/){
-    $cfg_ref->set('COMPILER', "USERDEFINED_required_build");
+  $cfg_ref->set('COMPILER', "USERDEFINED_required_build");
 } else { 
-    $compilers = $cfg_ref->get('COMPILERS');
-    my @compilers = split ",", $compilers, -1;
-    if ($compiler) {
-	if (! ($mach =~ "generic")){
-	    my $found = 0;
-	    foreach my $comp (@compilers) {
-		if ($compiler eq $comp) {
-		    $found = 1;
-		}
-	    }
-	    if (!$found) {
-		my $sysmod = "rm -rf $caseroot";
-		system($sysmod) == 0 or die "ERROR: $sysmod failed: $?\n";
-		die "ERROR: compiler setting of $compiler does not match supported values of $compilers \n";
-	    }
-	}
-	$cfg_ref->set('COMPILER', "$compiler");
-	if ($print>=2) { print "Machine compiler specifier: $compiler.$eol"; }
-    } else {
-	$compiler = $compilers[0];   
-	$cfg_ref->set('COMPILER', "$compiler");
-	if ($print>=2) { print "Machine compiler specifier: $compiler.$eol"; }
+  $compilers = $cfg_ref->get('COMPILERS');
+  my @compilers = split ",", $compilers, -1;
+  if ($compiler) {
+    if (! ($mach =~ "generic")){
+      my $found = 0;
+      foreach my $comp (@compilers) {
+        if ($compiler eq $comp) {
+            $found = 1;
+        }
+      }
+      if (!$found) {
+        my $sysmod = "rm -rf $caseroot";
+        system($sysmod) == 0 or die "ERROR: $sysmod failed: $?\n";
+        die "ERROR: compiler setting of $compiler does not match supported values of $compilers \n";
+      }
     }
+    $cfg_ref->set('COMPILER', "$compiler");
+    if ($print>=2) { print "Machine compiler specifier: $compiler.$eol"; }
+  } else {
+    $compiler = $compilers[0];
+    if ($camse_target eq "preqx_acc") {
+      if ($mach eq "titan") {
+        $compiler = "pgi_acc";
+      }
+    }
+    $cfg_ref->set('COMPILER', "$compiler");
+    if ($print>=2) { print "Machine compiler specifier: $compiler.$eol"; }
+  }
 }
 
 if ($print>=2) { print "Machine specifier: $mach.$eol"; }

diff --git a/components/cam/bld/cam.buildnml b/components/cam/bld/cam.buildnml
@@ -20,6 +20,7 @@ my $CAM_CONFIG_OPTS	= `./xmlquery  CAM_CONFIG_OPTS		-value`;
 my $CAM_NAMELIST_OPTS	= `./xmlquery  CAM_NAMELIST_OPTS	-value`;
 my $CAM_NML_USE_CASE	= `./xmlquery  CAM_NML_USE_CASE		-value`;
 my $CAM_DYCORE		= `./xmlquery  CAM_DYCORE		-value`;
+my $CAM_TARGET		= `./xmlquery  CAM_TARGET		-value`;
 my $CASEBUILD		= `./xmlquery  CASEBUILD		-value`;
 my $CCSM_CO2_PPMV	= `./xmlquery  CCSM_CO2_PPMV		-value`;
 my $CCSMROOT		= `./xmlquery  CCSMROOT			-value`;
@@ -84,7 +85,7 @@ if ($BUILD_COMPLETE eq 'FALSE') {
 	$ATM_GRID = $1;
 	$nlev="-nlev $3";
     }
-    $sysmod = "$CCSMROOT/components/cam/bld/configure -s -ccsm_seq -ice none -ocn $ocn -comp_intf $comp $scm -spmd $spmd -smp $smp -dyn $CAM_DYCORE -res $ATM_GRID $nlev $CAM_LIB_DIRS $CAM_CONFIG_OPTS";
+    $sysmod = "$CCSMROOT/components/cam/bld/configure -s -ccsm_seq -ice none -ocn $ocn -comp_intf $comp $scm -spmd $spmd -smp $smp -dyn $CAM_DYCORE -dyn_target $CAM_TARGET -res $ATM_GRID $nlev $CAM_LIB_DIRS $CAM_CONFIG_OPTS";
     system($sysmod) == 0 or die "ERROR cam.buildnml: $sysmod failed: $?\n";
 
 } else {

diff --git a/components/cam/bld/configure b/components/cam/bld/configure
@@ -324,6 +324,7 @@ GetOptions(
     "rain_evap_to_coarse_aero"  => \$opts{'rain_evap_to_coarse_aero'},
     "defaults=s"                => \$opts{'defaults'},
     "dyn=s"                     => \$opts{'dyn'},
+    "dyn_target=s"              => \$opts{'dyn_target'},
     "edit_chem_mech"            => \$opts{'edit_chem_mech'},
     "waccm_phys"                => \$opts{'waccm_phys'},
     "offline_dyn"               => \$opts{'offline_dyn'},
@@ -2764,8 +2765,13 @@ sub write_filepath
 
     print $fh "$camsrcdir/cam/src/dynamics/$dyn\n";
     if($dyn eq 'se') {
-	print $fh "$camsrcdir/homme/src/share\n";
-	print $fh "$camsrcdir/homme/src/share/openacc\n";
+	    print $fh "$camsrcdir/homme/src/share\n";
+      if ($opts{'dyn_target'} eq 'preqx') {
+	      print $fh "$camsrcdir/homme/src/preqx\n";
+      } 
+      elsif($opts{'dyn_target'} eq 'preqx_acc') {
+	      print $fh "$camsrcdir/homme/src/preqx_acc\n";
+      }
     }
 
     # Parallelization utilies

diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt
@@ -56,6 +56,7 @@ ENDIF ()
 # Some options for conditional builds
 OPTION(BUILD_HOMME_SWEQX "Shallow water equations FEM" ON)
 OPTION(BUILD_HOMME_PREQX "Primitive equations FEM" ON)
+OPTION(BUILD_HOMME_PREQX_ACC "Primitive equations FEM with OpenACC" ON)
 OPTION(BUILD_HOMME_SWDGX "Shallow water equations DG" OFF)
 OPTION(BUILD_HOMME_PRIMDGX "Primitive equations DG" OFF)
 OPTION(BUILD_HOMME_SWIM  "Shallow water equations implicit" OFF)
@@ -175,6 +176,10 @@ IF(${BUILD_HOMME_PREQX})
   ADD_SUBDIRECTORY(src/preqx)
 ENDIF()
 
+IF(${BUILD_HOMME_PREQX_ACC})
+  ADD_SUBDIRECTORY(src/preqx_acc)
+ENDIF()
+
 IF(${BUILD_HOMME_SWDGX})
   ADD_SUBDIRECTORY(src/swdgx)
 ENDIF()

diff --git a/components/homme/cmake/SetCompilerFlags.cmake b/components/homme/cmake/SetCompilerFlags.cmake
@@ -222,68 +222,7 @@ ENDIF ()
 ##############################################################################
 OPTION(ENABLE_OPENACC "Whether to build with OpenACC support" FALSE)
 IF (${ENABLE_OPENACC})
-  IF (${CMAKE_Fortran_COMPILER_ID} STREQUAL PGI)
-    # Need to add -acc to the Fortran FLAGS to see if it will compile 
-    # "call acc_init()"
-    SET(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc")
-    TRY_COMPILE(OPENACC_SUCCESS
-                ${CMAKE_BINARY_DIR}/tests/compilerTests/
-                ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compilerTests/openAccTest.f90
-                OUTPUT_VARIABLE COMPILE_OUTPUT)
-    IF (${OPENACC_SUCCESS})
-      MESSAGE(STATUS "Using OpenACC through PGI compiler")
-      SET(PREQX_USE_OPENACC TRUE)
-    ELSE()
-      MESSAGE(FATAL_ERROR "Unable to use OpenACC through the PGI compiler")
-    ENDIF()
-  ELSEIF (${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray)
-    SET(PREQX_USE_OPENACC TRUE)
-    MESSAGE(STATUS "Using OpenACC through Cray compiler")
-  ELSE ()
-    MESSAGE(FATAL_ERROR "OpenACC only supported through the PGI and Cray compilers")
-  ENDIF()
-ENDIF ()
-
-##############################################################################
-# CUDA Fortran specific flags - only supporting PGI compiler
-##############################################################################
-OPTION(ENABLE_CUDA_FORTRAN "Whether to build with CUDA Fortran support" FALSE)
-IF (${ENABLE_CUDA_FORTRAN})
-  IF (NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL PGI)
-    MESSAGE(FATAL_ERROR "CUDA Fortran only supported through the PGI compiler")
-  ELSE ()
-    # Set PGI CUDA Fortran flags
-
-    # Set defaults as lowest version of CUDA and device capability allowed
-    # To do: determine a way to generalize this
-    IF (NOT CUDA_VERSION)
-      SET(CUDA_VERSION "4.1")
-    ENDIF ()
-
-    # Compute capability: cc2x is for devices with compute capability >= 2.0 
-    IF (NOT CUDA_DEVICE_CAPABILITY)
-      SET(CUDA_DEVICE_CAPABILITY "cc2x")
-    ENDIF ()
-
-    SET(CMAKE_Fortran_FLAGS 
-        "${CMAKE_Fortran_FLAGS} -ta=nvidia -Mcuda=${CUDA_VERSION},${CUDA_DEVICE_CAPABILITY},ptxinfo,keepgpu")
-
-    MESSAGE(STATUS "Testing PGI CUDA Fortran Compilation with flags: ${CMAKE_Fortran_FLAGS}")
-
-    TRY_COMPILE(CUDAFOR
-                ${CMAKE_BINARY_DIR}/tests/compilerTests/
-                ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compilerTests/cudaFortranTest.f90
-                OUTPUT_VARIABLE COMPILE_OUTPUT)
-    IF (${CUDAFOR})
-      SET(PREQX_USE_CUDA_FORTRAN TRUE)
-      MESSAGE(STATUS "Succeeded. Using CUDA Fortran through PGI compiler")
-    ELSE()
-      SET(PREQX_USE_CUDA_FORTRAN FALSE)
-      MESSAGE(FATAL_ERROR "Unable to use CUDA Fortran through the PGI "
-              "compiler. Compilation failed with the following "
-              "output.\n${COMPILE_OUTPUT}")
-    ENDIF()
-  ENDIF()
+  SET(PREQX_USE_OPENACC TRUE)
 ENDIF ()
 
 ##############################################################################

diff --git a/components/homme/cmake/machineFiles/titan.cmake b/components/homme/cmake/machineFiles/titan.cmake
@@ -5,7 +5,16 @@ SET (CMAKE_CXX_COMPILER CC CACHE FILEPATH "")
 SET (NETCDF_DIR $ENV{NETCDF_DIR} CACHE FILEPATH "")
 SET (PNETCDF_DIR $ENV{PARALLEL_NETCDF_DIR} CACHE FILEPATH "")
 SET (HDF5_DIR $ENV{HDF5_DIR} CACHE FILEPATH "")
+SET (DEBUG_FLAGS " " CACHE STRING "")
+SET (Netcdf_NC_CONFIG_BIN "/opt/cray/netcdf/4.3.3.1/bin" CACHE FILEPATH "")
+SET (HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "")
+SET (USE_MPIEXEC "aprun" CACHE STRING "")
 
 # The following is required for cross compilation
 SET (CMAKE_SYSTEM_NAME Catamount CACHE FILEPATH "")
+
+#Regression test parameters
 SET (USE_QUEUING FALSE CACHE BOOL "")
+SET (USE_NUM_PROCS 64 CACHE STRING "")
+
+
diff --git a/components/homme/compile_scripts/thatchroof/compile.cpu b/components/homme/compile_scripts/thatchroof/compile.cpu