Merge branch 'rljacob/blues-nag' into next (PR #346)

Add NAG to Blues. Workaround for PGI build. Enable Blues to compile ACME with the NAG compiler version 6.0 Only mpich3 is supported because pnetcdf/nag was built with it. serial netcdf is pointing to a temporary location for now. Acme-developer does not completely pass yet. The serial netcdf softkey for pgi is currently not working and this commit also provides a workaround and reenables pgi buids on Blues.
E3SM-Project · Oct 20, 2015 · 598b82c · 598b82c
2 parents 334ddbb + c7c2ae8
commit 598b82c
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 4 deletions.
diff --git a/cime/machines-acme/config_compilers.xml b/cime/machines-acme/config_compilers.xml
@@ -474,7 +474,7 @@ for mct, etc.
   <!-- interfaces from certain argument checks. Should not be necessary in   -->
   <!-- libraries that only use the F90 module interface. mpibcast and        -->
   <!-- mpiscatterv are actually CAM wrappers for MPI.                        -->
-  <FFLAGS      > -kind=byte -wmismatch=mpi_send,mpi_recv,mpi_bcast,mpi_allreduce,mpi_reduce,mpi_isend,mpi_irecv,mpi_irsend,mpi_rsend,mpi_gatherv,mpi_gather,mpi_scatterv,mpi_allgather,mpi_alltoallv,mpi_file_read_all,mpi_file_write_all,mpibcast,mpiscatterv,mpi_alltoallw,nfmpi_get_vara_all,NFMPI_IPUT_VARA,NFMPI_GET_VAR_ALL,NFMPI_PUT_VARA,NFMPI_PUT_ATT_REAL,NFMPI_PUT_ATT_DOUBLE,NFMPI_PUT_ATT_INT,NFMPI_GET_ATT_REAL,NFMPI_GET_ATT_INT,NFMPI_GET_ATT_DOUBLE,NFMPI_PUT_VARA_DOUBLE_ALL,NFMPI_PUT_VARA_REAL_ALL,NFMPI_PUT_VARA_INT_ALL    -convert=BIG_ENDIAN </FFLAGS>
+  <FFLAGS      > -wmismatch=mpi_send,mpi_recv,mpi_bcast,mpi_allreduce,mpi_reduce,mpi_isend,mpi_irecv,mpi_irsend,mpi_rsend,mpi_gatherv,mpi_gather,mpi_scatterv,mpi_allgather,mpi_alltoallv,mpi_file_read_all,mpi_file_write_all,mpibcast,mpiscatterv,mpi_alltoallw,nfmpi_get_vara_all,NFMPI_IPUT_VARA,NFMPI_GET_VAR_ALL,NFMPI_PUT_VARA,NFMPI_PUT_ATT_REAL,NFMPI_PUT_ATT_DOUBLE,NFMPI_PUT_ATT_INT,NFMPI_GET_ATT_REAL,NFMPI_GET_ATT_INT,NFMPI_GET_ATT_DOUBLE,NFMPI_PUT_VARA_DOUBLE_ALL,NFMPI_PUT_VARA_REAL_ALL,NFMPI_PUT_VARA_INT_ALL    -convert=BIG_ENDIAN </FFLAGS>
   <FFLAGS_NOOPT> $(FFLAGS) </FFLAGS_NOOPT>
 
   <!-- DEBUG vs. non-DEBUG runs.                                             -->
@@ -819,6 +819,45 @@ for mct, etc.
   <PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
 </compiler>
 
+<compiler COMPILER="nag" MACH="blues">
+  <PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
+  <NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
+  <MPI_PATH MPILIB="mpich">/home/robl/soft/mpich-3.1.4-nag-6.0</MPI_PATH>
+  <MPI_LIB_NAME MPILIB="mpich"> mpi </MPI_LIB_NAME>
+  <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas</ADD_SLIBS>
+  <PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
+</compiler>
+
+<compiler COMPILER="pgi" MACH="eastwind">
+  <ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
+  <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
+  <NETCDF_PATH> $(NETCDF_HOME)</NETCDF_PATH>
+  <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
+  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
+  <ADD_CPPDEFS> -DLINUX </ADD_CPPDEFS>
+  <SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi </SLIBS>
+</compiler>
+
+<compiler COMPILER="pgi" MACH="olympus">
+  <ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
+  <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
+  <NETCDF_PATH> $(NETCDF_LIB)/..</NETCDF_PATH>
+  <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
+  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
+  <ADD_CPPDEFS> -DLINUX </ADD_CPPDEFS>
+  <SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi </SLIBS>
+</compiler>
+
+<compiler COMPILER="intel" MACH="olympus">
+  <ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
+  <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
+  <NETCDF_PATH> $(NETCDF_LIB)/..</NETCDF_PATH>
+  <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
+  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
+  <ADD_CPPDEFS> -DLINUX </ADD_CPPDEFS>
+  <SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi </SLIBS>
+</compiler>
+
 <compiler COMPILER="intel" MACH="sooty">
   <ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
   <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>

diff --git a/cime/machines-acme/config_machines.xml b/cime/machines-acme/config_machines.xml
@@ -368,7 +368,7 @@
          <DESC>ANL/LCRC Linux Cluster</DESC>
          <NODENAME_REGEX>blogin</NODENAME_REGEX>
          <TESTS>acme_developer</TESTS>
-         <COMPILERS>intel,gnu,pgi</COMPILERS>
+         <COMPILERS>intel,gnu,pgi,nag</COMPILERS>
          <MPILIBS>mpich,openmpi,mpi-serial</MPILIBS>
          <CESMSCRATCHROOT>/lcrc/project/$PROJECT/$USER/acme_scratch</CESMSCRATCHROOT>
          <RUNDIR>$CESMSCRATCHROOT/$CASE/run</RUNDIR>

diff --git a/cime/machines-acme/env_mach_specific.blues b/cime/machines-acme/env_mach_specific.blues
@@ -38,8 +38,13 @@ endif
 
 if ( $COMPILER == "pgi" ) then
   soft add +pgi-13.9
-  soft add +netcdf-4.3.1-serial-pgi
-  setenv NETCDFROOT /soft/netcdf/4.3.1-serial/pgi-13.9
+#  soft add +netcdf-4.3.1-serial-pgi
+  setenv NETCDFROOT /home/jacob/netcdf-4.3.3.1pg13.9
+  # Get newer netcdf
+  setenv PATH $NETCDFROOT/bin:$PATH
+  setenv LD_LIBRARY_PATH $NETCDFROOT/lib:$LD_LIBRARY_PATH
+  setenv NETCDF_INCLUDES $NETCDFROOT/include
+  setenv NETCDF_LIBS $NETCDFROOT/lib
   if ( $MPILIB == "mpi-serial") then
    setenv PNETCDFROOT ""
   else if ( $MPILIB == "openmpi") then
@@ -81,6 +86,34 @@ if ( $COMPILER == "gnu" ) then
     setenv PNETCDFROOT /soft/pnetcdf/1.5.0/gcc-4.7.2/mpich2-1.4.1
   endif
 endif
+
+if ( $COMPILER == "nag" ) then
+  soft add +nag-6.0
+  soft add +gcc-4.7.2
+#  soft add +netcdf-4.3.1-serial-nag
+  setenv NETCDFROOT /home/jacob/netcdf-4.3.3.1nag6
+  # Get newer netcdf
+  setenv PATH $NETCDFROOT/bin:$PATH
+  setenv LD_LIBRARY_PATH $NETCDFROOT/lib:$LD_LIBRARY_PATH
+  setenv NETCDF_INCLUDES $NETCDFROOT/include
+  setenv NETCDF_LIBS $NETCDFROOT/lib
+  if ( $MPILIB == "openmpi") then
+    echo "ERROR: OpenMPI+pnetcdf is currently not supported with nag compilers"
+    exit 1
+  endif
+  if ( $MPILIB == "mpi-serial") then
+   setenv PNETCDFROOT ""
+  else if ( $MPILIB == "mpich") then
+    soft add +mpich3-3.1.4-nag-6.0
+    soft add +pnetcdf-1.6.1-mpich-3.1.4-nag-6.0
+    setenv PNETCDFROOT /home/robl/soft/pnetcf-trunk-mpich-3.1.4-nag-6.0
+  else
+    # default - mpich + pnetcdf
+    soft add +mpich3-3.1.4-nag-6.0
+    soft add +pnetcdf-1.6.1-mpich-3.1.4-nag-6.0
+    setenv PNETCDFROOT /home/robl/soft/pnetcf-trunk-mpich-3.1.4-nag-6.0
+  endif
+endif
 if ( $?PERL ) then
   printenv
 endif