GitHub Issue NOAA-EMC/GSI#320. Replaced fv3gfs_ncio with hpc-stack bu…

…ilt ncio. * all CMakeLists.txt entries that use fv3gfs_ncio - updated to use hpc-stack built ncio library * cmake/Modules/setGNUFlags.cmake - remove fv3gfs_ncio flags * cmake/Modules/setIntelFlags.cmake - same * hpc-stack compliant modulefiles - add module load ncio/1.0.0 * src/fv3gfs_ncio - removed * all *90 which currently use module_fv3gfs_ncio - replace module_fv3gfs_ncio with module_ncio * ush/sub_hera - corrected issue that would cause regression tests to fail on Hera due to SLURM out-of-memory issues
MichaelLueken · Mar 22, 2022 · 2db5d3c · 2db5d3c
1 parent 47ee70b
commit 2db5d3c
Show file tree

Hide file tree

Showing 53 changed files with 130 additions and 2,051 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -76,7 +76,6 @@ project(GSI)
   option(BUILD_ENKF_PREPROCESS_ARW "Build enkf preprocess for ARW " OFF)
   option(BUILD_COV_CALC "Build the Desroziers utility" OFF)
   option(BUILD_NCDIAG "Build the NCDIAG libraries" ON)
-  option(BUILD_FV3GFS_NCIO "Build the FV3GFS_NCIO library" ON)
   option(BUILD_NCIO_UTIL "Build the NCIO Utilities" ON)
   option(BUILD_NCDIAG_SERIAL "Build the serial NCDIAG libraries" ON)
   option(BUILD_REG_TESTING "Build the Regression Testing Suite" ON)
@@ -233,11 +232,13 @@ project(GSI)
     add_subdirectory(src/ncdiag)
     set(NCDIAG_LIBRARIES ncdiag )
   endif(BUILD_NCDIAG)
-  if(BUILD_FV3GFS_NCIO)
-    set(FV3GFS_NCIO_INCS "${PROJECT_BINARY_DIR}/src/fv3gfs_ncio/include")
-    add_subdirectory(src/fv3gfs_ncio)
-    set(FV3GFS_NCIO_LIBRARIES fv3gfs_ncio )
-  endif(BUILD_FV3GFS_NCIO)
+# Set NCIO library from hpc-stack
+  if(DEFINED ENV{NCIO_LIB})
+    set(NCIO_LIB "$ENV{NCIO_LIB}" CACHE INTERNAL "NCIO library" )
+    set(NCIO_INC "$ENV{NCIO_INC}" CACHE INTERNAL "NCIO includes" )
+  else()
+    message("src/fv3gfs_ncio not pulled from git, please load ncio from hpc-stack")
+  endif()
   if(BUILD_NCIO_UTIL)
     add_subdirectory(util/netcdf_io)
   endif(BUILD_NCIO_UTIL)

diff --git a/cmake/Modules/setGNUFlags.cmake b/cmake/Modules/setGNUFlags.cmake
@@ -27,7 +27,6 @@ function (setGNU)
     set(W3NCO_C_FLAGS " -DLINUX -O3 -fconvert=big-endian -ffast-math -fno-second-underscore -frecord-marker=4 -funroll-loops -ggdb -static -Wall -fno-range-check  -D_REAL8_ -fopenmp" CACHE INTERNAL "")
     set(WRFLIB_Fortran_FLAGS " -O3 -fconvert=big-endian -ffast-math -fno-second-underscore -frecord-marker=4 -funroll-loops -ggdb -static -Wall -fno-range-check -D_REAL8_ -fopenmp -ffree-line-length-0" CACHE INTERNAL "")
     set( NCDIAG_Fortran_FLAGS "-ffree-line-length-none" CACHE INTERNAL "" )
-    set( FV3GFS_NCIO_Fortran_FLAGS "-ffree-line-length-none" CACHE INTERNAL "" )
     set( NDATE_Fortran_FLAGS "-fconvert=big-endian -DCOMMCODE -DLINUX -DUPPLITTLEENDIAN -O3 -Wl,-noinhibit-exec" CACHE INTERNAL "")
     set( COV_CALC_FLAGS "-c -O3 -fconvert=little-endian -ffast-math -ffree-form -fno-second-underscore -frecord-marker=4 -funroll-loops -ggdb -static -Wall -fopenmp" CACHE INTERNAL "")
     set(GSDCLOUD_Fortran_FLAGS "-O3 -fconvert=big-endian" CACHE INTERNAL "")
@@ -57,7 +56,6 @@ function (setGNU)
     set(W3NCO_C_FLAGS " -DLINUX -g -fbacktrace -fconvert=big-endian -ffast-math -fno-second-underscore -frecord-marker=4 -funroll-loops -g -ggdb -static -Wall -fno-range-check  -D_REAL8_ -fopenmp" CACHE INTERNAL "")
     set(WRFLIB_Fortran_FLAGS " -g -fbacktrace -fconvert=big-endian -ffast-math -fno-second-underscore -frecord-marker=4 -funroll-loops -g -ggdb -static -Wall -fno-range-check -D_REAL8_ -fopenmp -ffree-line-length-0" CACHE INTERNAL "")
     set( NCDIAG_Fortran_FLAGS "-ffree-line-length-none" CACHE INTERNAL "" )
-    set( FV3GFS_NCIO_Fortran_FLAGS "-ffree-line-length-none" CACHE INTERNAL "" )
     set( NDATE_Fortran_FLAGS "-fconvert=big-endian -DCOMMCODE -DLINUX -DUPPLITTLEENDIAN -g -fbacktrace -Wl,-noinhibit-exec" CACHE INTERNAL "")
     set( COV_CALC_FLAGS "-c -O3 -fconvert=little-endian -ffast-math -ffree-form -fno-second-underscore -frecord-marker=4 -funroll-loops -ggdb -static -Wall -fopenmp" CACHE INTERNAL "")
     set(GSDCLOUD_Fortran_FLAGS "-O3 -fconvert=big-endian" CACHE INTERNAL "")

diff --git a/cmake/Modules/setIntelFlags.cmake b/cmake/Modules/setIntelFlags.cmake
@@ -20,7 +20,6 @@ function(set_LIBRARY_UTIL_Intel)
     set (W3NCO_C_FLAGS "-O0 -DUNDERSCORE -DLINUX -D__linux__ " CACHE INTERNAL "" )
     set (NDATE_Fortran_FLAGS "${HOST_FLAG} -fp-model source -ftz -assume byterecl -convert big_endian -heap-arrays  -DCOMMCODE -DLINUX -DUPPLITTLEENDIAN -O3 -Wl,-noinhibit-exec" CACHE INTERNAL "" )
     set(NCDIAG_Fortran_FLAGS "-free -assume byterecl -convert big_endian" CACHE INTERNAL "" )
-    set(FV3GFS_NCIO_Fortran_FLAGS "-free" CACHE INTERNAL "" )
     set(UTIL_Fortran_FLAGS "-O3 ${HOST_FLAG} -warn all -implicitnone -traceback -fp-model strict -convert big_endian -DWRF -D_REAL8_ ${OpenMP_Fortran_FLAGS}" CACHE INTERNAL "")
     set(UTIL_COM_Fortran_FLAGS "-O3 -fp-model source -convert big_endian -assume byterecl -implicitnone" CACHE INTERNAL "")
 #    set(COV_CALC_FLAGS "-O3 ${HOST_FLAG} -warn all -implicitnone -traceback -fp-model strict -convert little_endian -D_REAL8_ -openmp -fpp -auto" CACHE INTERNAL "" )
@@ -46,7 +45,6 @@ function(set_LIBRARY_UTIL_Debug_Intel)
     set(W3NCO_4_Fortran_FLAGS " -g -auto -assume nocc_omp -i${intsize}     -convert big_endian -assume byterecl -fp-model strict ${OpenMP_Fortran_FLAGS} " CACHE INTERNAL "" )
     set(W3NCO_C_FLAGS "-O0 -g -DUNDERSCORE -DLINUX -D__linux__ " CACHE INTERNAL "" )
     set(NCDIAG_Fortran_FLAGS "-free -assume byterecl -convert big_endian" CACHE INTERNAL "" )
-    set(FV3GFS_NCIO_Fortran_FLAGS "-free" CACHE INTERNAL "" )
     set(WRFLIB_Fortran_FLAGS "-DPOUND_FOR_STRINGIFY -O1 -g -fp-model source -assume byterecl -convert big_endian -g -traceback -D_REAL8_ ${MPI_Fortran_COMPILE_FLAGS}" CACHE INTERNAL "")
     set(NDATE_Fortran_FLAGS "${HOST_FLAG} -fp-model source -ftz -assume byterecl -convert big_endian -heap-arrays  -DCOMMCODE -DLINUX -DUPPLITTLEENDIAN -g -Wl,-noinhibit-exec" CACHE INTERNAL "" )
     set(WRFLIB_C_FLAGS "-I. -DFortranByte=char -DFortranInt=int -DFortranLlong='long long'  -g  -Dfunder" CACHE INTERNAL "" )

diff --git a/modulefiles/modulefile.ProdGSI.hera b/modulefiles/modulefile.ProdGSI.hera
@@ -43,3 +43,4 @@ module load bacio/2.4.1
 module load crtm/2.3.0
 module load netcdf/4.7.4
 module load wrf_io/1.2.0
+module load ncio/1.0.0
diff --git a/modulefiles/modulefile.ProdGSI.jet b/modulefiles/modulefile.ProdGSI.jet
@@ -43,3 +43,4 @@ module load bacio/2.4.1
 module load crtm/2.3.0
 module load netcdf/4.7.4
 module load wrf_io/1.2.0
+module load ncio/1.0.0
diff --git a/modulefiles/modulefile.ProdGSI.orion b/modulefiles/modulefile.ProdGSI.orion
@@ -48,3 +48,4 @@ module load bacio/2.4.1
 module load crtm/2.3.0
 module load netcdf/4.7.4
 module load wrf_io/1.2.0
+module load ncio/1.0.0
diff --git a/modulefiles/modulefile.ProdGSI.s4 b/modulefiles/modulefile.ProdGSI.s4
@@ -30,3 +30,4 @@ module load bacio/2.4.1
 module load crtm/2.3.0
 module load netcdf/4.7.4
 module load wrf_io/1.2.0
+module load ncio/1.0.0
diff --git a/modulefiles/modulefile.ProdGSI.wcoss_d b/modulefiles/modulefile.ProdGSI.wcoss_d
@@ -44,3 +44,4 @@ module load bacio/2.4.1
 module load crtm/2.3.0
 module load netcdf/4.7.4
 module load wrf_io/1.2.0
+module load ncio/1.0.0
diff --git a/src/enkf/CMakeLists.txt b/src/enkf/CMakeLists.txt
@@ -57,7 +57,7 @@ if(BUILD_ENKF)
 
   add_definitions(${MPI_Fortran_FLAGS})
 
-  include_directories(${CMAKE_CURRENT_BINARY_DIR} "${PROJECT_BINARY_DIR}/include/wrf" "${PROJECT_BINARY_DIR}/include/global"  ${CMAKE_CURRENT_BINARY_DIR}/.. ${MPI_Fortran_INCLUDE_DIRS} ${MPI_Fortran_INCLUDE_PATH} ${CORE_INCS} ${NETCDF_INCLUDE_DIRS} ${NCDIAG_INCS} ${FV3GFS_NCIO_INCS})
+  include_directories(${CMAKE_CURRENT_BINARY_DIR} "${PROJECT_BINARY_DIR}/include/wrf" "${PROJECT_BINARY_DIR}/include/global"  ${CMAKE_CURRENT_BINARY_DIR}/.. ${MPI_Fortran_INCLUDE_DIRS} ${MPI_Fortran_INCLUDE_PATH} ${CORE_INCS} ${NETCDF_INCLUDE_DIRS} ${NCDIAG_INCS} ${NCIO_INC})
   link_directories(${MPI_Fortran_LIBRARIES})
 
   set_source_files_properties( ${ENKF_SRCS} PROPERTIES COMPILE_FLAGS ${ENKF_Fortran_FLAGS} )
@@ -74,7 +74,7 @@ if(BUILD_ENKF)
   add_executable(${ENKFEXEC} enkf_main.f90)
   target_link_libraries(${ENKFEXEC} enkflib enkfdeplib ${GSILIB} ${GSISHAREDLIB} ${CORE_LIBRARIES}
   ${MPI_Fortran_LIBRARIES} ${LAPACK_LIBRARIES} ${CORE_LIBRARIES} ${CORE_BUILT}
-  ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${FV3GFS_NCIO_LIBRARIES} 
+  ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${NCIO_LIB} 
   ${EXTRA_LINKER_FLAGS} ${GSI_LDFLAGS} ${NCDIAG_LIBRARIES})
   install(TARGETS ${ENKFEXEC} enkfdeplib enkflib 
         RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin

diff --git a/src/enkf/gridinfo_gfs.f90 b/src/enkf/gridinfo_gfs.f90
@@ -78,8 +78,8 @@ subroutine getgridinfo(fileprefix, reducedgrid)
 use nemsio_module, only: nemsio_gfile,nemsio_open,nemsio_close,&
                          nemsio_getfilehead,nemsio_getheadvar,&
                          nemsio_readrecv,nemsio_init, nemsio_realkind
-use module_fv3gfs_ncio, only: Dataset, Variable, Dimension, open_dataset,&
-                        read_attribute, close_dataset, get_dim, read_vardata 
+use module_ncio, only: Dataset, Variable, Dimension, open_dataset,&
+                       read_attribute, close_dataset, get_dim, read_vardata 
 implicit none
 
 type(Dataset) :: dset

diff --git a/src/enkf/gridio_gfs.f90 b/src/enkf/gridio_gfs.f90
@@ -61,7 +61,7 @@ module gridio
 
  subroutine readgriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,ntimes, &
                              fileprefixes,filesfcprefixes,reducedgrid,grdin,qsat)
-  use module_fv3gfs_ncio, only: Dataset, Variable, Dimension, open_dataset,&
+  use module_ncio, only: Dataset, Variable, Dimension, open_dataset,&
                  quantize_data,read_attribute, close_dataset, get_dim, read_vardata
   implicit none
 
@@ -423,7 +423,7 @@ subroutine readgriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,ntimes,
   use nemsio_module, only: nemsio_gfile,nemsio_open,nemsio_close,&
                            nemsio_getfilehead,nemsio_getheadvar,nemsio_realkind,nemsio_charkind,&
                            nemsio_readrecv,nemsio_init,nemsio_setheadvar,nemsio_writerecv
-  use module_fv3gfs_ncio, only: Dataset, Variable, Dimension, open_dataset,&
+  use module_ncio, only: Dataset, Variable, Dimension, open_dataset,&
                  quantize_data,read_attribute, close_dataset, get_dim, read_vardata
   implicit none
 
@@ -1032,11 +1032,11 @@ end subroutine readgriddata
 
  subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,grdin,no_inflate_flag)
   use netcdf
-  use module_fv3gfs_ncio, only: Dataset, Variable, Dimension, open_dataset,&
-                          read_attribute, close_dataset, get_dim, read_vardata,&
-                          create_dataset, get_idate_from_time_units, &
-                          get_time_units_from_idate, write_vardata, &
-                          write_attribute, quantize_data, has_var, has_attr
+  use module_ncio, only: Dataset, Variable, Dimension, open_dataset,&
+                         read_attribute, close_dataset, get_dim, read_vardata,&
+                         create_dataset, get_idate_from_time_units, &
+                         get_time_units_from_idate, write_vardata, &
+                         write_attribute, quantize_data, has_var, has_attr
   use constants, only: grav, zero
   use params, only: nbackgrounds,anlfileprefixes,fgfileprefixes,reducedgrid,&
                     nccompress
@@ -1842,11 +1842,11 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,grdin,n
                            nemsio_readrec,nemsio_writerec,nemsio_intkind,nemsio_charkind,&
                            nemsio_getheadvar,nemsio_realkind,nemsio_getfilehead,&
                            nemsio_readrecv,nemsio_init,nemsio_setheadvar,nemsio_writerecv
-  use module_fv3gfs_ncio, only: Dataset, Variable, Dimension, open_dataset,&
-                          read_attribute, close_dataset, get_dim, read_vardata,&
-                          create_dataset, get_idate_from_time_units, &
-                          get_time_units_from_idate, write_vardata, &
-                          write_attribute, quantize_data, has_var, has_attr
+  use module_ncio, only: Dataset, Variable, Dimension, open_dataset,&
+                         read_attribute, close_dataset, get_dim, read_vardata,&
+                         create_dataset, get_idate_from_time_units, &
+                         get_time_units_from_idate, write_vardata, &
+                         write_attribute, quantize_data, has_var, has_attr
   use constants, only: grav
   use params, only: nbackgrounds,anlfileprefixes,fgfileprefixes,reducedgrid,&
                     nccompress,write_ensmean
@@ -3308,11 +3308,11 @@ subroutine writeincrement(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,grdin,
                     datestring,nhr_anal,write_ensmean
   use constants, only: grav
   use mpi
-  use module_fv3gfs_ncio, only: Dataset, Variable, Dimension, open_dataset,&
-                          read_attribute, close_dataset, get_dim, read_vardata,&
-                          create_dataset, get_idate_from_time_units, &
-                          get_time_units_from_idate, write_vardata, &
-                          write_attribute, quantize_data, has_var, has_attr
+  use module_ncio, only: Dataset, Variable, Dimension, open_dataset,&
+                         read_attribute, close_dataset, get_dim, read_vardata,&
+                         create_dataset, get_idate_from_time_units, &
+                         get_time_units_from_idate, write_vardata, &
+                         write_attribute, quantize_data, has_var, has_attr
   implicit none
 
   integer, intent(in) :: nanal1,nanal2
@@ -3711,11 +3711,11 @@ subroutine writeincrement_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,grdin,no_inflate
                     datestring,nhr_anal
   use constants, only: grav
   use mpi
-  use module_fv3gfs_ncio, only: Dataset, Variable, Dimension, open_dataset,&
-                          read_attribute, close_dataset, get_dim, read_vardata,&
-                          create_dataset, get_idate_from_time_units, &
-                          get_time_units_from_idate, write_vardata, &
-                          write_attribute, quantize_data, has_var, has_attr
+  use module_ncio, only: Dataset, Variable, Dimension, open_dataset,&
+                         read_attribute, close_dataset, get_dim, read_vardata,&
+                         create_dataset, get_idate_from_time_units, &
+                         get_time_units_from_idate, write_vardata, &
+                         write_attribute, quantize_data, has_var, has_attr
   implicit none
 
   character(len=max_varname_length), dimension(n2d), intent(in) :: vars2d

diff --git a/src/fv3gfs_ncio/CMakeLists.txt b/src/fv3gfs_ncio/CMakeLists.txt