Update gsl develop from master

.gitmodules

@@ -9,7 +9,7 @@
 [submodule "FMS"]
   path = FMS
   url = https://github.com/NOAA-GFDL/FMS
-  branch = master
+  branch = release/2020.04
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3

CICE-interface/CMakeLists.txt

@@ -9,7 +9,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "-g -traceback")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -FR -convert big_endian -assume byterecl -ftz -align array64byte -xHOST")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fp-model precise")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
   set(CMAKE_Fortran_LINK_FLAGS "")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")

CMEPS-interface/CMakeLists.txt

@@ -16,17 +16,17 @@ else()
 endif()
 
 list(APPEND _nems_util_files
-  CMEPS/nems/util/shr_abort_mod.F90
-  CMEPS/nems/util/shr_log_mod.F90
-  CMEPS/nems/util/shr_pio_mod.F90
-  CMEPS/nems/util/shr_sys_mod.F90
-  CMEPS/nems/util/shr_flux_mod.F90
-  CMEPS/nems/util/shr_mpi_mod.F90
-  CMEPS/nems/util/glc_elevclass_mod.F90
-  CMEPS/nems/util/shr_mem_mod.F90
-  CMEPS/nems/util/shr_kind_mod.F90
-  CMEPS/nems/util/perf_mod.F90
-  CMEPS/nems/util/shr_const_mod.F90)
+  CMEPS/util/shr_abort_mod.F90
+  CMEPS/util/shr_log_mod.F90
+  CMEPS/util/shr_pio_mod.F90
+  CMEPS/util/shr_sys_mod.F90
+  CMEPS/util/shr_flux_mod.F90
+  CMEPS/util/shr_mpi_mod.F90
+  CMEPS/util/glc_elevclass_mod.F90
+  CMEPS/util/shr_mem_mod.F90
+  CMEPS/util/shr_kind_mod.F90
+  CMEPS/util/perf_mod.F90
+  CMEPS/util/shr_const_mod.F90)
 
 list(APPEND _mediator_files
   CMEPS/mediator/med_phases_restart_mod.F90
@@ -56,7 +56,14 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_fraction_mod.F90
   CMEPS/mediator/med_phases_aofluxes_mod.F90
   CMEPS/mediator/med_io_mod.F90
-  CMEPS/mediator/med_diag_mod.F90)
+  CMEPS/mediator/med_diag_mod.F90
+  CMEPS/mediator/med_phases_post_atm_mod.F90
+  CMEPS/mediator/med_phases_post_ice_mod.F90
+  CMEPS/mediator/med_phases_post_ocn_mod.F90
+  CMEPS/mediator/med_phases_post_lnd_mod.F90
+  CMEPS/mediator/med_phases_post_glc_mod.F90
+  CMEPS/mediator/med_phases_post_rof_mod.F90
+  CMEPS/mediator/med_phases_post_wav_mod.F90)
 
 list(APPEND _mediator_defs ESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}
                            ESMF_VERSION_MINOR=${ESMF_VERSION_MINOR}

CMakeLists.txt

@@ -21,7 +21,6 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 set(32BIT           OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore)")
 set(AVX2            ON  CACHE BOOL "Enable AVX2 instruction set")
 set(SIMDMULTIARCH   OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
-set(CCPP            ON  CACHE BOOL "Enable CCPP")
 set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
 set(DEBUG_LINKMPI   ON  CACHE BOOL "Enable linkmpi option when DEBUG mode is on")
 set(INLINE_POST     OFF CACHE BOOL "Enable inline post")
@@ -32,13 +31,16 @@ set(QUAD_PRECISION  ON  CACHE BOOL "Enable QUAD_PRECISION (for certain grid metr
 set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
 set(WW3             OFF CACHE BOOL "Enable WW3")
 set(S2S             OFF CACHE BOOL "Enable S2S")
+set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(DATM            OFF CACHE BOOL "Enable Data Atmosphere")
 
 ###############################################################################
 ### Set CMAKE_BUILD_TYPE for DEBUG mode
 ###############################################################################
 if(DEBUG)
   set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Set type of build to Debug." FORCE)
+else()
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Set type of build to Release." FORCE)
 endif()
 
 if(CMAKE_Platform)
@@ -49,13 +51,23 @@ if(CMAKE_Platform)
   else()
     message("Platform '${CMAKE_Platform}' configuration file does not exist")
   endif()
+
+  # DH* 20210208 temporary workaround for FMS issues on Cheyenne with GNU 9.1.0
+  if (CMAKE_Platform MATCHES "cheyenne.gnu")
+    message("Applying patch cheyenne_gnu_fms_mpp_util_mpi_inc.patch")
+    execute_process(COMMAND patch -N -p0 INPUT_FILE cheyenne_gnu_fms_mpp_util_mpi_inc.patch
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+                    OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/cheyenne_gnu_fms_mpp_util_mpi_inc.patch.out
+                    ERROR_FILE  ${CMAKE_CURRENT_BINARY_DIR}/cheyenne_gnu_fms_mpp_util_mpi_inc.patch.err
+                    RESULT_VARIABLE RC)
+    # *DH 20210208
+  endif()
 endif()
 
 message("")
 message("32BIT ............ ${32BIT}")
 message("AVX2 ............. ${AVX2}")
 message("SIMDMULTIARCH ...  ${SIMDMULTIARCH}")
-message("CCPP ............. ${CCPP}")
 message("DEBUG ............ ${DEBUG}")
 message("DEBUG_LINKMPI .... ${DEBUG_LINKMPI}")
 message("INLINE_POST ...... ${INLINE_POST}")
@@ -109,13 +121,9 @@ endif()
 # See https://github.com/NOAA-EMC/NCEPLIBS-nemsio/pull/22
 target_link_libraries(nemsio::nemsio INTERFACE w3emc::w3emc_d bacio::bacio_4)
 
-if(CCPP)
-  find_package(Python 3 QUIET COMPONENTS Interpreter)
-  if (NOT Python_Interpreter_FOUND)
-    find_package(Python 2.7 REQUIRED COMPONENTS Interpreter)
-  endif()
-  message("Found Python: ${Python_EXECUTABLE}")
-endif()
+# Configure Python
+find_package(Python 3 REQUIRED COMPONENTS Interpreter)
+message("Found Python: ${Python_EXECUTABLE}")
 
 ###############################################################################
 ### Checks
@@ -132,9 +140,7 @@ endif()
 ### FMS
 ###############################################################################
 include(fms_files.cmake)
-add_library(fms ${fms_src_files})
-# stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs
-set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0")
+add_library(fms ${fms_src_files} ${fms_headers})
 
 list(APPEND _fms_defs_public use_libMPI
                              use_netCDF
@@ -143,26 +149,39 @@ list(APPEND _fms_defs_public use_libMPI
 if(QUAD_PRECISION)
   list(APPEND _fms_defs_public ENABLE_QUAD_PRECISION)
 endif()
+
+# check gettid
+include(CheckFunctionExists)
+check_function_exists(gettid HAVE_GETTID)
+if(HAVE_GETTID)
+  list(APPEND _fms_defs_public HAVE_GETTID)
+endif()
+
 target_compile_definitions(fms PUBLIC "${_fms_defs_public}")
 
 if(32BIT)
   list(APPEND _fms_defs_private OVERLOAD_R4
                                 OVERLOAD_R8)
 endif()
+
 target_compile_definitions(fms PRIVATE "${_fms_defs_private}")
 
 target_include_directories(fms PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/include>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms2_io/include>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/mpp/include>)
+target_include_directories(fms INTERFACE
+                                      $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>
+                                      $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/mod>)
 target_link_libraries(fms PUBLIC MPI::MPI_Fortran
                                  NetCDF::NetCDF_Fortran)
 if(OpenMP_Fortran_FOUND)
   target_link_libraries(fms PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS)
 set_target_properties(fms PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)
-target_include_directories(fms INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
-                                         $<INSTALL_INTERFACE:FMS/mod>)
+set_target_properties(fms PROPERTIES PUBLIC_HEADER "${fms_headers}" )
 
 ###############################################################################
 ### stochastic_physics
@@ -189,6 +208,10 @@ if(WW3)
       set(WW3_COMP "hera")
     elseif(${CMAKE_Platform} STREQUAL "orion.intel")
       set(WW3_COMP "orion")
+    elseif(${CMAKE_Platform} STREQUAL "cheyenne.intel")
+      set(WW3_COMP "cheyenne")
+    elseif(${CMAKE_Platform} STREQUAL "wcoss2")
+      set(WW3_COMP "wcoss_cray")
     elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*")
       if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
         set(WW3_COMP "gnu")
@@ -232,8 +255,7 @@ list(APPEND _nems_srcs NEMS/src/module_NEMS_UTILS.F90
                        NEMS/src/module_NEMS_INTERNAL_STATE.F90
                        NEMS/src/module_NEMS_GRID_COMP.F90
                        NEMS/src/module_NEMS_Rusage.F90
-                       NEMS/src/nems_c_rusage.c
-                       NEMS/src/ENS_Cpl/ENS_CplComp_ESMFMod_STUB.F90)
+                       NEMS/src/nems_c_rusage.c)
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h
                ${CMAKE_CURRENT_BINARY_DIR}/NEMS/src/ESMFVersionDefine.h
@@ -259,7 +281,9 @@ else()
   list(APPEND _ufs_defs_private FRONT_FV3=fv3gfs_cap_mod)
   add_dependencies(ufs fv3atm)
 endif()
-
+if(JEDI_DRIVER)
+  list(APPEND _ufs_defs_private JEDI_DRIVER=ON)
+endif()
 set_target_properties(ufs PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(ufs PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/NEMS/src)
 
@@ -314,7 +338,7 @@ endif()
 target_compile_definitions(ufs_model PRIVATE "${_ufs_model_defs_private}")
 
 if(DATM)
-  target_link_libraries(ufs_model PRIVATE ufs w3nco::w3nco_d)
+  target_link_libraries(ufs_model PUBLIC ufs w3nco::w3nco_d)
 endif()
 target_link_libraries(ufs_model PRIVATE ufs
                                         esmf
@@ -325,17 +349,28 @@ target_link_libraries(ufs_model PRIVATE ufs
 ### Install
 ###############################################################################
 install(
-  TARGETS ufs_model ufs fms
-  EXPORT  ufs-config
-  RUNTIME DESTINATION bin
+  TARGETS fms
+  EXPORT fms-config
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  PUBLIC_HEADER DESTINATION include )
+
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+install(EXPORT fms-config
+  DESTINATION lib/cmake
+)
+install(
+  TARGETS ufs
+  EXPORT ufs-config
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
 
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
 
-install(EXPORT      ufs-config
-        DESTINATION lib/cmake)
-
+install(EXPORT ufs-config
+  DESTINATION lib/cmake
+)
 ###############################################################################
 ### Done
 ###############################################################################

MOM6-interface/CMakeLists.txt

@@ -11,7 +11,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -i4 -r8")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -sox")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal -fp-model source")
-  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -link_mpi=dbg")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv")
   set(CMAKE_Fortran_LINK_FLAGS "")
 else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")

cheyenne_gnu_fms_mpp_util_mpi_inc.patch

@@ -0,0 +1,12 @@
+--- FMS/mpp/include/mpp_util_mpi.inc	2021-02-08 08:24:21.000000000 -0700
++++ FMS/mpp/include/mpp_util_mpi.inc	2021-02-08 08:24:15.000000000 -0700
+@@ -169,7 +169,8 @@
+   integer, intent(in   ), optional :: msg_size(:)
+   integer, intent(in   ), optional :: msg_type(:)
+
+-  integer                       :: i, m, n, stride, my_check, rsize
++  integer                       :: i, m, n, stride, my_check
++  integer, volatile             :: rsize
+
+   if( debug .and. (current_clock.NE.0) )call SYSTEM_CLOCK(start_tick)
+   my_check = EVENT_SEND

cmake/Intel.cmake

@@ -21,11 +21,13 @@ elseif(DEBUG)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align")
 else()
     if(AVX2)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
     elseif(SIMDMULTIARCH)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")
+        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,CORE-AVX2 -qno-opt-dynamic-align")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,CORE-AVX2 -qno-opt-dynamic-align")
     else()
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qno-opt-dynamic-align")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align")

cmake/configure_wcoss2.cmake

@@ -0,0 +1,3 @@
+set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)