Merge PR MPAS-Dev#386 'jgfouca/cmake_refactor' into develop

This PR switches E3SM's default build-system to be cmake-based. It does
not alter the current standard MPAS make system.

src/CMakeLists.txt

@@ -0,0 +1,87 @@
+#
+# This is the interface between E3SM's new CMake-based build system and MPAS.
+#
+# The following CMake variables are expected to be defined:
+#  * CORES : A list of CORES to build, comma-separated (e.g. "ocean,seaice,landice")
+#  * Whatever CIME settings are setting to correctly resolve the ${CASEROOT}/Macros.cmake file
+#    - COMPILER, DEBUG, MPILIB, MACH, OS
+#
+
+# Source CIME-generated Macros
+include(${CASEROOT}/Macros.cmake)
+
+#
+# General setup
+#
+
+if (USE_ESMF_LIB)
+  set(ESMFDIR "esmf")
+else()
+  set(ESMFDIR "noesmf")
+endif()
+
+set(CMAKE_C_COMPILER ${MPICC})
+set(CMAKE_CXX_COMPILER ${MPICXX})
+set(CMAKE_Fortran_COMPILER ${MPIFC})
+set(CMAKE_EXE_LINKER_FLAGS "${LDFLAGS}")
+set(CMAKE_VERBOSE_MAKEFILE TRUE)
+
+# Set up CPPDEFS
+set(FILE_OFFSET "-DOFFSET64BIT")
+if (CPPDEFS)
+  separate_arguments(CPPDEFS UNIX_COMMAND "${CPPDEFS}")
+endif()
+list(APPEND CPPDEFS "-DMPAS_NO_LOG_REDIRECT" "-DUSE_PIO2" "-DMPAS_NO_ESMF_INIT" "-DMPAS_ESM_SHR_CONST" "-DMPAS_PERF_MOD_TIMERS" "${MODEL_FORMULATION}" "${FILE_OFFSET}" "${ZOLTAN_DEFINE}" "-D_MPI" "-DMPAS_NAMELIST_SUFFIX=${NAMELIST_SUFFIX}" "-DMPAS_EXE_NAME=${EXE_NAME}")
+if (DEBUG)
+  list(APPEND CPPDEFS "-DMPAS_DEBUG")
+endif()
+if (compile_threaded)
+  list(APPEND CPPDEFS "-DMPAS_OPENMP")
+endif()
+
+set(INCLUDES "${INSTALL_SHAREDPATH}/include" "${INSTALL_SHAREDPATH}/${COMP_INTERFACE}/${ESMFDIR}/${NINST_VALUE}/csm_share" "${INSTALL_SHAREDPATH}/pio" "${PNETCDF_PATH}/include" "${CMAKE_CURRENT_SOURCE_DIR}/external/ezxml" "${CMAKE_BINARY_DIR}/framework" "${CMAKE_BINARY_DIR}/operators")
+if (NETCDF_PATH)
+  list(APPEND INCLUDES ${NETCDF_PATH}/include)
+else()
+  if (NETCDF_C_PATH)
+    list(APPEND INCLUDES ${NETCDF_C_PATH}/include)
+  endif()
+  if (NETCDF_FORTRAN_PATH)
+    list(APPEND INCLUDES ${NETCDF_FORTRAN_PATH}/include)
+  endif()
+endif()
+
+if (USE_KOKKOS)
+  include(${INSTALL_SHAREDPATH}/kokkos_generated_settings.cmake)
+  string (REPLACE ";" " " KOKKOS_CXXFLAGS_STR "${KOKKOS_CXXFLAGS}")
+  set(CXXFLAGS "${CXXFLAGS} ${KOKKOS_CXXFLAGS_STR}")
+endif()
+
+set(CMAKE_Fortran_FLAGS "${FFLAGS}")
+set(CMAKE_C_FLAGS "${CFLAGS}")
+set(CMAKE_CXX_FLAGS "${CXXFLAGS}")
+
+# Include custom cmake libraries used for mpas
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake_utils.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/build_core.cmake)
+
+# Add tools
+add_subdirectory(tools)
+
+# Gather sources that are needed for all cores into "common" library
+
+set(COMMON_RAW_SOURCES external/ezxml/ezxml.c)
+include(${CMAKE_CURRENT_SOURCE_DIR}/framework/framework.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/operators/operators.cmake)
+
+add_library(common OBJECT)
+target_compile_definitions(common PRIVATE ${CPPDEFS})
+target_include_directories(common PRIVATE ${INCLUDES})
+
+genf90_targets("${COMMON_RAW_SOURCES}" "${INCLUDES}" "${CPPDEFS}" "" "")
+target_sources(common PRIVATE ${SOURCES})
+
+# Build cores!
+foreach(CORE IN LISTS CORES)
+  build_core(${CORE})
+endforeach()

src/build_core.cmake

@@ -0,0 +1,67 @@
+function(build_core CORE)
+  set(EXE_NAME ${CORE}_model)
+  set(NAMELIST_SUFFIX ${CORE})
+
+  # Map the ESM component corresponding to each MPAS core
+  if (CORE STREQUAL "ocean")
+    set(COMPONENT "ocn")
+  elseif(CORE STREQUAL "landice")
+    set(COMPONENT "glc")
+  elseif(CORE STREQUAL "seaice")
+    set(COMPONENT "ice")
+  else()
+    message(FATAL_ERROR "Unrecognized core: ${CORE}")
+  endif()
+
+  # Gather sources
+  set(CORE_BLDDIR ${CMAKE_BINARY_DIR}/core_${CORE})
+  if (NOT EXISTS ${CORE_BLDDIR})
+    file(MAKE_DIRECTORY ${CORE_BLDDIR})
+  endif()
+
+  set(CORE_INPUT_DIR ${CORE_BLDDIR}/default_inputs)
+  if (NOT EXISTS ${CORE_INPUT_DIR})
+    file(MAKE_DIRECTORY ${CORE_INPUT_DIR})
+  endif()
+
+  # Provides us RAW_SOURCES, CPPDEFS, and INCLUDES
+  include(${CMAKE_CURRENT_SOURCE_DIR}/core_${CORE}/${CORE}.cmake)
+
+  add_library(${COMPONENT})
+  target_compile_definitions(${COMPONENT} PRIVATE ${CPPDEFS})
+  target_include_directories(${COMPONENT} PRIVATE ${INCLUDES})
+
+  # Make .inc files
+  add_custom_command (
+    OUTPUT ${CORE_BLDDIR}/Registry_processed.xml
+    COMMAND cpp -P -traditional ${CPPDEFS} -Uvector
+    ${CMAKE_CURRENT_SOURCE_DIR}/core_${CORE}/Registry.xml > Registry_processed.xml
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/core_${CORE}/Registry.xml
+    WORKING_DIRECTORY ${CORE_BLDDIR}
+    )
+
+  set(INC_DIR ${CORE_BLDDIR}/inc)
+  if (NOT EXISTS ${INC_DIR})
+    file(MAKE_DIRECTORY ${INC_DIR})
+  endif()
+
+  add_custom_command(
+    OUTPUT ${INC_DIR}/core_variables.inc
+    COMMAND ${CMAKE_BINARY_DIR}/mpas-source/src/tools/parse < ${CORE_BLDDIR}/Registry_processed.xml
+    DEPENDS parse ${CORE_BLDDIR}/Registry_processed.xml
+    WORKING_DIRECTORY ${INC_DIR}
+  )
+
+  # Disable qsmp for some files
+  if (FFLAGS MATCHES ".*-qsmp.*")
+    foreach(DISABLE_QSMP_FILE IN LISTS DISABLE_QSMP)
+      get_filename_component(SOURCE_EXT ${DISABLE_QSMP_FILE} EXT)
+      string(REPLACE "${SOURCE_EXT}" ".f90" SOURCE_F90 ${DISABLE_QSMP_FILE})
+      set_property(SOURCE ${CMAKE_BINARY_DIR}/${SOURCE_F90} APPEND_STRING PROPERTY COMPILE_FLAGS " -nosmp")
+    endforeach()
+  endif()
+
+  genf90_targets("${RAW_SOURCES}" "${INCLUDES}" "${CPPDEFS}" "${NO_PREPROCESS}" "${INC_DIR}")
+  target_sources(${COMPONENT} PRIVATE ${SOURCES} $<TARGET_OBJECTS:common>)
+
+endfunction(build_core)

src/cmake_utils.cmake

@@ -0,0 +1,74 @@
+# Function for handling nl and st gen
+function(handle_st_nl_gen NL_GEN_ARGS ST_GEN_ARGS CORE_INPUT_DIR_ARG CORE_BLDDIR_ARG)
+  foreach(NL_GEN_ARG IN LISTS NL_GEN_ARGS)
+    separate_arguments(ITEMS UNIX_COMMAND "${NL_GEN_ARG}")
+    list(GET ITEMS 0 ITEM)
+    list(APPEND INPUTS ${ITEM})
+    add_custom_command(
+      OUTPUT ${CORE_INPUT_DIR_ARG}/${ITEM}
+      COMMAND ${CMAKE_BINARY_DIR}/tools/namelist_gen ${CORE_BLDDIR_ARG}/Registry_processed.xml ${NL_GEN_ARG}
+      DEPENDS namelist_gen ${CORE_BLDDIR_ARG}/Registry_processed.xml
+      WORKING_DIRECTORY ${CORE_INPUT_DIR_ARG}
+    )
+  endforeach()
+
+  foreach(ST_GEN_ARG IN LISTS ST_GEN_ARGS)
+    separate_arguments(ITEMS UNIX_COMMAND "${ST_GEN_ARG}")
+    list(GET ITEMS 0 ITEM)
+    list(APPEND INPUTS ${ITEM})
+    add_custom_command(
+      OUTPUT ${CORE_INPUT_DIR_ARG}/${ITEM}
+      COMMAND ${CMAKE_BINARY_DIR}/tools/streams_gen ${CORE_BLDDIR_ARG}/Registry_processed.xml ${ST_GEN_ARG}
+      DEPENDS streams_gen ${CORE_BLDDIR_ARG}/Registry_processed.xml
+      WORKING_DIRECTORY ${CORE_INPUT_DIR_ARG}
+    )
+  endforeach()
+
+  foreach(INPUT IN LISTS INPUTS)
+    add_custom_command(
+      OUTPUT ${CORE_BLDDIR_ARG}/${INPUT}
+      COMMAND ${CMAKE_COMMAND} -E copy ${CORE_INPUT_DIR_ARG}/${INPUT} ${CORE_BLDDIR_ARG}/${INPUT}
+      DEPENDS ${CORE_INPUT_DIR_ARG}/${INPUT}
+      WORKING_DIRECTORY ${CORE_BLDDIR_ARG}
+    )
+  endforeach()
+endfunction()
+
+# Function for generating f90 file targets, will add to parent's SOURCES var
+function(genf90_targets RAW_SOURCES_ARG INCLUDES_ARG CPPDEFS_ARG NO_PREPROCESS_ARG CORE_INC_DIR_ARG)
+  # Add -I to includes so that they can used for cpp command
+  foreach(ITEM IN LISTS INCLUDES_ARG)
+    list(APPEND INCLUDES_I "-I${ITEM}")
+  endforeach()
+
+  # Run all .F files through cpp to generate the f90 file
+  foreach(RAW_SOURCE_FILE IN LISTS RAW_SOURCES_ARG)
+    get_filename_component(SOURCE_EXT ${RAW_SOURCE_FILE} EXT)
+    if ( (SOURCE_EXT STREQUAL ".F" OR SOURCE_EXT STREQUAL ".F90") AND NOT RAW_SOURCE_FILE IN_LIST NO_PREPROCESS_ARG)
+      string(REPLACE "${SOURCE_EXT}" ".f90" SOURCE_F90 ${RAW_SOURCE_FILE})
+      get_filename_component(DIR_RELATIVE ${SOURCE_F90} DIRECTORY)
+      set(DIR_ABSOLUTE ${CMAKE_BINARY_DIR}/${DIR_RELATIVE})
+      if (NOT EXISTS ${DIR_ABSOLUTE})
+        file(MAKE_DIRECTORY ${DIR_ABSOLUTE})
+      endif()
+      if (CORE_INC_DIR_ARG)
+        add_custom_command (
+          OUTPUT ${CMAKE_BINARY_DIR}/${SOURCE_F90}
+          COMMAND cpp -P -traditional ${CPPDEFS_ARG} ${INCLUDES_I} -Uvector
+          ${CMAKE_CURRENT_SOURCE_DIR}/${RAW_SOURCE_FILE} > ${CMAKE_BINARY_DIR}/${SOURCE_F90}
+          DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${RAW_SOURCE_FILE} ${CORE_INC_DIR_ARG}/core_variables.inc)
+      else()
+        add_custom_command (
+          OUTPUT ${CMAKE_BINARY_DIR}/${SOURCE_F90}
+          COMMAND cpp -P -traditional ${CPPDEFS_ARG} ${INCLUDES_I} -Uvector
+          ${CMAKE_CURRENT_SOURCE_DIR}/${RAW_SOURCE_FILE} > ${CMAKE_BINARY_DIR}/${SOURCE_F90})
+      endif()
+      list(APPEND LOCAL_SOURCES ${CMAKE_BINARY_DIR}/${SOURCE_F90})
+    else()
+      list(APPEND LOCAL_SOURCES ${RAW_SOURCE_FILE})
+    endif()
+  endforeach()
+
+  set(SOURCES ${LOCAL_SOURCES} PARENT_SCOPE)
+
+endfunction(genf90_targets)

src/core_landice/landice.cmake

@@ -0,0 +1,79 @@
+
+# build_options.mk stuff handled here
+list(APPEND CPPDEFS "-DCORE_LANDICE")
+list(APPEND INCLUDES "${CMAKE_BINARY_DIR}/core_landice/shared" "${CMAKE_BINARY_DIR}/core_landice/analysis_members" "${CMAKE_BINARY_DIR}/core_landice/mode_forward")
+
+#
+# Check if building with LifeV, Albany, and/or PHG external libraries
+#
+
+if (LIFEV)
+  # LifeV can solve L1L2 or FO
+  list(APPEND CPPDEFS "-DLIFEV" "-DUSE_EXTERNAL_L1L2" "-DUSE_EXTERNAL_FIRSTORDER" "-DMPAS_LI_BUILD_INTERFACE")
+endif()
+
+# Albany can only solve FO at present
+if (ALBANY)
+  list(APPEND CPPDEFS "-DUSE_EXTERNAL_FIRSTORDER" "-DMPAS_LI_BUILD_INTERFACE")
+endif()
+
+if (LIFEV AND ALBANY)
+  message(FATAL "Compiling with both LifeV and Albany is not allowed at this time.")
+endif()
+
+# PHG currently requires LifeV
+if (PHG AND NOT LIFEV)
+  message(FATAL "Compiling with PHG requires LifeV at this time.")
+endif()
+
+# PHG can only Stokes at present
+if (PHG)
+  list(APPEND CPPDEFS "-DUSE_EXTERNAL_STOKES" "-DMPAS_LI_BUILD_INTERFACE")
+endif()
+
+# driver (files live in E3SM)
+list(APPEND RAW_SOURCES
+  ../../mpas-albany-landice/driver/glc_comp_mct.F
+  ../../mpas-albany-landice/driver/glc_cpl_indices.F
+  ../../mpas-albany-landice/driver/glc_mct_vars.F
+)
+
+# shared
+list(APPEND RAW_SOURCES
+  core_landice/shared/mpas_li_constants.F
+  core_landice/shared/mpas_li_mask.F
+  core_landice/shared/mpas_li_setup.F
+)
+
+# analysis members
+list(APPEND RAW_SOURCES
+  core_landice/analysis_members/mpas_li_analysis_driver.F
+  core_landice/analysis_members/mpas_li_global_stats.F
+  core_landice/analysis_members/mpas_li_regional_stats.F
+)
+
+# mode forward
+list(APPEND RAW_SOURCES
+  core_landice/mode_forward/mpas_li_core.F
+  core_landice/mode_forward/mpas_li_core_interface.F
+  core_landice/mode_forward/mpas_li_time_integration.F
+  core_landice/mode_forward/mpas_li_time_integration_fe.F
+  core_landice/mode_forward/mpas_li_diagnostic_vars.F
+  core_landice/mode_forward/mpas_li_advection.F
+  core_landice/mode_forward/mpas_li_calving.F
+  core_landice/mode_forward/mpas_li_statistics.F
+  core_landice/mode_forward/mpas_li_velocity.F
+  core_landice/mode_forward/mpas_li_thermal.F
+  core_landice/mode_forward/mpas_li_iceshelf_melt.F
+  core_landice/mode_forward/mpas_li_sia.F
+  core_landice/mode_forward/mpas_li_velocity_simple.F
+  core_landice/mode_forward/mpas_li_velocity_external.F
+  core_landice/mode_forward/mpas_li_subglacial_hydro.F
+)
+
+if (CPPFLAGS MATCHES ".*MPAS_LI_BUILD_INTERFACE.*")
+  list(APPEND RAW_SOURCES core_landice/mode_forward/Interface_velocity_solver.cpp)
+endif()
+
+# Generate core input
+handle_st_nl_gen("namelist.landice" "streams.landice stream_list.landice. listed" ${CORE_INPUT_DIR} ${CORE_BLDDIR})