Reads compiler/machine configs from E3SM CIME

This merge update the capability of using E3SM configurations including compiler and compiler options. * uses MPICXX from E3SM CIME as a default Omega compiler * User can specify compiler and compiler/linker options using OMEGA_CXX_COMPILER, OMEGA_CXX_FLAGS, and OMEGA_LINK_OPTIONS * adds gen_machine_info.py to read config_machines.xml in E3SM CIME * adds the unit testing for data types * added new cmake variables of OMEGA_INCLUDE_DIRECTORIES OMEGA_LINK_DIRECTORIES * better supports compilers that are not wrapped in a compiler wrapper such as hipcc * uses cmake variables from E3SM for GPU models including CUDA_FLAGS, HIP, and SYCL * supports older Python versions
E3SM-Project · Sep 20, 2023 · 1bb81df · 1bb81df
1 parent f085ec4
commit 1bb81df
Show file tree

Hide file tree

Showing 8 changed files with 424 additions and 75 deletions.
diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt
@@ -5,6 +5,15 @@
 #    2) e3sm build
 
 
+# Included scripts do automatic cmake_policy() PUSH and POP
+cmake_policy(SET CMP0011 NEW)
+
+# list command no longer ignores empty elements
+cmake_policy(SET CMP0007 NEW)
+
+# Only interpret if() arguments as variables or keywords when unquoted
+cmake_policy(SET CMP0054 NEW)
+
 ###########################################################
 # STEP 1: Setup                                           #
 #                                                         #

diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake
@@ -13,6 +13,8 @@ set_property(CACHE OMEGA_BUILD_MODE PROPERTY STRINGS ${OMEGA_BUILD_MODES})
 set(OMEGA_SOURCE_DIR              ${CMAKE_CURRENT_LIST_DIR})
 set(OMEGA_DEFAULT_BUILD_TYPE      Release) # Debug or Release
 
+set(E3SM_CIME_ROOT ${OMEGA_SOURCE_DIR}/../../cime)
+set(E3SM_CIMECONFIG_ROOT ${OMEGA_SOURCE_DIR}/../../cime_config)
 
 ###########################
 # Public variables        #
@@ -21,26 +23,63 @@ macro(setup_common_variables)
 
   option(OMEGA_DEBUG "Turn on error message throwing (default OFF)." OFF)
 
-  if(NOT DEFINED OMEGA_CXX_FLAGS )
-    # initialize cxx flags as an empty list
+  if(NOT DEFINED OMEGA_CXX_FLAGS)
     set(OMEGA_CXX_FLAGS "")
   endif()
 
+  if(NOT DEFINED OMEGA_INCLUDE_DIRECTORIES)
+    set(OMEGA_INCLUDE_DIRECTORIES "")
+  endif()
+
+  if(NOT DEFINED OMEGA_LINK_DIRECTORIES)
+    set(OMEGA_LINK_DIRECTORIES "")
+  endif()
+
+  if(NOT DEFINED OMEGA_LINK_OPTIONS)
+    set(OMEGA_LINK_OPTIONS "")
+  endif()
+
 endmacro()
 
 ###########################
 # Preset Standalone build #
 ###########################
 macro(preset)
 
-  # set CMAKE_CXX_COMPILER from OMEGA_CXX_COMPILER
-  if(OMEGA_CXX_COMPILER)
-    execute_process(COMMAND which ${OMEGA_CXX_COMPILER}
-                    OUTPUT_VARIABLE _OMEGA_CXX_COMPILER
-                    OUTPUT_STRIP_TRAILING_WHITESPACE)
-    set(CMAKE_CXX_COMPILER ${_OMEGA_CXX_COMPILER})
+  find_package (Python COMPONENTS Interpreter)
+
+  if(NOT Python_FOUND)
+    message(FATAL_ERROR "Python is not available, CMake will exit." )
+  endif()
+
+  set(_TMP_CMAKE_FILE ${CMAKE_CURRENT_BINARY_DIR}/_Omega.cmake)
+
+  if(DEFINED OMEGA_CIME_COMPILER)
+    set(_TMP_COMPILER "-c ${OMEGA_CIME_COMPILER}")
+  else()
+    set(_TMP_COMPILER "")
   endif()
 
+  execute_process(COMMAND ${Python_EXECUTABLE} gen_machine_info.py
+    -p ${E3SM_CIME_ROOT} -o ${_TMP_CMAKE_FILE} ${_TMP_COMPILER}
+    OUTPUT_QUIET ERROR_QUIET
+    WORKING_DIRECTORY ${OMEGA_SOURCE_DIR}
+    OUTPUT_VARIABLE _MACHINE_INFO)
+
+  include(${_TMP_CMAKE_FILE})
+  file(REMOVE ${_TMP_CMAKE_FILE})
+
+  if(NOT OMEGA_CXX_COMPILER)
+    if (MPILIB STREQUAL "mpi-serial")
+      set(OMEGA_CXX_COMPILER ${SCXX})
+    else()
+      set(OMEGA_CXX_COMPILER ${MPICXX})
+    endif()
+  endif()
+
+  set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER})
+  message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}")
+
 endmacro()
 
 macro(setup_standalone_build)
@@ -57,8 +96,6 @@ macro(setup_standalone_build)
       EXISTS ${OMEGA_SOURCE_DIR}/../../externals)
 
     set(E3SM_SOURCE_DIR ${OMEGA_SOURCE_DIR}/../../components)
-    set(E3SM_CIME_ROOT ${OMEGA_SOURCE_DIR}/../../cime)
-    set(E3SM_CIMECONFIG_ROOT ${OMEGA_SOURCE_DIR}/../../cime_config)
     set(E3SM_EXTERNALS_ROOT ${OMEGA_SOURCE_DIR}/../../externals)
 
   else()
@@ -77,8 +114,6 @@ macro(setup_e3sm_build)
   setup_common_variables()
 
   set(OMEGA_BUILD_TYPE ${E3SM_DEFAULT_BUILD_TYPE})
-  set(E3SM_CIME_ROOT ${CIMEROOT})
-  set(E3SM_CIMECONFIG_ROOT ${E3SM_SOURCE_DIR}/../cime_config)
   set(E3SM_EXTERNALS_ROOT ${E3SM_SOURCE_DIR}/../externals)
 
 
@@ -94,14 +129,76 @@ endmacro()
 ################################
 macro(update_variables)
 
+  # Set the build type
+  set(CMAKE_BUILD_TYPE ${OMEGA_BUILD_TYPE})
 
-  if(OMEGA_DEBUG)
-    add_compile_definitions(OMEGA_DEBUG)
-    add_compile_definitions(LOG_UNBUFFERED_LOGGING="1")
+  if (NETCDF_PATH)
+    list(APPEND OMEGA_INCLUDE_DIRECTORIES "${NETCDF_PATH}/include")
+    list(APPEND OMEGA_LINK_DIRECTORIES "${NETCDF_PATH}/lib")
+    list(APPEND OMEGA_LINK_OPTIONS "-lnetcdf")
   endif()
 
-  # Set the build type
-  set(CMAKE_BUILD_TYPE ${OMEGA_BUILD_TYPE})
+  if (PNETCDF_PATH)
+    list(APPEND OMEGA_INCLUDE_DIRECTORIES "${PNETCDF_PATH}/include")
+    list(APPEND OMEGA_LINK_DIRECTORIES "${PNETCDF_PATH}/lib")
+    list(APPEND OMEGA_LINK_OPTIONS "-lpnetcdf")
+  endif()
+
+  string(CONCAT _TestMPISource
+    "#include \"mpi.h\"\n"
+    "int main(int argc, char* argv[])\n"
+    "{MPI_Init(&argc, &argv)\; return 0\;}\n")
+  set(_TestMPISrcFile ${CMAKE_CURRENT_BINARY_DIR}/_testMPI.cpp)
+  set(_TestMPIObjFile ${CMAKE_CURRENT_BINARY_DIR}/_testMPI.o)
+  file(WRITE ${_TestMPISrcFile}  ${_TestMPISource})
+
+  execute_process(
+	COMMAND ${OMEGA_CXX_COMPILER} -c ${_TestMPISrcFile} -o ${_TestMPIObjFile}
+    OUTPUT_QUIET ERROR_QUIET
+    RESULT_VARIABLE _MPI_TEST_RESULT)
+
+  if(OMEGA_BUILD_TYPE EQUAL Release)
+	file(REMOVE ${_TestMPISrcFile})
+	file(REMOVE ${_TestMPIObjFile})
+  endif()
+
+  if (NOT _MPI_TEST_RESULT EQUAL 0)
+    if (_MPI_TEST_RESULT MATCHES "^[-]?[0-9]+$")
+      find_package(MPI)
+      if(MPI_CXX_FOUND)
+        list(APPEND OMEGA_INCLUDE_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}")
+        list(APPEND OMEGA_LINK_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}/../lib")
+        list(APPEND OMEGA_LINK_OPTIONS "-lmpi")
+      else()
+        message(FATAL_ERROR "MPI is not found" )
+      endif()
+    else()
+      message(FATAL_ERROR "MPI test failure: ${_MPI_TEST_RESULT}" )
+    endif()
+  endif()
+
+  if(USE_CUDA)
+    list(APPEND OMEGA_CXX_FLAGS ${CUDA_FLAGS})
+
+  elseif(USE_HIP)
+    list(APPEND OMEGA_CXX_FLAGS ${HIP_FLAGS})
+
+  elseif(USE_SYCL)
+    list(APPEND OMEGA_CXX_FLAGS ${SYCL_FLAGS})
+
+  else()
+    list(APPEND OMEGA_CXX_FLAGS ${CXXFLAGS})
+
+    if(CXX_LIBS)
+      list(APPEND OMEGA_LINK_OPTIONS  ${CXX_LIBS})
+    endif()
+
+  endif()
+
+  message(STATUS "OMEGA_CXX_FLAGS           = ${CMAKE_CXX_FLAGS}")
+  message(STATUS "OMEGA_INCLUDE_DIRECTORIES = ${OMEGA_INCLUDE_DIRECTORIES}")
+  message(STATUS "OMEGA_LINK_DIRECTORIES    = ${OMEGA_LINK_DIRECTORIES}")
+  message(STATUS "OMEGA_LINK_OPTIONS        = ${OMEGA_LINK_OPTIONS}")
 
   if(OMEGA_INSTALL_PREFIX)
     set(CMAKE_INSTALL_PREFIX ${OMEGA_INSTALL_PREFIX})
@@ -122,6 +219,7 @@ macro(update_variables)
     endif()
 
   else()
+
     set(YAKL_ARCH "")
 
   endif()
@@ -145,10 +243,10 @@ macro(check_setup)
   #message("OMEGA_BUILD_MODE = ${OMEGA_BUILD_MODE}")
 
   if(OMEGA_BUILD_MODE STREQUAL "E3SM")
-    message("*** Omega E3SM-component Build ***")
+    message(STATUS "*** Omega E3SM-component Build ***")
 
   elseif(${OMEGA_BUILD_MODE} STREQUAL "STANDALONE")
-    message("*** Omega Standalone Build ***")
+    message(STATUS "*** Omega Standalone Build ***")
 
   else()
 

diff --git a/components/omega/doc/devGuide/CMakeBuild.md b/components/omega/doc/devGuide/CMakeBuild.md
@@ -14,6 +14,8 @@ steps: Setup, Update, Build, and Output.
 The build step consists of adding three subdirectories that drive builds
 for external libraries, the Omega model, and optional tests.
 
+Python is required to use this build system.
+
 ## Step 1: Setup
 
 During this step, the build-controlling variables are configured.
@@ -45,7 +47,12 @@ OMEGA_SOURCE_DIR: Directory where the top-level Omega CMakeLists.txt is located
 OMEGA_INSTALL_PREFIX: User-defined output directory for the library and executable
 OMEGA_ARCH: User-defined programming framework (e.g., "CUDA", "HIP", "OPENMP", "SYCL", "")
 OMEGA_${YAKL_ARCH}_FLAGS: Framework-specific compiler flags
-OMEGA_CXX_FLAGS: C++ compiler flags
+OMEGA_CXX_COMPILER: C++ compiler
+OMEGA_CIME_COMPILER: E3SM compiler name defined in config_machines.xml
+OMEGA_CXX_FLAGS: a list for C++ compiler flags
+OMEGA_LINK_OPTIONS: a list for linker flags
+OMEGA_INCLUDE_DIRECTORIES: Directory where header files are located
+OMEGA_LINK_DIRECTORIES: Directory where library files are located
 OMEGA_BUILD_EXECUTABLE: Enable building the Omega executable
 OMEGA_BUILD_TEST: Enable building Omega tests
 ```
@@ -65,6 +72,8 @@ CMake variables
 ```
 CMAKE_CURRENT_SOURCE_DIR
 CMAKE_CXX_STANDARD
+CMAKE_CXX_COMPILER
+CMAKE_CXX_FLAGS
 CMAKE_CURRENT_LIST_DIR
 CMAKE_BUILD_TYPE
 CMAKE_INSTALL_PREFIX
@@ -75,8 +84,9 @@ CMAKE_VERSION
 ## Step 2: Update
 
 In this step, CMake is configured, and external library variables,
-such as YAKL, are set based on the settings defined in the Setup step.
-The integrity of the build setup is verified at the end of this step.
+such as YAKL, MPI, NetCDF, and PNetCDF, are set based on the settings
+defined in the Setup step. The integrity of the build setup is verified
+at the end of this step.
 
 ## Step 3: Build