From 294a5ac4d453c61999dee1e170597d80f73822ed Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Sun, 2 Jun 2024 10:27:12 -0400 Subject: [PATCH 1/3] Matches Nvidia GPU compute capability (CC) version to cc80 on Perlmutter-GPU. --- cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake b/cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake index 59758d59989c..852483663a09 100644 --- a/cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake +++ b/cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake @@ -6,8 +6,8 @@ if (COMP_NAME STREQUAL gptl) endif() string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK") string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc80 -Minfo=accel") set(SCC "cc") set(SCXX "CC") set(SFC "ftn") -string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel") +string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc80 -Minfo=accel") From 5fefa1ce3756f0146db9d0aade6448c00a5f529b Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Sun, 2 Jun 2024 10:41:55 -0400 Subject: [PATCH 2/3] Creates Omega compiler configurations for Frontier: amdomegagpu, crayomegagpu, and gnuomegagpu --- .../machines/cmake_macros/amdomegagpu.cmake | 20 ++++++ .../cmake_macros/amdomegagpu_frontier.cmake | 16 +++++ .../machines/cmake_macros/crayomegagpu.cmake | 42 +++++++++++++ .../cmake_macros/crayomegagpu_frontier.cmake | 62 +++++++++++++++++++ .../machines/cmake_macros/gnuomegagpu.cmake | 37 +++++++++++ .../cmake_macros/gnuomegagpu_frontier.cmake | 16 +++++ cime_config/machines/config_machines.xml | 33 +++++++++- 7 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 cime_config/machines/cmake_macros/amdomegagpu.cmake create mode 100644 cime_config/machines/cmake_macros/amdomegagpu_frontier.cmake create mode 100644 cime_config/machines/cmake_macros/crayomegagpu.cmake create mode 100644 cime_config/machines/cmake_macros/crayomegagpu_frontier.cmake create mode 100644 cime_config/machines/cmake_macros/gnuomegagpu.cmake create mode 100644 cime_config/machines/cmake_macros/gnuomegagpu_frontier.cmake diff --git a/cime_config/machines/cmake_macros/amdomegagpu.cmake b/cime_config/machines/cmake_macros/amdomegagpu.cmake new file mode 100644 index 000000000000..c3d01c2553c8 --- /dev/null +++ b/cime_config/machines/cmake_macros/amdomegagpu.cmake @@ -0,0 +1,20 @@ +string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") +string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") + +if (compile_threaded) + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + +string(APPEND CPPDEFS " -DNO_R16 -DCPRAMD -DFORTRANUNDERSCORE") + +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") diff --git a/cime_config/machines/cmake_macros/amdomegagpu_frontier.cmake b/cime_config/machines/cmake_macros/amdomegagpu_frontier.cmake new file mode 100644 index 000000000000..b5dc3d5ac3d0 --- /dev/null +++ b/cime_config/machines/cmake_macros/amdomegagpu_frontier.cmake @@ -0,0 +1,16 @@ +string(APPEND CPPDEFS " -DLINUX") + +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() + +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") + +string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") + +set(USE_HIP "TRUE") diff --git a/cime_config/machines/cmake_macros/crayomegagpu.cmake b/cime_config/machines/cmake_macros/crayomegagpu.cmake new file mode 100644 index 000000000000..307085258f3e --- /dev/null +++ b/cime_config/machines/cmake_macros/crayomegagpu.cmake @@ -0,0 +1,42 @@ +# Ran into Scorpio build issues using base and child compiler macro files. +# Especially specified compilers were not picked up properly. +# Had to replicate section below to get things working as expected. +# Commenting out until we figure out how to properly handle this. + +# if (compile_threaded) +# string(APPEND FFLAGS " -fopenmp") +# string(APPEND CFLAGS " -fopenmp") +# string(APPEND CXXFLAGS " -fopenmp") +# string(APPEND LDFLAGS " -fopenmp") +# endif() +# if (DEBUG) +# string(APPEND CFLAGS " -O0 -g") +# string(APPEND FFLAGS " -O0 -g") +# string(APPEND CXXFLAGS " -O0 -g") +# string(APPEND CPPDEFS " -DYAKL_DEBUG") +# endif() +# string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY") +# string(APPEND FFLAGS " -f free -em") +# if (NOT compile_threaded) +# # -M1077 flag used to suppress message about OpenMP directives +# # that are ignored for non-threaded builds. (-h omp inactive) +# # Details: `explain ftn-1077` +# string(APPEND FFLAGS " -M1077") +# endif() +# string(APPEND FFLAGS_NOOPT " -O0") +# set(HAS_F2008_CONTIGUOUS "TRUE") +# string(APPEND LDFLAGS " -Wl,--allow-multiple-definition") +# set(MPICC "cc") +# set(MPICXX "hipcc") +# set(MPIFC "ftn") +# set(SCC "cc") +# set(SCXX "hipcc") +# set(SFC "ftn") + +#if (NOT DEBUG) +# string(APPEND CFLAGS " -O2") +# string(APPEND CXXFLAGS " -O2") +# string(APPEND FFLAGS " -O2") +#endif() + + diff --git a/cime_config/machines/cmake_macros/crayomegagpu_frontier.cmake b/cime_config/machines/cmake_macros/crayomegagpu_frontier.cmake new file mode 100644 index 000000000000..39d8f2935776 --- /dev/null +++ b/cime_config/machines/cmake_macros/crayomegagpu_frontier.cmake @@ -0,0 +1,62 @@ +set(MPICC "cc") +set(MPICXX "CC") +set(MPIFC "ftn") +set(SCC "cc") +set(SCXX "CC") +set(SFC "ftn") + +string(APPEND CPPDEFS " -DLINUX") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() + +if (compile_threaded) + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + +string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") +string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") +string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY") +string(APPEND CMAKE_Fortran_FLAGS " -f free -em") + +if (NOT compile_threaded) + # -M1077 flag used to suppress message about OpenMP directives + # that are ignored for non-threaded builds. (-h omp inactive) + # Details: `explain ftn-1077` + string(APPEND CMAKE_Fortran_FLAGS " -M1077") +endif() + +set(HAS_F2008_CONTIGUOUS "TRUE") + +# -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: +# /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' +# AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") + +# Switching to O3 for performance benchmarking +# Will revisit any failing tests +string(APPEND CMAKE_C_FLAGS_RELEASE " -O3") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O3") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O3") + +if (COMP_NAME STREQUAL elm) + # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 + string(APPEND CMAKE_Fortran_FLAGS " -hfp0") +endif() + +# -em -ef generates modulename.mod (lowercase files) to support +# Scorpio installs +# Disable ipa and zero initialization are for other NaN isues: +# https://github.com/E3SM-Project/E3SM/pull/5208 +string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") + +string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") + +set(USE_HIP "TRUE") diff --git a/cime_config/machines/cmake_macros/gnuomegagpu.cmake b/cime_config/machines/cmake_macros/gnuomegagpu.cmake new file mode 100644 index 000000000000..f0b756cd083b --- /dev/null +++ b/cime_config/machines/cmake_macros/gnuomegagpu.cmake @@ -0,0 +1,37 @@ +string(APPEND CMAKE_C_FLAGS " -mcmodel=medium") +string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=medium -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none") + +if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10) + string(APPEND CMAKE_Fortran_FLAGS " -fallow-argument-mismatch") +endif() + +string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") + +if (compile_threaded) + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + +string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow") +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace") +string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") +string(APPEND CMAKE_C_FLAGS_RELEASE " -O") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O") + +if (COMP_NAME STREQUAL csm_share) + string(APPEND CMAKE_C_FLAGS " -std=c99") +endif() + +string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") +string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") + +set(MPICC "mpicc") +set(MPICXX "mpicxx") +set(MPIFC "mpif90") +set(SCC "gcc") +set(SCXX "g++") +set(SFC "gfortran") diff --git a/cime_config/machines/cmake_macros/gnuomegagpu_frontier.cmake b/cime_config/machines/cmake_macros/gnuomegagpu_frontier.cmake new file mode 100644 index 000000000000..403e4074caa6 --- /dev/null +++ b/cime_config/machines/cmake_macros/gnuomegagpu_frontier.cmake @@ -0,0 +1,16 @@ +string(APPEND CPPDEFS " -DLINUX") + +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() + +string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") + +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") +string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") + +set(USE_HIP "TRUE") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index bc7cf99f3d47..a295ca083aec 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -966,7 +966,7 @@ Frontier exascale supercomputer at ORNL. 9408 nodes, Node: 4 AMD MI250X GPUs (2 GCDs) ~ 8 GPUs, 512 GB HDB2E, AMD EPYC 64 cores, 512GB DDR4 .*frontier.* Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu + crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu,crayomegagpu,amdomegagpu,gnuomegagpu mpich cli115 /lustre/orion/cli115/world-shared/frontier @@ -986,6 +986,10 @@ 8 8 8 + 8 + 8 + 8 + TRUE srun @@ -1017,6 +1021,11 @@ craype-accel-amd-gfx90a rocm/5.4.0 + + craype-accel-amd-gfx90a + rocm/5.4.0 + + PrgEnv-cray PrgEnv-amd/8.3.3 @@ -1025,6 +1034,13 @@ craype-accel-amd-gfx90a + + + PrgEnv-cray PrgEnv-amd/8.3.3 + amd amd/5.4.0 + craype-accel-amd-gfx90a + + PrgEnv-cray PrgEnv-gnu/8.3.3 @@ -1035,6 +1051,14 @@ craype-accel-amd-gfx90a rocm/5.4.0 + + + PrgEnv-cray PrgEnv-gnu/8.3.3 + gcc gcc/11.2.0 + craype-accel-amd-gfx90a + rocm/5.4.0 + + cray-python/3.9.13.1 subversion/1.14.1 @@ -1085,6 +1109,13 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/amdclang-15.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/crayclang-15.0.1; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/amdclang-15.0.0; else echo "$ADIOS2_ROOT"; fi} + + From 2c541a9e22b4ebcee9b1e515a92e3c204b36fd25 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Sun, 2 Jun 2024 14:40:29 -0400 Subject: [PATCH 3/3] Refactors the Omega build to collect build configurations from the E3SM case --- cime_config/machines/config_machines.xml | 7 +- components/omega/OmegaBuild.cmake | 267 ++++++++---- components/omega/create_scripts.py | 411 ------------------- components/omega/doc/devGuide/CMakeBuild.md | 1 - components/omega/doc/userGuide/OmegaBuild.md | 11 +- components/omega/src/CMakeLists.txt | 4 +- components/omega/src/base/DataTypes.h | 6 +- components/omega/test/CMakeLists.txt | 28 +- 8 files changed, 221 insertions(+), 514 deletions(-) delete mode 100644 components/omega/create_scripts.py diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index a295ca083aec..708e0c12a50f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1022,6 +1022,10 @@ rocm/5.4.0 + + PrgEnv-cray PrgEnv-cray/8.3.3 + cce cce/15.0.1 + craype craype/2.7.20 craype-accel-amd-gfx90a rocm/5.4.0 @@ -1052,9 +1056,6 @@ rocm/5.4.0 - - PrgEnv-cray PrgEnv-gnu/8.3.3 - gcc gcc/11.2.0 craype-accel-amd-gfx90a rocm/5.4.0 diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index 81ec1212b404..d3727beac878 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -18,6 +18,8 @@ set(E3SM_CIME_ROOT "${E3SM_ROOT}/cime") set(E3SM_CIMECONFIG_ROOT "${E3SM_ROOT}/cime_config") set(E3SM_EXTERNALS_ROOT "${E3SM_ROOT}/externals") +set(CASEROOT "${OMEGA_BUILD_DIR}/e3smcase") + ########################### # Macros # ########################### @@ -34,64 +36,124 @@ macro(common) set(OMEGA_LINK_OPTIONS "") endif() - if(NOT Python_FOUND) - find_package (Python COMPONENTS Interpreter) +endmacro() - if(NOT Python_FOUND) - message(FATAL_ERROR "Python is not available, CMake will exit." ) - endif() - endif() +macro(run_bash_command command outvar) + + execute_process( + COMMAND bash -c "${command}" + OUTPUT_VARIABLE ${outvar} + OUTPUT_STRIP_TRAILING_WHITESPACE + ) endmacro() -# Collect machine and compiler info from CIME -# and detect OMEGA_ARCH and compilers -macro(init_standalone_build) +macro(cime_xmlquery query outvar) - # update CMake configuration with CIME configuration - set(_TMP_CMAKE_FILE ${OMEGA_BUILD_DIR}/_Omega.cmake) - set(_PY_OPTS "-p;${E3SM_CIME_ROOT};-o;${_TMP_CMAKE_FILE}") + run_bash_command("cd ${CASEROOT} && ./xmlquery ${query} --value" ${outvar}) - if(OMEGA_CIME_COMPILER) - list(APPEND _PY_OPTS "-c" "${OMEGA_CIME_COMPILER}") - endif() +endmacro() + +macro(read_cime_config) - if(OMEGA_CIME_MACHINE) - list(APPEND _PY_OPTS "-m" "${OMEGA_CIME_MACHINE}") + set(NEWCASE_COMMAND "${E3SM_ROOT}/cime/scripts/create_newcase \ + --res T62_oQU120 \ + --compset CMPASO-NYF \ + --handle-preexisting-dirs r \ + --case ${CASEROOT}") + + if(NOT "${OMEGA_CIME_MACHINE}" STREQUAL "") + set(NEWCASE_COMMAND "${NEWCASE_COMMAND} --machine ${OMEGA_CIME_MACHINE}") endif() - if(OMEGA_BUILD_TYPE STREQUAL "Debug") + if(NOT "${OMEGA_CIME_COMPILER}" STREQUAL "") + set(NEWCASE_COMMAND "${NEWCASE_COMMAND} --compiler ${OMEGA_CIME_COMPILER}") + endif() - list(APPEND _PY_OPTS "-d") + run_bash_command("${NEWCASE_COMMAND}" NEWCASE_OUTPUT) + run_bash_command("cd ${CASEROOT} && ./case.setup" CASESETUP_OUTPUT) + run_bash_command("source ${CASEROOT}/.env_mach_specific.sh && env" ENV_OUTPUT) - execute_process(COMMAND ${Python_EXECUTABLE} create_scripts.py ${_PY_OPTS} - WORKING_DIRECTORY ${OMEGA_SOURCE_DIR} - OUTPUT_VARIABLE _MACHINE_INFO - ERROR_VARIABLE _ERROR_INFO) + string(REPLACE "\n" ";" lines ${ENV_OUTPUT}) - message(STATUS "create_scripts.py output: ${_MACHINE_INFO}") - message(STATUS "create_scripts.py error: ${_ERROR_INFO}") + # set env. variables + foreach(line ${lines}) + string(REGEX MATCH "([A-Za-z_][A-Za-z0-9_]*)=(.*)" ENV_LINE ${line}) + set(ENV_VAR "${CMAKE_MATCH_1}") + set(ENV_VAL "${CMAKE_MATCH_2}") - else() + if(NOT "${ENV_VAR}" STREQUAL "") + set(ENV{${ENV_VAR}} "${ENV_VAL}") + #message(STATUS "${ENV_VAR}: ${ENV_VAL}") + endif() + endforeach() + + # Read .case.run.sh script in case directory + file(READ "${CASEROOT}/.case.run.sh" CASE_RUN) + + # Convert a string to a list + string(REPLACE "\n" ";" lines ${CASE_RUN}) + + # get mpi launch command-line arguments + foreach(line ${lines}) + string(FIND ${line} "e3sm.exe" _LINE_FOUND) + if(NOT _LINE_FOUND EQUAL -1) + string(REPLACE " " ";" args ${line}) + set(SKIP_ARG FALSE) + list(GET args 0 OMEGA_MPI_EXEC) + list(REMOVE_AT args 0) + set(OMEGA_MPI_ARGS) + foreach(arg ${args}) + if("${SKIP_ARG}" STREQUAL "TRUE") + set(SKIP_ARG FALSE) + continue() + endif() + + string(FIND "${arg}" "e3sm.exe" _ARG_FOUND) + + if(NOT _ARG_FOUND EQUAL -1) + break() + + elseif("${arg}" STREQUAL "-n" OR "${arg}" STREQUAL "-N" OR + "${arg}" STREQUAL "-c") + set(SKIP_ARG TRUE) + + else() + list(APPEND OMEGA_MPI_ARGS "${arg}") + endif() + endforeach() + endif() + endforeach() - execute_process(COMMAND ${Python_EXECUTABLE} create_scripts.py ${_PY_OPTS} - OUTPUT_QUIET ERROR_QUIET - WORKING_DIRECTORY ${OMEGA_SOURCE_DIR} - OUTPUT_VARIABLE _MACHINE_INFO) + cime_xmlquery("MPILIB" MPILIB_NAME) + cime_xmlquery("GMAKE_J" GMAKE_J) + cime_xmlquery("BUILD_THREADED" BUILD_THREADED) + cime_xmlquery("THREAD_COUNT" THREAD_COUNT) + cime_xmlquery("COMPILER" COMPILER) + cime_xmlquery("MACH" MACH) + if("${BUILD_THREADED}" STREQUAL "TRUE") + option(compile_threaded "" ON) endif() - include(${_TMP_CMAKE_FILE}) + set(SRCROOT "${E3SM_ROOT}") - if(NOT OMEGA_BUILD_TYPE STREQUAL "Debug") - file(REMOVE ${_TMP_CMAKE_FILE}) - endif() + include("${CASEROOT}/Macros.cmake") + +endmacro() + +# Collect machine and compiler info from CIME +# and detect OMEGA_ARCH and compilers +macro(init_standalone_build) + + # get cime configuration + read_cime_config() # find compilers if(OMEGA_C_COMPILER) find_program(_OMEGA_C_COMPILER ${OMEGA_C_COMPILER}) - elseif(MPILIB STREQUAL "mpi-serial") + elseif("${MPILIB}" STREQUAL "mpi-serial") find_program(_OMEGA_C_COMPILER ${SCC}) else() @@ -108,7 +170,7 @@ macro(init_standalone_build) if(OMEGA_CXX_COMPILER) find_program(_OMEGA_CXX_COMPILER ${OMEGA_CXX_COMPILER}) - elseif(MPILIB STREQUAL "mpi-serial") + elseif("${MPILIB}" STREQUAL "mpi-serial") find_program(_OMEGA_CXX_COMPILER ${SCXX}) else() @@ -125,7 +187,7 @@ macro(init_standalone_build) if(OMEGA_Fortran_COMPILER) find_program(_OMEGA_Fortran_COMPILER ${OMEGA_Fortran_COMPILER}) - elseif(MPILIB STREQUAL "mpi-serial") + elseif("${MPILIB}" STREQUAL "mpi-serial") find_program(_OMEGA_Fortran_COMPILER ${SFC}) else() @@ -144,7 +206,7 @@ macro(init_standalone_build) message(STATUS "OMEGA_Fortran_COMPILER = ${OMEGA_Fortran_COMPILER}") # detect OMEGA_ARCH if not provided - if(NOT OMEGA_ARCH) + if("${OMEGA_ARCH}" STREQUAL "") if(USE_CUDA) set(OMEGA_ARCH "CUDA") @@ -172,19 +234,62 @@ macro(init_standalone_build) elseif(_NVCC_CHECK AND _NVIDIA_CHECK) set(OMEGA_ARCH "CUDA") - else() + elseif(compile_threaded) set(OMEGA_ARCH "OPENMP") + else() + set(OMEGA_ARCH "SERIAL") + endif() - else() + + elseif(compile_threaded) set(OMEGA_ARCH "OPENMP") + else() + set(OMEGA_ARCH "SERIAL") + endif() endif() endif() message(STATUS "OMEGA_ARCH = ${OMEGA_ARCH}") + # create a env script + set(_EnvScript ${OMEGA_BUILD_DIR}/omega_env.sh) + file(WRITE ${_EnvScript} "#!/usr/bin/env bash\n\n") + file(APPEND ${_EnvScript} "source ./e3smcase/.env_mach_specific.sh\n\n") + if("${OMEGA_ARCH}" STREQUAL "OPENMP") + file(APPEND ${_EnvScript} "export OMP_NUM_THREADS=${THREAD_COUNT}\n\n") + if(DEFINED ENV{OMP_PROC_BIND}) + file(APPEND ${_EnvScript} "export OMP_PROC_BIND=$ENV{OMP_PROC_BIND}\n\n") + else() + file(APPEND ${_EnvScript} "export OMP_PROC_BIND=spread\n\n") + endif() + if(DEFINED ENV{OMP_PLACES}) + file(APPEND ${_EnvScript} "export OMP_PLACES=$ENV{OMP_PLACES}\n\n") + else() + file(APPEND ${_EnvScript} "export OMP_PLACES=threads\n\n") + endif() + endif() + + # create a build script + set(_BuildScript ${OMEGA_BUILD_DIR}/omega_build.sh) + file(WRITE ${_BuildScript} "#!/usr/bin/env bash\n\n") + file(APPEND ${_BuildScript} "source ./omega_env.sh\n\n") + file(APPEND ${_BuildScript} "make -j ${GMAKE_J}\n\n") + + # create a run script + set(_RunScript ${OMEGA_BUILD_DIR}/omega_run.sh) + file(WRITE ${_RunScript} "#!/usr/bin/env bash\n\n") + file(APPEND ${_RunScript} "source ./omega_env.sh\n\n") + file(APPEND ${_RunScript} "./src/omega.exe 1000\n\n") + + # create a ctest script + set(_CtestScript ${OMEGA_BUILD_DIR}/omega_ctest.sh) + file(WRITE ${_CtestScript} "#!/usr/bin/env bash\n\n") + file(APPEND ${_CtestScript} "source ./omega_env.sh\n\n") + file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n") + # create a profile script set(_ProfileScript ${OMEGA_BUILD_DIR}/omega_profile.sh) file(WRITE ${_ProfileScript} "#!/usr/bin/env bash\n\n") @@ -211,10 +316,18 @@ macro(init_standalone_build) # if (SLIBS) # separate_arguments(_SLIBS NATIVE_COMMAND ${SLIBS}) # list(APPEND OMEGA_LINK_OPTIONS ${_SLIBS}) +# endif() + + if(OMEGA_CXX_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_CXX_FLAGS}") + endif() + +# if(OMEGA_EXE_LINKER_FLAGS) +# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OMEGA_EXE_LINKER_FLAGS}") # endif() # set CXX compiler *before* calling CMake project() - if(OMEGA_ARCH STREQUAL "CUDA") + if("${OMEGA_ARCH}" STREQUAL "CUDA") if(NOT OMEGA_CUDA_COMPILER) find_program(OMEGA_CUDA_COMPILER @@ -233,29 +346,17 @@ macro(init_standalone_build) set(CMAKE_CXX_COMPILER ${OMEGA_CUDA_COMPILER}) set(CMAKE_CUDA_HOST_COMPILER ${OMEGA_CXX_COMPILER}) - # overwrite CMAKE_CXX_FLAGS and CMAKE_EXE_LINKER_FLAGS defined in - # cime configuration because those could break CUDA build - if(OMEGA_CXX_FLAGS) - set(CMAKE_CXX_FLAGS ${OMEGA_CXX_FLAGS}) - - else() - set(CMAKE_CXX_FLAGS "") - endif() - if(OMEGA_CUDA_FLAGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_CUDA_FLAGS}") endif() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ccbin ${CMAKE_CUDA_HOST_COMPILER}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-gpu-targets") - - if(OMEGA_EXE_LINKER_FLAGS) - set(CMAKE_EXE_LINKER_FLAGS ${OMEGA_EXE_LINKER_FLAGS}) - - else() - set(CMAKE_EXE_LINKER_FLAGS "") + string(FIND "${CMAKE_CXX_FLAGS}" "--ccbin" pos) + if(${pos} EQUAL -1) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ccbin ${CMAKE_CUDA_HOST_COMPILER}") endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-gpu-targets") + message(STATUS "CMAKE_CUDA_HOST_COMPILER = ${CMAKE_CUDA_HOST_COMPILER}") file(APPEND ${_ProfileScript} "OUTFILE=${OMEGA_BUILD_DIR}/nsys_output\n\n") @@ -266,7 +367,7 @@ macro(init_standalone_build) file(APPEND ${_ProfileScript} " --trace=cuda,nvtx,osrt \\\n") file(APPEND ${_ProfileScript} " ./src/omega.exe 1000") - elseif(OMEGA_ARCH STREQUAL "HIP") + elseif("${OMEGA_ARCH}" STREQUAL "HIP") if(NOT OMEGA_HIP_COMPILER) find_program(OMEGA_HIP_COMPILER "hipcc") @@ -282,20 +383,16 @@ macro(init_standalone_build) set(CMAKE_HIP_COMPILER ${OMEGA_HIP_COMPILER}) set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) - if(OMEGA_CXX_FLAGS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_CXX_FLAGS}") - endif() - if(OMEGA_HIP_FLAGS) set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} ${OMEGA_HIP_FLAGS}") endif() - if(${MPILIB_NAME} STREQUAL "mpich") + if("${MPILIB_NAME}" STREQUAL "mpich") if(NOT $ENV{MPICH_CXX}) set(ENV{MPICH_CXX} ${OMEGA_HIP_COMPILER}) endif() - elseif(${MPILIB_NAME} STREQUAL "openmpi") + elseif("${MPILIB_NAME}" STREQUAL "openmpi") if(NOT $ENV{OMPI_CXX}) set(ENV{OMPI_CXX} ${OMEGA_HIP_COMPILER}) endif() @@ -309,7 +406,7 @@ macro(init_standalone_build) file(APPEND ${_ProfileScript} "rocprof --hip-trace --hsa-trace --timestamp on \\\n") file(APPEND ${_ProfileScript} " -o \$OUTFILE ./src/omega.exe 1000") - elseif(OMEGA_ARCH STREQUAL "SYCL") + elseif("${OMEGA_ARCH}" STREQUAL "SYCL") set(CMAKE_CXX_COMPILER ${OMEGA_SYCL_COMPILER}) if(OMEGA_SYCL_FLAGS) @@ -321,16 +418,32 @@ macro(init_standalone_build) endif() + execute_process(COMMAND chmod +x ${_EnvScript}) + execute_process(COMMAND chmod +x ${_BuildScript}) + execute_process(COMMAND chmod +x ${_RunScript}) + execute_process(COMMAND chmod +x ${_CtestScript}) execute_process(COMMAND chmod +x ${_ProfileScript}) if(KOKKOS_OPTIONS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${KOKKOS_OPTIONS}") + + string(REPLACE " " ";" opts ${KOKKOS_OPTIONS}) + foreach(opt ${opts}) + string(REGEX MATCH "-D[ \t]*([A-Za-z_][A-Za-z0-9_]*)=(.*)" KOPT ${opt}) + set(KOPT_VAR "${CMAKE_MATCH_1}") + set(KOPT_VAL "${CMAKE_MATCH_2}") + + if(NOT "${KOPT_VAR}" STREQUAL "") + option(${KOPT_VAR} "" ${KOPT_VAL}) + endif() + endforeach() + unset(KOKKOS_OPTIONS) + endif() message(STATUS "CMAKE_CXX_COMPILER = ${CMAKE_CXX_COMPILER}") message(STATUS "CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}") - message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}") +# message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}") endmacro() @@ -382,7 +495,7 @@ macro(update_variables) # Set the build type set(CMAKE_BUILD_TYPE ${OMEGA_BUILD_TYPE}) - if(OMEGA_BUILD_TYPE STREQUAL "Debug") + if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug") set(OMEGA_DEBUG ON) endif() @@ -445,25 +558,23 @@ macro(update_variables) set(CMAKE_INSTALL_PREFIX ${OMEGA_INSTALL_PREFIX}) endif() - if(OMEGA_ARCH STREQUAL "CUDA") + if("${OMEGA_ARCH}" STREQUAL "CUDA") option(Kokkos_ENABLE_CUDA "" ON) option(Kokkos_ENABLE_CUDA_LAMBDA "" ON) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOMEGA_TARGET_DEVICE") - elseif(OMEGA_ARCH STREQUAL "HIP") + elseif("${OMEGA_ARCH}" STREQUAL "HIP") option(Kokkos_ENABLE_HIP "" ON) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOMEGA_TARGET_DEVICE") - elseif(OMEGA_ARCH STREQUAL "SYCL") + elseif("${OMEGA_ARCH}" STREQUAL "SYCL") option(Kokkos_ENABLE_SYCL "" ON) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOMEGA_TARGET_DEVICE") - elseif(OMEGA_ARCH STREQUAL "OPENMP") - set(ENV{OMP_PROC_BIND} "spread") - set(ENV{OMP_PLACES} "threads") + elseif("${OMEGA_ARCH}" STREQUAL "OPENMP") option(Kokkos_ENABLE_OPENMP "" ON) - elseif(OMEGA_ARCH STREQUAL "THREADS") + elseif("${OMEGA_ARCH}" STREQUAL "THREADS") option(Kokkos_ENABLE_THREADS "" ON) else() @@ -496,10 +607,10 @@ macro(check_setup) #message("OMEGA_BUILD_MODE = ${OMEGA_BUILD_MODE}") - if(OMEGA_BUILD_MODE STREQUAL "E3SM") + if("${OMEGA_BUILD_MODE}" STREQUAL "E3SM") message(STATUS "*** Omega E3SM-component Build ***") - elseif(${OMEGA_BUILD_MODE} STREQUAL "STANDALONE") + elseif("${OMEGA_BUILD_MODE}" STREQUAL "STANDALONE") message(STATUS "*** Omega Standalone Build ***") else() diff --git a/components/omega/create_scripts.py b/components/omega/create_scripts.py deleted file mode 100644 index 4364cd6f0d33..000000000000 --- a/components/omega/create_scripts.py +++ /dev/null @@ -1,411 +0,0 @@ -""" Collect compiler and machine info from CIME - -This script should be invoked by Omega cmake build system -""" - -import argparse -import getpass -import os -import re -import stat -import subprocess -import sys -import typing - -pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE) - - -def parse_cmdline(): - - here, progname = os.path.split(__file__) - - parser = argparse.ArgumentParser( - prog=progname, - description='generate machine specific infomation', - epilog='Contact: ') - - parser.add_argument('-p', '--cimepath', default=os.path.realpath( - os.path.join(here, "..", "..", "cime"))) # CIME root - parser.add_argument('-o', '--outpath', default="_Omega.cmake") # outfile - parser.add_argument('-m', '--machine') # machine - parser.add_argument('-c', '--compiler') # compiler - parser.add_argument('-d', '--debug', action='store_const', - const="TRUE", default="FALSE") # debug mode - parser.add_argument('-v', '--verbose') # verbose output - - return parser.parse_args() - - -args = parse_cmdline() - -# import Machines class from CIME -sys.path.insert(0, args.cimepath) -from CIME.XML.machines import Machines # noqa: E402 - -sys.path.pop(0) - - -# run a shell command -def run_cmd_no_fail(cmd, env): - out = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, env=env) - retval = str(out.stdout, 'UTF-8').strip() - return retval - - -# main class that extends Machines class -class OmegaMachines(Machines): - - def __init__(self, cimepath, outpath, machine, compiler, debug): - - if isinstance(machine, str): - machine = machine.strip() - - super(OmegaMachines, self).__init__(machine=machine) - - self.cimepath = cimepath - self.outpath = outpath - self.debug = debug - self.mpilib = "" - self.mpiexec = "" - - if compiler is None: - self.compiler = self.get_default_compiler().strip() - else: - self.compiler = compiler.strip() - - self.machpath = os.path.join(self.cimepath, "..", "cime_config", - "machines") - self.macrospath = os.path.join(self.machpath, "cmake_macros", - "Macros.cmake") - - self.machname = self.get_machine_name() - self.machos = self.get_value("OS") - self.mpilibs = self.get_value("MPILIBS").split(",") - - # modified based on generic_xml.py in CIME - def get_processed_value(self, raw_value, outvar): - - reference_re = re.compile(r"\${?(\w+)}?") - env_ref_re = re.compile(r"\$ENV\{(\w+)\}") - shell_ref_re = re.compile(r"\$SHELL\{([^}]+)\}") - math_re = re.compile(r"\s[+-/*]\s") - item_data = raw_value - - if item_data is None: - return None - - if not isinstance(item_data, str): - return item_data - - for m in env_ref_re.finditer(item_data): - env_var = m.groups()[0] - - if env_var in outvar: - item_data = item_data.replace(m.group(), outvar[env_var]) - - elif env_var in os.environ: - item_data = item_data.replace(m.group(), os.environ[env_var]) - - else: - raise Exception(f"Undefined env var '{env_var}'") - - for s in shell_ref_re.finditer(item_data): - shell_cmd = s.groups()[0] - item_data = item_data.replace(s.group(), - run_cmd_no_fail(shell_cmd, outvar)) - - for m in reference_re.finditer(item_data): - var = m.groups()[0] - ref = self.get_value(var) - - if ref is not None: - procval = self.get_processed_value(str(ref), outvar) - item_data = item_data.replace(m.group(), procval) - elif var == "CIMEROOT": - item_data = item_data.replace(m.group(), self.cimepath) - elif var == "SRCROOT": - cpath = os.path.join(self.cimepath, "..") - item_data = item_data.replace(m.group(), cpath) - elif var == "USER": - item_data = item_data.replace(m.group(), getpass.getuser()) - - if math_re.search(item_data): - try: - tmp = eval(item_data) - except Exception: - tmp = item_data - item_data = str(tmp) - - return item_data - - # get module info - def get_modules(self, outvar, exclude_envs): - - modcmds: typing.List[str] = [] - self.__OMEGA_MODULE_COMMANDS__ = modcmds - - module_system_node = self.get_child("module_system") - module_system_type = self.get(module_system_node, "type") - - if module_system_type != "module": - print((f"ERROR: '{module_system_type}' " - "module system is not supported.")) - exit(-1) - - # get env. variables *before* applying module configurations - # specified in config_machines.xml - - out1 = subprocess.check_output("env", shell=True) - env1 = str(out1, 'UTF-8') - - module_nodes = self.get_children( - "modules", root=module_system_node - ) - - modcmd = "module" - - shcmds = [] - - # read module commands for the specified compiler and mpi library - - for module_node in module_nodes: - compiler = self.get(module_node, "compiler") - mpilib = self.get(module_node, "mpilib") - debug = self.get(module_node, "DEBUG") - - if not (compiler is None or - re.match("^" + compiler + "$", self.compiler)): - continue - - if not (mpilib is None or - re.match("^" + mpilib + "$", self.mpilib)): - continue - - if not (debug is None or - re.match("^" + debug + "$", self.debug)): - continue - - command_nodes = self.get_children("command", - root=module_node) - for command_node in command_nodes: - name = self.get(command_node, "name") - module = self.text(command_node) - if module is None: - shcmd = f"{modcmd} {name}" - else: - shcmd = f"{modcmd} {name} {module}" - - modcmds.append(shcmd) - shcmds.append(shcmd) - - shcmds.append("env") - - # get env. variables *after* applying module configurations - # specified in config_machines.xml - - out2 = subprocess.check_output(";".join(shcmds), shell=True) - env2 = str(out2, 'UTF-8') - - parsed1 = {} - parsed2 = {} - - for (name, value) in pat_envvar.findall(env1): - parsed1[name] = value - - for (name, value) in pat_envvar.findall(env2): - parsed2[name] = value - - for (name, value) in parsed2.items(): - if name in parsed1.keys(): - if parsed1[name] != value: - outvar[name] = value - else: - outvar[name] = value - - for (name, value) in parsed1.items(): - if name not in parsed2.keys(): - exclude_envs.append(name) - - # get environmental variables info - def get_envs(self, outvar): - - exports: typing.Dict[str, str] = {} - self.__OMEGA_SCRIPT_EXPORTS__ = exports - - envvar_nodes = self.get_children("environment_variables", - root=self.machine_node) - - # possible attribs : compiler, DEBUG, SMP_PRESENT, mpilib - for envvar_node in envvar_nodes: - compiler = self.get(envvar_node, "compiler") - mpilib = self.get(envvar_node, "mpilib") - debug = self.get(envvar_node, "DEBUG") - - if not (compiler is None or - re.match("^" + compiler + "$", self.compiler)): - continue - - if not (mpilib is None or - re.match("^" + mpilib + "$", self.mpilib)): - continue - - if not (debug is None or - re.match("^" + debug + "$", self.debug)): - continue - - env_nodes = self.get_children("env", root=envvar_node) - for env_node in env_nodes: - name = self.get(env_node, "name") - value = self.get_processed_value(self.text(env_node).strip(), - outvar) - if value is None: - outvar[name] = "" - exports[name] = outvar[name] - elif value.startswith("$ENV{") and value.endswith("}"): - vname = value[5:-1] - if vname in outvar: - outvar[name] = outvar[vname] - elif vname in os.environ: - outvar[name] = os.environ[vname] - else: - outvar[name] = "" - exports[name] = outvar[name] - elif value.startswith("$SHELL{") and value.endswith("}"): - print("Warning: SHELL evaluation is not supported.") - - else: - outvar[name] = value - exports[name] = outvar[name] - - # get mpirun info - def get_mpirun(self, outvar): - - mpirun_node = self.get_child("mpirun") - mpirun_mpilib = self.get(mpirun_node, "mpilib") - - exec_nodes = self.get_children( - "executable", root=mpirun_node - ) - - if len(exec_nodes) != 1: - print("ERROR: 'more than one executable nodes in mpirun node") - exit(-1) - - self.mpilib = mpirun_mpilib - if self.mpilib == "default": - self.mpilib = self.mpilibs[0] - - self.mpiexec = self.text(exec_nodes[0]) - - # collect machine info - def gen_machinfo(self): - - outvar: typing.Dict[str, str] = {} - exclude_envs: typing.List[str] = [] - - self.get_mpirun(outvar) - self.get_modules(outvar, exclude_envs) - self.get_envs(outvar) - self.write_output(outvar, exclude_envs) - self.generate_scripts(outvar) - - # create a temporary cmake script to be included - # in the main Omega cmake build system - def write_output(self, outvar, exclude_envs): - - with open(self.outpath, "w") as f: - f.write("message(STATUS \"Reading E3SM machine info\")\n") - - for key, value in outvar.items(): - if not key.startswith("__OMEGA_"): - f.write("set(ENV{%s} \"%s\")\n" % (key, value)) - - for name in exclude_envs: - f.write("unset(ENV{%s})\n" % name) - - f.write(f"set(MACH {self.machname})\n") - f.write(f"set(OS {self.machos})\n") - f.write(f"set(COMPILER {self.compiler})\n") - f.write(f"set(MPI_EXEC {self.mpiexec})\n") - f.write(f"set(MPILIB_NAME {self.mpilib})\n") - f.write(f"set(CASEROOT {self.machpath})\n") - f.write(f"include({self.macrospath})\n") - - # create scripts - def generate_scripts(self, outvar): - - omega_env = os.path.join(os.path.dirname(self.outpath), - "omega_env.sh") - omega_build = os.path.join(os.path.dirname(self.outpath), - "omega_build.sh") - omega_run = os.path.join(os.path.dirname(self.outpath), - "omega_run.sh") - omega_ctest = os.path.join(os.path.dirname(self.outpath), - "omega_ctest.sh") - - with open(omega_env, "w") as f: - f.write("#!/usr/bin/env bash\n\n") - - f.write("# module commands\n") - for cmd in self.__OMEGA_MODULE_COMMANDS__: - f.write(cmd + "\n") - - f.write("\n# env. variables\n") - for key, value in self.__OMEGA_SCRIPT_EXPORTS__.items(): - f.write(f"export {key}=\"{value}\"\n") - - if "OMP_NUM_THREADS" not in self.__OMEGA_SCRIPT_EXPORTS__: - f.write("export OMP_NUM_THREADS=\"1\"\n") - - if "OMP_PROC_BIND" not in self.__OMEGA_SCRIPT_EXPORTS__: - f.write("export OMP_PROC_BIND=\"spread\"\n") - - if "OMP_PLACES" not in self.__OMEGA_SCRIPT_EXPORTS__: - f.write("export OMP_PLACES=\"threads\"\n") - - with open(omega_build, "w") as f: - f.write("#!/usr/bin/env bash\n\n") - - f.write("source ./omega_env.sh\n") - - if self.debug == "TRUE": - f.write("make\n") - - else: - nthreads_build = self.get_value("GMAKE_J") - f.write(f"make -j{nthreads_build}\n") - - with open(omega_run, "w") as f: - f.write("#!/usr/bin/env bash\n\n") - - f.write("source ./omega_env.sh\n") - f.write("./src/omega.exe 1000\n") - - with open(omega_ctest, "w") as f: - f.write("#!/usr/bin/env bash\n\n") - - f.write("source ./omega_env.sh\n") - f.write("ctest --output-on-failure $* # --rerun-failed \n") - - st = os.stat(omega_env) - os.chmod(omega_env, st.st_mode | stat.S_IEXEC) - - st = os.stat(omega_build) - os.chmod(omega_build, st.st_mode | stat.S_IEXEC) - - st = os.stat(omega_run) - os.chmod(omega_run, st.st_mode | stat.S_IEXEC) - - st = os.stat(omega_ctest) - os.chmod(omega_ctest, st.st_mode | stat.S_IEXEC) - - -def main(): - - mach = OmegaMachines(args.cimepath, args.outpath, args.machine, - args.compiler, args.debug) - mach.gen_machinfo() - - -if __name__ == "__main__": - main() diff --git a/components/omega/doc/devGuide/CMakeBuild.md b/components/omega/doc/devGuide/CMakeBuild.md index d8942fac0507..ee625948e768 100644 --- a/components/omega/doc/devGuide/CMakeBuild.md +++ b/components/omega/doc/devGuide/CMakeBuild.md @@ -55,7 +55,6 @@ OMEGA_CIME_COMPILER: E3SM compiler name defined in config_machines.xml OMEGA_CIME_MACHINE: E3SM machine name defined in config_machines.xml OMEGA_CXX_FLAGS: a list for C++ compiler flags OMEGA_LINK_OPTIONS: a list for linker flags -OMEGA_EXE_LINKER_FLAGS: linker flags for building the Omega executable OMEGA_BUILD_EXECUTABLE: Enable building the Omega executable OMEGA_BUILD_TEST: Enable building Omega tests OMEGA_PARMETIS_ROOT: Parmetis installtion directory diff --git a/components/omega/doc/userGuide/OmegaBuild.md b/components/omega/doc/userGuide/OmegaBuild.md index 5ba7e13f6801..9d1724007754 100644 --- a/components/omega/doc/userGuide/OmegaBuild.md +++ b/components/omega/doc/userGuide/OmegaBuild.md @@ -7,11 +7,12 @@ offering a strong foundation for managing the building process. The Omega build system has two modes: standalone and E3SM component. -At the start of the Omega build, the system reads the E3SM machine file -(`config_machines.xml`) for both standalone and E3SM component builds. -Following this, it configures CMake variables and environment variables -based on the computing system where the build is taking place, as well as -user input from the CMake command-line. +For standalone build, the Omega build creates a generic E3SM case and +reads build configurations from the case. In E3SM component build, the +Omega build reads build configrations from the E3SM case created by +CIME build system. Following this, it configures CMake variables and +environment variables based on the computing system where the build is +taking place, as well as user input from the CMake command-line. For the Omega build system to function, a Python interpreter is necessary. The minimum version of CMake is 3.21. diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt index dfb2e61f1275..cde26f1e4a19 100644 --- a/components/omega/src/CMakeLists.txt +++ b/components/omega/src/CMakeLists.txt @@ -69,6 +69,8 @@ if(OMEGA_BUILD_EXECUTABLE) PRIVATE ${OMEGA_LIB_NAME} OmegaLibFlags -) + ) + + set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE C) endif() diff --git a/components/omega/src/base/DataTypes.h b/components/omega/src/base/DataTypes.h index 3098a381bfde..bff58d381d3a 100644 --- a/components/omega/src/base/DataTypes.h +++ b/components/omega/src/base/DataTypes.h @@ -45,9 +45,11 @@ using MemSpace = Kokkos::CudaSpace; using MemSpace = Kokkos::Experimental::HIPSpace; #elif OMEGA_ENABLE_OPENMP using MemSpace = Kokkos::HostSpace; +#elif OMEGA_ENABLE_SERIAL +using MemSpace = Kokkos::HostSpace; #else -#error \ - "None of OMEGA_ENABLE_CUDA, OMEGA_ENABLE_HIP, and OMEGA_ENABLE_OPENMP is defined." +#error "None of OMEGA_ENABLE_CUDA, OMEGA_ENABLE_HIP, OMEGA_ENABLE_OPENMP, \ +OMEGA_ENABLE_SERIAL is defined." #endif // Set default tile length diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 9e863b00bcfe..e85f9eaab9a7 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -18,11 +18,13 @@ function(add_omega_test test_name exe_name source_files mpi_args) OmegaLibFlags ) + set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE C) + # Add the test command if (mpi_args) add_test( NAME ${test_name} - COMMAND ${MPI_EXEC} ${mpi_args} -- ./${exe_name} + COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name} ) else() @@ -42,7 +44,7 @@ add_omega_test( DATA_TYPES_TEST testDataTypes.exe base/DataTypesTest.cpp - "-n 1;--cpu-bind=cores" + "-n;1" ) ################## @@ -53,7 +55,7 @@ add_omega_test( MACHINE_ENV_TEST testMachEnv.exe base/MachEnvTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ################## @@ -64,7 +66,7 @@ add_omega_test( BROADCAST_TEST testBroadcast.exe base/BroadcastTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ################## @@ -75,7 +77,7 @@ add_omega_test( LOGGING_TEST testLogging.exe infra/LoggingTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ############# @@ -86,7 +88,7 @@ add_omega_test( DECOMP_TEST testDecomp.exe base/DecompTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ################## @@ -97,7 +99,7 @@ add_omega_test( HALO_TEST testHalo.exe base/HaloTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ################ @@ -108,7 +110,7 @@ add_omega_test( HORZMESH_TEST testHorzMesh.exe ocn/HorzMeshTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ####################### @@ -119,7 +121,7 @@ add_omega_test( HORZOPERATORS_PLANE_TEST testHorzOperatorsPlane.exe ocn/HorzOperatorsTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) target_compile_definitions( testHorzOperatorsPlane.exe @@ -131,7 +133,7 @@ add_omega_test( HORZOPERATORS_SPHERE_TEST testHorzOperatorsSphere.exe ocn/HorzOperatorsTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) target_compile_definitions( testHorzOperatorsSphere.exe @@ -147,7 +149,7 @@ add_omega_test( IO_TEST testIO.exe base/IOTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ################## @@ -158,7 +160,7 @@ add_omega_test( CONFIG_TEST testConfig.exe infra/ConfigTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ################## @@ -180,7 +182,7 @@ add_omega_test( IOFIELD_TEST testIOField.exe infra/IOFieldTest.cpp - "-n 8;--cpu-bind=cores" + "-n;8" ) ##################