Merge pull request E3SM-Project#85 from grnydawn/ykim/omega/cmakebuild

Refactors the Omega build to collect build configurations from the E3SM case

cime_config/machines/cmake_macros/amdomegagpu.cmake

@@ -0,0 +1,20 @@
+string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
+string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG")
+
+if (compile_threaded)
+  string(APPEND CMAKE_Fortran_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_C_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_CXX_FLAGS " -fopenmp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS  " -fopenmp")
+endif()
+
+string(APPEND CPPDEFS " -DNO_R16 -DCPRAMD -DFORTRANUNDERSCORE")
+
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")

cime_config/machines/cmake_macros/amdomegagpu_frontier.cmake

@@ -0,0 +1,16 @@
+string(APPEND CPPDEFS " -DLINUX")
+
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
+
+string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
+
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
+
+set(USE_HIP "TRUE")

cime_config/machines/cmake_macros/crayomegagpu.cmake

@@ -0,0 +1,42 @@
+# Ran into Scorpio build issues using base and child compiler macro files.
+# Especially specified compilers were not picked up properly.
+# Had to replicate section below to get things working as expected.
+# Commenting out until we figure out how to properly handle this. 
+
+# if (compile_threaded)
+#   string(APPEND FFLAGS   " -fopenmp")
+#   string(APPEND CFLAGS   " -fopenmp")
+#   string(APPEND CXXFLAGS " -fopenmp")
+#   string(APPEND LDFLAGS  " -fopenmp")
+# endif()
+# if (DEBUG)
+#   string(APPEND CFLAGS   " -O0 -g")
+#   string(APPEND FFLAGS   " -O0 -g")
+#   string(APPEND CXXFLAGS " -O0 -g")
+#   string(APPEND CPPDEFS " -DYAKL_DEBUG")
+# endif()
+# string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY")
+# string(APPEND FFLAGS " -f free  -em")
+# if (NOT compile_threaded)
+#   # -M1077 flag used to suppress message about OpenMP directives
+#   # that are ignored for non-threaded builds. (-h omp inactive)
+#   # Details: `explain ftn-1077`
+#   string(APPEND FFLAGS " -M1077")
+# endif()
+# string(APPEND FFLAGS_NOOPT " -O0")
+# set(HAS_F2008_CONTIGUOUS "TRUE")
+# string(APPEND LDFLAGS " -Wl,--allow-multiple-definition")
+# set(MPICC "cc")
+# set(MPICXX "hipcc")
+# set(MPIFC "ftn")
+# set(SCC "cc")
+# set(SCXX "hipcc")
+# set(SFC "ftn")
+
+#if (NOT DEBUG)
+#  string(APPEND CFLAGS   " -O2")
+#  string(APPEND CXXFLAGS " -O2")
+#  string(APPEND FFLAGS   " -O2")
+#endif()
+
+

cime_config/machines/cmake_macros/crayomegagpu_frontier.cmake

@@ -0,0 +1,62 @@
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+
+if (compile_threaded)
+  string(APPEND CMAKE_Fortran_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_C_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_CXX_FLAGS " -fopenmp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS  " -fopenmp")
+endif()
+
+string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
+string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG")
+string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY")
+string(APPEND CMAKE_Fortran_FLAGS " -f free  -em")
+
+if (NOT compile_threaded)
+  # -M1077 flag used to suppress message about OpenMP directives
+  # that are ignored for non-threaded builds. (-h omp inactive)
+  # Details: `explain ftn-1077`
+  string(APPEND CMAKE_Fortran_FLAGS " -M1077")
+endif()
+
+set(HAS_F2008_CONTIGUOUS "TRUE")
+
+# -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue:
+# /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock<std::mutex>&)@GLIBCXX_3.4.30'
+# AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain.
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined")
+
+# Switching to O3 for performance benchmarking
+# Will revisit any failing tests
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O3")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O3")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O3")
+
+if (COMP_NAME STREQUAL elm)
+  # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996
+  string(APPEND CMAKE_Fortran_FLAGS " -hfp0")
+endif()
+
+# -em -ef generates modulename.mod (lowercase files) to support
+# Scorpio installs
+# Disable ipa and zero initialization are for other NaN isues:
+# https://github.com/E3SM-Project/E3SM/pull/5208
+string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc")
+
+string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
+
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
+
+set(USE_HIP "TRUE")

cime_config/machines/cmake_macros/gnuomegagpu.cmake

@@ -0,0 +1,37 @@
+string(APPEND CMAKE_C_FLAGS " -mcmodel=medium")
+string(APPEND CMAKE_Fortran_FLAGS " -mcmodel=medium -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none")
+
+if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
+   string(APPEND CMAKE_Fortran_FLAGS " -fallow-argument-mismatch")
+endif()
+
+string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU")
+
+if (compile_threaded)
+  string(APPEND CMAKE_C_FLAGS " -fopenmp")
+  string(APPEND CMAKE_CXX_FLAGS " -fopenmp")
+  string(APPEND CMAKE_Fortran_FLAGS " -fopenmp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp")
+endif()
+
+string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow")
+string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace")
+string(APPEND CPPDEFS_DEBUG  " -DYAKL_DEBUG")
+string(APPEND CMAKE_C_FLAGS_RELEASE " -O")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O")
+
+if (COMP_NAME STREQUAL csm_share)
+  string(APPEND CMAKE_C_FLAGS " -std=c99")
+endif()
+
+string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form")
+string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form")
+
+set(MPICC "mpicc")
+set(MPICXX "mpicxx")
+set(MPIFC "mpif90")
+set(SCC "gcc")
+set(SCXX "g++")
+set(SFC "gfortran")

cime_config/machines/cmake_macros/gnuomegagpu_frontier.cmake

@@ -0,0 +1,16 @@
+string(APPEND CPPDEFS " -DLINUX")
+
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+
+string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface")
+
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
+string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
+
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off")
+
+set(USE_HIP "TRUE")

cime_config/machines/cmake_macros/nvidiagpu_pm-gpu.cmake

@@ -6,8 +6,8 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
 string(APPEND CMAKE_CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -acc -gpu=cc80 -Minfo=accel")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
-string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc70,cc60 -Minfo=accel")
+string(APPEND CMAKE_Fortran_FLAGS " -acc -gpu=cc80 -Minfo=accel")

cime_config/machines/config_machines.xml

@@ -966,7 +966,7 @@
     <DESC>Frontier exascale supercomputer at ORNL. 9408 nodes, Node: 4 AMD MI250X GPUs (2 GCDs) ~ 8 GPUs, 512 GB HDB2E, AMD EPYC 64 cores, 512GB DDR4 </DESC>
     <NODENAME_REGEX>.*frontier.*</NODENAME_REGEX>
     <OS>Linux</OS>
-    <COMPILERS>crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu</COMPILERS>
+    <COMPILERS>crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu,crayomegagpu,amdomegagpu,gnuomegagpu</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>cli115</PROJECT>
     <SAVE_TIMING_DIR>/lustre/orion/cli115/world-shared/frontier</SAVE_TIMING_DIR>
@@ -986,6 +986,10 @@
     <MAX_MPITASKS_PER_NODE compiler="gnugpu">8</MAX_MPITASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE compiler="crayclanggpu">8</MAX_MPITASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE compiler="amdclanggpu">8</MAX_MPITASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE compiler="gnuomegagpu">8</MAX_MPITASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE compiler="crayomegagpu">8</MAX_MPITASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE compiler="amdomegagpu">8</MAX_MPITASKS_PER_NODE>
+
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>srun</executable>
@@ -1017,6 +1021,15 @@
         <command name="load">craype-accel-amd-gfx90a</command>
         <command name="load">rocm/5.4.0</command>
       </modules>
+      <modules compiler="crayomegagpu">
+        <command name="reset"></command>
+        <command name="switch">PrgEnv-cray PrgEnv-cray/8.3.3</command>
+        <command name="switch">cce cce/15.0.1</command>
+        <command name="switch">craype craype/2.7.20</command>
+        <command name="load">craype-accel-amd-gfx90a</command>
+        <command name="load">rocm/5.4.0</command>
+      </modules>
+
       <modules compiler="amdclang.*">
         <command name="reset"></command>
         <command name="switch">PrgEnv-cray PrgEnv-amd/8.3.3</command>
@@ -1025,6 +1038,13 @@
       <modules compiler="amdclanggpu">
         <command name="load">craype-accel-amd-gfx90a</command>
       </modules>
+      <modules compiler="amdomegagpu">
+        <command name="reset"></command>
+        <command name="switch">PrgEnv-cray PrgEnv-amd/8.3.3</command>
+        <command name="switch">amd amd/5.4.0</command>
+        <command name="load">craype-accel-amd-gfx90a</command>
+      </modules>
+
       <modules compiler="gnu.*">
         <command name="reset"></command>
         <command name="switch">PrgEnv-cray PrgEnv-gnu/8.3.3</command>
@@ -1035,6 +1055,11 @@
         <command name="load">craype-accel-amd-gfx90a</command>
         <command name="load">rocm/5.4.0</command>
       </modules>
+      <modules compiler="gnuomegagpu">
+        <command name="load">craype-accel-amd-gfx90a</command>
+        <command name="load">rocm/5.4.0</command>
+      </modules>
+
       <modules>
         <command name="load">cray-python/3.9.13.1</command>
         <command name="load">subversion/1.14.1</command>
@@ -1085,6 +1110,13 @@
     <environment_variables compiler="amdclang.*" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/amdclang-15.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
     </environment_variables>
+    <environment_variables compiler="crayomegagpu" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/crayclang-15.0.1; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
+    <environment_variables compiler="amdomegagpu" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.23/amdclang-15.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
+
   </machine>
 
   <machine MACH="crusher">