CMake: New AMReX Options

Anticipate new AMReX options in CMake.
ECP-WarpX · Nov 5, 2020 · 6fadd57 · 6fadd57
1 parent 5d1b88e
commit 6fadd57
Show file tree

Hide file tree

Showing 7 changed files with 92 additions and 96 deletions.
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -29,5 +29,5 @@ jobs:
         cd ..
 
         mkdir build_sp && cd build_sp
-        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single
+        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE
         make -j 2
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -34,7 +34,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         mkdir build_sp && cd build_sp
-        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_COMPUTE=CUDA -DCUDA_ARCH=6.0 -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single -DCUDA_ERROR_CAPTURE_THIS=ON
+        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_COMPUTE=CUDA -DAMReX_CUDA_ARCH=6.0 -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE -DCUDA_ERROR_CAPTURE_THIS=ON
         make -j 2
 
 # Ref.: https://github.com/rscohn2/oneapi-ci
@@ -76,7 +76,7 @@ jobs:
         cd ..
 
         mkdir build_sp && cd build_sp
-        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single
+        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE
         make -j 2
 
   build_dpcc:
@@ -117,7 +117,7 @@ jobs:
         export CC=$(which clang)
 
         mkdir build_sp && cd build_sp
-        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_COMPUTE=DPCPP -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single
+        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_COMPUTE=DPCPP -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE
         make -j 2
 
   build_hip:
@@ -137,5 +137,5 @@ jobs:
         export CC=$(which hipcc)
 
         mkdir build_sp && cd build_sp
-        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=ON -DWarpX_COMPUTE=HIP -DAMD_ARCH=gfx900 -DWarpX_OPENPMD=ON -DWarpX_PRECISION=single
+        cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=ON -DWarpX_COMPUTE=HIP -DAMReX_AMD_ARCH=gfx900 -DWarpX_OPENPMD=ON -DWarpX_PRECISION=SINGLE
         make -j 2
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -56,8 +56,8 @@ if(NOT WarpX_DIMS IN_LIST WarpX_DIMS_VALUES)
     message(FATAL_ERROR "WarpX_DIMS (${WarpX_DIMS}) must be one of ${WarpX_DIMS_VALUES}")
 endif()
 
-set(WarpX_PRECISION_VALUES double single)
-set(WarpX_PRECISION double CACHE STRING "Floating point precision (single/double)")
+set(WarpX_PRECISION_VALUES SINGLE DOUBLE)
+set(WarpX_PRECISION DOUBLE CACHE STRING "Floating point precision (SINGLE/DOUBLE)")
 set_property(CACHE WarpX_PRECISION PROPERTY STRINGS ${WarpX_PRECISION_VALUES})
 if(NOT WarpX_PRECISION IN_LIST WarpX_PRECISION_VALUES)
     message(FATAL_ERROR "WarpX_PRECISION (${WarpX_PRECISION}) must be one of ${WarpX_PRECISION_VALUES}")
@@ -67,7 +67,7 @@ set(WarpX_COMPUTE_VALUES NOACC OMP CUDA DPCPP HIP)
 set(WarpX_COMPUTE OMP CACHE STRING "On-node, accelerated computing backend (NOACC/OMP/CUDA/DPCPP/HIP)")
 set_property(CACHE WarpX_COMPUTE PROPERTY STRINGS ${WarpX_COMPUTE_VALUES})
 if(NOT WarpX_COMPUTE IN_LIST WarpX_COMPUTE_VALUES)
-    message(FATAL_ERROR "WarpX_PRECISION (${WarpX_COMPUTE}) must be one of ${WarpX_COMPUTE_VALUES}")
+    message(FATAL_ERROR "WarpX_COMPUTE (${WarpX_COMPUTE}) must be one of ${WarpX_COMPUTE_VALUES}")
 endif()
 
 option(WarpX_MPI_THREAD_MULTIPLE "MPI thread-multiple support, i.e. for async_io" ON)
@@ -106,9 +106,9 @@ include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake)
 # PSATD
 if(WarpX_PSATD)
     # FFTW (non-GPU) and cuFFT (GPU)
-    if(NOT ENABLE_CUDA)
+    if(NOT AMReX_CUDA)
         find_package(PkgConfig REQUIRED QUIET)
-        if(WarpX_PRECISION STREQUAL "double")
+        if(WarpX_PRECISION STREQUAL "DOUBLE")
             pkg_check_modules(fftw3 REQUIRED IMPORTED_TARGET fftw3)
         else()
             pkg_check_modules(fftw3f REQUIRED IMPORTED_TARGET fftw3f)
@@ -169,11 +169,11 @@ set_target_properties(WarpX PROPERTIES
 target_link_libraries(WarpX PUBLIC WarpX::thirdparty::AMReX)
 
 if(WarpX_PSATD)
-    if(ENABLE_CUDA)
+    if(AMReX_CUDA)
         # CUDA_ADD_CUFFT_TO_TARGET(WarpX)
         target_link_libraries(WarpX PUBLIC cufft)
     else()
-        if(WarpX_PRECISION STREQUAL "double")
+        if(WarpX_PRECISION STREQUAL "DOUBLE")
             target_link_libraries(WarpX PUBLIC PkgConfig::fftw3)
         else()
             target_link_libraries(WarpX PUBLIC PkgConfig::fftw3f)
@@ -198,7 +198,7 @@ if(WarpX_QED)
 endif()
 
 # AMReX helper function: propagate CUDA specific target & source properties
-if(ENABLE_CUDA)
+if(AMReX_CUDA)
     setup_target_for_cuda_compilation(WarpX)
     if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17)
         target_compile_features(WarpX PUBLIC cuda_std_14)

diff --git a/Docs/source/building/cmake.rst b/Docs/source/building/cmake.rst
@@ -129,7 +129,7 @@ CMake Option                  Default & Values                             Descr
 ``WarpX_MPI_THREAD_MULTIPLE`` **ON**/OFF                                   MPI thread-multiple support, i.e. for ``async_io``
 ``WarpX_OPENPMD``             ON/**OFF**                                   openPMD I/O (HDF5, ADIOS)
 ``WarpX_PARSER_DEPTH``        **24**                                       Maximum parser depth for input file functions
-``WarpX_PRECISION``           **double**/single                            Floating point precision (single/double)
+``WarpX_PRECISION``           SINGLE/**DOUBLE**                            Floating point precision (single/double)
 ``WarpX_PSATD``               ON/**OFF**                                   Spectral solver
 ``WarpX_QED``                 ON/**OFF**                                   PICSAR QED (requires Boost and PICSAR)
 ``WarpX_amrex_repo``          ``https://github.com/AMReX-Codes/amrex.git`` Repository URI to pull and build AMReX from

diff --git a/Docs/source/building/summit.rst b/Docs/source/building/summit.rst
@@ -136,7 +136,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman
 
    mkdir -p build
    cd build
-   cmake .. -DWarpX_OPENPMD=ON -DWarpX_DIMS=3 -DWarpX_COMPUTE=CUDA -DCMAKE_CUDA_ARCHITECTURES=70 -DCUDA_ARCH=7.0
+   cmake .. -DWarpX_OPENPMD=ON -DWarpX_DIMS=3 -DWarpX_COMPUTE=CUDA -DCMAKE_CUDA_ARCHITECTURES=70 -DAMReX_CUDA_ARCH=7.0
    make -j 10
 
 The general :ref:`cmake compile-time options <building-cmake>` apply as usual.

diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake
@@ -4,7 +4,7 @@ macro(set_ccache)
     find_program(CCACHE_PROGRAM ccache)
     if(CCACHE_PROGRAM)
         set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
-        if(ENABLE_CUDA)
+        if(AMReX_CUDA)
             set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
         endif()
     endif()
@@ -146,7 +146,7 @@ function(set_warpx_binary_name)
 
     set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".${WarpX_COMPUTE}")
 
-    if(WarpX_PRECISION STREQUAL "double")
+    if(WarpX_PRECISION STREQUAL "DOUBLE")
         set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".DP")
     else()
         set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".SP")

diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake
@@ -6,84 +6,79 @@ macro(find_amrex)
 
         # see https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#customization-options
         if(WarpX_ASCENT)
-            set(ENABLE_ASCENT ON CACHE INTERNAL "")
-            set(ENABLE_CONDUIT ON CACHE INTERNAL "")
+            set(AMReX_ASCENT ON CACHE INTERNAL "")
+            set(AMReX_CONDUIT ON CACHE INTERNAL "")
         endif()
 
         if("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
-            set(ENABLE_ASSERTIONS ON CACHE BOOL "")
+            set(AMReX_ASSERTIONS ON CACHE BOOL "")
             # note: floating-point exceptions can slow down debug runs a lot
-            set(ENABLE_FPE ON CACHE BOOL "")
+            set(AMReX_FPE ON CACHE BOOL "")
         else()
-            set(ENABLE_ASSERTIONS OFF CACHE BOOL "")
-            set(ENABLE_FPE OFF CACHE BOOL "")
+            set(AMReX_ASSERTIONS OFF CACHE BOOL "")
+            set(AMReX_FPE OFF CACHE BOOL "")
         endif()
 
         if(WarpX_COMPUTE STREQUAL CUDA)
-            set(ENABLE_ACC   OFF CACHE INTERNAL "")
-            set(ENABLE_CUDA  ON  CACHE INTERNAL "")
-            set(ENABLE_DPCPP OFF CACHE BOOL "")
-            set(ENABLE_HIP   OFF CACHE BOOL "")
-            set(ENABLE_OMP   OFF CACHE INTERNAL "")
+            set(AMReX_CUDA  ON  CACHE INTERNAL "")
+            set(AMReX_DPCPP OFF CACHE BOOL "")
+            set(AMReX_HIP   OFF CACHE BOOL "")
+            set(AMReX_OMP   OFF CACHE INTERNAL "")
         elseif(WarpX_COMPUTE STREQUAL OMP)
-            set(ENABLE_ACC   OFF CACHE INTERNAL "")
-            set(ENABLE_CUDA  OFF CACHE INTERNAL "")
-            set(ENABLE_DPCPP OFF CACHE BOOL "")
-            set(ENABLE_HIP   OFF CACHE BOOL "")
-            set(ENABLE_OMP   ON  CACHE INTERNAL "")
+            set(AMReX_CUDA  OFF CACHE INTERNAL "")
+            set(AMReX_DPCPP OFF CACHE BOOL "")
+            set(AMReX_HIP   OFF CACHE BOOL "")
+            set(AMReX_OMP   ON  CACHE INTERNAL "")
         elseif(WarpX_COMPUTE STREQUAL DPCPP)
-            set(ENABLE_ACC   OFF CACHE INTERNAL "")
-            set(ENABLE_CUDA  OFF CACHE INTERNAL "")
-            set(ENABLE_DPCPP ON  CACHE BOOL "")
-            set(ENABLE_HIP   OFF CACHE BOOL "")
-            set(ENABLE_OMP   OFF CACHE INTERNAL "")
+            set(AMReX_CUDA  OFF CACHE INTERNAL "")
+            set(AMReX_DPCPP ON  CACHE BOOL "")
+            set(AMReX_HIP   OFF CACHE BOOL "")
+            set(AMReX_OMP   OFF CACHE INTERNAL "")
         elseif(WarpX_COMPUTE STREQUAL HIP)
-            set(ENABLE_ACC   OFF CACHE INTERNAL "")
-            set(ENABLE_CUDA  OFF CACHE INTERNAL "")
-            set(ENABLE_DPCPP OFF  CACHE BOOL "")
-            set(ENABLE_HIP   ON CACHE BOOL "")
-            set(ENABLE_OMP   OFF CACHE INTERNAL "")
+            set(AMReX_CUDA  OFF CACHE INTERNAL "")
+            set(AMReX_DPCPP OFF  CACHE BOOL "")
+            set(AMReX_HIP   ON CACHE BOOL "")
+            set(AMReX_OMP   OFF CACHE INTERNAL "")
         else()
-            set(ENABLE_ACC   OFF CACHE INTERNAL "")
-            set(ENABLE_CUDA  OFF CACHE INTERNAL "")
-            set(ENABLE_DPCPP OFF CACHE BOOL "")
-            set(ENABLE_HIP   OFF CACHE BOOL "")
-            set(ENABLE_OMP   OFF CACHE INTERNAL "")
+            set(AMReX_CUDA  OFF CACHE INTERNAL "")
+            set(AMReX_DPCPP OFF CACHE BOOL "")
+            set(AMReX_HIP   OFF CACHE BOOL "")
+            set(AMReX_OMP   OFF CACHE INTERNAL "")
         endif()
 
         if(WarpX_MPI)
-            set(ENABLE_MPI ON CACHE INTERNAL "")
+            set(AMReX_MPI ON CACHE INTERNAL "")
             if(WarpX_MPI_THREAD_MULTIPLE)
-                set(ENABLE_MPI_THREAD_MULTIPLE ON CACHE INTERNAL "")
+                set(AMReX_MPI_THREAD_MULTIPLE ON CACHE INTERNAL "")
             else()
-                set(ENABLE_MPI_THREAD_MULTIPLE OFF CACHE INTERNAL "")
+                set(AMReX_MPI_THREAD_MULTIPLE OFF CACHE INTERNAL "")
             endif()
         else()
-            set(ENABLE_MPI OFF CACHE INTERNAL "")
+            set(AMReX_MPI OFF CACHE INTERNAL "")
         endif()
 
-        if(WarpX_PRECISION STREQUAL "double")
-            set(ENABLE_DP ON CACHE INTERNAL "")
-            set(ENABLE_DP_PARTICLES ON CACHE INTERNAL "")
+        if(WarpX_PRECISION STREQUAL "DOUBLE")
+            set(AMReX_PRECISION "DOUBLE" CACHE INTERNAL "")
+            set(AMReX_PRECISION_PARTICLES "DOUBLE" CACHE INTERNAL "")
         else()
-            set(ENABLE_DP OFF CACHE INTERNAL "")
-            set(ENABLE_DP_PARTICLES OFF CACHE INTERNAL "")
+            set(AMReX_PRECISION "SINGLE" CACHE INTERNAL "")
+            set(AMReX_PRECISION_PARTICLES "SINGLE" CACHE INTERNAL "")
         endif()
 
-        set(ENABLE_FORTRAN OFF CACHE INTERNAL "")
-        set(ENABLE_FORTRAN_INTERFACES OFF CACHE INTERNAL "")
-        set(ENABLE_TUTORIALS OFF CACHE INTERNAL "")
-        set(ENABLE_PARTICLES ON CACHE INTERNAL "")
-        set(ENABLE_TINY_PROFILE ON CACHE BOOL "")
+        set(AMReX_FORTRAN OFF CACHE INTERNAL "")
+        set(AMReX_FORTRAN_INTERFACES OFF CACHE INTERNAL "")
+        set(AMReX_BUILD_TUTORIALS OFF CACHE INTERNAL "")
+        set(AMReX_PARTICLES ON CACHE INTERNAL "")
+        set(AMReX_TINY_PROFILE ON CACHE BOOL "")
 
-        # ENABLE_SENSEI
+        # AMReX_SENSEI
         # we'll need this for Python bindings
-        #set(ENABLE_PIC ON CACHE INTERNAL "")
+        #set(AMReX_PIC ON CACHE INTERNAL "")
 
         if(WarpX_DIMS STREQUAL RZ)
-            set(DIM 2 CACHE INTERNAL "")
+            set(AMReX_SPACEDIM 2 CACHE INTERNAL "")
         else()
-            set(DIM ${WarpX_DIMS} CACHE INTERNAL "")
+            set(AMReX_SPACEDIM ${WarpX_DIMS} CACHE INTERNAL "")
         endif()
 
         FetchContent_Declare(fetchedamrex
@@ -96,7 +91,7 @@ macro(find_amrex)
         if(NOT fetchedamrex_POPULATED)
             FetchContent_Populate(fetchedamrex)
             list(APPEND CMAKE_MODULE_PATH "${fetchedamrex_SOURCE_DIR}/Tools/CMake")
-            if(ENABLE_CUDA)
+            if(AMReX_CUDA)
                 enable_language(CUDA)
                 include(AMReX_SetupCUDA)
             endif()
@@ -113,47 +108,48 @@ macro(find_amrex)
 
         # AMReX options not relevant to WarpX users
         mark_as_advanced(AMREX_BUILD_DATETIME)
-        mark_as_advanced(DIM)
-        mark_as_advanced(ENABLE_ACC)
-        mark_as_advanced(ENABLE_ASSERTIONS)
-        mark_as_advanced(ENABLE_AMRDATA)
-        mark_as_advanced(ENABLE_BASE_PROFILE) # mutually exclusive to tiny profile
-        mark_as_advanced(ENABLE_CONDUIT)
-        mark_as_advanced(ENABLE_CUDA)
-        mark_as_advanced(ENABLE_DP)
-        mark_as_advanced(ENABLE_DP_PARTICLES)
-        mark_as_advanced(ENABLE_DPCPP)
-        mark_as_advanced(ENABLE_EB)
-        mark_as_advanced(ENABLE_FPE)
-        mark_as_advanced(ENABLE_FORTRAN)
-        mark_as_advanced(ENABLE_FORTRAN_INTERFACES)
-        mark_as_advanced(ENABLE_HDF5)  # we do HDF5 I/O (and more) via openPMD-api
-        mark_as_advanced(ENABLE_LINEAR_SOLVERS)
-        mark_as_advanced(ENABLE_MEM_PROFILE)
-        mark_as_advanced(ENABLE_MPI)
-        mark_as_advanced(ENABLE_MPI_THREAD_MULTIPLE)
-        mark_as_advanced(ENABLE_OMP)
-        mark_as_advanced(ENABLE_PIC)
-        mark_as_advanced(ENABLE_SENSEI)
-        mark_as_advanced(ENABLE_TINY_PROFILE)
-        mark_as_advanced(TP_PROFILE)
+        mark_as_advanced(AMReX_SPACEDIM)
+        mark_as_advanced(AMReX_ASSERTIONS)
+        mark_as_advanced(AMReX_AMRDATA)
+        mark_as_advanced(AMReX_BASE_PROFILE) # mutually exclusive to tiny profile
+        mark_as_advanced(AMReX_CONDUIT)
+        mark_as_advanced(AMReX_CUDA)
+        mark_as_advanced(AMReX_PARTICLES)
+        mark_as_advanced(AMReX_PARTICLES_PRECISION)
+        mark_as_advanced(AMReX_DPCPP)
+        mark_as_advanced(AMReX_EB)
+        mark_as_advanced(AMReX_FPE)
+        mark_as_advanced(AMReX_FORTRAN)
+        mark_as_advanced(AMReX_FORTRAN_INTERFACES)
+        mark_as_advanced(AMReX_HDF5)  # we do HDF5 I/O (and more) via openPMD-api
+        mark_as_advanced(AMReX_HIP)
+        mark_as_advanced(AMReX_LINEAR_SOLVERS)
+        mark_as_advanced(AMReX_MEM_PROFILE)
+        mark_as_advanced(AMReX_MPI)
+        mark_as_advanced(AMReX_MPI_THREAD_MULTIPLE)
+        mark_as_advanced(AMReX_OMP)
+        mark_as_advanced(AMReX_PIC)
+        mark_as_advanced(AMReX_SENSEI)
+        mark_as_advanced(AMReX_TINY_PROFILE)
+        mark_as_advanced(AMReX_TP_PROFILE)
         mark_as_advanced(USE_XSDK_DEFAULTS)
 
         message(STATUS "AMReX: Using INTERNAL version '${AMREX_PKG_VERSION}' (${AMREX_GIT_VERSION})")
     else()
         # https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#importing-amrex-into-your-cmake-project
         if(WarpX_ASCENT)
-            set(COMP_ASCENT ENABLE_ASCENT ENABLE_CONDUIT)
+            set(COMPONENT_ASCENT AMReX_ASCENT AMReX_CONDUIT)
         else()
-            set(COMP_ASCENT)
+            set(COMPONENT_ASCENT)
         endif()
         if(WarpX_DIMS STREQUAL RZ)
-            set(COMP_DIM 2D)
+            set(COMPONENT_DIM 2D)
         else()
-            set(COMP_DIM ${WarpX_DIMS}D)
+            set(COMPONENT_DIM ${WarpX_DIMS}D)
         endif()
+        set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PRECISION})
 
-        find_package(AMReX 20.05 CONFIG REQUIRED COMPONENTS ${COMP_ASCENT} ${COMP_DIM} PARTICLES DPARTICLES DP TINYP LSOLVERS FINTERFACES)
+        find_package(AMReX 20.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIM} PARTICLES ${COMPONENT_PRECISION} TINYP LSOLVERS)
         message(STATUS "AMReX: Found version '${AMReX_VERSION}'")
     endif()
 endmacro()