Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CMake: New AMReX Options #1492

Merged
merged 2 commits into from
Nov 5, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,5 @@ jobs:
cd ..

mkdir build_sp && cd build_sp
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE
make -j 2
8 changes: 4 additions & 4 deletions .github/workflows/ubuntu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
which nvcc || echo "nvcc not in PATH!"

mkdir build_sp && cd build_sp
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_COMPUTE=CUDA -DCUDA_ARCH=6.0 -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single -DCUDA_ERROR_CAPTURE_THIS=ON
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_COMPUTE=CUDA -DAMReX_CUDA_ARCH=6.0 -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE -DCUDA_ERROR_CAPTURE_THIS=ON
make -j 2

# Ref.: https://github.com/rscohn2/oneapi-ci
Expand Down Expand Up @@ -76,7 +76,7 @@ jobs:
cd ..

mkdir build_sp && cd build_sp
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE
make -j 2

build_dpcc:
Expand Down Expand Up @@ -117,7 +117,7 @@ jobs:
export CC=$(which clang)

mkdir build_sp && cd build_sp
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_COMPUTE=DPCPP -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=single
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=OFF -DWarpX_COMPUTE=DPCPP -DWarpX_OPENPMD=ON -DWarpX_openpmd_internal=OFF -DWarpX_PRECISION=SINGLE
make -j 2

build_hip:
Expand All @@ -137,5 +137,5 @@ jobs:
export CC=$(which hipcc)

mkdir build_sp && cd build_sp
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=ON -DWarpX_COMPUTE=HIP -DAMD_ARCH=gfx900 -DWarpX_OPENPMD=ON -DWarpX_PRECISION=single
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DWarpX_MPI=ON -DWarpX_COMPUTE=HIP -DAMReX_AMD_ARCH=gfx900 -DWarpX_OPENPMD=ON -DWarpX_PRECISION=SINGLE
make -j 2
16 changes: 8 additions & 8 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ if(NOT WarpX_DIMS IN_LIST WarpX_DIMS_VALUES)
message(FATAL_ERROR "WarpX_DIMS (${WarpX_DIMS}) must be one of ${WarpX_DIMS_VALUES}")
endif()

set(WarpX_PRECISION_VALUES double single)
set(WarpX_PRECISION double CACHE STRING "Floating point precision (single/double)")
set(WarpX_PRECISION_VALUES SINGLE DOUBLE)
set(WarpX_PRECISION DOUBLE CACHE STRING "Floating point precision (SINGLE/DOUBLE)")
set_property(CACHE WarpX_PRECISION PROPERTY STRINGS ${WarpX_PRECISION_VALUES})
if(NOT WarpX_PRECISION IN_LIST WarpX_PRECISION_VALUES)
message(FATAL_ERROR "WarpX_PRECISION (${WarpX_PRECISION}) must be one of ${WarpX_PRECISION_VALUES}")
Expand All @@ -67,7 +67,7 @@ set(WarpX_COMPUTE_VALUES NOACC OMP CUDA DPCPP HIP)
set(WarpX_COMPUTE OMP CACHE STRING "On-node, accelerated computing backend (NOACC/OMP/CUDA/DPCPP/HIP)")
set_property(CACHE WarpX_COMPUTE PROPERTY STRINGS ${WarpX_COMPUTE_VALUES})
if(NOT WarpX_COMPUTE IN_LIST WarpX_COMPUTE_VALUES)
message(FATAL_ERROR "WarpX_PRECISION (${WarpX_COMPUTE}) must be one of ${WarpX_COMPUTE_VALUES}")
message(FATAL_ERROR "WarpX_COMPUTE (${WarpX_COMPUTE}) must be one of ${WarpX_COMPUTE_VALUES}")
endif()

option(WarpX_MPI_THREAD_MULTIPLE "MPI thread-multiple support, i.e. for async_io" ON)
Expand Down Expand Up @@ -106,9 +106,9 @@ include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake)
# PSATD
if(WarpX_PSATD)
# FFTW (non-GPU) and cuFFT (GPU)
if(NOT ENABLE_CUDA)
if(NOT AMReX_CUDA)
find_package(PkgConfig REQUIRED QUIET)
if(WarpX_PRECISION STREQUAL "double")
if(WarpX_PRECISION STREQUAL "DOUBLE")
pkg_check_modules(fftw3 REQUIRED IMPORTED_TARGET fftw3)
else()
pkg_check_modules(fftw3f REQUIRED IMPORTED_TARGET fftw3f)
Expand Down Expand Up @@ -169,11 +169,11 @@ set_target_properties(WarpX PROPERTIES
target_link_libraries(WarpX PUBLIC WarpX::thirdparty::AMReX)

if(WarpX_PSATD)
if(ENABLE_CUDA)
if(AMReX_CUDA)
# CUDA_ADD_CUFFT_TO_TARGET(WarpX)
target_link_libraries(WarpX PUBLIC cufft)
else()
if(WarpX_PRECISION STREQUAL "double")
if(WarpX_PRECISION STREQUAL "DOUBLE")
target_link_libraries(WarpX PUBLIC PkgConfig::fftw3)
else()
target_link_libraries(WarpX PUBLIC PkgConfig::fftw3f)
Expand All @@ -198,7 +198,7 @@ if(WarpX_QED)
endif()

# AMReX helper function: propagate CUDA specific target & source properties
if(ENABLE_CUDA)
if(AMReX_CUDA)
setup_target_for_cuda_compilation(WarpX)
if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17)
target_compile_features(WarpX PUBLIC cuda_std_14)
Expand Down
2 changes: 1 addition & 1 deletion Docs/source/building/cmake.rst
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ CMake Option Default & Values Descr
``WarpX_MPI_THREAD_MULTIPLE`` **ON**/OFF MPI thread-multiple support, i.e. for ``async_io``
``WarpX_OPENPMD`` ON/**OFF** openPMD I/O (HDF5, ADIOS)
``WarpX_PARSER_DEPTH`` **24** Maximum parser depth for input file functions
``WarpX_PRECISION`` **double**/single Floating point precision (single/double)
``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double)
``WarpX_PSATD`` ON/**OFF** Spectral solver
``WarpX_QED`` ON/**OFF** PICSAR QED (requires Boost and PICSAR)
``WarpX_amrex_repo`` ``https://github.com/AMReX-Codes/amrex.git`` Repository URI to pull and build AMReX from
Expand Down
3 changes: 3 additions & 0 deletions Docs/source/building/juwels.rst
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ We use the following modules and environments on the system.
export GPUS_PER_SOCKET=2
export GPUS_PER_NODE=4

# optimize CUDA compilation for V100 (CMake hint)
export AMReX_CUDA_ARCH=7.0

Note that for now WarpX must rely on OpenMPI instead of the recommended MPI implementation on this platform MVAPICH2.

We recommend to store the above lines in a file, such as ``$HOME/warpx.profile``, and load it into your shell after a login:
Expand Down
3 changes: 3 additions & 0 deletions Docs/source/building/lassen.rst
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ We use the following modules and environments on the system.
# fix system defaults: do not escape $ with a \ on tab completion
shopt -s direxpand

# optimize CUDA compilation for V100
export AMReX_CUDA_ARCH=7.0

# compiler environment hints
export CC=$(which gcc)
export CXX=$(which g++)
Expand Down
5 changes: 4 additions & 1 deletion Docs/source/building/summit.rst
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ We use the following modules and environments on the system.
# fix system defaults: do not escape $ with a \ on tab completion
shopt -s direxpand

# optimize CUDA compilation for V100
export AMReX_CUDA_ARCH=7.0

# compiler environment hints
export CC=$(which gcc)
export CXX=$(which g++)
Expand Down Expand Up @@ -136,7 +139,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman

mkdir -p build
cd build
cmake .. -DWarpX_OPENPMD=ON -DWarpX_DIMS=3 -DWarpX_COMPUTE=CUDA -DCMAKE_CUDA_ARCHITECTURES=70 -DCUDA_ARCH=7.0
cmake .. -DWarpX_OPENPMD=ON -DWarpX_DIMS=3 -DWarpX_COMPUTE=CUDA
make -j 10

The general :ref:`cmake compile-time options <building-cmake>` apply as usual.
Expand Down
4 changes: 2 additions & 2 deletions cmake/WarpXFunctions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ macro(set_ccache)
find_program(CCACHE_PROGRAM ccache)
if(CCACHE_PROGRAM)
set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
if(ENABLE_CUDA)
if(AMReX_CUDA)
set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
endif()
endif()
Expand Down Expand Up @@ -146,7 +146,7 @@ function(set_warpx_binary_name)

set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".${WarpX_COMPUTE}")

if(WarpX_PRECISION STREQUAL "double")
if(WarpX_PRECISION STREQUAL "DOUBLE")
set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".DP")
else()
set_property(TARGET WarpX APPEND_STRING PROPERTY OUTPUT_NAME ".SP")
Expand Down
154 changes: 75 additions & 79 deletions cmake/dependencies/AMReX.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -6,84 +6,79 @@ macro(find_amrex)

# see https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#customization-options
if(WarpX_ASCENT)
set(ENABLE_ASCENT ON CACHE INTERNAL "")
set(ENABLE_CONDUIT ON CACHE INTERNAL "")
set(AMReX_ASCENT ON CACHE INTERNAL "")
set(AMReX_CONDUIT ON CACHE INTERNAL "")
endif()

if("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
set(ENABLE_ASSERTIONS ON CACHE BOOL "")
set(AMReX_ASSERTIONS ON CACHE BOOL "")
# note: floating-point exceptions can slow down debug runs a lot
set(ENABLE_FPE ON CACHE BOOL "")
set(AMReX_FPE ON CACHE BOOL "")
else()
set(ENABLE_ASSERTIONS OFF CACHE BOOL "")
set(ENABLE_FPE OFF CACHE BOOL "")
set(AMReX_ASSERTIONS OFF CACHE BOOL "")
set(AMReX_FPE OFF CACHE BOOL "")
endif()

if(WarpX_COMPUTE STREQUAL CUDA)
set(ENABLE_ACC OFF CACHE INTERNAL "")
set(ENABLE_CUDA ON CACHE INTERNAL "")
set(ENABLE_DPCPP OFF CACHE BOOL "")
set(ENABLE_HIP OFF CACHE BOOL "")
set(ENABLE_OMP OFF CACHE INTERNAL "")
set(AMReX_CUDA ON CACHE INTERNAL "")
set(AMReX_DPCPP OFF CACHE BOOL "")
set(AMReX_HIP OFF CACHE BOOL "")
set(AMReX_OMP OFF CACHE INTERNAL "")
elseif(WarpX_COMPUTE STREQUAL OMP)
set(ENABLE_ACC OFF CACHE INTERNAL "")
set(ENABLE_CUDA OFF CACHE INTERNAL "")
set(ENABLE_DPCPP OFF CACHE BOOL "")
set(ENABLE_HIP OFF CACHE BOOL "")
set(ENABLE_OMP ON CACHE INTERNAL "")
set(AMReX_CUDA OFF CACHE INTERNAL "")
set(AMReX_DPCPP OFF CACHE BOOL "")
set(AMReX_HIP OFF CACHE BOOL "")
set(AMReX_OMP ON CACHE INTERNAL "")
elseif(WarpX_COMPUTE STREQUAL DPCPP)
set(ENABLE_ACC OFF CACHE INTERNAL "")
set(ENABLE_CUDA OFF CACHE INTERNAL "")
set(ENABLE_DPCPP ON CACHE BOOL "")
set(ENABLE_HIP OFF CACHE BOOL "")
set(ENABLE_OMP OFF CACHE INTERNAL "")
set(AMReX_CUDA OFF CACHE INTERNAL "")
set(AMReX_DPCPP ON CACHE BOOL "")
set(AMReX_HIP OFF CACHE BOOL "")
set(AMReX_OMP OFF CACHE INTERNAL "")
elseif(WarpX_COMPUTE STREQUAL HIP)
set(ENABLE_ACC OFF CACHE INTERNAL "")
set(ENABLE_CUDA OFF CACHE INTERNAL "")
set(ENABLE_DPCPP OFF CACHE BOOL "")
set(ENABLE_HIP ON CACHE BOOL "")
set(ENABLE_OMP OFF CACHE INTERNAL "")
set(AMReX_CUDA OFF CACHE INTERNAL "")
set(AMReX_DPCPP OFF CACHE BOOL "")
set(AMReX_HIP ON CACHE BOOL "")
set(AMReX_OMP OFF CACHE INTERNAL "")
else()
set(ENABLE_ACC OFF CACHE INTERNAL "")
set(ENABLE_CUDA OFF CACHE INTERNAL "")
set(ENABLE_DPCPP OFF CACHE BOOL "")
set(ENABLE_HIP OFF CACHE BOOL "")
set(ENABLE_OMP OFF CACHE INTERNAL "")
set(AMReX_CUDA OFF CACHE INTERNAL "")
set(AMReX_DPCPP OFF CACHE BOOL "")
set(AMReX_HIP OFF CACHE BOOL "")
set(AMReX_OMP OFF CACHE INTERNAL "")
endif()

if(WarpX_MPI)
set(ENABLE_MPI ON CACHE INTERNAL "")
set(AMReX_MPI ON CACHE INTERNAL "")
if(WarpX_MPI_THREAD_MULTIPLE)
set(ENABLE_MPI_THREAD_MULTIPLE ON CACHE INTERNAL "")
set(AMReX_MPI_THREAD_MULTIPLE ON CACHE INTERNAL "")
else()
set(ENABLE_MPI_THREAD_MULTIPLE OFF CACHE INTERNAL "")
set(AMReX_MPI_THREAD_MULTIPLE OFF CACHE INTERNAL "")
endif()
else()
set(ENABLE_MPI OFF CACHE INTERNAL "")
set(AMReX_MPI OFF CACHE INTERNAL "")
endif()

if(WarpX_PRECISION STREQUAL "double")
set(ENABLE_DP ON CACHE INTERNAL "")
set(ENABLE_DP_PARTICLES ON CACHE INTERNAL "")
if(WarpX_PRECISION STREQUAL "DOUBLE")
set(AMReX_PRECISION "DOUBLE" CACHE INTERNAL "")
set(AMReX_PRECISION_PARTICLES "DOUBLE" CACHE INTERNAL "")
else()
set(ENABLE_DP OFF CACHE INTERNAL "")
set(ENABLE_DP_PARTICLES OFF CACHE INTERNAL "")
set(AMReX_PRECISION "SINGLE" CACHE INTERNAL "")
set(AMReX_PRECISION_PARTICLES "SINGLE" CACHE INTERNAL "")
endif()

set(ENABLE_FORTRAN OFF CACHE INTERNAL "")
set(ENABLE_FORTRAN_INTERFACES OFF CACHE INTERNAL "")
set(ENABLE_TUTORIALS OFF CACHE INTERNAL "")
set(ENABLE_PARTICLES ON CACHE INTERNAL "")
set(ENABLE_TINY_PROFILE ON CACHE BOOL "")
set(AMReX_FORTRAN OFF CACHE INTERNAL "")
set(AMReX_FORTRAN_INTERFACES OFF CACHE INTERNAL "")
set(AMReX_BUILD_TUTORIALS OFF CACHE INTERNAL "")
set(AMReX_PARTICLES ON CACHE INTERNAL "")
set(AMReX_TINY_PROFILE ON CACHE BOOL "")

# ENABLE_SENSEI
# AMReX_SENSEI
# we'll need this for Python bindings
#set(ENABLE_PIC ON CACHE INTERNAL "")
#set(AMReX_PIC ON CACHE INTERNAL "")

if(WarpX_DIMS STREQUAL RZ)
set(DIM 2 CACHE INTERNAL "")
set(AMReX_SPACEDIM 2 CACHE INTERNAL "")
else()
set(DIM ${WarpX_DIMS} CACHE INTERNAL "")
set(AMReX_SPACEDIM ${WarpX_DIMS} CACHE INTERNAL "")
endif()

FetchContent_Declare(fetchedamrex
Expand All @@ -96,7 +91,7 @@ macro(find_amrex)
if(NOT fetchedamrex_POPULATED)
FetchContent_Populate(fetchedamrex)
list(APPEND CMAKE_MODULE_PATH "${fetchedamrex_SOURCE_DIR}/Tools/CMake")
if(ENABLE_CUDA)
if(AMReX_CUDA)
enable_language(CUDA)
include(AMReX_SetupCUDA)
endif()
Expand All @@ -113,47 +108,48 @@ macro(find_amrex)

# AMReX options not relevant to WarpX users
mark_as_advanced(AMREX_BUILD_DATETIME)
mark_as_advanced(DIM)
mark_as_advanced(ENABLE_ACC)
mark_as_advanced(ENABLE_ASSERTIONS)
mark_as_advanced(ENABLE_AMRDATA)
mark_as_advanced(ENABLE_BASE_PROFILE) # mutually exclusive to tiny profile
mark_as_advanced(ENABLE_CONDUIT)
mark_as_advanced(ENABLE_CUDA)
mark_as_advanced(ENABLE_DP)
mark_as_advanced(ENABLE_DP_PARTICLES)
mark_as_advanced(ENABLE_DPCPP)
mark_as_advanced(ENABLE_EB)
mark_as_advanced(ENABLE_FPE)
mark_as_advanced(ENABLE_FORTRAN)
mark_as_advanced(ENABLE_FORTRAN_INTERFACES)
mark_as_advanced(ENABLE_HDF5) # we do HDF5 I/O (and more) via openPMD-api
mark_as_advanced(ENABLE_LINEAR_SOLVERS)
mark_as_advanced(ENABLE_MEM_PROFILE)
mark_as_advanced(ENABLE_MPI)
mark_as_advanced(ENABLE_MPI_THREAD_MULTIPLE)
mark_as_advanced(ENABLE_OMP)
mark_as_advanced(ENABLE_PIC)
mark_as_advanced(ENABLE_SENSEI)
mark_as_advanced(ENABLE_TINY_PROFILE)
mark_as_advanced(TP_PROFILE)
mark_as_advanced(AMReX_SPACEDIM)
mark_as_advanced(AMReX_ASSERTIONS)
mark_as_advanced(AMReX_AMRDATA)
mark_as_advanced(AMReX_BASE_PROFILE) # mutually exclusive to tiny profile
mark_as_advanced(AMReX_CONDUIT)
mark_as_advanced(AMReX_CUDA)
mark_as_advanced(AMReX_PARTICLES)
mark_as_advanced(AMReX_PARTICLES_PRECISION)
mark_as_advanced(AMReX_DPCPP)
mark_as_advanced(AMReX_EB)
mark_as_advanced(AMReX_FPE)
mark_as_advanced(AMReX_FORTRAN)
mark_as_advanced(AMReX_FORTRAN_INTERFACES)
mark_as_advanced(AMReX_HDF5) # we do HDF5 I/O (and more) via openPMD-api
mark_as_advanced(AMReX_HIP)
mark_as_advanced(AMReX_LINEAR_SOLVERS)
mark_as_advanced(AMReX_MEM_PROFILE)
mark_as_advanced(AMReX_MPI)
mark_as_advanced(AMReX_MPI_THREAD_MULTIPLE)
mark_as_advanced(AMReX_OMP)
mark_as_advanced(AMReX_PIC)
mark_as_advanced(AMReX_SENSEI)
mark_as_advanced(AMReX_TINY_PROFILE)
mark_as_advanced(AMReX_TP_PROFILE)
mark_as_advanced(USE_XSDK_DEFAULTS)

message(STATUS "AMReX: Using INTERNAL version '${AMREX_PKG_VERSION}' (${AMREX_GIT_VERSION})")
else()
# https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#importing-amrex-into-your-cmake-project
if(WarpX_ASCENT)
set(COMP_ASCENT ENABLE_ASCENT ENABLE_CONDUIT)
set(COMPONENT_ASCENT AMReX_ASCENT AMReX_CONDUIT)
else()
set(COMP_ASCENT)
set(COMPONENT_ASCENT)
endif()
if(WarpX_DIMS STREQUAL RZ)
set(COMP_DIM 2D)
set(COMPONENT_DIM 2D)
else()
set(COMP_DIM ${WarpX_DIMS}D)
set(COMPONENT_DIM ${WarpX_DIMS}D)
endif()
set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PRECISION})

find_package(AMReX 20.05 CONFIG REQUIRED COMPONENTS ${COMP_ASCENT} ${COMP_DIM} PARTICLES DPARTICLES DP TINYP LSOLVERS FINTERFACES)
find_package(AMReX 20.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIM} PARTICLES ${COMPONENT_PRECISION} TINYP LSOLVERS)
message(STATUS "AMReX: Found version '${AMReX_VERSION}'")
endif()
endmacro()
Expand Down