diff --git a/CMakeLists.txt b/CMakeLists.txt index ff352b268d..93d76a790a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,7 +237,7 @@ endif() set(ADIOS2_CONFIG_OPTS BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ - Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP + Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) diff --git a/bindings/CXX11/adios2/cxx11/KokkosView.h b/bindings/CXX11/adios2/cxx11/KokkosView.h index 1d0e70b12a..f092e51c9f 100644 --- a/bindings/CXX11/adios2/cxx11/KokkosView.h +++ b/bindings/CXX11/adios2/cxx11/KokkosView.h @@ -9,55 +9,16 @@ namespace detail { template -struct memspace_kokkos_to_adios2; - -template <> -struct memspace_kokkos_to_adios2 -{ - static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host; -}; - -#if defined(KOKKOS_ENABLE_CUDA) && defined(ADIOS2_HAVE_GPU_SUPPORT) - -template <> -struct memspace_kokkos_to_adios2 +struct memspace_kokkos_to_adios2 { static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; }; template <> -struct memspace_kokkos_to_adios2 -{ - static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; -}; - -template <> -struct memspace_kokkos_to_adios2 -{ - static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; -}; - -#endif - -#if defined(KOKKOS_ENABLE_HIP) && defined(ADIOS2_HAVE_GPU_SUPPORT) -template <> -struct memspace_kokkos_to_adios2 -{ - static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; -}; - -template <> -struct memspace_kokkos_to_adios2 -{ - static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; -}; - -template <> -struct memspace_kokkos_to_adios2 +struct memspace_kokkos_to_adios2 { - static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; + static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host; }; -#endif } // namespace detail diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 871ad0c42f..b3c271c647 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -192,6 +192,9 @@ if(ADIOS2_USE_Kokkos) set(ADIOS2_HAVE_Kokkos_HIP TRUE) enable_language(HIP) endif() + if(Kokkos_ENABLE_SYCL) + set(ADIOS2_HAVE_Kokkos_SYCL TRUE) + endif() set(ADIOS2_HAVE_GPU_Support TRUE) endif() endif() diff --git a/scripts/build_scripts/build-adios2-sycl-polaris.sh b/scripts/build_scripts/build-adios2-sycl-polaris.sh new file mode 100644 index 0000000000..34487c98c0 --- /dev/null +++ b/scripts/build_scripts/build-adios2-sycl-polaris.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# shellcheck disable=SC2191 + +module load oneapi +module load cmake/3.23.2 +module refresh + +######## User Configurations ######## +Kokkos_HOME=$HOME/kokkos/kokkos +ADIOS2_HOME=$(pwd) +BUILD_DIR=${ADIOS2_HOME}/build-kokkos-polaris +INSTALL_DIR=${ADIOS2_HOME}/install-kokkos-polaris + +num_build_procs=4 + +######## Kokkos ######## +mkdir -p "${BUILD_DIR}/kokkos" +rm -f "${BUILD_DIR}/kokkos/CMakeCache.txt" +rm -rf "${BUILD_DIR}/kokkos/CMakeFiles" + +ARGS=( + -D CMAKE_BUILD_TYPE=RelWithDebInfo + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D CMAKE_CXX_COMPILER=clang++ + + -D Kokkos_ENABLE_SERIAL=ON + -D Kokkos_ARCH_ZEN3=ON + -D Kokkos_ENABLE_SYCL=ON + -D Kokkos_ARCH_AMPERE80=ON + -D Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON + + -D CMAKE_CXX_STANDARD=17 + -D CMAKE_CXX_EXTENSIONS=OFF + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS[@]}" -S "${Kokkos_HOME}" -B "${BUILD_DIR}/kokkos" +cmake --build "${BUILD_DIR}/kokkos" -j${num_build_procs} +cmake --install "${BUILD_DIR}/kokkos" + +######## ADIOS2 ######## +mkdir -p "${BUILD_DIR}/adios2" +rm -f "${BUILD_DIR}/adios2/CMakeCache.txt" +rm -rf "${BUILD_DIR}/adios2/CMakeFiles" + +ARGS_ADIOS=( + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D BUILD_TESTING=OFF + #-D ADIOS2_BUILD_EXAMPLES=OFF + -D CMAKE_CXX_COMPILER=g++ + -D CMAKE_C_COMPILER=gcc + + -D ADIOS2_USE_SST=OFF + -D ADIOS2_USE_Kokkos=ON + -D Kokkos_ROOT="${INSTALL_DIR}" + + -D CMAKE_CXX_STANDARD=17 + -D CMAKE_CXX_EXTENSIONS=OFF + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON + -D ADIOS2_USE_Fortran=OFF +) +cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}"/adios2 +cmake --build "${BUILD_DIR}/adios2" -j${num_build_procs} +cmake --install "${BUILD_DIR}/adios2" diff --git a/source/adios2/helper/kokkos/adiosKokkos.cpp b/source/adios2/helper/kokkos/adiosKokkos.cpp index 7a9f53c66b..dffc18e29e 100644 --- a/source/adios2/helper/kokkos/adiosKokkos.cpp +++ b/source/adios2/helper/kokkos/adiosKokkos.cpp @@ -13,10 +13,10 @@ namespace { -template void KokkosDeepCopy(const char *src, char *dst, size_t byteCount) { - Kokkos::View> srcView(src, byteCount); Kokkos::View * /*values*/, const size_t /*size*/, std::complex & /*min*/, std::complex & /*max*/) @@ -62,22 +66,12 @@ namespace helper { void MemcpyGPUToBuffer(char *dst, const char *GPUbuffer, size_t byteCount) { -#ifdef ADIOS2_HAVE_KOKKOS_CUDA - KokkosDeepCopy(GPUbuffer, dst, byteCount); -#endif -#ifdef ADIOS2_HAVE_KOKKOS_HIP - KokkosDeepCopy(GPUbuffer, dst, byteCount); -#endif + KokkosDeepCopy(GPUbuffer, dst, byteCount); } void MemcpyBufferToGPU(char *GPUbuffer, const char *src, size_t byteCount) { -#ifdef ADIOS2_HAVE_KOKKOS_CUDA - KokkosDeepCopy(src, GPUbuffer, byteCount); -#endif -#ifdef ADIOS2_HAVE_KOKKOS_HIP - KokkosDeepCopy(src, GPUbuffer, byteCount); -#endif + KokkosDeepCopy(src, GPUbuffer, byteCount); } bool IsGPUbuffer(const void *ptr) @@ -98,6 +92,15 @@ bool IsGPUbuffer(const void *ptr) { return true; } +#endif +#ifdef ADIOS2_HAVE_KOKKOS_SYCL + auto ret = + sycl::address_space_cast(ptr); + if (ret != nullptr) + { + return true; + } #endif return false; } @@ -121,6 +124,7 @@ void KokkosInit() settings.set_device_id(device_id); } #endif + // GetDevice not supported for SYCL, use the default device Kokkos::initialize(settings); }