First version of the DPC++ executor.

+ Add a new `Dpcpp` executor. Use it in all `gko::Operation` and
  wherever appropriate.
+ Add the general architecture with all kernels set as `GKO_NOT_IMPLEMENTED`
+ Cope with unsupported copies between devices by going through the
  master executors. This (should) allow to copy from an AMD GPU to a
  NVIDIA GPU, or from Intel to AMD/NVIDIA (untested).
+ Implement a few simple kernels in the components and `stop/criterion`
  to ensure kernel execution works properly.
+ Implement DPC++ specific executor tests.
+ Circumvent some issues with `abs` and `sqrt` my using the sycl
  specific functions when appropriate. Note that it's still needed to
  namespace `abs` (and probably `sqrt` as well) calls inside DPC++
  kernels, otherwise the default (std) version is still used.

CMakeLists.txt

@@ -5,7 +5,7 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12)
     cmake_policy(SET CMP0074 NEW)
 endif()
 
-# Let CAS handle the CUDA architecture flags (for now)
+# Let CAS handle the CUDA architecture flags(for now)
 # Windows still gives CMP0104 warning if putting it in cuda.
 if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
     cmake_policy(SET CMP0104 OLD)
@@ -27,6 +27,7 @@ option(GINKGO_BUILD_EXAMPLES "Build Ginkgo's examples" ON)
 option(GINKGO_BUILD_BENCHMARKS "Build Ginkgo's benchmarks" ON)
 option(GINKGO_BUILD_REFERENCE "Compile reference CPU kernels" ON)
 option(GINKGO_BUILD_OMP "Compile OpenMP kernels for CPU" ${GINKGO_HAS_OMP})
+option(GINKGO_BUILD_DPCPP "Compile DPC++ kernels for CPU" ${GINKGO_HAS_DPCPP})
 option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
@@ -72,7 +73,8 @@ option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
 if(BUILD_SHARED_LIBS AND (WIN32 OR CYGWIN) AND (GINKGO_BUILD_TESTS OR GINKGO_BUILD_EXAMPLES OR GINKGO_BUILD_BENCHMARKS))
-    # Change shared libraries output only if this build has executable program with shared libraries.
+    # Change shared libraries output only if this build has executable program
+    # with shared libraries.
     set(GINKGO_CHANGED_SHARED_LIBRARY TRUE)
     option(GINKGO_CHECK_PATH "Tell Ginkgo to check if the environment variable PATH is available for this build." ON)
     set(GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH "windows_shared_library" CACHE STRING
@@ -83,7 +85,7 @@ else()
     set(GINKGO_CHANGED_SHARED_LIBRARY FALSE)
 endif()
 
-if(GINKGO_BUILD_TESTS AND (GINKGO_BUILD_CUDA OR GINKGO_BUILD_OMP OR GINKGO_BUILD_HIP))
+if(GINKGO_BUILD_TESTS AND (GINKGO_BUILD_CUDA OR GINKGO_BUILD_OMP OR GINKGO_BUILD_HIP OR GINKGO_BUILD_DPCPP))
     message(STATUS "GINKGO_BUILD_TESTS is ON, enabling GINKGO_BUILD_REFERENCE")
     set(GINKGO_BUILD_REFERENCE ON CACHE BOOL "Compile reference CPU kernels" FORCE)
 endif()
@@ -105,12 +107,12 @@ if(NOT DEFINED CMAKE_DEBUG_POSTFIX)
 endif()
 
 if(GINKGO_BUILD_TESTS)
-    # Configure CTest
+# Configure CTest
     configure_file(
         ${CMAKE_CURRENT_LIST_DIR}/cmake/CTestCustom.cmake.in
         ${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake @ONLY)
 
-    # For testing, we need some special matrices
+#For testing, we need some special matrices
     add_subdirectory(matrices)
 
     enable_testing()
@@ -127,8 +129,9 @@ endif()
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
-# Find important header files, store the definitions in include/ginkgo/config.h.in
-# For details, see https://gitlab.kitware.com/cmake/community/wikis/doc/tutorials/How-To-Write-Platform-Checks
+# Find important header files, store the definitions in
+# include/ginkgo/config.h.in For details, see
+# https://gitlab.kitware.com/cmake/community/wikis/doc/tutorials/How-To-Write-Platform-Checks
 include(CheckIncludeFileCXX)
 check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H)
 
@@ -143,11 +146,12 @@ set(GINKGO_HIP_PLATFORM_NVCC 0)
 set(GINKGO_HIP_PLATFORM_HCC 0)
 
 if(GINKGO_BUILD_HIP)
-    # GINKGO_HIPCONFIG_PATH and HIP_PATH are set in cmake/hip_path.cmake
+# GINKGO_HIPCONFIG_PATH and HIP_PATH are set in cmake/hip_path.cmake
     if(DEFINED ENV{HIP_PLATFORM})
         set(GINKGO_HIP_PLATFORM "$ENV{HIP_PLATFORM}")
     elseif(GINKGO_HIPCONFIG_PATH)
-        execute_process(COMMAND ${GINKGO_HIPCONFIG_PATH} --platform OUTPUT_VARIABLE GINKGO_HIP_PLATFORM)
+        execute_process(COMMAND ${
+    GINKGO_HIPCONFIG_PATH} --platform OUTPUT_VARIABLE GINKGO_HIP_PLATFORM)
     else()
         message(FATAL_ERROR "No platform could be found for HIP. "
             "Set and export the environment variable HIP_PLATFORM.")
@@ -170,7 +174,8 @@ include(cmake/hip_helpers.cmake)
 include(cmake/install_helpers.cmake)
 include(cmake/windows_helpers.cmake)
 
-# This is modified from https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
+# This is modified from
+# https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
 if(MSVC)
     if(BUILD_SHARED_LIBS)
         ginkgo_switch_to_windows_dynamic("CXX")
@@ -204,12 +209,16 @@ endif()
 if (GINKGO_BUILD_OMP)
     add_subdirectory(omp)        # High-performance omp kernels
 endif()
+if (GINKGO_BUILD_DPCPP)
+    add_subdirectory(dpcpp)        # High-performance DPC++ kernels
+endif()
 # HIP needs to be last because it builds the GINKGO_RPATH_FOR_HIP variable
 # which needs to know the `ginkgo` target.
 if(GINKGO_BUILD_HIP)
     add_subdirectory(hip)        # High-performance kernels for AMD or NVIDIA GPUs
 endif()
 
+
 # Non core directories and targets
 if(GINKGO_BUILD_EXAMPLES)
     add_subdirectory(examples)
@@ -226,8 +235,8 @@ if(GINKGO_DEVEL_TOOLS)
     add_dependencies(format add_license)
 endif()
 
-# MacOS needs to install bash, gnu-sed, findutils and coreutils
-# format_header needs clang-format 6.0.0+
+# MacOS needs to install bash, gnu - sed, findutils and coreutils
+# format_header needs clang - format 6.0.0 +
 find_program(BASH bash)
 if(NOT "${BASH}" STREQUAL "BASH-NOTFOUND" AND GINKGO_DEVEL_TOOLS)
     add_custom_target(generate_ginkgo_header ALL
@@ -271,14 +280,15 @@ endif()
 configure_file(${Ginkgo_SOURCE_DIR}/cmake/ginkgo.pc.in
     ${Ginkgo_BINARY_DIR}/ginkgo.pc @ONLY)
 
-# WINDOWS NVCC has " inside the string, add escape charater to avoid config problem.
+# WINDOWS NVCC has " inside the string, add escape character
+# to avoid config problem.
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS)
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS_DEBUG)
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS_RELEASE)
 ginkgo_install()
 
 if(MSVC)
-    # Set path/command with $<CONFIG>
+# Set path / command with $ < CONFIG>
     set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install")
     if(GINKGO_BUILD_CUDA)
         set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_COMMAND}" "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install_cuda")
@@ -295,8 +305,9 @@ add_custom_target(test_install
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
-    # `--config cfg` is ignored by single-configuration generator.
-    # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in single-configuration generator.
+# `-- config cfg` is ignored by single - configuration generator.
+# `$ <CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in single - \
+    configuration generator.
     COMMAND ${CMAKE_COMMAND} --build ${Ginkgo_BINARY_DIR}/test_install --config $<CONFIG>
     COMMAND ${GINKGO_TEST_INSTALL_COMMAND}
     COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
@@ -309,7 +320,6 @@ set(CPACK_PACKAGE_CONTACT "ginkgo.library@gmail.com")
 include(CPack)
 
 # And finally, print the configuration to screen:
-#
 if(GINKGO_CONFIG_LOG_DETAILED)
     FILE(READ ${PROJECT_BINARY_DIR}/detailed.log GINKGO_LOG_SUMMARY)
 else()

CONTRIBUTING.md

@@ -68,8 +68,8 @@ look at our coding guidelines before proposing a pull request.
 
 Ginkgo is divided into a `core` module with common functionalities independent
 of the architecture, and several kernel modules (`reference`, `omp`, `cuda`,
-`hip`) which contain low-level computational routines for each supported
-architecture.
+`hip`, `dpcpp`) which contain low-level computational routines for each
+supported architecture.
 
 ### Extended header files
 

INSTALL.md

@@ -32,6 +32,10 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_BUILD_CUDA={ON, OFF}` builds optimized cuda versions of the kernels
     (requires CUDA), default is `ON` if a CUDA compiler could be detected,
     `OFF` otherwise.
+*   `-DGINKGO_BUILD_DPCPP={ON, OFF}` builds optimized DPC++ versions of the
+    kernels (requires `CMAKE_CXX_COMPILER` to be set to the `dpcpp` compiler).
+    The default is `ON` if `CMAKE_CXX_COMPILER` is a DPC++ compiler, `OFF`
+    otherwise.
 *   `-DGINKGO_BUILD_HIP={ON, OFF}` builds optimized HIP versions of the kernels
     (requires HIP), default is `ON` if an installation of HIP could be detected,
     `OFF` otherwise.

benchmark/utils/overhead_linop.hpp

@@ -109,6 +109,15 @@ GKO_DECLARE_ALL;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace overhead {
+
+GKO_DECLARE_ALL;
+
+}  // namespace overhead
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL
 
 

cmake/GinkgoConfig.cmake.in

@@ -36,6 +36,7 @@ set(GINKGO_BUILD_REFERENCE @GINKGO_BUILD_REFERENCE@)
 set(GINKGO_BUILD_OMP @GINKGO_BUILD_OMP@)
 set(GINKGO_BUILD_CUDA @GINKGO_BUILD_CUDA@)
 set(GINKGO_BUILD_HIP @GINKGO_BUILD_HIP@)
+set(GINKGO_BUILD_DPCPP @GINKGO_BUILD_HIP@)
 
 set(GINKGO_DEVEL_TOOLS @GINKGO_DEVEL_TOOLS@)
 set(GINKGO_BUILD_TESTS @GINKGO_BUILD_TESTS@)

cmake/autodetect_executors.cmake

@@ -1,9 +1,13 @@
 set(GINKGO_HAS_OMP OFF)
 set(GINKGO_HAS_CUDA OFF)
+set(GINKGO_HAS_DPCPP OFF)
 set(GINKGO_HAS_HIP OFF)
 find_package(OpenMP)
 include(CheckLanguage)
 check_language(CUDA)
+try_compile(GKO_CAN_COMPILE_DPCPP ${PROJECT_BINARY_DIR}/dpcpp
+    SOURCES ${PROJECT_SOURCE_DIR}/dpcpp/test_dpcpp.dp.cpp
+    CXX_STANDARD 17)
 
 if(OpenMP_CXX_FOUND)
     if(NOT DEFINED GINKGO_BUILD_OMP)
@@ -25,3 +29,10 @@ if(GINKGO_HIPCONFIG_PATH)
     endif()
     set(GINKGO_HAS_HIP ON)
 endif()
+
+if (GKO_CAN_COMPILE_DPCPP)
+    if(NOT DEFINED GINKGO_BUILD_DPCPP)
+        message(STATUS "Enabling DPCPP executor")
+    endif()
+    set(GINKGO_HAS_DPCPP ON)
+endif()

cmake/create_test.cmake

@@ -17,6 +17,26 @@ function(ginkgo_create_test test_name)
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_test)
 
+function(ginkgo_create_dpcpp_test test_name)
+    file(RELATIVE_PATH REL_BINARY_DIR
+        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
+    add_executable(${TEST_TARGET_NAME} ${test_name}.dp.cpp)
+    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_17)
+    target_compile_options("${TEST_TARGET_NAME}" PRIVATE "${GINKGO_DPCPP_FLAGS}")
+    target_include_directories("${TEST_TARGET_NAME}"
+        PRIVATE
+        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
+        )
+    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
+        OUTPUT_NAME ${test_name})
+    if (GINKGO_CHECK_CIRCULAR_DEPS)
+        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
+    endif()
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
+    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+endfunction(ginkgo_create_dpcpp_test)
+
 function(ginkgo_create_thread_test test_name)
     set(THREADS_PREFER_PTHREAD_FLAG ON)
     find_package(Threads REQUIRED)

cmake/get_info.cmake

@@ -72,6 +72,12 @@ function(ginkgo_print_variable log_type var_name)
     FILE(APPEND ${log_type} "${upd_string}")
 endfunction()
 
+macro(ginkgo_print_foreach_variable variables)
+    foreach(var ${variables})
+        ginkgo_print_variable(${${log_type}} ${var} )
+    endforeach()
+endmacro()
+
 IF("${GINKGO_GIT_SHORTREV}" STREQUAL "")
     set(to_print "Summary of Configuration for Ginkgo (version ${Ginkgo_VERSION} with tag ${Ginkgo_VERSION_TAG})
 --"
@@ -88,12 +94,9 @@ ENDIF()
 set(log_types "detailed_log;minimal_log")
 foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "Ginkgo configuration:")
-    set(print_var
-        "CMAKE_BUILD_TYPE;BUILD_SHARED_LIBS;CMAKE_INSTALL_PREFIX;PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR"
-        )
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
+    ginkgo_print_foreach_variable(
+        "CMAKE_BUILD_TYPE;BUILD_SHARED_LIBS;CMAKE_INSTALL_PREFIX"
+        "PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR")
     string(SUBSTRING
         "
 --        CMAKE_CXX_COMPILER:                                                   " 0 55 print_string)
@@ -108,31 +111,17 @@ foreach(log_type ${log_types})
     FILE(APPEND ${${log_type}} "${print_string}")
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
-    set(print_var
-        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP"
-        )
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
+    ginkgo_print_foreach_variable(
+        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
-    set(print_var
+    ginkgo_print_foreach_variable(
         "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS")
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
     ginkgo_print_module_footer(${${log_type}} "  Documentation:")
-    set(print_var
-        "GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
+    ginkgo_print_foreach_variable("GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
     ginkgo_print_module_footer(${${log_type}} "  Developer helpers:")
-    set(print_var
+    ginkgo_print_foreach_variable(
         "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
         "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH")
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
     ginkgo_print_module_footer(${${log_type}} "")
 endforeach()
 
@@ -160,6 +149,10 @@ IF(GINKGO_BUILD_HIP)
     include(hip/get_info.cmake)
 ENDIF()
 
+IF(GINKGO_BUILD_DPCPP)
+    include(dpcpp/get_info.cmake)
+ENDIF()
+
 ginkgo_print_generic_header(${detailed_log} "Optional Components:")
 ginkgo_print_variable(${detailed_log} "GKO_HAVE_PAPI_SDE")
 if(PAPI_sde_FOUND)

core/CMakeLists.txt

@@ -57,7 +57,7 @@ target_compile_options(ginkgo PRIVATE "${GINKGO_COMPILER_FLAGS}")
 # regardless of whether it is installed or added as a subdirectory
 add_library(Ginkgo::ginkgo ALIAS ginkgo)
 target_link_libraries(ginkgo
-    PUBLIC ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip)
+    PUBLIC ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
 # The PAPI dependency needs to be exposed to the user.
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)

core/base/executor.cpp

@@ -53,6 +53,10 @@ void Operation::run(std::shared_ptr<const HipExecutor> executor) const
     GKO_NOT_IMPLEMENTED;
 
 
+void Operation::run(std::shared_ptr<const DpcppExecutor> executor) const
+    GKO_NOT_IMPLEMENTED;
+
+
 void Operation::run(std::shared_ptr<const ReferenceExecutor> executor) const
 {
     this->run(static_cast<std::shared_ptr<const OmpExecutor>>(executor));

core/components/absolute_array.hpp

@@ -99,6 +99,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 

core/components/fill_array.hpp

@@ -91,6 +91,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 

core/components/precision_conversion.hpp

@@ -93,6 +93,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES