Revert "Changes to enable OneAPI usage with Trilinos build (kokkos#2185…

…)" This reverts commit b87dc95.
ndellingwood · Aug 6, 2024 · a47f8d7 · a47f8d7
1 parent 4cf78e9
commit a47f8d7
Show file tree

Hide file tree

Showing 15 changed files with 45 additions and 28 deletions.
diff --git a/blas/tpls/KokkosBlas1_nrm1_tpl_spec_avail.hpp b/blas/tpls/KokkosBlas1_nrm1_tpl_spec_avail.hpp
@@ -95,7 +95,8 @@ KOKKOSBLAS1_NRM1_TPL_SPEC_AVAIL_ROCBLAS(Kokkos::complex<float>, Kokkos::LayoutLe
 // oneMKL
 #ifdef KOKKOSKERNELS_ENABLE_TPL_MKL
 
-#if defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOS_ENABLE_SYCL) && \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
 
 #define KOKKOSBLAS1_NRM1_TPL_SPEC_AVAIL_MKL_SYCL(SCALAR, LAYOUT, MEMSPACE)                                             \
   template <class ExecSpace>                                                                                           \

diff --git a/blas/tpls/KokkosBlas1_nrm1_tpl_spec_decl.hpp b/blas/tpls/KokkosBlas1_nrm1_tpl_spec_decl.hpp
@@ -263,7 +263,8 @@ KOKKOSBLAS1_NRM1_TPL_SPEC_DECL_ROCBLAS(Kokkos::complex<double>, Kokkos::LayoutLe
 // oneMKL
 #ifdef KOKKOSKERNELS_ENABLE_TPL_MKL
 
-#if defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOS_ENABLE_SYCL) && \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
 
 #include <KokkosBlas_tpl_spec.hpp>
 #include <oneapi/mkl/blas.hpp>

diff --git a/blas/tpls/KokkosBlas1_nrm2_tpl_spec_avail.hpp b/blas/tpls/KokkosBlas1_nrm2_tpl_spec_avail.hpp
@@ -75,7 +75,9 @@ KOKKOSBLAS1_NRM2_TPL_SPEC_AVAIL(Kokkos::LayoutLeft, Kokkos::Cuda, Kokkos::CudaSp
 KOKKOSBLAS1_NRM2_TPL_SPEC_AVAIL(Kokkos::LayoutLeft, Kokkos::HIP, Kokkos::HIPSpace)
 #endif
 
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) &&                \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE) && \
+    defined(KOKKOS_ENABLE_SYCL)
 KOKKOSBLAS1_NRM2_TPL_SPEC_AVAIL(Kokkos::LayoutLeft, Kokkos::Experimental::SYCL,
                                 Kokkos::Experimental::SYCLDeviceUSMSpace)
 #endif

diff --git a/blas/tpls/KokkosBlas1_nrm2_tpl_spec_decl.hpp b/blas/tpls/KokkosBlas1_nrm2_tpl_spec_decl.hpp
@@ -283,7 +283,9 @@ KOKKOSBLAS1_NRM2_TPL_SPEC_DECL_ROCBLAS_EXT(false)
 
 #endif
 
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) &&                \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE) && \
+    defined(KOKKOS_ENABLE_SYCL)
 #include <mkl.h>
 #include <oneapi/mkl/blas.hpp>
 #include <KokkosBlas_tpl_spec.hpp>

diff --git a/blas/tpls/KokkosBlas2_gemv_tpl_spec_avail.hpp b/blas/tpls/KokkosBlas2_gemv_tpl_spec_avail.hpp
@@ -129,7 +129,8 @@ KOKKOSBLAS2_GEMV_TPL_SPEC_AVAIL_ROCBLAS(Kokkos::complex<float>, Kokkos::LayoutRi
 
 #ifdef KOKKOSKERNELS_ENABLE_TPL_MKL
 
-#if defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOS_ENABLE_SYCL) && \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
 
 #define KOKKOSBLAS2_GEMV_TPL_SPEC_AVAIL_ONEMKL(SCALAR, LAYOUT)                                           \
   template <class ExecSpace>                                                                             \

diff --git a/blas/tpls/KokkosBlas2_gemv_tpl_spec_decl.hpp b/blas/tpls/KokkosBlas2_gemv_tpl_spec_decl.hpp
@@ -594,7 +594,9 @@ KOKKOSBLAS2_CGEMV_ROCBLAS(Kokkos::LayoutRight, Kokkos::HIPSpace, false)
 #endif  // KOKKOSKERNELS_ENABLE_TPL_ROCBLAS
 
 // ONEMKL
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) &&                \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE) && \
+    defined(KOKKOS_ENABLE_SYCL)
 #include <mkl.h>
 #include <oneapi/mkl/blas.hpp>
 #include <KokkosBlas_tpl_spec.hpp>

diff --git a/blas/tpls/KokkosBlas_Host_tpl.hpp b/blas/tpls/KokkosBlas_Host_tpl.hpp
@@ -32,7 +32,7 @@
 namespace KokkosBlas {
 namespace Impl {
 
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) && defined(MKL_PROVIDES_BLAS_LAPACK)
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL)
 using KK_INT = MKL_INT;
 #else
 using KK_INT = int;

diff --git a/cmake/KokkosKernels_config.h.in b/cmake/KokkosKernels_config.h.in
@@ -29,6 +29,7 @@
    requires (a) header file(s) as well, and may use functions other
    than just BLAS and LAPACK functions.  */
 #cmakedefine HAVE_KOKKOSKERNELS_MKL
+#cmakedefine KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE
 
 #cmakedefine KOKKOSKERNELS_ENABLE_TESTS_AND_PERFSUITE
 #cmakedefine KOKKOSKERNELS_ENABLE_BENCHMARK
@@ -153,9 +154,6 @@
 #endif
 #endif
 
-/* Whether MKL is providing the BLAS and LAPACK implementation */
-#cmakedefine MKL_PROVIDES_BLAS_LAPACK
-
 #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && \
     !defined(KOKKOS_ENABLE_SYCL) && !defined(KOKKOS_ENABLE_OPENMPTARGET)
 #define KOKKOSKERNELS_ENABLE_HOST_ONLY

diff --git a/cmake/Modules/FindTPLBLAS.cmake b/cmake/Modules/FindTPLBLAS.cmake
@@ -8,3 +8,4 @@ ELSE()
   FIND_PACKAGE(BLAS REQUIRED)
   KOKKOSKERNELS_CREATE_IMPORTED_TPL(BLAS INTERFACE LINK_LIBRARIES ${BLAS_LIBRARIES})
 ENDIF()
+
diff --git a/cmake/Modules/FindTPLLAPACK.cmake b/cmake/Modules/FindTPLLAPACK.cmake
@@ -8,3 +8,4 @@ ELSE()
   FIND_PACKAGE(LAPACK REQUIRED)
   KOKKOSKERNELS_CREATE_IMPORTED_TPL(LAPACK INTERFACE LINK_LIBRARIES ${LAPACK_LIBRARIES})
 ENDIF()
+
diff --git a/cmake/Modules/FindTPLMKL.cmake b/cmake/Modules/FindTPLMKL.cmake
@@ -74,7 +74,3 @@ ELSE()
     )
     ENDIF()
 ENDIF()
-# This logic to find MKL is only used in non-Trilinos builds.
-# In this case, MKL can always be used as the host BLAS/LAPACK implementation
-# (whether MKL_INT is 32- or 64-bit).
-set (MKL_PROVIDES_BLAS_LAPACK ON INTERNAL)
diff --git a/cmake/kokkoskernels_tpls.cmake b/cmake/kokkoskernels_tpls.cmake
@@ -31,6 +31,10 @@ MACRO(KOKKOSKERNELS_ADD_TPL_OPTION NAME DEFAULT_VALUE DOCSTRING)
   SET(ROOT_DEFAULT $ENV{${_NAME_ORIG}_ROOT})
   KOKKOSKERNELS_ADD_OPTION(${_NAME_ORIG}_ROOT "${ROOT_DEFAULT}" PATH "Location of ${_NAME} install root. Default: None or the value of the environment variable ${_NAME}_ROOT if set")
   IF (DEFINED TPL_ENABLE_${_NAME})
+    IF (${_NAME} STREQUAL MKL AND KOKKOSKERNELS_HAS_TRILINOS)
+      MESSAGE("Trilinos has enabled MKL and SYCL but it does not detect oneMKL correctly so we disable it!")
+      SET(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE ON)
+    ENDIF ()
     IF (TPL_ENABLE_${_NAME} AND NOT KOKKOSKERNELS_ENABLE_TPL_${_NAME})
       MESSAGE("Overriding KOKKOSKERNELS_ENABLE_TPL_${_NAME_ORIG}=OFF with TPL_ENABLE_${_NAME}=ON")
       SET(KOKKOSKERNELS_ENABLE_TPL_${_NAME_ORIG} ON)

diff --git a/sparse/src/KokkosSparse_spmv_handle.hpp b/sparse/src/KokkosSparse_spmv_handle.hpp
@@ -190,7 +190,8 @@ struct MKL_SpMV_Data : public TPL_SpMV_Data<ExecutionSpace> {
 };
 #endif
 
-#if defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOS_ENABLE_SYCL) && \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
 struct OneMKL_SpMV_Data : public TPL_SpMV_Data<Kokkos::Experimental::SYCL> {
   OneMKL_SpMV_Data(const Kokkos::Experimental::SYCL& exec_) : TPL_SpMV_Data(exec_) {}
   ~OneMKL_SpMV_Data() {

diff --git a/sparse/tpls/KokkosSparse_spmv_tpl_spec_avail.hpp b/sparse/tpls/KokkosSparse_spmv_tpl_spec_avail.hpp
@@ -180,18 +180,24 @@ KOKKOSSPARSE_SPMV_TPL_SPEC_AVAIL_MKL(Kokkos::complex<double>, Kokkos::OpenMP)
 #endif
 
 #if defined(KOKKOS_ENABLE_SYCL)
-#define KOKKOSSPARSE_SPMV_TPL_SPEC_AVAIL_ONEMKL(SCALAR, ORDINAL, MEMSPACE)                                       \
-  template <>                                                                                                    \
-  struct spmv_tpl_spec_avail<                                                                                    \
-      Kokkos::Experimental::SYCL,                                                                                \
-      KokkosSparse::Impl::SPMVHandleImpl<Kokkos::Experimental::SYCL, MEMSPACE, SCALAR, ORDINAL, ORDINAL>,        \
-      KokkosSparse::CrsMatrix<const SCALAR, const ORDINAL, Kokkos::Device<Kokkos::Experimental::SYCL, MEMSPACE>, \
-                              Kokkos::MemoryTraits<Kokkos::Unmanaged>, const ORDINAL>,                           \
-      Kokkos::View<const SCALAR*, Kokkos::LayoutLeft, Kokkos::Device<Kokkos::Experimental::SYCL, MEMSPACE>,      \
-                   Kokkos::MemoryTraits<Kokkos::Unmanaged | Kokkos::RandomAccess>>,                              \
-      Kokkos::View<SCALAR*, Kokkos::LayoutLeft, Kokkos::Device<Kokkos::Experimental::SYCL, MEMSPACE>,            \
-                   Kokkos::MemoryTraits<Kokkos::Unmanaged>>> {                                                   \
-    enum : bool { value = true };                                                                                \
+#define KOKKOSSPARSE_SPMV_TPL_SPEC_AVAIL_ONEMKL(SCALAR, ORDINAL, MEMSPACE)     \
+  template <>                                                                  \
+  struct spmv_tpl_spec_avail<                                                  \
+      Kokkos::Experimental::SYCL,                                              \
+      KokkosSparse::Impl::SPMVHandleImpl<Kokkos::Experimental::SYCL, MEMSPACE, \
+                                         SCALAR, ORDINAL, ORDINAL>,            \
+      KokkosSparse::CrsMatrix<                                                 \
+          const SCALAR, const ORDINAL,                                         \
+          Kokkos::Device<Kokkos::Experimental::SYCL, MEMSPACE>,                \
+          Kokkos::MemoryTraits<Kokkos::Unmanaged>, const ORDINAL>,             \
+      Kokkos::View<                                                            \
+          const SCALAR*, Kokkos::LayoutLeft,                                   \
+          Kokkos::Device<Kokkos::Experimental::SYCL, MEMSPACE>,                \
+          Kokkos::MemoryTraits<Kokkos::Unmanaged | Kokkos::RandomAccess>>,     \
+      Kokkos::View<SCALAR*, Kokkos::LayoutLeft,                                \
+                   Kokkos::Device<Kokkos::Experimental::SYCL, MEMSPACE>,       \
+                   Kokkos::MemoryTraits<Kokkos::Unmanaged>>> {                 \
+    enum : bool { value = true };                                              \
   };
 
 // intel-oneapi-mkl/2023.2.0: spmv with complex data types produce:

diff --git a/sparse/tpls/KokkosSparse_spmv_tpl_spec_decl.hpp b/sparse/tpls/KokkosSparse_spmv_tpl_spec_decl.hpp
@@ -548,7 +548,8 @@ KOKKOSSPARSE_SPMV_MKL(Kokkos::complex<double>, Kokkos::OpenMP)
 #undef KOKKOSSPARSE_SPMV_MKL
 #endif
 
-#if defined(KOKKOS_ENABLE_SYCL)
+#if defined(KOKKOS_ENABLE_SYCL) && \
+    !defined(KOKKOSKERNELS_ENABLE_TPL_MKL_SYCL_OVERRIDE)
 inline oneapi::mkl::transpose mode_kk_to_onemkl(char mode_kk) {
   switch (toupper(mode_kk)) {
     case 'N': return oneapi::mkl::transpose::nontrans;
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,3 +8,4 @@ ELSE()
		FIND_PACKAGE(BLAS REQUIRED)
		KOKKOSKERNELS_CREATE_IMPORTED_TPL(BLAS INTERFACE LINK_LIBRARIES ${BLAS_LIBRARIES})
		ENDIF()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,3 +8,4 @@ ELSE()
		FIND_PACKAGE(LAPACK REQUIRED)
		KOKKOSKERNELS_CREATE_IMPORTED_TPL(LAPACK INTERFACE LINK_LIBRARIES ${LAPACK_LIBRARIES})
		ENDIF()