onnxruntime: add with_cuda option

recipes/onnxruntime/all/conandata.yml

@@ -22,6 +22,9 @@
     - patch_file: "patches/1.18.0-0004-abseil-no-string-view.patch"
       patch_description: "allow to build with abseil built without c++17 support"
       patch_type: "portability"
+    - patch_file: "patches/1.18.1-0005-fix-cutlass-cuda-provider.patch"
+      patch_description: "use cutlass from Conan"
+      patch_type: "portability"
   "1.17.3":
     - patch_file: "patches/1.17.3-0001-patch-macos-cpp20-date-compat.patch"
       patch_description: "allow to build with macos c++20 support"
@@ -33,6 +36,9 @@
     - patch_file: "patches/1.14.1-0004-abseil-no-string-view.patch"
       patch_description: "allow to build with abseil built without c++17 support"
       patch_type: "portability"
+    - patch_file: "patches/1.17.3-0003-fix-cutlass-cuda-provider.patch"
+      patch_description: "use cutlass from Conan"
+      patch_type: "portability"
   "1.16.3":
     - patch_file: "patches/1.14.1-0004-abseil-no-string-view.patch"
       patch_description: "allow to build with abseil built without c++17 support"
@@ -41,8 +47,12 @@
     - patch_file: "patches/1.14.1-0004-abseil-no-string-view.patch"
       patch_description: "allow to build with abseil built without c++17 support"
       patch_type: "portability"
+    - patch_file: "patches/1.15.1-0002-fix-attention-15983.patch"
+      patch_description: "Fix attention.cc"
+      patch_source: "https://github.com/microsoft/onnxruntime/pull/15983"
+      patch_type: "backport"
   "1.14.1":
     - patch_file: "patches/1.14.1-0001-cmake-dependencies.patch"
       patch_description: "CMake: ensure conan dependencies are used (upstreamed future versions)"
       patch_type: "conan"
     - patch_file: "patches/1.14.1-0003-amx-gas-version.patch"
@@ -56,6 +66,7 @@
       patch_description: "Ensures the forward compatibility with the newest versions of re2 library."
       patch_type: "portability"
       patch_source: "https://github.com/microsoft/onnxruntime/commit/126e7bf15fa4af8621814b82a3f7bd0d786f0239.patch"
+
 # ONNX versions are based on the minor version used at
 # https://github.com/microsoft/onnxruntime/tree/main/cmake/external
 onnx_version_map:

recipes/onnxruntime/all/conanfile.py

@@ -27,11 +27,13 @@ class OnnxRuntimeConan(ConanFile):
         "shared": [True, False],
         "fPIC": [True, False],
         "with_xnnpack": [True, False],
+        "with_cuda": [True, False],
     }
     default_options = {
         "shared": False,
         "fPIC": True,
         "with_xnnpack": False,
+        "with_cuda": False,
     }
     short_paths = True
 
@@ -70,6 +72,9 @@ def config_options(self):
     def configure(self):
         if self.options.shared:
             self.options.rm_safe("fPIC")
+        # onnxruntime forces this to be True
+        # https://github.com/microsoft/onnxruntime/blob/be76e1e1b8e2914e448d12a0cc683c00014c0490/cmake/external/onnxruntime_external_deps.cmake#L542
+        self.options["onnx"].disable_static_registration = True
 
     def layout(self):
         cmake_layout(self, src_folder="src")
@@ -105,6 +110,8 @@ def requirements(self):
                 self.requires("xnnpack/cci.20230715")
             else:
                 self.requires("xnnpack/cci.20220801")
+        if self.options.with_cuda:
+            self.requires("cutlass/3.5.0")
 
     def validate(self):
         if self.settings.compiler.get_safe("cppstd"):
@@ -114,6 +121,10 @@ def validate(self):
             raise ConanInvalidConfiguration(
                 f"{self.ref} requires minimum compiler version {minimum_version}."
             )
+        if not self.dependencies["onnx"].options.disable_static_registration:
+            raise ConanInvalidConfiguration(
+                f"{self.ref} requires onnx compiled with `-o onnx:disable_static_registration=True`."
+            )
 
     def validate_build(self):
         if self.version >= Version("1.15.0") and self.options.shared and sys.version_info[:2] < (3, 8):
@@ -143,8 +154,10 @@ def generate(self):
         tc.variables["onnxruntime_USE_FULL_PROTOBUF"] = not self.dependencies["protobuf"].options.lite
         tc.variables["onnxruntime_USE_XNNPACK"] = self.options.with_xnnpack
 
+        tc.variables["onnxruntime_USE_CUDA"] = self.options.with_cuda
         tc.variables["onnxruntime_BUILD_UNIT_TESTS"] = False
         tc.variables["onnxruntime_DISABLE_CONTRIB_OPS"] = False
+        tc.variables["onnxruntime_USE_FLASH_ATTENTION"] = False
         tc.variables["onnxruntime_DISABLE_RTTI"] = False
         tc.variables["onnxruntime_DISABLE_EXCEPTIONS"] = False
 
@@ -157,7 +170,7 @@ def generate(self):
         if Version(self.version) >= "1.17":
             tc.variables["onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS"] = False
             tc.variables["onnxruntime_USE_NEURAL_SPEED"] = False
-            tc.variables["onnxruntime_USE_MEMORY_EFFICIENT_ATTENTION"] = True
+            tc.variables["onnxruntime_USE_MEMORY_EFFICIENT_ATTENTION"] = False
 
         # Disable a warning that gets converted to an error
         tc.preprocessor_definitions["_SILENCE_ALL_CXX23_DEPRECATION_WARNINGS"] = "1"
@@ -180,6 +193,11 @@ def _patch_sources(self):
         if Version(self.version) >= "15.0":
             replace_in_file(self, os.path.join(self.source_folder, "cmake", "CMakeLists.txt"),
                             "if (Git_FOUND)", "if (FALSE)")
+        if Version(self.version) >= "1.17":
+            # https://github.com/microsoft/onnxruntime/commit/5bfca1dc576720627f3af8f65e25af408271079b
+            replace_in_file(self, os.path.join(self.source_folder, "cmake", "onnxruntime_providers_cuda.cmake"),
+                            'option(onnxruntime_NVCC_THREADS "Number of threads that NVCC can use for compilation." 1)', 
+                            'set(onnxruntime_NVCC_THREADS "1" CACHE STRING "Number of threads that NVCC can use for compilation.")')
 
     def build(self):
         self._patch_sources()
@@ -252,6 +270,8 @@ def package_info(self):
             self.cpp_info.requires.append("wil::wil")
         if self.options.with_xnnpack:
             self.cpp_info.requires.append("xnnpack::xnnpack")
+        if self.options.with_cuda:
+            self.cpp_info.requires.append("cutlass::cutlass")
 
         # https://github.com/microsoft/onnxruntime/blob/v1.16.0/cmake/CMakeLists.txt#L1759-L1763
         self.cpp_info.set_property("cmake_file_name", "onnxruntime")

recipes/onnxruntime/all/patches/1.15.1-0002-fix-attention-15983.patch

@@ -0,0 +1,11 @@
+--- a/onnxruntime/contrib_ops/cuda/bert/attention.cc
++++ b/onnxruntime/contrib_ops/cuda/bert/attention.cc
+@@ -164,7 +164,7 @@
+                                         has_memory_efficient_attention(sm, sizeof(T) == 2);
+ #else
+   constexpr bool use_memory_efficient_attention = false;
+-  ORT_UNUSED_VARIABLE(is_mask_1d_key_seq_len_start);
++  ORT_UNUSED_PARAMETER(is_mask_1d_key_seq_len_start);
+ #endif
+
+   cublasHandle_t cublas = GetCublasHandle(context);

recipes/onnxruntime/all/patches/1.17.3-0003-fix-cutlass-cuda-provider.patch

@@ -0,0 +1,15 @@
+diff --git a/cmake/onnxruntime_providers_cuda.cmake b/cmake/onnxruntime_providers_cuda.cmake
+index b5c3100865..129c4f19f6 100644
+--- a/cmake/onnxruntime_providers_cuda.cmake
++++ b/cmake/onnxruntime_providers_cuda.cmake
+@@ -180,8 +180,8 @@
+       target_link_libraries(${target} PRIVATE CUDA::cuda_driver)
+     endif()
+
+-    include(cutlass)
+-    target_include_directories(${target} PRIVATE ${cutlass_SOURCE_DIR}/include ${cutlass_SOURCE_DIR}/examples)
++    find_package(NvidiaCutlass)
++    target_link_libraries(${target} PRIVATE nvidia::cutlass::cutlass)
+
+     target_include_directories(${target} PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR}  ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES}
+      PUBLIC ${CUDAToolkit_INCLUDE_DIRS})

recipes/onnxruntime/all/patches/1.18.1-0005-fix-cutlass-cuda-provider.patch

@@ -0,0 +1,15 @@
+diff --git a/cmake/onnxruntime_providers_cuda.cmake b/cmake/onnxruntime_providers_cuda.cmake
+index 1346a9ce96..f40661d4bd 100644
+--- a/cmake/onnxruntime_providers_cuda.cmake
++++ b/cmake/onnxruntime_providers_cuda.cmake
+@@ -211,8 +211,8 @@
+       target_link_libraries(${target} PRIVATE CUDA::cuda_driver)
+     endif()
+
+-    include(cutlass)
+-    target_include_directories(${target} PRIVATE ${cutlass_SOURCE_DIR}/include ${cutlass_SOURCE_DIR}/examples ${cutlass_SOURCE_DIR}/tools/util/include)
++    find_package(NvidiaCutlass)
++    target_link_libraries(${target} PRIVATE nvidia::cutlass::cutlass)
+
+     target_include_directories(${target} PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR}  ${eigen_INCLUDE_DIRS} ${TVM_INCLUDES}
+      PUBLIC ${CUDAToolkit_INCLUDE_DIRS})

recipes/onnxruntime/all/test_package/CMakeLists.txt

@@ -5,5 +5,8 @@ project(test_package CXX)
 find_package(onnxruntime REQUIRED CONFIG)
 
 add_executable(${PROJECT_NAME} test_package.cpp)
+if(WITH_CUDA)
+  target_compile_definitions(${PROJECT_NAME} PRIVATE WITH_CUDA)
+endif()
 target_link_libraries(${PROJECT_NAME} PRIVATE onnxruntime::onnxruntime)
 target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17)

recipes/onnxruntime/all/test_package/conanfile.py

@@ -1,20 +1,30 @@
 from conan import ConanFile
 from conan.tools.build import can_run
-from conan.tools.cmake import cmake_layout, CMake
+from conan.tools.cmake import cmake_layout, CMake, CMakeToolchain
+from conan.tools.files import copy
 import os
 
 
 # It will become the standard on Conan 2.x
 class TestPackageConan(ConanFile):
     settings = "os", "arch", "compiler", "build_type"
-    generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv"
+    generators = "CMakeDeps", "VirtualRunEnv"
     test_type = "explicit"
 
     def requirements(self):
         self.requires(self.tested_reference_str)
 
     def layout(self):
         cmake_layout(self, src_folder=".")
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+        tc.cache_variables["WITH_CUDA"] = self.dependencies["onnxruntime"].options.with_cuda
+        tc.generate()
+        if self.settings.os == "Windows":
+            # on windows the system dll C:\WINDOWS\system32\onnxruntime.dll may be loaded instead even if the conan lib is first in the PATH, see https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order
+            for bindir in self.dependencies[self.tested_reference_str].cpp_info.bindirs:
+                copy(self, "*.dll", bindir, os.path.join(self.build_folder, str(self.settings.build_type)))
 
     def build(self):
         cmake = CMake(self)
@@ -23,5 +33,5 @@ def build(self):
 
     def test(self):
         if can_run(self):
-            bin_path = os.path.join(self.cpp.build.bindirs[0], "test_package")
+            bin_path = os.path.join(self.cpp.build.bindir, "test_package")
             self.run(bin_path, env="conanrun")

recipes/onnxruntime/all/test_package/test_package.cpp

@@ -2,8 +2,22 @@
 #include <onnxruntime_cxx_api.h>
 #include <iostream>
 
+#ifdef WITH_CUDA
+#include <onnxruntime_c_api.h>
+#endif
+
 int main() {
   const auto& api = Ort::GetApi();
-  std::cout << OrtGetApiBase()->GetVersionString() << std::endl;
+  std::cout << "Version: " << OrtGetApiBase()->GetVersionString() << std::endl;
+  std::cout << "Providers: " << std::endl;
+  for(const auto& provider: Ort::GetAvailableProviders())
+    std::cout << provider << ", " << std::endl;
+
+#ifdef WITH_CUDA
+  Ort::SessionOptions session_options;
+  OrtSessionOptionsAppendExecutionProvider_CUDA(session_options, 1);
+  std::cout << "with cuda!" << std::endl;
+#endif
+
   return 0;
 }

recipes/onnxruntime/all/test_v1_package/CMakeLists.txt

@@ -4,5 +4,19 @@ project(test_package)
 include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
 conan_basic_setup(TARGETS)
 
-add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../test_package
-                 ${CMAKE_CURRENT_BINARY_DIR}/test_package)
+find_package(onnxruntime REQUIRED CONFIG)
+add_executable(${PROJECT_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/../test_package/test_package.cpp)
+if(WITH_CUDA)
+    target_compile_definitions(${PROJECT_NAME} PRIVATE WITH_CUDA)
+endif()
+
+target_link_libraries(${PROJECT_NAME} PRIVATE onnxruntime::onnxruntime)
+target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17)
+
+if(WIN32 AND DEFINED CONAN_BIN_DIRS_ONNXRUNTIME AND EXISTS ${CONAN_BIN_DIRS_ONNXRUNTIME})
+    add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_directory
+        ${CONAN_BIN_DIRS_ONNXRUNTIME}
+        $<TARGET_FILE_DIR:${PROJECT_NAME}>
+    )
+endif()