From badc0625a9c6096dfa2d9d47d8fe55be3df53bbe Mon Sep 17 00:00:00 2001 From: Martin Valgur Date: Mon, 2 Sep 2024 17:42:11 +0300 Subject: [PATCH] (#18979) libfabric: migrate to Conan v2, add missing dependencies * libfabric: migrate to Conan v2 * libfabric: explicitly disable providers that cannot be built by default * libfabric: add v1.18.1 * libfabric: opx also requires rdma-core * libfabric: add libatomic system dependency * libfabric: macOS does not support `shm` * libfabric: macOS does not support `sm2` * libfabric: fix libatomic system dep * libfabric: do not ignore libcxx for libatomic * libfabric: workaround for linter error * libfabric: fix typo * libfabric: add libatomic for both gcc and clang * libfabric: fix_apple_shared_install_name() * libfabric: bump to v1.21.0, drop v1.12.1, remove 'auto' options * libfabric: disable 'verbs' * libfabric: fix invalid default configurations on macOS * libfabric: fix a linter error * libfabric: fix a missing option error * libfabric: fix a linter error * libfabric: fix a minor recipe bug * libfabric: add VirtualRunEnv for Conan v1 * libfabric: use get_safe() Co-authored-by: PerseoGI * Improve Windows invalid config message Signed-off-by: Uilian Ries * Use None ANY for providers Signed-off-by: Uilian Ries * Remove psm option Signed-off-by: Uilian Ries * Remove xpmem and cxi Signed-off-by: Uilian Ries * Improve verbs invalid configuration message Signed-off-by: Uilian Ries * Remove opx Signed-off-by: Uilian Ries * Disable bgq Signed-off-by: Uilian Ries * Enable verbs and disable others Signed-off-by: Uilian Ries --------- Signed-off-by: Uilian Ries Co-authored-by: PerseoGI Co-authored-by: Uilian Ries --- recipes/libfabric/all/conandata.yml | 6 +- recipes/libfabric/all/conanfile.py | 232 +++++++++++++----- .../libfabric/all/test_package/CMakeLists.txt | 9 +- .../libfabric/all/test_package/conanfile.py | 21 +- .../all/test_v1_package/CMakeLists.txt | 8 + .../all/test_v1_package/conanfile.py | 17 ++ recipes/libfabric/config.yml | 2 +- 7 files changed, 212 insertions(+), 83 deletions(-) create mode 100644 recipes/libfabric/all/test_v1_package/CMakeLists.txt create mode 100644 recipes/libfabric/all/test_v1_package/conanfile.py diff --git a/recipes/libfabric/all/conandata.yml b/recipes/libfabric/all/conandata.yml index 1eedb4da16c00..1488ae656448e 100644 --- a/recipes/libfabric/all/conandata.yml +++ b/recipes/libfabric/all/conandata.yml @@ -1,4 +1,4 @@ sources: - "1.12.1": - url: https://github.com/ofiwg/libfabric/releases/download/v1.12.1/libfabric-1.12.1.tar.bz2 - sha256: db3c8e0a495e6e9da6a7436adab905468aedfbd4579ee3da5232a5c111ba642c + "1.21.0": + url: "https://github.com/ofiwg/libfabric/releases/download/v1.21.0/libfabric-1.21.0.tar.bz2" + sha256: "0c1b7b830d9147f661e5d7f359250b85b5a9885c330464cd3b5e5d35b86551c7" diff --git a/recipes/libfabric/all/conanfile.py b/recipes/libfabric/all/conanfile.py index 96dbea392e3a6..9d5b8417f1cbd 100644 --- a/recipes/libfabric/all/conanfile.py +++ b/recipes/libfabric/all/conanfile.py @@ -1,99 +1,195 @@ -from conans import ConanFile, AutoToolsBuildEnvironment, tools -from conans.errors import ConanInvalidConfiguration import os -required_conan_version = ">=1.35.0" +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.apple import is_apple_os, fix_apple_shared_install_name +from conan.tools.env import VirtualBuildEnv, VirtualRunEnv +from conan.tools.files import copy, get, rm, rmdir +from conan.tools.gnu import Autotools, AutotoolsToolchain, AutotoolsDeps +from conan.tools.layout import basic_layout + +required_conan_version = ">=1.53.0" + class LibfabricConan(ConanFile): name = "libfabric" - description = "Open Fabric Interfaces" - topics = ("fabric", "communication", "framework", "service") + description = ("Libfabric, also known as Open Fabrics Interfaces (OFI), " + "defines a communication API for high-performance parallel and distributed applications.") + license = ("BSD-2-Clause", "GPL-2.0-or-later") url = "https://github.com/conan-io/conan-center-index" homepage = "http://libfabric.org" - license = "BSD-2-Clause", "GPL-2.0-or-later" + topics = ("fabric", "communication", "framework", "service") + + package_type = "library" settings = "os", "arch", "compiler", "build_type" - _providers = ['gni', 'psm', 'psm2', 'psm3', 'rxm', 'sockets', 'tcp', 'udp', 'usnic', 'verbs', 'bgq'] + _providers = [ + "dmabuf_peer_mem", + "efa", + "hook_debug", + "hook_hmem", + "mrail", + "perf", + "profile", + "rxd", + "rxm", + "shm", + "sm2", + "sockets", + "tcp", + "trace", + "ucx", + "udp", + "usnic", + "verbs", + ] options = { - **{ p: "ANY" for p in _providers }, - **{ - "shared": [True, False], - "fPIC": [True, False], - "with_libnl": "ANY", - "with_bgq_progress": [None, "auto", "manual"], - "with_bgq_mr": [None, "basic", "scalable"] - } + "shared": [True, False], + "fPIC": [True, False], + **{ p: [None, "ANY"] for p in _providers }, } default_options = { - **{ p: "auto" for p in _providers }, - **{ - "shared": False, - "fPIC": True, - "with_libnl": None, - "with_bgq_progress": None, - "with_bgq_mr": None - } + "shared": False, + "fPIC": True, + "dmabuf_peer_mem": "yes", + "efa": "yes", + "hook_debug": "yes", + "hook_hmem": "yes", + "mrail": "yes", + "perf": "no", + "profile": "yes", + "rxd": "yes", + "rxm": "yes", + "shm": "yes", + "sm2": "yes", + "sockets": "yes", + "tcp": "yes", + "trace": "yes", + "ucx": "no", + "udp": "yes", + "usnic": "no", + "verbs": "yes" } - @property - def _source_subfolder(self): - return "source_subfolder" - - _autotools = None - def config_options(self): - if self.settings.os == 'Windows': - del self.options.fPIC + if is_apple_os(self): + # Requires libnl, which is not available on macOS + del self.options.usnic + # Require Linux-specific process_vm_readv syscall + del self.options.shm + del self.options.sm2 + # rdma-core is not available on macOS + del self.options.efa + del self.options.verbs def configure(self): if self.options.shared: - del self.options.fPIC - del self.settings.compiler.libcxx - del self.settings.compiler.cppstd + self.options.rm_safe("fPIC") + self.settings.rm_safe("compiler.libcxx") + self.settings.rm_safe("compiler.cppstd") + + def _is_enabled(self, opt): + return str(self.options.get_safe(opt)) == "yes" or str(self.options.get_safe(opt)).startswith("dl") + + def requirements(self): + if self._is_enabled("usnic"): + self.requires("libnl/3.8.0") + if self._is_enabled("efa") or self._is_enabled("usnic") or self._is_enabled("verbs"): + self.requires("rdma-core/52.0") + + def layout(self): + basic_layout(self, src_folder="src") def validate(self): - if self.settings.os not in ["Linux", "FreeBSD", "Macos"]: - raise ConanInvalidConfiguration("libfabric only builds on Linux, Macos, and FreeBSD.") - for p in self._providers: - if self.options.get_safe(p) not in ["auto", "yes", "no", "dl"] and not os.path.isdir(str(self.options.get_safe(p))): - raise ConanInvalidConfiguration("Option {} can only be one of 'auto', 'yes', 'no', 'dl' or a directory path") - if self.options.get_safe('with_libnl') and not os.path.isdir(str(self.options.with_libnl)): - raise ConanInvalidConfiguration("Value of with_libnl must be an existing directory") + if self.settings.os == "Windows": + # FIXME: libfabric provides msbuild project files. + raise ConanInvalidConfiguration(f"{self.ref} Conan recipes is not supported on Windows. Contributions are welcome.") + + for provider in self._providers: + provider = str(self.options.get_safe(provider)) + if provider.lower() not in ["yes", "no", "dl", "none"] and \ + not os.path.isdir(provider) and \ + (not provider.startswith("dl:") and not os.path.isdir(provider[3:])): + raise ConanInvalidConfiguration(f"{self.ref} provider option '{provider}' is not valid. It must be 'yes', 'no', 'dl', 'dl:' or a directory path.") + + if self._is_enabled("verbs"): + if not self.dependencies["rdma-core"].options.build_librdmacm: + raise ConanInvalidConfiguration(f"{self.ref} '-o rdma-core/*:build_librdmacm=True' is required when '-o &:verbs=True'") + + def build_requirements(self): + # Used in ./configure tests and build + self.tool_requires("libtool/2.4.7") def source(self): - tools.get(**self.conan_data["sources"][self.version], - destination=self._source_subfolder, strip_root=True) - - def _configure_autotools(self): - if self._autotools: - return self._autotools - self._autotools = AutoToolsBuildEnvironment(self) - args = [] + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = AutotoolsToolchain(self) for p in self._providers: - args.append('--enable-{}={}'.format(p, self.options.get_safe(p))) - if self.options.with_libnl: - args.append('--with-libnl={}'.format(self.options.with_libnl)) - if self.options.with_bgq_progress: - args.append('--with-bgq-progress={}'.format(self.options.with_bgq_progress)) - if self.options.with_bgq_mr: - args.append('--with-bgq-mr={}'.format(self.options.with_bgq_mr)) - self._autotools.configure(args=args, configure_dir=self._source_subfolder) - return self._autotools + if p == "verbs" and self.options.get_safe(p, "no") != "no": + path = self.dependencies["rdma-core"].package_folder + if self.options.get_safe("verbs") == "dl": + tc.configure_args.append(f"--enable-verbs=dl:{path}") + else: + tc.configure_args.append(f"--enable-verbs={path}") + else: + tc.configure_args.append(f"--enable-{p}={self.options.get_safe(p, 'no')}") + if self.settings.build_type == "Debug": + tc.configure_args.append("--enable-debug") + tc.configure_args.append(f"--with-bgq-progress=no") + tc.configure_args.append(f"--with-bgq-mr=no") + tc.configure_args.append("--with-cassin-headers=no") + tc.configure_args.append("--with-cuda=no") # TODO + tc.configure_args.append("--with-curl=no") # TODO + tc.configure_args.append("--with-cxi-uapi-headers=no") + tc.configure_args.append("--with-dsa=no") + tc.configure_args.append("--with-gdrcopy=no") + tc.configure_args.append("--with-json-c=no") # TODO + if self._is_enabled("usnic"): + tc.configure_args.append(f"--with-libnl={self.dependencies['libnl'].package_folder}") + else: + tc.configure_args.append("--with-libnl=no") + tc.configure_args.append("--with-lttng=no") + tc.configure_args.append("--with-neuron=no") + tc.configure_args.append(f"--with-numa=no") + tc.configure_args.append("--with-psm2-src=no") + tc.configure_args.append("--with-psm3-rv=no") + tc.configure_args.append("--with-rocr=no") + tc.configure_args.append("--with-synapseai=no") + tc.configure_args.append("--with-uring=no") # TODO + tc.configure_args.append("--with-ze=no") + tc.configure_args.append("-enable-psm=no") + tc.configure_args.append("--enable-psm2=no") + tc.configure_args.append("--enable-psm3=no") + tc.configure_args.append("--enable-xpmem=no") + tc.configure_args.append("--enable-cxi=no") + tc.configure_args.append("--enable-opx=no") + tc.configure_args.append("--enable-bgq=no") + tc.generate() + + deps = AutotoolsDeps(self) + deps.generate() + + VirtualBuildEnv(self).generate() + VirtualRunEnv(self).generate(scope="build") def build(self): - autotools = self._configure_autotools() + autotools = Autotools(self) + autotools.configure() autotools.make() def package(self): - self.copy(pattern="COPYING", dst="licenses", src=self._source_subfolder) - autotools = self._configure_autotools() + copy(self, "COPYING", dst=os.path.join(self.package_folder, "licenses"), src=self.source_folder) + autotools = Autotools(self) autotools.install() - - tools.rmdir(os.path.join(self.package_folder, "share")) - tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig")) - tools.remove_files_by_mask(os.path.join(self.package_folder, "lib"), "*.la") + rmdir(self, os.path.join(self.package_folder, "share")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + rm(self, "*.la", self.package_folder, recursive=True) + fix_apple_shared_install_name(self) def package_info(self): - self.cpp_info.names["pkg_config"] = "libfabric" - self.cpp_info.libs = self.collect_libs() + self.cpp_info.set_property("pkg_config_name", "libfabric") + self.cpp_info.libs = ["fabric"] if self.settings.os in ["Linux", "FreeBSD"]: - self.cpp_info.system_libs = ["pthread", "m"] + self.cpp_info.system_libs = ["pthread", "m", "rt", "dl"] + if self.settings.compiler in ["gcc", "clang"]: + self.cpp_info.system_libs.append("atomic") diff --git a/recipes/libfabric/all/test_package/CMakeLists.txt b/recipes/libfabric/all/test_package/CMakeLists.txt index 196188113685c..09b7943e0c2de 100644 --- a/recipes/libfabric/all/test_package/CMakeLists.txt +++ b/recipes/libfabric/all/test_package/CMakeLists.txt @@ -1,8 +1,7 @@ -cmake_minimum_required(VERSION 3.1) -project(test_package) +cmake_minimum_required(VERSION 3.15) +project(test_package LANGUAGES CXX) -include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) -conan_basic_setup() +find_package(libfabric REQUIRED CONFIG) add_executable(${PROJECT_NAME} test_package.cpp) -target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS}) +target_link_libraries(${PROJECT_NAME} PRIVATE libfabric::libfabric) diff --git a/recipes/libfabric/all/test_package/conanfile.py b/recipes/libfabric/all/test_package/conanfile.py index bd7165a553cf4..ef5d7042163ec 100644 --- a/recipes/libfabric/all/test_package/conanfile.py +++ b/recipes/libfabric/all/test_package/conanfile.py @@ -1,10 +1,19 @@ -from conans import ConanFile, CMake, tools +from conan import ConanFile +from conan.tools.build import can_run +from conan.tools.cmake import cmake_layout, CMake import os class TestPackageConan(ConanFile): - settings = "os", "compiler", "build_type", "arch" - generators = "cmake" + settings = "os", "arch", "compiler", "build_type" + generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv" + test_type = "explicit" + + def requirements(self): + self.requires(self.tested_reference_str) + + def layout(self): + cmake_layout(self) def build(self): cmake = CMake(self) @@ -12,6 +21,6 @@ def build(self): cmake.build() def test(self): - if not tools.cross_building(self.settings): - bin_path = os.path.join("bin", "test_package") - self.run(bin_path, run_environment=True) + if can_run(self): + bin_path = os.path.join(self.cpp.build.bindir, "test_package") + self.run(bin_path, env="conanrun") diff --git a/recipes/libfabric/all/test_v1_package/CMakeLists.txt b/recipes/libfabric/all/test_v1_package/CMakeLists.txt new file mode 100644 index 0000000000000..91630d79f4abb --- /dev/null +++ b/recipes/libfabric/all/test_v1_package/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.15) +project(test_package) + +include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) +conan_basic_setup(TARGETS) + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../test_package/ + ${CMAKE_CURRENT_BINARY_DIR}/test_package/) diff --git a/recipes/libfabric/all/test_v1_package/conanfile.py b/recipes/libfabric/all/test_v1_package/conanfile.py new file mode 100644 index 0000000000000..7e2dfe859bb27 --- /dev/null +++ b/recipes/libfabric/all/test_v1_package/conanfile.py @@ -0,0 +1,17 @@ +from conans import ConanFile, CMake, tools +import os + + +class TestPackageConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + generators = "cmake", "cmake_find_package_multi" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def test(self): + if not tools.cross_building(self.settings): + bin_path = os.path.join("bin", "test_package") + self.run(bin_path, run_environment=True) diff --git a/recipes/libfabric/config.yml b/recipes/libfabric/config.yml index daa3a213a56fa..09d8df06c28ff 100644 --- a/recipes/libfabric/config.yml +++ b/recipes/libfabric/config.yml @@ -1,3 +1,3 @@ versions: - "1.12.1": + "1.21.0": folder: all