(#18979) libfabric: migrate to Conan v2, add missing dependencies

* libfabric: migrate to Conan v2

* libfabric: explicitly disable providers that cannot be built by default

* libfabric: add v1.18.1

* libfabric: opx also requires rdma-core

* libfabric: add libatomic system dependency

* libfabric: macOS does not support `shm`

* libfabric: macOS does not support `sm2`

* libfabric: fix libatomic system dep

* libfabric: do not ignore libcxx for libatomic

* libfabric: workaround for linter error

* libfabric: fix typo

* libfabric: add libatomic for both gcc and clang

* libfabric: fix_apple_shared_install_name()

* libfabric: bump to v1.21.0, drop v1.12.1, remove 'auto' options

* libfabric: disable 'verbs'

* libfabric: fix invalid default configurations on macOS

* libfabric: fix a linter error

* libfabric: fix a missing option error

* libfabric: fix a linter error

* libfabric: fix a minor recipe bug

* libfabric: add VirtualRunEnv for Conan v1

* libfabric: use get_safe()

Co-authored-by: PerseoGI <perseog@jfrog.com>

* Improve Windows invalid config message

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Use None ANY for providers

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Remove psm option

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Remove xpmem and cxi

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Improve verbs invalid configuration message

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Remove opx

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Disable bgq

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Enable verbs and disable others

Signed-off-by: Uilian Ries <uilianries@gmail.com>

---------

Signed-off-by: Uilian Ries <uilianries@gmail.com>
Co-authored-by: PerseoGI <perseog@jfrog.com>
Co-authored-by: Uilian Ries <uilianries@gmail.com>

recipes/libfabric/all/conandata.yml

@@ -1,4 +1,4 @@
 sources:
-  "1.12.1":
-    url: https://github.com/ofiwg/libfabric/releases/download/v1.12.1/libfabric-1.12.1.tar.bz2
-    sha256: db3c8e0a495e6e9da6a7436adab905468aedfbd4579ee3da5232a5c111ba642c
+  "1.21.0":
+    url: "https://github.com/ofiwg/libfabric/releases/download/v1.21.0/libfabric-1.21.0.tar.bz2"
+    sha256: "0c1b7b830d9147f661e5d7f359250b85b5a9885c330464cd3b5e5d35b86551c7"

recipes/libfabric/all/conanfile.py

@@ -1,99 +1,195 @@
-from conans import ConanFile, AutoToolsBuildEnvironment, tools
-from conans.errors import ConanInvalidConfiguration
 import os
 
-required_conan_version = ">=1.35.0"
+from conan import ConanFile
+from conan.errors import ConanInvalidConfiguration
+from conan.tools.apple import is_apple_os, fix_apple_shared_install_name
+from conan.tools.env import VirtualBuildEnv, VirtualRunEnv
+from conan.tools.files import copy, get, rm, rmdir
+from conan.tools.gnu import Autotools, AutotoolsToolchain, AutotoolsDeps
+from conan.tools.layout import basic_layout
+
+required_conan_version = ">=1.53.0"
+
 
 class LibfabricConan(ConanFile):
     name = "libfabric"
-    description = "Open Fabric Interfaces"
-    topics = ("fabric", "communication", "framework", "service")
+    description = ("Libfabric, also known as Open Fabrics Interfaces (OFI), "
+                   "defines a communication API for high-performance parallel and distributed applications.")
+    license = ("BSD-2-Clause", "GPL-2.0-or-later")
     url = "https://github.com/conan-io/conan-center-index"
     homepage = "http://libfabric.org"
-    license = "BSD-2-Clause", "GPL-2.0-or-later"
+    topics = ("fabric", "communication", "framework", "service")
+
+    package_type = "library"
     settings = "os", "arch", "compiler", "build_type"
-    _providers = ['gni', 'psm', 'psm2', 'psm3', 'rxm', 'sockets', 'tcp', 'udp', 'usnic', 'verbs', 'bgq']
+    _providers = [
+        "dmabuf_peer_mem",
+        "efa",
+        "hook_debug",
+        "hook_hmem",
+        "mrail",
+        "perf",
+        "profile",
+        "rxd",
+        "rxm",
+        "shm",
+        "sm2",
+        "sockets",
+        "tcp",
+        "trace",
+        "ucx",
+        "udp",
+        "usnic",
+        "verbs",
+    ]
     options = {
-        **{ p: "ANY" for p in _providers },
-        **{
-            "shared": [True, False],
-            "fPIC": [True, False],
-            "with_libnl": "ANY",
-            "with_bgq_progress": [None, "auto", "manual"],
-            "with_bgq_mr": [None, "basic", "scalable"]
-        }
+        "shared": [True, False],
+        "fPIC": [True, False],
+        **{ p: [None, "ANY"] for p in _providers },
     }
     default_options = {
-        **{ p: "auto" for p in _providers },
-        **{
-            "shared": False,
-            "fPIC": True,
-            "with_libnl": None,
-            "with_bgq_progress": None,
-            "with_bgq_mr": None
-        }
+        "shared": False,
+        "fPIC": True,
+        "dmabuf_peer_mem": "yes",
+        "efa": "yes",
+        "hook_debug": "yes",
+        "hook_hmem": "yes",
+        "mrail": "yes",
+        "perf": "no",
+        "profile": "yes",
+        "rxd": "yes",
+        "rxm": "yes",
+        "shm": "yes",
+        "sm2": "yes",
+        "sockets": "yes",
+        "tcp": "yes",
+        "trace": "yes",
+        "ucx": "no",
+        "udp": "yes",
+        "usnic": "no",
+        "verbs": "yes"
     }
 
-    @property
-    def _source_subfolder(self):
-        return "source_subfolder"
-
-    _autotools = None
-
     def config_options(self):
-        if self.settings.os == 'Windows':
-            del self.options.fPIC
+        if is_apple_os(self):
+            # Requires libnl, which is not available on macOS
+            del self.options.usnic
+            # Require Linux-specific process_vm_readv syscall
+            del self.options.shm
+            del self.options.sm2
+            # rdma-core is not available on macOS
+            del self.options.efa
+            del self.options.verbs
 
     def configure(self):
         if self.options.shared:
-            del self.options.fPIC
-        del self.settings.compiler.libcxx
-        del self.settings.compiler.cppstd
+            self.options.rm_safe("fPIC")
+        self.settings.rm_safe("compiler.libcxx")
+        self.settings.rm_safe("compiler.cppstd")
+
+    def _is_enabled(self, opt):
+        return str(self.options.get_safe(opt)) == "yes" or str(self.options.get_safe(opt)).startswith("dl")
+
+    def requirements(self):
+        if self._is_enabled("usnic"):
+            self.requires("libnl/3.8.0")
+        if self._is_enabled("efa") or self._is_enabled("usnic") or self._is_enabled("verbs"):
+            self.requires("rdma-core/52.0")
+
+    def layout(self):
+        basic_layout(self, src_folder="src")
 
     def validate(self):
-        if self.settings.os not in ["Linux", "FreeBSD", "Macos"]:
-            raise ConanInvalidConfiguration("libfabric only builds on Linux, Macos, and FreeBSD.")
-        for p in self._providers:
-            if self.options.get_safe(p) not in ["auto", "yes", "no", "dl"] and not os.path.isdir(str(self.options.get_safe(p))):
-                raise ConanInvalidConfiguration("Option {} can only be one of 'auto', 'yes', 'no', 'dl' or a directory path")
-        if self.options.get_safe('with_libnl') and not os.path.isdir(str(self.options.with_libnl)):
-            raise ConanInvalidConfiguration("Value of with_libnl must be an existing directory")
+        if self.settings.os == "Windows":
+            # FIXME: libfabric provides msbuild project files.
+            raise ConanInvalidConfiguration(f"{self.ref} Conan recipes is not supported on Windows. Contributions are welcome.")
+
+        for provider in self._providers:
+            provider = str(self.options.get_safe(provider))
+            if provider.lower() not in ["yes", "no", "dl", "none"] and \
+                    not os.path.isdir(provider) and \
+                    (not provider.startswith("dl:") and not os.path.isdir(provider[3:])):
+                raise ConanInvalidConfiguration(f"{self.ref} provider option '{provider}' is not valid. It must be 'yes', 'no', 'dl', 'dl:<dir_path>' or a directory path.")
+
+        if self._is_enabled("verbs"):
+            if not self.dependencies["rdma-core"].options.build_librdmacm:
+                raise ConanInvalidConfiguration(f"{self.ref} '-o rdma-core/*:build_librdmacm=True' is required when '-o &:verbs=True'")
+
+    def build_requirements(self):
+        # Used in ./configure tests and build
+        self.tool_requires("libtool/2.4.7")
 
     def source(self):
-        tools.get(**self.conan_data["sources"][self.version],
-                  destination=self._source_subfolder, strip_root=True)
-
-    def _configure_autotools(self):
-        if self._autotools:
-            return self._autotools
-        self._autotools = AutoToolsBuildEnvironment(self)
-        args = []
+        get(self, **self.conan_data["sources"][self.version], strip_root=True)
+
+    def generate(self):
+        tc = AutotoolsToolchain(self)
         for p in self._providers:
-            args.append('--enable-{}={}'.format(p, self.options.get_safe(p)))
-        if self.options.with_libnl:
-            args.append('--with-libnl={}'.format(self.options.with_libnl))
-        if self.options.with_bgq_progress:
-            args.append('--with-bgq-progress={}'.format(self.options.with_bgq_progress))
-        if self.options.with_bgq_mr:
-            args.append('--with-bgq-mr={}'.format(self.options.with_bgq_mr))
-        self._autotools.configure(args=args, configure_dir=self._source_subfolder)
-        return self._autotools
+            if p == "verbs" and self.options.get_safe(p, "no") != "no":
+                path = self.dependencies["rdma-core"].package_folder
+                if self.options.get_safe("verbs") == "dl":
+                    tc.configure_args.append(f"--enable-verbs=dl:{path}")
+                else:
+                    tc.configure_args.append(f"--enable-verbs={path}")
+            else:
+                tc.configure_args.append(f"--enable-{p}={self.options.get_safe(p, 'no')}")
+        if self.settings.build_type == "Debug":
+            tc.configure_args.append("--enable-debug")
+        tc.configure_args.append(f"--with-bgq-progress=no")
+        tc.configure_args.append(f"--with-bgq-mr=no")
+        tc.configure_args.append("--with-cassin-headers=no")
+        tc.configure_args.append("--with-cuda=no")  # TODO
+        tc.configure_args.append("--with-curl=no")  # TODO
+        tc.configure_args.append("--with-cxi-uapi-headers=no")
+        tc.configure_args.append("--with-dsa=no")
+        tc.configure_args.append("--with-gdrcopy=no")
+        tc.configure_args.append("--with-json-c=no")  # TODO
+        if self._is_enabled("usnic"):
+            tc.configure_args.append(f"--with-libnl={self.dependencies['libnl'].package_folder}")
+        else:
+            tc.configure_args.append("--with-libnl=no")
+        tc.configure_args.append("--with-lttng=no")
+        tc.configure_args.append("--with-neuron=no")
+        tc.configure_args.append(f"--with-numa=no")
+        tc.configure_args.append("--with-psm2-src=no")
+        tc.configure_args.append("--with-psm3-rv=no")
+        tc.configure_args.append("--with-rocr=no")
+        tc.configure_args.append("--with-synapseai=no")
+        tc.configure_args.append("--with-uring=no")  # TODO
+        tc.configure_args.append("--with-ze=no")
+        tc.configure_args.append("-enable-psm=no")
+        tc.configure_args.append("--enable-psm2=no")
+        tc.configure_args.append("--enable-psm3=no")
+        tc.configure_args.append("--enable-xpmem=no")
+        tc.configure_args.append("--enable-cxi=no")
+        tc.configure_args.append("--enable-opx=no")
+        tc.configure_args.append("--enable-bgq=no")
+        tc.generate()
+
+        deps = AutotoolsDeps(self)
+        deps.generate()
+
+        VirtualBuildEnv(self).generate()
+        VirtualRunEnv(self).generate(scope="build")
 
     def build(self):
-        autotools = self._configure_autotools()
+        autotools = Autotools(self)
+        autotools.configure()
         autotools.make()
 
     def package(self):
-        self.copy(pattern="COPYING", dst="licenses", src=self._source_subfolder)
-        autotools = self._configure_autotools()
+        copy(self, "COPYING", dst=os.path.join(self.package_folder, "licenses"), src=self.source_folder)
+        autotools = Autotools(self)
         autotools.install()
-
-        tools.rmdir(os.path.join(self.package_folder, "share"))
-        tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig"))
-        tools.remove_files_by_mask(os.path.join(self.package_folder, "lib"), "*.la")
+        rmdir(self, os.path.join(self.package_folder, "share"))
+        rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig"))
+        rm(self, "*.la", self.package_folder, recursive=True)
+        fix_apple_shared_install_name(self)
 
     def package_info(self):
-        self.cpp_info.names["pkg_config"] = "libfabric"
-        self.cpp_info.libs = self.collect_libs()
+        self.cpp_info.set_property("pkg_config_name", "libfabric")
+        self.cpp_info.libs = ["fabric"]
         if self.settings.os in ["Linux", "FreeBSD"]:
-            self.cpp_info.system_libs = ["pthread", "m"]
+            self.cpp_info.system_libs = ["pthread", "m", "rt", "dl"]
+        if self.settings.compiler in ["gcc", "clang"]:
+            self.cpp_info.system_libs.append("atomic")

recipes/libfabric/all/test_package/CMakeLists.txt

@@ -1,8 +1,7 @@
-cmake_minimum_required(VERSION 3.1)
-project(test_package)
+cmake_minimum_required(VERSION 3.15)
+project(test_package LANGUAGES CXX)
 
-include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
-conan_basic_setup()
+find_package(libfabric REQUIRED CONFIG)
 
 add_executable(${PROJECT_NAME} test_package.cpp)
-target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS})
+target_link_libraries(${PROJECT_NAME} PRIVATE libfabric::libfabric)

recipes/libfabric/all/test_package/conanfile.py

@@ -1,17 +1,26 @@
-from conans import ConanFile, CMake, tools
+from conan import ConanFile
+from conan.tools.build import can_run
+from conan.tools.cmake import cmake_layout, CMake
 import os
 
 
 class TestPackageConan(ConanFile):
-    settings = "os", "compiler", "build_type", "arch"
-    generators = "cmake"
+    settings = "os", "arch", "compiler", "build_type"
+    generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv"
+    test_type = "explicit"
+
+    def requirements(self):
+        self.requires(self.tested_reference_str)
+
+    def layout(self):
+        cmake_layout(self)
 
     def build(self):
         cmake = CMake(self)
         cmake.configure()
         cmake.build()
 
     def test(self):
-        if not tools.cross_building(self.settings):
-            bin_path = os.path.join("bin", "test_package")
-            self.run(bin_path, run_environment=True)
+        if can_run(self):
+            bin_path = os.path.join(self.cpp.build.bindir, "test_package")
+            self.run(bin_path, env="conanrun")

recipes/libfabric/all/test_v1_package/CMakeLists.txt

@@ -0,0 +1,8 @@
+cmake_minimum_required(VERSION 3.15)
+project(test_package)
+
+include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
+conan_basic_setup(TARGETS)
+
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../test_package/
+                 ${CMAKE_CURRENT_BINARY_DIR}/test_package/)

recipes/libfabric/all/test_v1_package/conanfile.py

@@ -0,0 +1,17 @@
+from conans import ConanFile, CMake, tools
+import os
+
+
+class TestPackageConan(ConanFile):
+    settings = "os", "compiler", "build_type", "arch"
+    generators = "cmake", "cmake_find_package_multi"
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def test(self):
+        if not tools.cross_building(self.settings):
+            bin_path = os.path.join("bin", "test_package")
+            self.run(bin_path, run_environment=True)

recipes/libfabric/config.yml

@@ -1,3 +1,3 @@
 versions:
-  "1.12.1":
+  "1.21.0":
     folder: all