From 5a3aea567d91d9b9824ba1588e6d220d5c46f750 Mon Sep 17 00:00:00 2001 From: Martin Valgur Date: Thu, 5 Sep 2024 11:42:13 +0300 Subject: [PATCH] (#18980) openmpi: migrate to Conan v2 * openmpi: migrate to Conan v2 * openmpi: bump deps * openmpi: do not print created env vars * openmpi: add v4.1.6 * openmpi: improve options coverage * openmpi: fix path env vars * openmpi: fix a PMIX error * openmpi: add MPI::MPI_CXX only if `cxx` is enabled * openmpi: fix static build issues, add external_hwloc option * openmpi: fix libltdl issue, replace AutotoolsDeps with PkgConfigDeps * openmpi: disable buildign of docs * openmpi: drop libltdl dependency * openmpi: external libevent does not really work * openmpi: model components correctly * openmpi: fix pkg-config names * openmpi: disable armv8 for v4.1.0 * openmpi: bump libnl * openmpi: enable with_verbs * openmpi: use REQUIRED CONFIG in test_package for legacy generators * bump rdma-core * openmpi: fix libibverbs.so not being found * openmpi: drop test_v1_package * openmpi: fix apple-clang cross-compilation * openmpi: run autoreconf * openmpi: drop v4.1.0 * openmpi: Revert "run autoreconf" This reverts commit 6f678dab6d734cb6b5723b2ea8ea407465fcd485. * openmpi: disable Clang on Conan v1 * openmpi: restore v4.1.0 This reverts commit 04611660ba75fa06edbcd58ff28ca3c9021bac0e. * openmpi: add --allow-run-as-root to test_package Co-authored-by: Maksim Petukhov <6967052+maksim-petukhov@users.noreply.github.com> * openmpi: adjust deps * openmpi: enable cxx by default * openmpi: drop external_hwloc option * openmpi: fix an invalid require name * openmpi: add missing requires * openmpi: add VirtualRunEnv to fix ./configure * openmpi: workaround for macOS build failure * openmpi: re-enable Clang/AppleClang for Conan v1 * openmpi: fix_apple_shared_install_name() * openmpi: apply review suggestions * openmpi: drop --with-libevent=internal * openmpi: drop --allow-run-as-root in test_package * openmpi: add a comment for pkg_config_name Co-authored-by: Uilian Ries * openmpi: get_safe("with_verbs") * openmpi: Revert "re-enable Clang/AppleClang for Conan v1" This reverts commit 40d407ee66231a2df8efc376ff6e5acf77a80090. * openmpi: VirtualRunEnv is required to find libhwloc.so during ./configure * openmpi: macOS armv8 fails when shared=True * Simplify test package to avoid networking usage The current openmpi test package tries to use networking, and in OS like Mac, the GUI asks about permission to execute the test package. It's too much for a simple package validation. It would be better keep a version check for library linkage validation. Signed-off-by: Uilian Ries * Revert "openmpi: macOS armv8 fails when shared=True" Enable again dynamic library build for Mac. With the simplified test package, I see no errors locally for Mac M1. This reverts commit d79c757c2f05864b6decedc835ad08c693386374. --------- Signed-off-by: Uilian Ries Co-authored-by: Maksim Petukhov <6967052+maksim-petukhov@users.noreply.github.com> Co-authored-by: Uilian Ries --- recipes/openmpi/all/conandata.yml | 7 +- recipes/openmpi/all/conanfile.py | 284 ++++++++++++++---- .../openmpi/all/test_package/CMakeLists.txt | 13 +- recipes/openmpi/all/test_package/conanfile.py | 24 +- .../openmpi/all/test_package/test_package.c | 13 + .../openmpi/all/test_package/test_package.cpp | 23 -- recipes/openmpi/config.yml | 2 + 7 files changed, 265 insertions(+), 101 deletions(-) create mode 100644 recipes/openmpi/all/test_package/test_package.c delete mode 100644 recipes/openmpi/all/test_package/test_package.cpp diff --git a/recipes/openmpi/all/conandata.yml b/recipes/openmpi/all/conandata.yml index 4580e17ba76a3f..7211bc311a7194 100644 --- a/recipes/openmpi/all/conandata.yml +++ b/recipes/openmpi/all/conandata.yml @@ -1,4 +1,7 @@ sources: + "4.1.6": + url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.bz2" + sha256: "f740994485516deb63b5311af122c265179f5328a0d857a567b85db00b11e415" "4.1.0": - sha256: 73866fb77090819b6a8c85cb8539638d37d6877455825b74e289d647a39fd5b5 - url: https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.bz2 + url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.bz2" + sha256: "73866fb77090819b6a8c85cb8539638d37d6877455825b74e289d647a39fd5b5" diff --git a/recipes/openmpi/all/conanfile.py b/recipes/openmpi/all/conanfile.py index 26cc0e027159c1..785217bcba3dcc 100644 --- a/recipes/openmpi/all/conanfile.py +++ b/recipes/openmpi/all/conanfile.py @@ -1,97 +1,259 @@ -from conans import ConanFile, tools, AutoToolsBuildEnvironment -from conans.errors import ConanInvalidConfiguration import os -required_conan_version = ">=1.29.1" +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.apple import is_apple_os, fix_apple_shared_install_name +from conan.tools.env import VirtualRunEnv +from conan.tools.files import copy, get, rm, rmdir, save, replace_in_file +from conan.tools.gnu import Autotools, AutotoolsToolchain, AutotoolsDeps +from conan.tools.layout import basic_layout +from conan.tools.microsoft import unix_path + +required_conan_version = ">=1.53.0" class OpenMPIConan(ConanFile): name = "openmpi" - homepage = "https://www.open-mpi.org" - url = "https://github.com/conan-io/conan-center-index" - topics = ("conan", "mpi", "openmpi") description = "A High Performance Message Passing Library" license = "BSD-3-Clause" + url = "https://github.com/conan-io/conan-center-index" + homepage = "https://www.open-mpi.org" + topics = ("mpi", "openmpi") + provides = "mpi" + package_type = "library" settings = "os", "arch", "compiler", "build_type" options = { "shared": [True, False], "fPIC": [True, False], - "fortran": ["yes", "mpifh", "usempi", "usempi80", "no"] + "fortran": ["yes", "mpifh", "usempi", "usempi80", "no"], + "enable_cxx": [True, False], + "enable_cxx_exceptions": [True, False], + "with_verbs": [True, False], } default_options = { "shared": False, "fPIC": True, - "fortran": "no" + "fortran": "no", + "enable_cxx": False, + "enable_cxx_exceptions": False, + "with_verbs": False, } - _autotools = None - - @property - def _source_subfolder(self): - return "source_subfolder" + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC def configure(self): if self.options.shared: - del self.options.fPIC - del self.settings.compiler.libcxx - del self.settings.compiler.cppstd + self.options.rm_safe("fPIC") + if not self.options.enable_cxx: + self.settings.rm_safe("compiler.libcxx") + self.settings.rm_safe("compiler.cppstd") + del self.options.enable_cxx_exceptions + if is_apple_os(self): + # Unavailable due to dependency on libnl + del self.options.with_verbs + + def layout(self): + basic_layout(self, src_folder="src") + + def requirements(self): + # OpenMPI public headers don't include anything besides stddef.h. + # transitive_headers=True is not needed for any dependencies. + self.requires("hwloc/2.10.0") + self.requires("zlib/[>=1.2.11 <2]") + if self.settings.os == "Linux": + self.requires("libnl/3.8.0") + if self.options.get_safe("with_verbs"): + self.requires("rdma-core/52.0") + + def validate(self): if self.settings.os == "Windows": + # Requires Cygwin or WSL raise ConanInvalidConfiguration("OpenMPI doesn't support Windows") - def requirements(self): - # FIXME : self.requires("libevent/2.1.12") - try to use libevent from conan - self.requires("zlib/1.2.11") + if self.version == "4.1.0" and is_apple_os(self) and self.settings.arch == "armv8": + # INFO: https://github.com/open-mpi/ompi/issues/8410 + raise ConanInvalidConfiguration(f"{self.ref} is not supported in Mac M1. Use a newer version.") def source(self): - tools.get(**self.conan_data["sources"][self.version]) - extracted_dir = self.name + "-" + self.version - os.rename(extracted_dir, self._source_subfolder) - - def _configure_autotools(self): - if self._autotools: - return self._autotools - self._autotools = AutoToolsBuildEnvironment(self) - args = ["--disable-wrapper-rpath", "--disable-wrapper-runpath"] - if self.settings.build_type == "Debug": - args.append("--enable-debug") - if self.options.shared: - args.extend(["--enable-shared", "--disable-static"]) - else: - args.extend(["--enable-static", "--disable-shared"]) - args.append("--with-pic" if self.options.get_safe("fPIC", True) else "--without-pic") - args.append("--enable-mpi-fortran={}".format(str(self.options.fortran))) - args.append("--with-zlib={}".format(self.deps_cpp_info["zlib"].rootpath)) - args.append("--with-zlib-libdir={}".format(self.deps_cpp_info["zlib"].lib_paths[0])) - args.append("--datarootdir=${prefix}/res") - self._autotools.configure(args=args) - return self._autotools + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + def root(pkg): + return unix_path(self, self.dependencies[pkg].package_folder) + + def yes_no(v): + return "yes" if v else "no" + + tc = AutotoolsToolchain(self) + tc.configure_args += [ + f"--enable-mpi-fortran={self.options.fortran}", + f"--enable-mpi-cxx={yes_no(self.options.enable_cxx)}", + f"--enable-cxx-exceptions={yes_no(self.options.get_safe('enable_cxx_exceptions'))}", + f"--with-hwloc={root('hwloc')}", + f"--with-libnl={root('libnl') if not is_apple_os(self) else 'no'}", + f"--with-verbs={root('rdma-core') if self.options.get_safe('with_verbs') else 'no'}", + f"--with-zlib={root('zlib')}", + "--with-pic" if self.options.get_safe("fPIC", True) else "--without-pic", + "--disable-wrapper-rpath", + "--disable-wrapper-runpath", + "--exec-prefix=/", + "--datarootdir=${prefix}/res", + # Disable other external libraries explicitly + "--with-alps=no", # ALPS + "--with-cuda=no", # CUDA + "--with-fca=no", # FCA + "--with-gpfs=no", # Gpfs + "--with-hcoll=no", # hcoll + "--with-ime=no", # IME + "--with-lsf=no", # LSF + "--with-lustre=no", # Lustre + "--with-memkind=no", # memkind + "--with-moab=no", # Moab + "--with-mxm=no", # Mellanox MXM + "--with-ofi=no", # libfabric, TODO: enable once libfabric is available + "--with-pmi=no", # PMI + "--with-pmix=internal", # PMIx + "--with-portals4=no", # Portals4 + "--with-psm2=no", # PSM2 + "--with-psm=no", # PSM + "--with-pvfs2=no", # Pvfs2 + "--with-treematch=no", # TreeMatch + "--with-ucx=no", # UCX + "--with-valgrind=no", # Valgrind + "--with-x=no", # X11 + "--with-xpmem=no", # XPMEM + ] + if is_apple_os(self): + if self.settings.arch == "armv8": + tc.configure_args.append("--host=aarch64-apple-darwin") + tc.extra_ldflags.append("-arch arm64") + # macOS has no libnl + tc.configure_args.append("--enable-mca-no-build=reachable-netlink") + # libtool's libltdl is not really needed, OpenMPI provides its own equivalent. + # Not adding it as it fails to be detected by ./configure in some cases. + # https://github.com/open-mpi/ompi/blob/v4.1.6/opal/mca/dl/dl.h#L20-L25 + tc.configure_args.append("--with-libltdl=no") + tc.generate() + + deps = AutotoolsDeps(self) + deps.generate() + + # Needed for ./configure to find libhwloc.so and libibnetdisc.so + VirtualRunEnv(self).generate(scope="build") + + # TODO: might want to enable reproducible builds by setting + # $SOURCE_DATE_EPOCH, $USER and $HOSTNAME + + def _patch_sources(self): + # Not needed and fails with v5.0 due to additional Python dependencies + save(self, os.path.join(self.source_folder, "docs", "Makefile.in"), "all:\ninstall:\n") + # Workaround for trying to include VERSION from source dir due to a case-insensitive filesystem on macOS + # Based on https://github.com/macports/macports-ports/blob/22dded99ae76a287f04a9685bbc820ecaa397fea/science/openmpi/files/patch-configure.diff + replace_in_file(self, os.path.join(self.source_folder, "configure"), + "-I$(top_srcdir) ", "-idirafter$(top_srcdir) ") def build(self): - with tools.chdir(self._source_subfolder): - autotools = self._configure_autotools() - autotools.make() + self._patch_sources() + autotools = Autotools(self) + autotools.configure() + autotools.make() def package(self): - self.copy(pattern="LICENSE", src=self._source_subfolder, dst="licenses") - with tools.chdir(self._source_subfolder): - autotools = self._configure_autotools() - autotools.install() - tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig")) - tools.rmdir(os.path.join(self.package_folder, "share")) - tools.rmdir(os.path.join(self.package_folder, "etc")) - tools.remove_files_by_mask(os.path.join(self.package_folder, "lib"), "*.la") + copy(self, "LICENSE", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + autotools = Autotools(self) + autotools.install() + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + rmdir(self, os.path.join(self.package_folder, "etc")) + rmdir(self, os.path.join(self.package_folder, "res", "man")) + rm(self, "*.la", self.package_folder, recursive=True) + fix_apple_shared_install_name(self) def package_info(self): - self.cpp_info.libs = ['mpi', 'open-rte', 'open-pal'] + # Based on https://cmake.org/cmake/help/latest/module/FindMPI.html#variables-for-using-mpi + self.cpp_info.set_property("cmake_find_mode", "both") + self.cpp_info.set_property("cmake_file_name", "MPI") + # TODO: Use None when available as Conan feature. + self.cpp_info.set_property("pkg_config_name", "_ompi-do-not-use") + # TODO: export a .cmake module to correctly set all variables set by CMake's FindMPI.cmake + + requires = [ + "hwloc::hwloc", + "zlib::zlib", + ] if self.settings.os == "Linux": - self.cpp_info.system_libs = ["dl", "pthread", "rt", "util"] + requires.append("libnl::libnl") + if self.options.get_safe("with_verbs"): + requires.extend(["rdma-core::libibverbs", "rdma-core::librdmacm"]) + + # The components are modelled based on OpenMPI's pkg-config files - self.output.info("Creating MPI_HOME environment variable: {}".format(self.package_folder)) + # Run-time environment library + self.cpp_info.components["orte"].set_property("pkg_config_name", "orte") + self.cpp_info.components["orte"].libs = ["open-rte", "open-pal"] + self.cpp_info.components["orte"].includedirs.append(os.path.join("include", "openmpi")) + if self.settings.os in ["Linux", "FreeBSD"]: + self.cpp_info.components["orte"].system_libs = ["dl", "pthread", "rt", "util"] + self.cpp_info.components["orte"].cflags = ["-pthread"] + if self.options.get_safe("enable_cxx_exceptions"): + self.cpp_info.components["orte"].cflags.append("-fexceptions") + self.cpp_info.components["orte"].requires = requires + + self.cpp_info.components["ompi"].set_property("pkg_config_name", "ompi") + self.cpp_info.components["ompi"].libs = ["mpi"] + self.cpp_info.components["ompi"].requires = ["orte"] + + self.cpp_info.components["ompi-c"].set_property("pkg_config_name", "ompi-c") + self.cpp_info.components["ompi-c"].set_property("cmake_target_name", "MPI::MPI_C") + self.cpp_info.components["ompi-c"].requires = ["ompi"] + + self.cpp_info.components["ompitrace"].set_property("pkg_config_name", "ompitrace") + self.cpp_info.components["ompitrace"].libs = ["ompitrace"] + self.cpp_info.components["ompitrace"].requires = ["ompi"] + + if self.options.enable_cxx: + self.cpp_info.components["ompi-cxx"].set_property("pkg_config_name", "ompi-cxx") + self.cpp_info.components["ompi-cxx"].set_property("cmake_target_name", "MPI::MPI_CXX") + self.cpp_info.components["ompi-cxx"].libs = ["mpi_cxx"] + self.cpp_info.components["ompi-cxx"].requires = ["ompi"] + + if self.options.fortran != "no": + self.cpp_info.components["ompi-fort"].set_property("pkg_config_name", "ompi-fort") + self.cpp_info.components["ompi-fort"].set_property("cmake_target_name", "MPI::MPI_Fortran") + self.cpp_info.components["ompi-fort"].libs = ["mpi_mpifh"] + self.cpp_info.components["ompi-fort"].requires = ["ompi"] + # Aliases + self.cpp_info.components["ompi-f77"].set_property("pkg_config_name", "ompi-f77") + self.cpp_info.components["ompi-f77"].requires = ["ompi-fort"] + self.cpp_info.components["ompi-f90"].set_property("pkg_config_name", "ompi-f90") + self.cpp_info.components["ompi-f90"].requires = ["ompi-fort"] + + bin_folder = os.path.join(self.package_folder, "bin") + # Prepend to PATH to avoid a conflict with system MPI + self.runenv_info.prepend_path("PATH", bin_folder) + self.runenv_info.define_path("MPI_BIN", bin_folder) + self.runenv_info.define_path("MPI_HOME", self.package_folder) + self.runenv_info.define_path("OPAL_PREFIX", self.package_folder) + self.runenv_info.define_path("OPAL_EXEC_PREFIX", self.package_folder) + self.runenv_info.define_path("OPAL_LIBDIR", os.path.join(self.package_folder, "lib")) + self.runenv_info.define_path("OPAL_DATADIR", os.path.join(self.package_folder, "res")) + self.runenv_info.define_path("OPAL_DATAROOTDIR", os.path.join(self.package_folder, "res")) + + # TODO: Legacy, to be removed on Conan 2.0 + self.env_info.PATH.append(bin_folder) + self.env_info.MPI_BIN = bin_folder self.env_info.MPI_HOME = self.package_folder - self.output.info("Creating OPAL_PREFIX environment variable: {}".format(self.package_folder)) self.env_info.OPAL_PREFIX = self.package_folder - mpi_bin = os.path.join(self.package_folder, 'bin') - self.output.info("Creating MPI_BIN environment variable: {}".format(mpi_bin)) - self.env_info.MPI_BIN = mpi_bin - self.output.info("Appending PATH environment variable: {}".format(mpi_bin)) - self.env_info.PATH.append(mpi_bin) + self.env_info.OPAL_EXEC_PREFIX = self.package_folder + self.env_info.OPAL_LIBDIR = os.path.join(self.package_folder, "lib") + self.env_info.OPAL_DATADIR = os.path.join(self.package_folder, "res") + self.env_info.OPAL_DATAROOTDIR = os.path.join(self.package_folder, "res") + + self.cpp_info.names["cmake_find_package"] = "MPI" + self.cpp_info.names["cmake_find_package_multi"] = "MPI" + self.cpp_info.components["ompi-c"].names["cmake_find_package"] = "MPI_C" + if self.options.enable_cxx: + self.cpp_info.components["ompi-cxx"].names["cmake_find_package"] = "MPI_CXX" + if self.options.fortran != "no": + self.cpp_info.components["ompi-fort"].names["cmake_find_package"] = "MPI_Fortran" diff --git a/recipes/openmpi/all/test_package/CMakeLists.txt b/recipes/openmpi/all/test_package/CMakeLists.txt index d61f3c4bef33a9..a166065f8b6628 100644 --- a/recipes/openmpi/all/test_package/CMakeLists.txt +++ b/recipes/openmpi/all/test_package/CMakeLists.txt @@ -1,10 +1,7 @@ -cmake_minimum_required(VERSION 3.1) -project(test_package) +cmake_minimum_required(VERSION 3.15) +project(test_package LANGUAGES C) -include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) -conan_basic_setup() +find_package(MPI REQUIRED CONFIG) -find_package(MPI REQUIRED) - -add_executable(${PROJECT_NAME} test_package.cpp) -target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS}) +add_executable(${PROJECT_NAME} test_package.c) +target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_C) diff --git a/recipes/openmpi/all/test_package/conanfile.py b/recipes/openmpi/all/test_package/conanfile.py index c63b947b1cb955..09ff6fcc3cc716 100644 --- a/recipes/openmpi/all/test_package/conanfile.py +++ b/recipes/openmpi/all/test_package/conanfile.py @@ -1,10 +1,20 @@ -from conans import ConanFile, CMake, tools import os +from conan import ConanFile +from conan.tools.build import can_run +from conan.tools.cmake import cmake_layout, CMake + class TestPackageConan(ConanFile): - settings = "os", "compiler", "build_type", "arch" - generators = "cmake" + settings = "os", "arch", "compiler", "build_type" + generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv" + test_type = "explicit" + + def requirements(self): + self.requires(self.tested_reference_str) + + def layout(self): + cmake_layout(self) def build(self): cmake = CMake(self) @@ -12,7 +22,7 @@ def build(self): cmake.build() def test(self): - if not tools.cross_building(self.settings): - mpiexec = os.path.join(os.environ['MPI_BIN'], 'mpiexec') - command = '%s -mca plm_rsh_agent yes -np 2 %s' % (mpiexec, os.path.join("bin", "test_package")) - self.run(command, run_environment=True) + if can_run(self): + bin_path = os.path.join(self.cpp.build.bindir, "test_package") + command = f"mpiexec -mca plm_rsh_agent yes {bin_path}" + self.run(command, env="conanrun") diff --git a/recipes/openmpi/all/test_package/test_package.c b/recipes/openmpi/all/test_package/test_package.c new file mode 100644 index 00000000000000..540ba2f9beaccd --- /dev/null +++ b/recipes/openmpi/all/test_package/test_package.c @@ -0,0 +1,13 @@ +#include +#include + +#include + +int main(int argc, char* argv[]) +{ + char version[MPI_MAX_LIBRARY_VERSION_STRING] = {0}; + int len = 0; + MPI_Get_library_version(version, &len); + printf("MPI version: %s\n", version); + return EXIT_SUCCESS; +} diff --git a/recipes/openmpi/all/test_package/test_package.cpp b/recipes/openmpi/all/test_package/test_package.cpp deleted file mode 100644 index d6fa9065fdf2a7..00000000000000 --- a/recipes/openmpi/all/test_package/test_package.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include - -int main(int argc, char* argv[]) -{ - MPI_Init(&argc, &argv); - - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (rank == 0) { - int value = 17; - int result = MPI_Send(&value, 1, MPI_INT, 1, 0, MPI_COMM_WORLD); - if (result == MPI_SUCCESS) - std::cout << "Rank 0 OK!" << std::endl; - } else if (rank == 1) { - int value; - int result = MPI_Recv(&value, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - if (result == MPI_SUCCESS && value == 17) - std::cout << "Rank 1 OK!" << std::endl; - } - MPI_Finalize(); - return 0; -} diff --git a/recipes/openmpi/config.yml b/recipes/openmpi/config.yml index 702888c1595948..b6ed9888d01769 100644 --- a/recipes/openmpi/config.yml +++ b/recipes/openmpi/config.yml @@ -1,3 +1,5 @@ versions: + "4.1.6": + folder: all "4.1.0": folder: all