require-replaces proposal (#15136)

* wip

* test passing

* wip

* wip

* wip

* wip

* wip

* wip

* new approach

* extra checks

* more tests

* review

conan/cli/printers/graph.py

@@ -56,6 +56,11 @@ def _format_resolved(title, reqs_to_print):
         for k, v in sorted(reqs_to_print.items()):
             output.info("    {}: {}".format(k, v), Color.BRIGHT_CYAN)
 
+    if graph.replaced_requires:
+        output.info("Replaced requires", Color.BRIGHT_YELLOW)
+        for k, v in graph.replaced_requires.items():
+            output.info("    {}: {}".format(k, v), Color.BRIGHT_CYAN)
+
     _format_resolved("Resolved alias", graph.aliased)
     if graph.aliased:
         output.warning("'alias' is a Conan 1.X legacy feature, no longer recommended and "

conans/client/graph/graph.py

@@ -319,6 +319,7 @@ def __init__(self):
         self.nodes = []
         self.aliased = {}
         self.resolved_ranges = {}
+        self.replaced_requires = {}
         self.error = False
 
     def overrides(self):
@@ -391,4 +392,5 @@ def serialize(self):
         result["root"] = {self.root.id: repr(self.root.ref)}  # TODO: ref of consumer/virtual
         result["overrides"] = self.overrides().serialize()
         result["resolved_ranges"] = {repr(r): s.repr_notime() for r, s in self.resolved_ranges.items()}
+        result["replaced_requires"] = {k: v for k, v in self.replaced_requires.items()}
         return result

conans/client/graph/graph_builder.py

@@ -38,7 +38,7 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None):
         dep_graph = DepsGraph()
 
         self._prepare_node(root_node, profile_host, profile_build, Options())
-        self._initialize_requires(root_node, dep_graph, graph_lock)
+        self._initialize_requires(root_node, dep_graph, graph_lock, profile_build, profile_host)
         dep_graph.add_node(root_node)
 
         open_requires = deque((r, root_node) for r in root_node.conanfile.requires.values())
@@ -51,7 +51,8 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None):
                 new_node = self._expand_require(require, node, dep_graph, profile_host,
                                                 profile_build, graph_lock)
                 if new_node:
-                    self._initialize_requires(new_node, dep_graph, graph_lock)
+                    self._initialize_requires(new_node, dep_graph, graph_lock, profile_build,
+                                              profile_host)
                     open_requires.extendleft((r, new_node)
                                              for r in reversed(new_node.conanfile.requires.values()))
             self._remove_overrides(dep_graph)
@@ -160,7 +161,7 @@ def _prepare_node(node, profile_host, profile_build, down_options):
                     node.conanfile.requires.tool_require(tool_require.repr_notime(),
                                                          raise_if_duplicated=False)
 
-    def _initialize_requires(self, node, graph, graph_lock):
+    def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_host):
         for require in node.conanfile.requires.values():
             alias = require.alias  # alias needs to be processed this early
             if alias is not None:
@@ -170,6 +171,7 @@ def _initialize_requires(self, node, graph, graph_lock):
                 # if partial, we might still need to resolve the alias
                 if not resolved:
                     self._resolve_alias(node, require, alias, graph)
+            self._resolve_replace_requires(node, require, profile_build, profile_host, graph)
             node.transitive_deps[require] = TransitiveRequirement(require, node=None)
 
     def _resolve_alias(self, node, require, alias, graph):
@@ -233,6 +235,40 @@ def _resolved_system(node, require, profile_build, profile_host, resolve_prerele
                             require.ref.revision = d.revision
                             return d, ConanFile(str(d)), RECIPE_PLATFORM, None
 
+    def _resolve_replace_requires(self, node, require, profile_build, profile_host, graph):
+        profile = profile_build if node.context == CONTEXT_BUILD else profile_host
+        replacements = profile.replace_tool_requires if require.build else profile.replace_requires
+        if not replacements:
+            return
+
+        for pattern, alternative_ref in replacements.items():
+            if pattern.name != require.ref.name:
+                continue  # no match in name
+            if pattern.version != "*":  # we need to check versions
+                rrange = require.version_range
+                valid = rrange.contains(pattern.version, self._resolve_prereleases) if rrange else \
+                    require.ref.version == pattern.version
+                if not valid:
+                    continue
+            if pattern.user != "*" and pattern.user != require.ref.user:
+                continue
+            if pattern.channel != "*" and pattern.channel != require.ref.channel:
+                continue
+            original_require = repr(require.ref)
+            if alternative_ref.version != "*":
+                require.ref.version = alternative_ref.version
+            if alternative_ref.user != "*":
+                require.ref.user = alternative_ref.user
+            if alternative_ref.channel != "*":
+                require.ref.channel = alternative_ref.channel
+            if alternative_ref.revision != "*":
+                require.ref.revision = alternative_ref.revision
+            if require.ref.name != alternative_ref.name:  # This requires re-doing dict!
+                node.conanfile.requires.reindex(require, alternative_ref.name)
+            require.ref.name = alternative_ref.name
+            graph.replaced_requires[original_require] = repr(require.ref)
+            break  # First match executes the alternative and finishes checking others
+
     def _create_new_node(self, node, require, graph, profile_host, profile_build, graph_lock):
         if require.ref.version == "<host_version>":
             if not require.build or require.visible:

conans/client/profile_loader.py

@@ -223,7 +223,8 @@ def get_profile(profile_text, base_profile=None):
                                                          "system_tools",  # DEPRECATED: platform_tool_requires
                                                          "platform_requires",
                                                          "platform_tool_requires", "settings",
-                                                         "options", "conf", "buildenv", "runenv"])
+                                                         "options", "conf", "buildenv", "runenv",
+                                                         "replace_requires", "replace_tool_requires"])
 
         # Parse doc sections into Conan model, Settings, Options, etc
         settings, package_settings = _ProfileValueParser._parse_settings(doc)
@@ -238,6 +239,26 @@ def get_profile(profile_text, base_profile=None):
         platform_tool_requires = [RecipeReference.loads(r) for r in doc_platform_tool_requires.splitlines()]
         platform_requires = [RecipeReference.loads(r) for r in doc_platform_requires.splitlines()]
 
+        def load_replace(doc_replace_requires):
+            result = {}
+            for r in doc_replace_requires.splitlines():
+                r = r.strip()
+                if not r or r.startswith("#"):
+                    continue
+                try:
+                    src, target = r.split(":")
+                    target = RecipeReference.loads(target.strip())
+                    src = RecipeReference.loads(src.strip())
+                except Exception as e:
+                    raise ConanException(f"Error in [replace_xxx] '{r}'.\nIt should be in the form"
+                                         f" 'pattern: replacement', without package-ids.\n"
+                                         f"Original error: {str(e)}")
+                result[src] = target
+            return result
+
+        replace_requires = load_replace(doc.replace_requires) if doc.replace_requires else {}
+        replace_tool = load_replace(doc.replace_tool_requires) if doc.replace_tool_requires else {}
+
         if doc.conf:
             conf = ConfDefinition()
             conf.loads(doc.conf, profile=True)
@@ -248,6 +269,9 @@ def get_profile(profile_text, base_profile=None):
 
         # Create or update the profile
         base_profile = base_profile or Profile()
+        base_profile.replace_requires.update(replace_requires)
+        base_profile.replace_tool_requires.update(replace_tool)
+
         current_platform_tool_requires = {r.name: r for r in base_profile.platform_tool_requires}
         current_platform_tool_requires.update({r.name: r for r in platform_tool_requires})
         base_profile.platform_tool_requires = list(current_platform_tool_requires.values())

conans/model/profile.py

@@ -17,6 +17,8 @@ def __init__(self):
         self.package_settings = defaultdict(OrderedDict)
         self.options = Options()
         self.tool_requires = OrderedDict()  # ref pattern: list of ref
+        self.replace_requires = {}
+        self.replace_tool_requires = {}
         self.platform_tool_requires = []
         self.platform_requires = []
         self.conf = ConfDefinition()
@@ -120,6 +122,9 @@ def compose_profile(self, other):
                 existing[r.name] = req
             self.tool_requires[pattern] = list(existing.values())
 
+        self.replace_requires.update(other.replace_requires)
+        self.replace_tool_requires.update(other.replace_tool_requires)
+
         current_platform_tool_requires = {r.name: r for r in self.platform_tool_requires}
         current_platform_tool_requires.update({r.name: r for r in other.platform_tool_requires})
         self.platform_tool_requires = list(current_platform_tool_requires.values())

conans/model/requires.py

@@ -467,6 +467,18 @@ def __init__(self, declared=None, declared_build=None, declared_test=None,
                     raise ConanException("Wrong 'tool_requires' definition, "
                                          "did you mean 'build_requirements()'?")
 
+    def reindex(self, require, new_name):
+        """ This operation is necessary when the reference name of a package is changed
+        as a result of an "alternative" replacement of the package name, otherwise the dictionary
+        gets broken by modified key
+        """
+        result = OrderedDict()
+        for k, v in self._requires.items():
+            if k is require:
+                k.ref.name = new_name
+            result[k] = v
+        self._requires = result
+
     def values(self):
         return self._requires.values()
 

conans/test/integration/graph/test_replace_requires.py

@@ -0,0 +1,144 @@
+import json
+
+import pytest
+
+from conans.model.recipe_ref import RecipeReference
+from conans.test.assets.genconanfile import GenConanfile
+from conans.test.utils.tools import TestClient
+
+
+@pytest.mark.parametrize("require, pattern, alternative, pkg", [
+    # PATTERN VERSIONS
+    # override all dependencies to "dep" to a specific version,user and channel)
+    # TODO: This is a version override, is this really wanted?
+    ("dep/1.3", "dep/*", "dep/1.1", "dep/1.1"),
+    ("dep/[>=1.0 <2]", "dep/*", "dep/1.1", "dep/1.1"),
+    # override all dependencies to "dep" to the same version with other user, remove channel)
+    ("dep/1.3", "dep/*", "dep/*@system", "dep/1.3@system"),
+    ("dep/[>=1.0 <2]", "dep/*", "dep/*@system", "dep/1.1@system"),
+    # override all dependencies to "dep" to the same version with other user, same channel)
+    ("dep/1.3@comp/stable", "dep/*@*/*", "dep/*@system/*", "dep/1.3@system/stable"),
+    ("dep/[>=1.0 <2]@comp/stable", "dep/*@*/*", "dep/*@system/*", "dep/1.1@system/stable"),
+    # EXACT VERSIONS
+    # replace exact dependency version for one in the system
+    ("dep/1.1", "dep/1.1", "dep/1.1@system", "dep/1.1@system"),
+    ("dep/[>=1.0 <2]", "dep/1.1", "dep/1.1@system", "dep/1.1@system"),
+    ("dep/[>=1.0 <2]@comp", "dep/1.1@*", "dep/1.1@*/stable", "dep/1.1@comp/stable"),
+    ("dep/1.1@comp", "dep/1.1@*", "dep/1.1@*/stable", "dep/1.1@comp/stable"),
+    # PACKAGE ALTERNATIVES (zlib->zlibng)
+    ("dep/1.0", "dep/*", "depng/*", "depng/1.0"),
+    ("dep/[>=1.0 <2]", "dep/*", "depng/*", "depng/1.1"),
+    ("dep/[>=1.0 <2]", "dep/1.1", "depng/1.2", "depng/1.2"),
+    # NON MATCHING
+    ("dep/1.3", "dep/1.1", "dep/1.1@system", "dep/1.3"),
+    ("dep/1.3", "dep/*@comp", "dep/*@system", "dep/1.3"),
+    ("dep/[>=1.0 <2]", "dep/2.1", "dep/2.1@system", "dep/1.1"),
+])
+@pytest.mark.parametrize("tool_require", [False, True])
+class TestReplaceRequires:
+    def test_alternative(self, tool_require, require, pattern, alternative, pkg):
+        c = TestClient()
+        conanfile = GenConanfile().with_tool_requires(require) if tool_require else \
+            GenConanfile().with_requires(require)
+        profile_tag = "replace_requires" if not tool_require else "replace_tool_requires"
+        c.save({"dep/conanfile.py": GenConanfile(),
+                "pkg/conanfile.py": conanfile,
+                "profile": f"[{profile_tag}]\n{pattern}: {alternative}"})
+        ref = RecipeReference.loads(pkg)
+        user = f"--user={ref.user}" if ref.user else ""
+        channel = f"--channel={ref.channel}" if ref.channel else ""
+        c.run(f"create dep --name={ref.name} --version={ref.version} {user} {channel}")
+        rrev = c.exported_recipe_revision()
+        c.run("install pkg -pr=profile")
+        c.assert_listed_require({f"{pkg}#{rrev}": "Cache"}, build=tool_require)
+
+        # Check lockfile
+        c.run("lock create pkg -pr=profile")
+        lock = c.load("pkg/conan.lock")
+        assert f"{pkg}#{rrev}" in lock
+
+        # c.run("create dep2 --version=1.2")
+        # with lockfile
+        c.run("install pkg -pr=profile")
+        c.assert_listed_require({f"{pkg}#{rrev}": "Cache"}, build=tool_require)
+
+    def test_diamond(self, tool_require, require, pattern, alternative, pkg):
+        c = TestClient()
+        conanfile = GenConanfile().with_tool_requires(require) if tool_require else \
+            GenConanfile().with_requires(require)
+        profile_tag = "replace_requires" if not tool_require else "replace_tool_requires"
+
+        c.save({"dep/conanfile.py": GenConanfile(),
+                "libb/conanfile.py": conanfile,
+                "libc/conanfile.py": conanfile,
+                "app/conanfile.py": GenConanfile().with_requires("libb/0.1", "libc/0.1"),
+                "profile": f"[{profile_tag}]\n{pattern}: {alternative}"})
+        ref = RecipeReference.loads(pkg)
+        user = f"--user={ref.user}" if ref.user else ""
+        channel = f"--channel={ref.channel}" if ref.channel else ""
+        c.run(f"create dep --name={ref.name} --version={ref.version} {user} {channel}")
+        rrev = c.exported_recipe_revision()
+
+        c.run("export libb --name=libb --version=0.1")
+        c.run("export libc --name=libc --version=0.1")
+
+        c.run("install app -pr=profile", assert_error=True)
+        assert "ERROR: Missing binary: libb/0.1" in c.out
+        assert "ERROR: Missing binary: libc/0.1" in c.out
+
+        c.run("install app -pr=profile --build=missing")
+        c.assert_listed_require({f"{pkg}#{rrev}": "Cache"}, build=tool_require)
+
+        # Check lockfile
+        c.run("lock create app -pr=profile")
+        lock = c.load("app/conan.lock")
+        assert f"{pkg}#{rrev}" in lock
+
+        # with lockfile
+        c.run("install app -pr=profile")
+        c.assert_listed_require({f"{pkg}#{rrev}": "Cache"}, build=tool_require)
+
+
+@pytest.mark.parametrize("pattern, replace", [
+    ("pkg", "pkg/0.1"),
+    ("pkg/*", "pkg"),
+    ("pkg/*:pid1", "pkg/0.1"),
+    ("pkg/*:pid1", "pkg/0.1:pid2"),
+    ("pkg/*", "pkg/0.1:pid2"),
+    (":", ""),
+    ("pkg/version:pid", ""),
+    ("pkg/version:pid", ":")
+])
+def test_replace_requires_errors(pattern, replace):
+    c = TestClient()
+    c.save({"pkg/conanfile.py": GenConanfile("pkg", "0.1"),
+            "app/conanfile.py": GenConanfile().with_requires("pkg/0.2"),
+            "profile": f"[replace_requires]\n{pattern}: {replace}"})
+    c.run("create pkg")
+    c.run("install app -pr=profile", assert_error=True)
+    assert "ERROR: Error reading 'profile' profile: Error in [replace_xxx]" in c.out
+
+
+def test_replace_requires_invalid_requires_errors():
+    """
+    replacing for something incorrect not existing is not an error per-se, it is valid that
+    a recipe requires("pkg/2.*"), and then it will fail because such package doesn't exist
+    """
+    c = TestClient()
+    c.save({"app/conanfile.py": GenConanfile().with_requires("pkg/0.2"),
+            "profile": f"[replace_requires]\npkg/0.2: pkg/2.*"})
+    c.run("install app -pr=profile", assert_error=True)
+    assert "pkg/0.2: pkg/2.*" in c.out  # The replacement happens
+    assert "ERROR: Package 'pkg/2.*' not resolved" in c.out
+
+
+def test_replace_requires_json_format():
+    c = TestClient()
+    c.save({"pkg/conanfile.py": GenConanfile("pkg", "0.2"),
+            "app/conanfile.py": GenConanfile().with_requires("pkg/0.1"),
+            "profile": f"[replace_requires]\npkg/0.1: pkg/0.2"})
+    c.run("create pkg")
+    c.run("install app -pr=profile --format=json")
+    assert "pkg/0.1: pkg/0.2" in c.out  # The replacement happens
+    graph = json.loads(c.stdout)
+    assert graph["graph"]["replaced_requires"] == {"pkg/0.1": "pkg/0.2"}