New conan graph explain command to search, compare and explain missing binaries

conan/api/subapi/list.py

@@ -1,8 +1,10 @@
 from typing import Dict
 
 from conan.api.model import PackagesList
+from conan.api.output import ConanOutput
 from conan.internal.conan_app import ConanApp
 from conans.errors import ConanException, NotFoundException
+from conans.model.info import load_binary_info
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
 from conans.search.search import get_cache_packages_binary_info, filter_packages
@@ -164,3 +166,113 @@ def select(self, pattern, package_query=None, remote=None, lru=None):
                 select_bundle.add_prefs(rrev, prefs)
                 select_bundle.add_configurations(packages)
         return select_bundle
+
+    def explain_missing_binaries(self, ref, conaninfo, remotes):
+        ConanOutput().info(f"Missing binary: {ref}")
+        ConanOutput().info(f"With conaninfo.txt (package_id):\n{conaninfo.dumps()}")
+        conaninfo = load_binary_info(conaninfo.dumps())
+        # Collect all configurations
+        candidates = []
+        ConanOutput().info(f"Finding binaries in the cache")
+        pkg_configurations = self.packages_configurations(ref)
+        candidates.extend(_BinaryDistance(pref, data, conaninfo)
+                          for pref, data in pkg_configurations.items())
+
+        for remote in remotes:
+            try:
+                ConanOutput().info(f"Finding binaries in remote {remote.name}")
+                pkg_configurations = self.packages_configurations(ref, remote=remote)
+            except Exception as e:
+                pass
+                ConanOutput(f"ERROR IN REMOTE {remote.name}: {e}")
+            else:
+                candidates.extend(_BinaryDistance(pref, data, conaninfo, remote)
+                                  for pref, data in pkg_configurations.items())
+
+        candidates.sort()
+        pkglist = PackagesList()
+        pkglist.add_refs([ref])
+        # If there are exact matches, only return the matches
+        # else, limit to the number specified
+        candidate_distance = None
+        for candidate in candidates:
+            if candidate_distance and candidate.distance != candidate_distance:
+                break
+            candidate_distance = candidate.distance
+            pref = candidate.pref
+            pkglist.add_prefs(ref, [pref])
+            pkglist.add_configurations({pref: candidate.binary_config})
+            # Add the diff data
+            rev_dict = pkglist.recipes[str(pref.ref)]["revisions"][pref.ref.revision]
+            rev_dict["packages"][pref.package_id]["diff"] = candidate.serialize()
+            remote = candidate.remote.name if candidate.remote else "Local Cache"
+            rev_dict["packages"][pref.package_id]["remote"] = remote
+        return pkglist
+
+
+class _BinaryDistance:
+    def __init__(self, pref, binary_config, expected_config, remote=None):
+        self.remote = remote
+        self.pref = pref
+        self.binary_config = binary_config
+
+        # Settings
+        self.platform_diff = {}
+        self.settings_diff = {}
+        binary_settings = binary_config.get("settings", {})
+        expected_settings = expected_config.get("settings", {})
+        for k, v in expected_settings.items():
+            value = binary_settings.get(k)
+            if value is not None and value != v:
+                diff = self.platform_diff if k in ("os", "arch") else self.settings_diff
+                diff.setdefault("expected", []).append(f"{k}={v}")
+                diff.setdefault("existing", []).append(f"{k}={value}")
+
+        # Options
+        self.options_diff = {}
+        binary_options = binary_config.get("options", {})
+        expected_options = expected_config.get("options", {})
+        for k, v in expected_options.items():
+            value = binary_options.get(k)
+            if value is not None and value != v:
+                self.options_diff.setdefault("expected", []).append(f"{k}={v}")
+                self.options_diff.setdefault("existing", []).append(f"{k}={value}")
+
+        # Requires
+        self.deps_diff = {}
+        binary_requires = binary_config.get("requires", [])
+        expected_requires = expected_config.get("requires", [])
+        binary_requires = [RecipeReference.loads(r) for r in binary_requires]
+        expected_requires = [RecipeReference.loads(r) for r in expected_requires]
+        binary_requires = {r.name: r for r in binary_requires}
+        for r in expected_requires:
+            existing = binary_requires.get(r.name)
+            if not existing or r != existing:
+                self.deps_diff.setdefault("expected", []).append(repr(r))
+                self.deps_diff.setdefault("existing", []).append(repr(existing))
+
+    def __lt__(self, other):
+        return self.distance < other.distance
+
+    def explanation(self):
+        if self.platform_diff:
+            return "This binary belongs to another OS or Architecture, highly incompatible."
+        if self.settings_diff:
+            return "This binary was built with different settings."
+        if self.options_diff:
+            return "This binary was built with the same settings, but different options"
+        if self.deps_diff:
+            return "This binary has same settings and options, but different dependencies"
+        return "This binary is an exact match for the defined inputs"
+
+    @property
+    def distance(self):
+        return len(self.platform_diff), len(self.settings_diff), \
+               len(self.options_diff), len(self.deps_diff)
+
+    def serialize(self):
+        return {"platform": self.platform_diff,
+                "settings": self.settings_diff,
+                "options": self.options_diff,
+                "dependencies": self.deps_diff,
+                "explanation": self.explanation()}

conan/cli/commands/graph.py

@@ -1,15 +1,33 @@
 import json
 import os
+
+from conan.api.model import MultiPackagesList
 from conan.api.output import ConanOutput, cli_out_write, Color
 from conan.cli import make_abs_path
 from conan.cli.args import common_graph_args, validate_common_graph_args
 from conan.cli.command import conan_command, conan_subcommand
+from conan.cli.commands.list import print_list_compact, print_list_json
 from conan.cli.formatters.graph import format_graph_html, format_graph_json, format_graph_dot
 from conan.cli.formatters.graph.graph_info_text import format_graph_info
 from conan.cli.printers.graph import print_graph_packages, print_graph_basic
+from conan.errors import ConanException
 from conan.internal.deploy import do_deploys
+from conans.client.graph.graph import BINARY_MISSING
 from conans.client.graph.install_graph import InstallGraph
-from conan.errors import ConanException
+from conans.model.recipe_ref import ref_matches
+
+
+def explain_formatter_text(data):
+    if "Closest binaries" in data:
+        # To be able to reuse the print_list_compact method,
+        # we need to wrap this in a MultiPackagesList
+        multipackage = MultiPackagesList()
+        multipackage.add("Closest binaries", data["Closest binaries"])
+        print_list_compact({"results": multipackage.serialize()})
+
+
+def explain_formatter_json(data):
+    print_list_json({"results": data})
 
 
 @conan_command(group="Consumer")
@@ -179,3 +197,75 @@ def graph_info(conan_api, parser, subparser, *args):
             "field_filter": args.filter,
             "package_filter": args.package_filter,
             "conan_api": conan_api}
+
+
+@conan_subcommand(formatters={"text": explain_formatter_text,
+                              "json": explain_formatter_json})
+def graph_explain(conan_api, parser,  subparser, *args):
+    """
+    Explain what is wrong with the dependency graph, like report missing binaries closest
+    alternatives, trying to explain why the existing binaries do not match
+    """
+    common_graph_args(subparser)
+    subparser.add_argument("--check-updates", default=False, action="store_true",
+                           help="Check if there are recipe updates")
+    subparser.add_argument("--build-require", action='store_true', default=False,
+                           help='Whether the provided reference is a build-require')
+    subparser.add_argument('--missing', nargs="?",
+                           help="A pattern in the form 'pkg/version#revision:package_id#revision', "
+                                "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. "
+                                "If revision is not specified, it is assumed latest one.")
+
+    args = parser.parse_args(*args)
+    # parameter validation
+    validate_common_graph_args(args)
+
+    cwd = os.getcwd()
+    path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None
+
+    # Basic collaborators, remotes, lockfile, profiles
+    remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None
+    lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile,
+                                               conanfile_path=path,
+                                               cwd=cwd,
+                                               partial=args.lockfile_partial,
+                                               overrides=overrides)
+    profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args)
+
+    if path:
+        deps_graph = conan_api.graph.load_graph_consumer(path, args.name, args.version,
+                                                         args.user, args.channel,
+                                                         profile_host, profile_build, lockfile,
+                                                         remotes, args.update,
+                                                         check_updates=args.check_updates,
+                                                         is_build_require=args.build_require)
+    else:
+        deps_graph = conan_api.graph.load_graph_requires(args.requires, args.tool_requires,
+                                                         profile_host, profile_build, lockfile,
+                                                         remotes, args.update,
+                                                         check_updates=args.check_updates)
+    print_graph_basic(deps_graph)
+    deps_graph.report_graph_error()
+    conan_api.graph.analyze_binaries(deps_graph, args.build, remotes=remotes, update=args.update,
+                                     lockfile=lockfile)
+    print_graph_packages(deps_graph)
+
+    ConanOutput().title("Retrieving and computing closest binaries")
+    # compute ref and conaninfo
+    missing = args.missing
+    for node in deps_graph.ordered_iterate():
+        if node.binary == BINARY_MISSING:
+            if not missing or ref_matches(node.ref, missing, is_consumer=None):
+                ref = node.ref
+                conaninfo = node.conanfile.info
+                break
+    else:
+        raise ConanException("There is no missing binary")
+
+    pkglist = conan_api.list.explain_missing_binaries(ref, conaninfo, remotes)
+
+    ConanOutput().title("Closest binaries")
+    return {
+        "Closest binaries": pkglist.serialize(),
+    }

conan/cli/commands/list.py

@@ -35,6 +35,8 @@ def print_serial(item, indent=None, color_index=None):
                 elif k.lower() == "warning":
                     color = Color.BRIGHT_YELLOW
                     k = "WARN"
+                color = Color.BRIGHT_RED if k == "expected" else color
+                color = Color.BRIGHT_GREEN if k == "existing" else color
                 cli_out_write(f"{indent}{k}: {v}", fg=color)
             else:
                 cli_out_write(f"{indent}{k}", fg=color)
@@ -157,6 +159,20 @@ def compact_format_info(local_info, common_options=None):
                 result[k] = v
         return result
 
+    def compact_diff(diffinfo):
+        """ return a compact and red/green diff for binary differences
+        """
+        result = {}
+        for k, v in diffinfo.items():
+            if not v:
+                continue
+            if isinstance(v, dict):
+                result[k] = {"expected": ", ".join(value for value in v["expected"]),
+                             "existing": ", ".join(value for value in v["existing"])}
+            else:
+                result[k] = v
+        return result
+
     for remote, remote_info in info.items():
         for ref, revisions in remote_info.items():
             if not isinstance(revisions, dict):
@@ -167,6 +183,9 @@ def compact_format_info(local_info, common_options=None):
                 for pref, pref_contents in prefs.items():
                     pref_info = pref_contents.pop("info")
                     pref_contents.update(compact_format_info(pref_info, pkg_common_options))
+                    diff_info = pref_contents.pop("diff", None)
+                    if diff_info is not None:
+                        pref_contents["diff"] = compact_diff(diff_info)
 
     print_serial(info)
 

conans/client/graph/install_graph.py

@@ -326,12 +326,13 @@ def _raise_missing(self, missing):
             else:
                 build_str = " ".join(list(sorted(["--build=%s" % str(pref.ref)
                                                   for pref in missing_prefs])))
-            build_msg = f"or try to build locally from sources using the '{build_str}' argument"
+            build_msg = f"Try to build locally from sources using the '{build_str}' argument"
 
         raise ConanException(textwrap.dedent(f'''\
-           Missing prebuilt package for '{missing_pkgs}'
-           Check the available packages using 'conan list {ref}:* -r=remote'
-           {build_msg}
+           Missing prebuilt package for '{missing_pkgs}'. You can try:
+               - List all available packages using 'conan list {ref}:* -r=remote'
+               - Explain missing binaries: replace 'conan install ...' with 'conan graph explain ...'
+               - {build_msg}
 
            More Info at 'https://docs.conan.io/2/knowledge/faq.html#error-missing-prebuilt-package'
            '''))

conans/test/functional/only_source_test.py

@@ -26,18 +26,18 @@ def test_conan_test(self):
         # Will Fail because hello0/0.0 and hello1/1.1 has not built packages
         # and by default no packages are built
         client.run("create . --user=lasote --channel=stable", assert_error=True)
-        self.assertIn("or try to build locally from sources using the '--build=hello0/0.0@lasote/stable "
+        self.assertIn("Try to build locally from sources using the '--build=hello0/0.0@lasote/stable "
                       "--build=hello1/1.1@lasote/stable'",
                       client.out)
         # Only 1 reference!
-        assert "Check the available packages using 'conan list hello0/0.0@lasote/stable:* -r=remote'" in client.out
+        assert "List all available packages using 'conan list hello0/0.0@lasote/stable:* -r=remote'" in client.out
 
         # We generate the package for hello0/0.0
         client.run("install --requires=hello0/0.0@lasote/stable --build hello0*")
 
         # Still missing hello1/1.1
         client.run("create . --user=lasote --channel=stable", assert_error=True)
-        self.assertIn("or try to build locally from sources using the "
+        self.assertIn("Try to build locally from sources using the "
                       "'--build=hello1/1.1@lasote/stable'", client.out)
 
         # We generate the package for hello1/1.1

conans/test/integration/command/install/install_missing_dep_test.py

@@ -52,4 +52,4 @@ def test_missing_multiple_dep(self):
         client.save({"conanfile.py": conanfile}, clean_first=True)
         client.run("create . --name=pkg --version=1.0", assert_error=True)
         self.assertIn("ERROR: Missing prebuilt package for 'dep1/1.0', 'dep2/1.0'", client.out)
-        self.assertIn("or try to build locally from sources using the '--build=dep1/1.0 --build=dep2/1.0'", client.out)
+        self.assertIn("Try to build locally from sources using the '--build=dep1/1.0 --build=dep2/1.0'", client.out)