From b51227b744c3db5276280198a85caa3b0ddc49e3 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 13:51:56 +0200 Subject: [PATCH 01/12] fix test_package --build=missing, now it works --- conans/client/graph/graph_binaries.py | 1 + .../test/integration/command/create_test.py | 29 +++++++++++++++++++ .../integration/command/test_package_test.py | 10 +++---- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 3e1050ff3aa..ff22f9b0673 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -308,6 +308,7 @@ def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): self._selected_remotes = remotes or [] # TODO: A bit dirty interfaz, pass as arg instead self._update = update # TODO: Dirty, fix it test_package = deps_graph.root.conanfile.tested_reference_str is not None + test_package = False # HARDCODED disabled feature, need to properly remove it if test_package: main_mode = BuildMode(["never"]) test_mode = BuildMode(build_mode) diff --git a/conans/test/integration/command/create_test.py b/conans/test/integration/command/create_test.py index da88dc82891..5c462a45253 100644 --- a/conans/test/integration/command/create_test.py +++ b/conans/test/integration/command/create_test.py @@ -702,3 +702,32 @@ def test_name_never(): c.save({"conanfile.py": GenConanfile("never", "0.1")}) c.run("create .") assert "never/0.1: Created package" in c.out + + +def test_create_both_host_build_require(): + c = TestClient() + c.save({"conanfile.py": GenConanfile("protobuf", "0.1").with_settings("build_type"), + "test_package/conanfile.py": GenConanfile().with_build_requires("protobuf/0.1") + .with_test("pass")}) + c.run("create . -s:b build_type=Release -s:h build_type=Debug", assert_error=True) + print(c.out) + # The main "host" Debug binary will be correctly build + c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Build")}) + # But test_package will fail because of the missing "tool_require" in Release + c.assert_listed_binary({"protobuf/0.1": ("efa83b160a55b033c4ea706ddb980cd708e3ba1b", "Missing")}, + build=True, test_package=True) + + c.run("remove * -c") # make sure that previous binary is removed + c.run("create . -s:b build_type=Release -s:h build_type=Debug --build=missing") + c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Build")}) + # it used to fail, now it works and builds the test_package "tools_requires" in Release + c.assert_listed_binary({"protobuf/0.1": ("efa83b160a55b033c4ea706ddb980cd708e3ba1b", "Build")}, + build=True, test_package=True) + + # we can be more explicit about the current package only with "missing:protobuf/*" + c.run("remove * -c") # make sure that previous binary is removed + c.run("create . -s:b build_type=Release -s:h build_type=Debug --build=missing:protobuf/*") + c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Build")}) + # it used to fail, now it works and builds the test_package "tools_requires" in Release + c.assert_listed_binary({"protobuf/0.1": ("efa83b160a55b033c4ea706ddb980cd708e3ba1b", "Build")}, + build=True, test_package=True) diff --git a/conans/test/integration/command/test_package_test.py b/conans/test/integration/command/test_package_test.py index 48a03bcfa5f..18c74b565f7 100644 --- a/conans/test/integration/command/test_package_test.py +++ b/conans/test/integration/command/test_package_test.py @@ -131,14 +131,15 @@ def test_build_all(self): .with_test("pass")}) c.run("export tool") c.run("export dep") + # This will rebuild everything, including test_package requires c.run("create pkg --build=*") c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) c.assert_listed_require({"tool/0.1": "(tp) Cache"}, build=True, test_package=True) c.assert_listed_binary({"tool/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}, build=True, test_package=True) - c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"), - "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Cache")}, + c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), + "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}, test_package=True) def test_build_missing(self): @@ -399,6 +400,5 @@ def test_package_missing_binary_msg(): c.run("test test_package dep/0.1", assert_error=True) assert "ERROR: Missing binary: dep/0.1" in c.out assert "'conan test' tested packages must exist" in c.out - c.run("test test_package dep/0.1 --build=dep/0.1", assert_error=True) - assert "ERROR: Missing binary: dep/0.1" in c.out - assert "'conan test' tested packages must exist" in c.out + c.run("test test_package dep/0.1 --build=dep/0.1") + c.assert_listed_binary({"dep/0.1": (NO_SETTINGS_PACKAGE_ID, "Build")}) From 271c9752298e7f8707398ab35dde3a0faacd80e4 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 17:21:42 +0200 Subject: [PATCH 02/12] wip --- conans/client/graph/graph_binaries.py | 15 ++++++++------- .../test/integration/command/test_package_test.py | 6 +++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index ff22f9b0673..2e7a1819032 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -307,13 +307,14 @@ def _evaluate_package_id(self, node): def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): self._selected_remotes = remotes or [] # TODO: A bit dirty interfaz, pass as arg instead self._update = update # TODO: Dirty, fix it - test_package = deps_graph.root.conanfile.tested_reference_str is not None - test_package = False # HARDCODED disabled feature, need to properly remove it - if test_package: - main_mode = BuildMode(["never"]) - test_mode = BuildMode(build_mode) - else: - main_mode = test_mode = BuildMode(build_mode) + + main_mode = test_mode = None + if build_mode: + main_mode = [m for m in build_mode if not m.startswith("test:")] + test_mode = [m for m in build_mode if not m.startswith("test:")] + main_mode = BuildMode(main_mode) + test_mode = BuildMode(test_mode) + if main_mode.cascade: ConanOutput().warning("Using build-mode 'cascade' is generally inefficient and it " "shouldn't be used. Use 'package_id' and 'package_id_modes' for" diff --git a/conans/test/integration/command/test_package_test.py b/conans/test/integration/command/test_package_test.py index 18c74b565f7..28496ff4a0e 100644 --- a/conans/test/integration/command/test_package_test.py +++ b/conans/test/integration/command/test_package_test.py @@ -131,15 +131,15 @@ def test_build_all(self): .with_test("pass")}) c.run("export tool") c.run("export dep") - # This will rebuild everything, including test_package requires c.run("create pkg --build=*") c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) c.assert_listed_require({"tool/0.1": "(tp) Cache"}, build=True, test_package=True) c.assert_listed_binary({"tool/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}, build=True, test_package=True) - c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), - "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}, + # FIXME: Re-building already built things is unaceptable + c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"), + "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Cache")}, test_package=True) def test_build_missing(self): From a5d6dac4aad1f1aa2537ca3203403099d04d6078 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 17:26:28 +0200 Subject: [PATCH 03/12] removed unused .all in BuildMode --- conans/client/graph/build_mode.py | 58 ++++++++++++++----------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/conans/client/graph/build_mode.py b/conans/client/graph/build_mode.py index fd556fbcd6a..ad70da848af 100644 --- a/conans/client/graph/build_mode.py +++ b/conans/client/graph/build_mode.py @@ -4,11 +4,9 @@ class BuildMode: - """ build_mode => ["*"] if user wrote "--build" - => ["hello*", "bye*"] if user wrote "--build hello --build bye" + """ build_mode => ["*"] if user wrote "--build=*" + => ["hello", "bye"] if user wrote "--build hello --build bye" => ["hello/0.1@foo/bar"] if user wrote "--build hello/0.1@foo/bar" - => False if user wrote "never" - => True if user wrote "missing" => ["!foo"] or ["~foo"] means exclude when building all from sources """ def __init__(self, params): @@ -20,38 +18,36 @@ def __init__(self, params): self.build_missing_patterns = [] self._unused_patterns = [] self._excluded_patterns = [] - self.all = False if params is None: return assert isinstance(params, list) - if len(params) == 0: - self.all = True - else: - for param in params: - if param == "missing": - self.missing = True - elif param == "editable": - self.editable = True - elif param == "never": - self.never = True - elif param == "cascade": - self.cascade = True + assert len(params) > 0 # Not empty list + + for param in params: + if param == "missing": + self.missing = True + elif param == "editable": + self.editable = True + elif param == "never": + self.never = True + elif param == "cascade": + self.cascade = True + else: + if param.startswith("missing:"): + clean_pattern = param[len("missing:"):] + clean_pattern = clean_pattern[:-1] if param.endswith("@") else clean_pattern + clean_pattern = clean_pattern.replace("@#", "#") + self.build_missing_patterns.append(clean_pattern) else: - if param.startswith("missing:"): - clean_pattern = param[len("missing:"):] - clean_pattern = clean_pattern[:-1] if param.endswith("@") else clean_pattern - clean_pattern = clean_pattern.replace("@#", "#") - self.build_missing_patterns.append(clean_pattern) + # Remove the @ at the end, to match for + # "conan install --requires=pkg/0.1@ --build=pkg/0.1@" + clean_pattern = param[:-1] if param.endswith("@") else param + clean_pattern = clean_pattern.replace("@#", "#") + if clean_pattern and clean_pattern[0] in ["!", "~"]: + self._excluded_patterns.append(clean_pattern[1:]) else: - # Remove the @ at the end, to match for - # "conan install --requires=pkg/0.1@ --build=pkg/0.1@" - clean_pattern = param[:-1] if param.endswith("@") else param - clean_pattern = clean_pattern.replace("@#", "#") - if clean_pattern and clean_pattern[0] in ["!", "~"]: - self._excluded_patterns.append(clean_pattern[1:]) - else: - self.patterns.append(clean_pattern) + self.patterns.append(clean_pattern) if self.never and (self.missing or self.patterns or self.cascade): raise ConanException("--build=never not compatible with other options") @@ -74,8 +70,6 @@ def forced(self, conan_file, ref, with_deps_to_build=False): if self.never: return False - if self.all: - return True if conan_file.build_policy == "always": raise ConanException("{}: build_policy='always' has been removed. " From f6eccefd9b9d792db93019a0df4bac1baaa83f71 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 17:33:32 +0200 Subject: [PATCH 04/12] fix unittest --- conans/test/unittests/client/graph/build_mode_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/test/unittests/client/graph/build_mode_test.py b/conans/test/unittests/client/graph/build_mode_test.py index c65a19b3113..3e54d78205a 100644 --- a/conans/test/unittests/client/graph/build_mode_test.py +++ b/conans/test/unittests/client/graph/build_mode_test.py @@ -134,7 +134,7 @@ def test_allowed(conanfile): build_mode = BuildMode(["missing"]) assert build_mode.allowed(conanfile) is True - build_mode = BuildMode([]) + build_mode = BuildMode(None) assert build_mode.allowed(conanfile) is False From 4784414718ec8063e55fb1b1c9564f5402d65c46 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 17:39:38 +0200 Subject: [PATCH 05/12] wip --- conan/cli/printers/graph.py | 10 +++---- conans/client/graph/graph.py | 1 - conans/client/graph/graph_binaries.py | 2 +- conans/client/graph/graph_builder.py | 29 ------------------- .../integration/command/test_package_test.py | 4 +-- 5 files changed, 7 insertions(+), 39 deletions(-) diff --git a/conan/cli/printers/graph.py b/conan/cli/printers/graph.py index 07efd738a69..293975c768e 100644 --- a/conan/cli/printers/graph.py +++ b/conan/cli/printers/graph.py @@ -22,12 +22,12 @@ def print_graph_basic(graph): if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL): continue if node.context == CONTEXT_BUILD: - build_requires[node.ref] = node.recipe, node.remote, node.test_package + build_requires[node.ref] = node.recipe, node.remote else: if node.test: - test_requires[node.ref] = node.recipe, node.remote, node.test_package + test_requires[node.ref] = node.recipe, node.remote else: - requires[node.ref] = node.recipe, node.remote, node.test_package + requires[node.ref] = node.recipe, node.remote if node.conanfile.deprecated: deprecated[node.ref] = node.conanfile.deprecated @@ -39,11 +39,9 @@ def _format_requires(title, reqs_to_print): if not reqs_to_print: return output.info(title, Color.BRIGHT_YELLOW) - for ref, (recipe, remote, test_package) in sorted(reqs_to_print.items()): + for ref, (recipe, remote) in sorted(reqs_to_print.items()): if remote is not None: recipe = "{} ({})".format(recipe, remote.name) - if test_package: - recipe = f"(tp) {recipe}" output.info(" {} - {}".format(ref.repr_notime(), recipe), Color.BRIGHT_CYAN) _format_requires("Requirements", requires) diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index ff2aa3b0c12..91f588712e0 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -57,7 +57,6 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None, test=False): self.binary_remote = None self.context = context self.test = test - self.test_package = False # True if it is a test_package only package # real graph model self.transitive_deps = OrderedDict() # of _TransitiveRequirement diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 2e7a1819032..266443f3392 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -311,7 +311,7 @@ def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): main_mode = test_mode = None if build_mode: main_mode = [m for m in build_mode if not m.startswith("test:")] - test_mode = [m for m in build_mode if not m.startswith("test:")] + test_mode = [m[5:] for m in build_mode if m.startswith("test:")] main_mode = BuildMode(main_mode) test_mode = BuildMode(test_mode) diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index 5449e637ff0..f57d2914975 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -55,7 +55,6 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None): for r in reversed(new_node.conanfile.requires.values())) self._remove_overrides(dep_graph) check_graph_provides(dep_graph) - self._compute_test_package_deps(dep_graph) except GraphError as e: dep_graph.error = e dep_graph.resolved_ranges = self._resolver.resolved_ranges @@ -310,31 +309,3 @@ def _remove_overrides(dep_graph): to_remove = [r for r in node.transitive_deps if r.override] for r in to_remove: node.transitive_deps.pop(r) - - @staticmethod - def _compute_test_package_deps(graph): - """ compute and tag the graph nodes that belong exclusively to test_package - dependencies but not the main graph - """ - root_node = graph.root - tested_ref = root_node.conanfile.tested_reference_str - if tested_ref is None: - return - tested_ref = RecipeReference.loads(root_node.conanfile.tested_reference_str) - tested_ref = str(tested_ref) - # We classify direct dependencies in the "tested" main ones and the "test_package" specific - direct_nodes = [n.node for n in root_node.transitive_deps.values() if n.require.direct] - main_nodes = [n for n in direct_nodes if tested_ref == str(n.ref)] - test_package_nodes = [n for n in direct_nodes if tested_ref != str(n.ref)] - - # Accumulate the transitive dependencies of the 2 subgraphs ("main", and "test_package") - main_graph_nodes = set(main_nodes) - for n in main_nodes: - main_graph_nodes.update(t.node for t in n.transitive_deps.values()) - test_graph_nodes = set(test_package_nodes) - for n in test_package_nodes: - test_graph_nodes.update(t.node for t in n.transitive_deps.values()) - # Some dependencies in "test_package" might be "main" graph too, "main" prevails - test_package_only = test_graph_nodes.difference(main_graph_nodes) - for t in test_package_only: - t.test_package = True diff --git a/conans/test/integration/command/test_package_test.py b/conans/test/integration/command/test_package_test.py index 28496ff4a0e..9ae610303c2 100644 --- a/conans/test/integration/command/test_package_test.py +++ b/conans/test/integration/command/test_package_test.py @@ -134,7 +134,7 @@ def test_build_all(self): c.run("create pkg --build=*") c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) - c.assert_listed_require({"tool/0.1": "(tp) Cache"}, build=True, test_package=True) + c.assert_listed_require({"tool/0.1": "Cache"}, build=True, test_package=True) c.assert_listed_binary({"tool/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}, build=True, test_package=True) # FIXME: Re-building already built things is unaceptable @@ -154,7 +154,7 @@ def test_build_missing(self): c.run("create pkg --build=missing") c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) - c.assert_listed_require({"tool/0.1": "(tp) Cache"}, build=True, test_package=True) + c.assert_listed_require({"tool/0.1": "Cache"}, build=True, test_package=True) c.assert_listed_binary({"tool/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}, build=True, test_package=True) c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"), From 4c7b1c4b7e8a1b775b6b80190c9b81e3faa20cfa Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 17:40:29 +0200 Subject: [PATCH 06/12] change order build_mode --- conans/client/graph/graph_binaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 3e1050ff3aa..0fc973ad4c1 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -318,7 +318,6 @@ def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): "shouldn't be used. Use 'package_id' and 'package_id_modes' for" "more efficient re-builds") for node in deps_graph.ordered_iterate(): - build_mode = test_mode if node.test_package else main_mode if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL): if node.path is not None and node.path.endswith(".py"): # For .py we keep evaluating the package_id, validate(), etc @@ -331,6 +330,7 @@ def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): node.conanfile.layout() else: self._evaluate_package_id(node) + build_mode = test_mode if node.test_package else main_mode if lockfile: locked_prev = lockfile.resolve_prev(node) if locked_prev: From 935954f4f3fc4ad27f58781a4aeee9deef0ce00b Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 17:58:00 +0200 Subject: [PATCH 07/12] wip --- conan/api/subapi/graph.py | 6 ++++-- conan/cli/commands/create.py | 3 ++- conan/cli/commands/test.py | 4 ++-- conans/client/graph/graph_binaries.py | 6 ++++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index 92794fa2e6b..0a6ab2fd019 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -170,7 +170,8 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) return deps_graph - def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, lockfile=None): + def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, lockfile=None, + tested_graph=None): """ Given a dependency graph, will compute the package_ids of all recipes in the graph, and evaluate if they should be built from sources, downloaded from a remote server, of if the packages are already in the local Conan cache @@ -181,8 +182,9 @@ def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, lo :param remotes: list of remotes :param update: (False by default), if Conan should look for newer versions or revisions for already existing recipes in the Conan cache + :param tested_graph: In case of a "test_package", the graph being tested """ ConanOutput().title("Computing necessary packages") conan_app = ConanApp(self.conan_api.cache_folder) binaries_analyzer = GraphBinariesAnalyzer(conan_app) - binaries_analyzer.evaluate_graph(graph, build_mode, lockfile, remotes, update) + binaries_analyzer.evaluate_graph(graph, build_mode, lockfile, remotes, update, tested_graph) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 3c0aefa66de..84553c8e878 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -91,7 +91,8 @@ def create(conan_api, parser, *args): # The test_package do not make the "conan create" command return a different graph or # produce a different lockfile. The result is always the same, irrespective of test_package run_test(conan_api, test_conanfile_path, ref, profile_host, profile_build, remotes, lockfile, - update=False, build_modes=args.build, tested_python_requires=tested_python_requires) + update=False, build_modes=args.build, tested_python_requires=tested_python_requires, + tested_graph=deps_graph) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) return {"graph": deps_graph, diff --git a/conan/cli/commands/test.py b/conan/cli/commands/test.py index 1f9af9fdd20..f3113420dd3 100644 --- a/conan/cli/commands/test.py +++ b/conan/cli/commands/test.py @@ -48,7 +48,7 @@ def test(conan_api, parser, *args): def run_test(conan_api, path, ref, profile_host, profile_build, remotes, lockfile, update, - build_modes, tested_python_requires=None): + build_modes, tested_python_requires=None, tested_graph=None): root_node = conan_api.graph.load_root_test_conanfile(path, ref, profile_host, profile_build, remotes=remotes, @@ -68,7 +68,7 @@ def run_test(conan_api, path, ref, profile_host, profile_build, remotes, lockfil deps_graph.report_graph_error() conan_api.graph.analyze_binaries(deps_graph, build_modes, remotes=remotes, update=update, - lockfile=lockfile) + lockfile=lockfile, tested_graph=tested_graph) print_graph_packages(deps_graph) conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 998bf10be46..ea97f7ac662 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -304,7 +304,7 @@ def _evaluate_package_id(self, node): with conanfile_exception_formatter(conanfile, "layout"): conanfile.layout() - def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): + def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update, tested_graph=None): self._selected_remotes = remotes or [] # TODO: A bit dirty interfaz, pass as arg instead self._update = update # TODO: Dirty, fix it @@ -314,6 +314,7 @@ def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): test_mode = [m[5:] for m in build_mode if m.startswith("test:")] main_mode = BuildMode(main_mode) test_mode = BuildMode(test_mode) + mainprefs = [str(n.pref) for n in tested_graph.nodes] if tested_graph is not None else None if main_mode.cascade: ConanOutput().warning("Using build-mode 'cascade' is generally inefficient and it " @@ -332,7 +333,8 @@ def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update): node.conanfile.layout() else: self._evaluate_package_id(node) - build_mode = test_mode if node.test_package else main_mode + build_mode = main_mode if mainprefs is None or str(node.pref) in mainprefs \ + else test_mode if lockfile: locked_prev = lockfile.resolve_prev(node) if locked_prev: From 007c54e3f7b961517bd44e3e6d956625ae303c5d Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 18:00:15 +0200 Subject: [PATCH 08/12] fix tests --- conans/test/integration/graph/core/graph_manager_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/test/integration/graph/core/graph_manager_base.py b/conans/test/integration/graph/core/graph_manager_base.py index 9cafafcd694..f6b559545e5 100644 --- a/conans/test/integration/graph/core/graph_manager_base.py +++ b/conans/test/integration/graph/core/graph_manager_base.py @@ -115,7 +115,7 @@ def build_consumer(self, path, profile_build_requires=None, ref=None, create_ref profile_build.options = Options(options_values=options_build) profile_host.process_settings(self.cache) profile_build.process_settings(self.cache) - build_mode = [] # Means build all + build_mode = ["*"] # Means build all conan_api = ConanAPI(cache_folder=self.cache_folder) From 605c8126a7650e08e51b0c7b659b8dae5833d690 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 21 Jul 2023 19:15:10 +0200 Subject: [PATCH 09/12] working --- conan/api/subapi/graph.py | 6 ++++-- conan/cli/commands/create.py | 6 ++++-- conan/cli/commands/test.py | 5 +++-- conan/cli/printers/graph.py | 2 +- conans/client/graph/graph_binaries.py | 19 +++++++++++-------- .../test/integration/command/create_test.py | 7 ++++--- .../integration/command/test_package_test.py | 7 ++++--- 7 files changed, 31 insertions(+), 21 deletions(-) diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index 0a6ab2fd019..4b663342c9f 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -171,7 +171,7 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo return deps_graph def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, lockfile=None, - tested_graph=None): + build_modes_test=None, tested_graph=None): """ Given a dependency graph, will compute the package_ids of all recipes in the graph, and evaluate if they should be built from sources, downloaded from a remote server, of if the packages are already in the local Conan cache @@ -182,9 +182,11 @@ def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, lo :param remotes: list of remotes :param update: (False by default), if Conan should look for newer versions or revisions for already existing recipes in the Conan cache + :param build_modes_test: the --build-test argument :param tested_graph: In case of a "test_package", the graph being tested """ ConanOutput().title("Computing necessary packages") conan_app = ConanApp(self.conan_api.cache_folder) binaries_analyzer = GraphBinariesAnalyzer(conan_app) - binaries_analyzer.evaluate_graph(graph, build_mode, lockfile, remotes, update, tested_graph) + binaries_analyzer.evaluate_graph(graph, build_mode, lockfile, remotes, update, + build_modes_test, tested_graph) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 84553c8e878..1fc0b4d2a33 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -25,6 +25,8 @@ def create(conan_api, parser, *args): parser.add_argument("-tf", "--test-folder", action=OnceArgument, help='Alternative test folder name. By default it is "test_package". ' 'Use "" to skip the test stage') + parser.add_argument("-bt", "--build-test", action="append", + help="Same as '--build' but only for the test_package requires") args = parser.parse_args(*args) cwd = os.getcwd() @@ -91,8 +93,8 @@ def create(conan_api, parser, *args): # The test_package do not make the "conan create" command return a different graph or # produce a different lockfile. The result is always the same, irrespective of test_package run_test(conan_api, test_conanfile_path, ref, profile_host, profile_build, remotes, lockfile, - update=False, build_modes=args.build, tested_python_requires=tested_python_requires, - tested_graph=deps_graph) + update=False, build_modes=args.build, build_modes_test=args.build_test, + tested_python_requires=tested_python_requires, tested_graph=deps_graph) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) return {"graph": deps_graph, diff --git a/conan/cli/commands/test.py b/conan/cli/commands/test.py index f3113420dd3..d8f2093fb78 100644 --- a/conan/cli/commands/test.py +++ b/conan/cli/commands/test.py @@ -48,7 +48,7 @@ def test(conan_api, parser, *args): def run_test(conan_api, path, ref, profile_host, profile_build, remotes, lockfile, update, - build_modes, tested_python_requires=None, tested_graph=None): + build_modes, tested_python_requires=None, build_modes_test=None, tested_graph=None): root_node = conan_api.graph.load_root_test_conanfile(path, ref, profile_host, profile_build, remotes=remotes, @@ -68,7 +68,8 @@ def run_test(conan_api, path, ref, profile_host, profile_build, remotes, lockfil deps_graph.report_graph_error() conan_api.graph.analyze_binaries(deps_graph, build_modes, remotes=remotes, update=update, - lockfile=lockfile, tested_graph=tested_graph) + lockfile=lockfile, build_modes_test=build_modes_test, + tested_graph=tested_graph) print_graph_packages(deps_graph) conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) diff --git a/conan/cli/printers/graph.py b/conan/cli/printers/graph.py index 293975c768e..d5aa8248305 100644 --- a/conan/cli/printers/graph.py +++ b/conan/cli/printers/graph.py @@ -18,7 +18,7 @@ def print_graph_basic(graph): for node in graph.nodes: if hasattr(node.conanfile, "python_requires"): for r in node.conanfile.python_requires._pyrequires.values(): # TODO: improve interface - python_requires[r.ref] = r.recipe, r.remote, False + python_requires[r.ref] = r.recipe, r.remote if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL): continue if node.context == CONTEXT_BUILD: diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index ea97f7ac662..3c4588a6ea4 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -304,17 +304,20 @@ def _evaluate_package_id(self, node): with conanfile_exception_formatter(conanfile, "layout"): conanfile.layout() - def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update, tested_graph=None): + def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update, build_mode_test=None, + tested_graph=None): self._selected_remotes = remotes or [] # TODO: A bit dirty interfaz, pass as arg instead self._update = update # TODO: Dirty, fix it - main_mode = test_mode = None - if build_mode: - main_mode = [m for m in build_mode if not m.startswith("test:")] - test_mode = [m[5:] for m in build_mode if m.startswith("test:")] - main_mode = BuildMode(main_mode) - test_mode = BuildMode(test_mode) - mainprefs = [str(n.pref) for n in tested_graph.nodes] if tested_graph is not None else None + if tested_graph is None: + main_mode = BuildMode(build_mode) + test_mode = None # Should not be used at all + mainprefs = None + else: + main_mode = BuildMode(["never"]) + test_mode = BuildMode(build_mode_test) + mainprefs = [str(n.pref) for n in tested_graph.nodes + if n.recipe not in (RECIPE_CONSUMER, RECIPE_VIRTUAL)] if main_mode.cascade: ConanOutput().warning("Using build-mode 'cascade' is generally inefficient and it " diff --git a/conans/test/integration/command/create_test.py b/conans/test/integration/command/create_test.py index 5c462a45253..68f69c2c9a5 100644 --- a/conans/test/integration/command/create_test.py +++ b/conans/test/integration/command/create_test.py @@ -710,7 +710,6 @@ def test_create_both_host_build_require(): "test_package/conanfile.py": GenConanfile().with_build_requires("protobuf/0.1") .with_test("pass")}) c.run("create . -s:b build_type=Release -s:h build_type=Debug", assert_error=True) - print(c.out) # The main "host" Debug binary will be correctly build c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Build")}) # But test_package will fail because of the missing "tool_require" in Release @@ -718,15 +717,17 @@ def test_create_both_host_build_require(): build=True, test_package=True) c.run("remove * -c") # make sure that previous binary is removed - c.run("create . -s:b build_type=Release -s:h build_type=Debug --build=missing") + c.run("create . -s:b build_type=Release -s:h build_type=Debug --build-test=missing") c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Build")}) # it used to fail, now it works and builds the test_package "tools_requires" in Release + c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Cache")}, + test_package=True) c.assert_listed_binary({"protobuf/0.1": ("efa83b160a55b033c4ea706ddb980cd708e3ba1b", "Build")}, build=True, test_package=True) # we can be more explicit about the current package only with "missing:protobuf/*" c.run("remove * -c") # make sure that previous binary is removed - c.run("create . -s:b build_type=Release -s:h build_type=Debug --build=missing:protobuf/*") + c.run("create . -s:b build_type=Release -s:h build_type=Debug --build-test=missing:protobuf/*") c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Build")}) # it used to fail, now it works and builds the test_package "tools_requires" in Release c.assert_listed_binary({"protobuf/0.1": ("efa83b160a55b033c4ea706ddb980cd708e3ba1b", "Build")}, diff --git a/conans/test/integration/command/test_package_test.py b/conans/test/integration/command/test_package_test.py index 9ae610303c2..5c590665076 100644 --- a/conans/test/integration/command/test_package_test.py +++ b/conans/test/integration/command/test_package_test.py @@ -131,13 +131,14 @@ def test_build_all(self): .with_test("pass")}) c.run("export tool") c.run("export dep") - c.run("create pkg --build=*") + c.run("create pkg --build=* --build-test=*") + c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) c.assert_listed_require({"tool/0.1": "Cache"}, build=True, test_package=True) c.assert_listed_binary({"tool/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}, build=True, test_package=True) - # FIXME: Re-building already built things is unaceptable + # Note we do NOT rebuild the already built binaries c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Cache")}, test_package=True) @@ -151,7 +152,7 @@ def test_build_missing(self): .with_test("pass")}) c.run("export tool") c.run("create dep") - c.run("create pkg --build=missing") + c.run("create pkg --build=missing --build-test=missing") c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) c.assert_listed_require({"tool/0.1": "Cache"}, build=True, test_package=True) From a2bfda4f7480902b00b22d8b1b8003f2892a6dd6 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 25 Jul 2023 18:09:07 +0200 Subject: [PATCH 10/12] new test --- .../integration/command/test_package_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/conans/test/integration/command/test_package_test.py b/conans/test/integration/command/test_package_test.py index 5c590665076..a0bb803cf38 100644 --- a/conans/test/integration/command/test_package_test.py +++ b/conans/test/integration/command/test_package_test.py @@ -162,6 +162,24 @@ def test_build_missing(self): "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Cache")}, test_package=True) + def test_build_test_package_dep(self): + c = TestClient() + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "pkg/conanfile.py": GenConanfile("pkg", "0.1"), + "pkg/test_package/conanfile.py": GenConanfile().with_requires("dep/0.1") + .with_test("pass")}) + c.run("export dep") + c.run("create pkg --build=missing", assert_error=True) + c.assert_listed_binary({"pkg/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}) + c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Missing"), + "pkg/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache")}, + test_package=True) + c.run("create pkg --build-test=missing") + c.assert_listed_binary({"pkg/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}) + c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), + "pkg/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache")}, + test_package=True) + class ConanTestTest(unittest.TestCase): From 855bdb75333381850d4f386d75ea89e4095571e3 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 26 Jul 2023 14:42:16 +0200 Subject: [PATCH 11/12] better defaults --- conan/cli/commands/create.py | 2 ++ conans/client/graph/install_graph.py | 6 +++--- conans/test/integration/command/create_test.py | 1 + conans/test/integration/command/test_package_test.py | 8 ++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 1fc0b4d2a33..4635288a0dc 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -53,6 +53,8 @@ def create(conan_api, parser, *args): args.build_require) print_profiles(profile_host, profile_build) + if args.build is not None and args.build_test is None: + args.build_test = args.build deps_graph = None if not is_python_require: diff --git a/conans/client/graph/install_graph.py b/conans/client/graph/install_graph.py index 9fa5fef074a..f9e962e7613 100644 --- a/conans/client/graph/install_graph.py +++ b/conans/client/graph/install_graph.py @@ -301,9 +301,9 @@ def _raise_missing(self, missing): conanfile.output.warning(msg) missing_pkgs = "', '".join(list(sorted([str(pref.ref) for pref in missing_prefs]))) if self._is_test_package: - build_msg = "'conan test' tested packages must exist, and '--build' argument " \ - "is used only for the 'test_package' dependencies, not for the tested " \ - "dependencies" + build_msg = "This is a **test_package** missing binary. You can use --build (for " \ + "all dependencies) or --build-test (exclusive for 'test_package' " \ + "dependencies) to define what can be built from sources" else: if len(missing_prefs) >= 5: build_str = "--build=missing" diff --git a/conans/test/integration/command/create_test.py b/conans/test/integration/command/create_test.py index 68f69c2c9a5..8507c909f95 100644 --- a/conans/test/integration/command/create_test.py +++ b/conans/test/integration/command/create_test.py @@ -710,6 +710,7 @@ def test_create_both_host_build_require(): "test_package/conanfile.py": GenConanfile().with_build_requires("protobuf/0.1") .with_test("pass")}) c.run("create . -s:b build_type=Release -s:h build_type=Debug", assert_error=True) + print(c.out) # The main "host" Debug binary will be correctly build c.assert_listed_binary({"protobuf/0.1": ("9e186f6d94c008b544af1569d1a6368d8339efc5", "Build")}) # But test_package will fail because of the missing "tool_require" in Release diff --git a/conans/test/integration/command/test_package_test.py b/conans/test/integration/command/test_package_test.py index a0bb803cf38..98e6915bf7c 100644 --- a/conans/test/integration/command/test_package_test.py +++ b/conans/test/integration/command/test_package_test.py @@ -131,7 +131,7 @@ def test_build_all(self): .with_test("pass")}) c.run("export tool") c.run("export dep") - c.run("create pkg --build=* --build-test=*") + c.run("create pkg --build=*") c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) @@ -152,7 +152,7 @@ def test_build_missing(self): .with_test("pass")}) c.run("export tool") c.run("create dep") - c.run("create pkg --build=missing --build-test=missing") + c.run("create pkg --build=missing") c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache"), "pkg/0.1": ("59205ba5b14b8f4ebc216a6c51a89553021e82c1", "Build")}) c.assert_listed_require({"tool/0.1": "Cache"}, build=True, test_package=True) @@ -169,7 +169,7 @@ def test_build_test_package_dep(self): "pkg/test_package/conanfile.py": GenConanfile().with_requires("dep/0.1") .with_test("pass")}) c.run("export dep") - c.run("create pkg --build=missing", assert_error=True) + c.run('create pkg --build=missing --build-test=""', assert_error=True) c.assert_listed_binary({"pkg/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}) c.assert_listed_binary({"dep/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Missing"), "pkg/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Cache")}, @@ -418,6 +418,6 @@ def test_package_missing_binary_msg(): c.run("export .") c.run("test test_package dep/0.1", assert_error=True) assert "ERROR: Missing binary: dep/0.1" in c.out - assert "'conan test' tested packages must exist" in c.out + assert "This is a **test_package** missing binary." in c.out c.run("test test_package dep/0.1 --build=dep/0.1") c.assert_listed_binary({"dep/0.1": (NO_SETTINGS_PACKAGE_ID, "Build")}) From c34636314e01aa21fbbdeeeea828ac49b3e17e25 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 26 Jul 2023 16:18:39 +0200 Subject: [PATCH 12/12] minor -h improvements --- conan/cli/args.py | 2 +- conan/cli/commands/create.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conan/cli/args.py b/conan/cli/args.py index 82771e78159..3313d3280af 100644 --- a/conan/cli/args.py +++ b/conan/cli/args.py @@ -7,7 +7,7 @@ --build="*" Force build from source for all packages. --build=never Disallow build for all packages, use binary packages or fail if a binary - package is not found. Cannot be combined with other '--build' options. + package is not found, it cannot be combined with other '--build' options. --build=missing Build packages from source whose binary package is not found. --build=cascade Build packages from source that have at least one dependency being built from source. diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 4635288a0dc..c37c235af86 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -21,12 +21,14 @@ def create(conan_api, parser, *args): add_lockfile_args(parser) add_common_install_arguments(parser) parser.add_argument("--build-require", action='store_true', default=False, - help='Whether the provided reference is a build-require') + help='Whether the package being created is a build-require (to be used' + 'as tool_requires() by other packages)') parser.add_argument("-tf", "--test-folder", action=OnceArgument, help='Alternative test folder name. By default it is "test_package". ' 'Use "" to skip the test stage') parser.add_argument("-bt", "--build-test", action="append", - help="Same as '--build' but only for the test_package requires") + help="Same as '--build' but only for the test_package requires. By default" + " if not specified it will take the '--build' value if specified") args = parser.parse_args(*args) cwd = os.getcwd()