From c38521f54133c4c6abe8d189769576c3b107408a Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 22 May 2023 12:11:25 +0200 Subject: [PATCH 01/29] proposal for adding a conan-center-index clone as remote --- conans/client/remote_manager.py | 16 +- conans/client/remote_manager_git.py | 167 ++++++++++++++++++ .../remote/local_tree_remote_test.py | 151 ++++++++++++++++ 3 files changed, 332 insertions(+), 2 deletions(-) create mode 100644 conans/client/remote_manager_git.py create mode 100644 conans/test/integration/remote/local_tree_remote_test.py diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 534cb659164..811406d23d7 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -7,6 +7,7 @@ from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import METADATA from conans.client.cache.remote_registry import Remote +from conans.client.remote_manager_git import GitRemoteManager from conans.client.pkg_sign import PkgSignaturesPlugin from conans.errors import ConanConnectionError, ConanException, NotFoundException, \ PackageNotFoundException @@ -18,14 +19,18 @@ from conans.util.files import mkdir, tar_extract -class RemoteManager(object): +class RemoteManager: """ Will handle the remotes to get recipes, packages etc """ - def __init__(self, cache, auth_manager): self._cache = cache self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache) + @staticmethod + def _git_remote(remote): + if remote.url.startswith("file://"): # FIXME: remote type? + return GitRemoteManager(remote) + def check_credentials(self, remote): self._call_remote(remote, "check_credentials") @@ -88,6 +93,10 @@ def get_recipe_sources(self, ref, layout, remote): download_folder = layout.download_export() export_sources_folder = layout.export_sources() + git_remote = self._git_remote(remote) + if git_remote is not None: + return git_remote.get_recipe_sources(ref, export_sources_folder) + zipped_files = self._call_remote(remote, "get_recipe_sources", ref, download_folder) if not zipped_files: mkdir(export_sources_folder) # create the folder even if no source files @@ -201,6 +210,9 @@ def _call_remote(self, remote, method, *args, **kwargs): enforce_disabled = kwargs.pop("enforce_disabled", True) if remote.disabled and enforce_disabled: raise ConanException("Remote '%s' is disabled" % remote.name) + git_remote = self._git_remote(remote) + if git_remote is not None: + return git_remote.call_method(method, *args, **kwargs) try: return self._auth_manager.call_rest_api_method(remote, method, *args, **kwargs) except ConnectionError as exc: diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py new file mode 100644 index 00000000000..154d4b35045 --- /dev/null +++ b/conans/client/remote_manager_git.py @@ -0,0 +1,167 @@ +import os +import sys +from distutils.dir_util import copy_tree +from fnmatch import fnmatch +from io import StringIO + +import yaml + +from conan.tools.files import load +from conans.client.cmd.export import cmd_export +from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException +from conans.model.recipe_ref import RecipeReference + + +class GitRemoteManager: + + def __init__(self, remote): + self._remote = remote + cache_folder = self._remote.url.replace("file://", "") + self._remote_cache_dir = "{}/.cache".format(cache_folder) + from conan.internal.conan_app import ConanApp + self._app = ConanApp(self._remote_cache_dir) + self.layout = _ConanCenterIndexLayout(cache_folder) + + def call_method(self, method_name, *args, **kwargs): + return getattr(self, method_name)(*args, **kwargs) + + def get_recipe(self, ref, dest_folder): + export_folder = self._app.cache.ref_layout(ref).export() + return self._copy_files(export_folder, dest_folder) + + def get_recipe_sources(self, ref, dest_folder): + export_sources = self._app.cache.ref_layout(ref).export_sources() + return self._copy_files(export_sources, dest_folder) + + def get_package(self, pref, dest_folder): + raise ConanException(f"The remote '{self._remote.name}' doesn't support binary packages") + + def upload_recipe(self, ref, files_to_upload): + raise ConanException(f"Git remote '{self._remote.name}' do not support upload") + + def upload_package(self, pref, files_to_upload): + raise ConanException(f"Git remote '{self._remote.name}' do not support upload") + + def authenticate(self, user, password): + raise ConanException(f"Git remote '{self._remote.name}' do not support authentication") + + def check_credentials(self): + raise ConanException(f"Git remote '{self._remote.name}' do not support check credentials") + + def search(self, pattern=None, ignorecase=True): + ret = [] + for ref in self.layout.get_recipes_references(): + # TODO: Check the search pattern is the same as remotes and cache + if fnmatch(str(ref), pattern): + ret.append(ref) + return ret + + def search_packages(self, reference): + return {} + + def remove_recipe(self, ref): + raise ConanException(f"Git remote '{self._remote.name}' do not support remove") + + def remove_all_packages(self, ref): + raise ConanException(f"Git remote '{self._remote.name}' do not support remove") + + def remove_packages(self, prefs): + raise ConanException(f"Git remote '{self._remote.name}' do not support remove") + + def get_recipe_revisions_references(self, ref): + ref = self._export_recipe(ref) + return [ref] + + def get_package_revisions_references(self, pref, headers=None): + raise PackageNotFoundException(pref) + + def get_latest_recipe_reference(self, ref): + ref = self._export_recipe(ref) + return ref + + def get_latest_package_reference(self, pref, headers): + raise PackageNotFoundException(pref) + + def get_recipe_revision_reference(self, ref): + ref = self._export_recipe(ref) + return ref + + def get_package_revision_reference(self, pref): + raise PackageNotFoundException(pref) + + ####################################################################################### + ####################################################################################### + ####################################################################################### + def _export_recipe(self, ref): + folder = self.layout.get_recipe_folder(ref) + conanfile_path = os.path.join(folder, "conanfile.py") + original_stderr = sys.stderr + sys.stderr = StringIO() + try: + ref, _ = cmd_export(self._app, conanfile_path, ref.name, ref.version, None, None) + finally: + sys.stderr = original_stderr + + return ref + + @staticmethod + def _copy_files(source_folder, dest_folder): + if not os.path.exists(source_folder): + return {} + copy_tree(source_folder, dest_folder) + ret = {} + for root, _, _files in os.walk(dest_folder): + for _f in _files: + rel = os.path.relpath(os.path.join(root, _f), dest_folder) + ret[rel] = os.path.join(dest_folder, root, _f) + return ret + + ''' + def get_recipe_revisions_references(self, ref): + ref = self._export_recipe(ref) + tmp = copy.copy(ref) + tmp.revision = None + return self._conan_api.list.recipe_revisions(tmp) + + def get_recipe_revision_reference(self, ref): + ref = self._export_recipe(ref) + if ref in self.get_recipe_revisions_references(ref): + return ref + else: + raise RecipeNotFoundException(ref) + ''' + + +class _ConanCenterIndexLayout: + + def __init__(self, base_folder): + self._base_folder = base_folder + + def get_base_folder(self, recipe_name): + return os.path.join(self._base_folder, "recipes", recipe_name) + + def _load_config_yml(self, recipe_name): + content = load(None, os.path.join(self.get_base_folder(recipe_name), "config.yml")) + return yaml.safe_load(content) + + def get_versions(self, recipe_name): + data = self._load_config_yml(recipe_name)["versions"] + return data.keys() + + def get_recipes_references(self): + recipes_dir = os.path.join(self._base_folder, "recipes") + recipes = os.listdir(recipes_dir) + recipes.sort() + ret = [] + for r in recipes: + for v in self.get_versions(r): + ret.append(RecipeReference.loads("{}/{}".format(r, v))) + return ret + + def get_recipe_folder(self, ref): + data = self._load_config_yml(ref.name) + versions = data["versions"] + if str(ref.version) not in versions: + raise RecipeNotFoundException(ref) + subfolder = versions[str(ref.version)]["folder"] + return os.path.join(self.get_base_folder(ref.name), subfolder) diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/local_tree_remote_test.py new file mode 100644 index 00000000000..8adeb3b5aa6 --- /dev/null +++ b/conans/test/integration/remote/local_tree_remote_test.py @@ -0,0 +1,151 @@ +import json +import os +import textwrap + +import pytest + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient +from conans.util.files import mkdir, save, save_files + + +@pytest.fixture(scope="module") +def c3i_folder(): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + zlib_config = textwrap.dedent(""" + versions: + "1.2.8": + folder: all + "1.2.11": + folder: all + """) + zlib = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import load + class Zlib(ConanFile): + name = "zlib" + exports_sources = "*" + def build(self): + self.output.info(f"CONANDATA: {self.conan_data}") + self.output.info(f"BUILDING: {load(self, 'file.h')}") + """) + save_files(recipes_folder, + {"zlib/config.yml": zlib_config, + "zlib/all/conanfile.py": zlib, + "zlib/all/conandata.yml": "", + "zlib/all/file.h": "//myheader"}) + mkdir(os.path.join(recipes_folder, "openssl", "1.X")) + mkdir(os.path.join(recipes_folder, "openssl", "2.X")) + save(os.path.join(recipes_folder, "openssl", "config.yml"), textwrap.dedent(""" + versions: + "1.0": + folder: "1.X" + "1.1": + folder: "1.X" + "2.0": + folder: "2.X" + """)) + save(os.path.join(recipes_folder, "openssl", "1.X", "conanfile.py"), + str(GenConanfile().with_require("zlib/1.2.8"))) + save(os.path.join(recipes_folder, "openssl", "2.X", "conanfile.py"), + str(GenConanfile().with_require("zlib/1.2.11"))) + mkdir(os.path.join(recipes_folder, "libcurl", "all")) + save(os.path.join(recipes_folder, "libcurl", "config.yml"), textwrap.dedent(""" + versions: + "1.0": + folder: "all" + """)) + save(os.path.join(recipes_folder, "libcurl", "all", "conanfile.py"), + str(GenConanfile().with_require("openssl/2.0"))) + return folder + + +class TestSearchList: + def test_basic_search(self, c3i_folder): + client = TestClient() + client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("search *") + assert textwrap.dedent("""\ + ccifork + libcurl + libcurl/1.0 + openssl + openssl/1.0 + openssl/1.1 + openssl/2.0 + zlib + zlib/1.2.8 + zlib/1.2.11 + """) in client.out + + def test_list_refs(self, c3i_folder): + client = TestClient() + client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("list *#* -r=ccifork --format=json") + listjson = json.loads(client.stdout) + revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] + assert len(revs) == 1 and "e468388f0e4e098d5b62ad68979aebd5" in revs + revs = listjson["ccifork"]["openssl/1.0"]["revisions"] + assert len(revs) == 1 and "b35ffb31b6d5a9d8af39f5de3cf4fd63" in revs + revs = listjson["ccifork"]["openssl/1.1"]["revisions"] + assert len(revs) == 1 and "b35ffb31b6d5a9d8af39f5de3cf4fd63" in revs + revs = listjson["ccifork"]["openssl/2.0"]["revisions"] + assert len(revs) == 1 and "e50e871efca149f160fa6354c8534449" in revs + revs = listjson["ccifork"]["zlib/1.2.8"]["revisions"] + assert len(revs) == 1 and "6f5c31bb1219e9393743d1fbf2ee1b52" in revs + revs = listjson["ccifork"]["zlib/1.2.11"]["revisions"] + assert len(revs) == 1 and "6f5c31bb1219e9393743d1fbf2ee1b52" in revs + + def test_list_rrevs(self, c3i_folder): + client = TestClient() + client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("list libcurl/1.0#* -r=ccifork --format=json") + listjson = json.loads(client.stdout) + revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] + assert len(revs) == 1 and "e468388f0e4e098d5b62ad68979aebd5" in revs + + def test_list_binaries(self, c3i_folder): + client = TestClient() + client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("list libcurl/1.0:* -r=ccifork --format=json") + listjson = json.loads(client.stdout) + rev = listjson["ccifork"]["libcurl/1.0"]["revisions"]["e468388f0e4e098d5b62ad68979aebd5"] + assert rev["packages"] == {} + + +class TestInstall: + def test_install(self, c3i_folder): + c = TestClient() + c.run("remote add ccifork 'file://{}'".format(c3i_folder)) + c.run("install --requires=libcurl/1.0 --build missing") + assert "zlib/1.2.11: CONANDATA: {}" in c.out + assert "zlib/1.2.11: BUILDING: //myheader" in c.out + bins = {"libcurl/1.0": ("aa69c1e1e39a18fe70001688213dbb7ada95f890", "Build"), + "openssl/2.0": ("594ed0eb2e9dfcc60607438924c35871514e6c2a", "Build"), + "zlib/1.2.11": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")} + c.assert_listed_binary(bins) + + # Already installed in the cache + c.run("install --requires=libcurl/1.0") + assert "zlib/1.2.11: Already installed!" in c.out + assert "openssl/2.0: Already installed!" in c.out + assert "libcurl/1.0: Already installed!" in c.out + + # Can update + c.run("install --requires libcurl/1.0 --update") + bins = {"libcurl/1.0": "Cache (Updated date) (ccifork)", + "openssl/2.0": "Cache (Updated date) (ccifork)", + "zlib/1.2.11": "Cache (Updated date) (ccifork)"} + c.assert_listed_require(bins) + assert "zlib/1.2.11: Already installed!" in c.out + assert "openssl/2.0: Already installed!" in c.out + assert "libcurl/1.0: Already installed!" in c.out + + # Doing local changes creates a new revision + # New recipe revision for the zlib library + save(os.path.join(c3i_folder, "recipes", "zlib", "all", "conanfile.py"), + str(GenConanfile()) + "\n") + c.run("install --requires=libcurl/1.0 --build missing --update") + assert "lib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (ccifork)" in c.out From 4cbd200a0c642a63dc3cb729b34e5580eb03cfa2 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 22 May 2023 12:23:17 +0200 Subject: [PATCH 02/29] urls --- conans/client/remote_manager.py | 2 +- conans/client/remote_manager_git.py | 2 +- .../test/integration/remote/local_tree_remote_test.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 811406d23d7..3bfaad61a13 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -28,7 +28,7 @@ def __init__(self, cache, auth_manager): @staticmethod def _git_remote(remote): - if remote.url.startswith("file://"): # FIXME: remote type? + if remote.url.startswith("file:///"): # FIXME: remote type? return GitRemoteManager(remote) def check_credentials(self, remote): diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py index 154d4b35045..a250be5e900 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/remote_manager_git.py @@ -16,7 +16,7 @@ class GitRemoteManager: def __init__(self, remote): self._remote = remote - cache_folder = self._remote.url.replace("file://", "") + cache_folder = self._remote.url.replace("file:///", "") self._remote_cache_dir = "{}/.cache".format(cache_folder) from conan.internal.conan_app import ConanApp self._app = ConanApp(self._remote_cache_dir) diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/local_tree_remote_test.py index 8adeb3b5aa6..7b919041f9c 100644 --- a/conans/test/integration/remote/local_tree_remote_test.py +++ b/conans/test/integration/remote/local_tree_remote_test.py @@ -65,7 +65,7 @@ def build(self): class TestSearchList: def test_basic_search(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) client.run("search *") assert textwrap.dedent("""\ ccifork @@ -82,7 +82,7 @@ def test_basic_search(self, c3i_folder): def test_list_refs(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) client.run("list *#* -r=ccifork --format=json") listjson = json.loads(client.stdout) revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] @@ -100,7 +100,7 @@ def test_list_refs(self, c3i_folder): def test_list_rrevs(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) client.run("list libcurl/1.0#* -r=ccifork --format=json") listjson = json.loads(client.stdout) revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] @@ -108,7 +108,7 @@ def test_list_rrevs(self, c3i_folder): def test_list_binaries(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file://{}'".format(c3i_folder)) + client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) client.run("list libcurl/1.0:* -r=ccifork --format=json") listjson = json.loads(client.stdout) rev = listjson["ccifork"]["libcurl/1.0"]["revisions"]["e468388f0e4e098d5b62ad68979aebd5"] @@ -118,7 +118,7 @@ def test_list_binaries(self, c3i_folder): class TestInstall: def test_install(self, c3i_folder): c = TestClient() - c.run("remote add ccifork 'file://{}'".format(c3i_folder)) + c.run("remote add ccifork 'file:///{}'".format(c3i_folder)) c.run("install --requires=libcurl/1.0 --build missing") assert "zlib/1.2.11: CONANDATA: {}" in c.out assert "zlib/1.2.11: BUILDING: //myheader" in c.out From 2bd20d819a1ee3861a2b60af7c2f7ae2804d363b Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 May 2023 01:04:58 +0200 Subject: [PATCH 03/29] wip --- conans/client/remote_manager_git.py | 98 ++++++++++--------- .../remote/local_tree_remote_test.py | 16 ++- 2 files changed, 62 insertions(+), 52 deletions(-) diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py index a250be5e900..add5cc75595 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/remote_manager_git.py @@ -1,3 +1,4 @@ +import json import os import sys from distutils.dir_util import copy_tree @@ -6,10 +7,11 @@ import yaml -from conan.tools.files import load +from conan.api.output import ConanOutput from conans.client.cmd.export import cmd_export from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException from conans.model.recipe_ref import RecipeReference +from conans.util.files import load, save class GitRemoteManager: @@ -18,6 +20,7 @@ def __init__(self, remote): self._remote = remote cache_folder = self._remote.url.replace("file:///", "") self._remote_cache_dir = "{}/.cache".format(cache_folder) + self._exported_file = os.path.join(self._remote_cache_dir, "exported.json") from conan.internal.conan_app import ConanApp self._app = ConanApp(self._remote_cache_dir) self.layout = _ConanCenterIndexLayout(cache_folder) @@ -37,36 +40,31 @@ def get_package(self, pref, dest_folder): raise ConanException(f"The remote '{self._remote.name}' doesn't support binary packages") def upload_recipe(self, ref, files_to_upload): - raise ConanException(f"Git remote '{self._remote.name}' do not support upload") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") def upload_package(self, pref, files_to_upload): - raise ConanException(f"Git remote '{self._remote.name}' do not support upload") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") def authenticate(self, user, password): - raise ConanException(f"Git remote '{self._remote.name}' do not support authentication") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support authentication") def check_credentials(self): - raise ConanException(f"Git remote '{self._remote.name}' do not support check credentials") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support check credentials") def search(self, pattern=None, ignorecase=True): - ret = [] - for ref in self.layout.get_recipes_references(): - # TODO: Check the search pattern is the same as remotes and cache - if fnmatch(str(ref), pattern): - ret.append(ref) - return ret + return self.layout.get_recipes_references(pattern) def search_packages(self, reference): return {} def remove_recipe(self, ref): - raise ConanException(f"Git remote '{self._remote.name}' do not support remove") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support remove") def remove_all_packages(self, ref): - raise ConanException(f"Git remote '{self._remote.name}' do not support remove") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support remove") def remove_packages(self, prefs): - raise ConanException(f"Git remote '{self._remote.name}' do not support remove") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support remove") def get_recipe_revisions_references(self, ref): ref = self._export_recipe(ref) @@ -83,26 +81,35 @@ def get_latest_package_reference(self, pref, headers): raise PackageNotFoundException(pref) def get_recipe_revision_reference(self, ref): - ref = self._export_recipe(ref) - return ref + new_ref = self._export_recipe(ref) + if new_ref != ref: + raise RecipeNotFoundException(ref) + return new_ref def get_package_revision_reference(self, pref): raise PackageNotFoundException(pref) - ####################################################################################### - ####################################################################################### - ####################################################################################### + # Helper methods to implement the interface def _export_recipe(self, ref): + exported = load(self._exported_file) if os.path.isfile(self._exported_file) else "{}" + exported = json.loads(exported) + existing = exported.get(str(ref)) + if existing is not None: + return RecipeReference.loads(existing["ref"]) + folder = self.layout.get_recipe_folder(ref) conanfile_path = os.path.join(folder, "conanfile.py") original_stderr = sys.stderr sys.stderr = StringIO() try: - ref, _ = cmd_export(self._app, conanfile_path, ref.name, ref.version, None, None) + original_stderr.write(f"Git remote processing {ref}\n") + new_ref, _ = cmd_export(self._app, conanfile_path, ref.name, ref.version, None, None) finally: sys.stderr = original_stderr - - return ref + # Cache the result, so it is not constantly re-exported + exported[str(ref)] = {"ref": repr(new_ref)} + save(self._exported_file, json.dumps(exported)) + return new_ref @staticmethod def _copy_files(source_folder, dest_folder): @@ -116,46 +123,43 @@ def _copy_files(source_folder, dest_folder): ret[rel] = os.path.join(dest_folder, root, _f) return ret - ''' - def get_recipe_revisions_references(self, ref): - ref = self._export_recipe(ref) - tmp = copy.copy(ref) - tmp.revision = None - return self._conan_api.list.recipe_revisions(tmp) - - def get_recipe_revision_reference(self, ref): - ref = self._export_recipe(ref) - if ref in self.get_recipe_revisions_references(ref): - return ref - else: - raise RecipeNotFoundException(ref) - ''' - class _ConanCenterIndexLayout: def __init__(self, base_folder): self._base_folder = base_folder - def get_base_folder(self, recipe_name): + def _get_base_folder(self, recipe_name): return os.path.join(self._base_folder, "recipes", recipe_name) def _load_config_yml(self, recipe_name): - content = load(None, os.path.join(self.get_base_folder(recipe_name), "config.yml")) + content = load(os.path.join(self._get_base_folder(recipe_name), "config.yml")) return yaml.safe_load(content) - def get_versions(self, recipe_name): - data = self._load_config_yml(recipe_name)["versions"] - return data.keys() - - def get_recipes_references(self): + def get_recipes_references(self, pattern): recipes_dir = os.path.join(self._base_folder, "recipes") recipes = os.listdir(recipes_dir) recipes.sort() ret = [] + excluded = set() for r in recipes: - for v in self.get_versions(r): - ret.append(RecipeReference.loads("{}/{}".format(r, v))) + config_yml = self._load_config_yml(r) + versions = config_yml["versions"] + for v in versions: + # TODO: Check the search pattern is the same as remotes and cache + ref = f"{r}/{v}" + if not fnmatch(ref, pattern): + continue + subfolder = versions[v]["folder"] + # This check can be removed after compatibility with 2.0 + conanfile = os.path.join(recipes_dir, r, subfolder, "conanfile.py") + conanfile_content = load(conanfile) + if "from conans" in conanfile_content or "import conants" in conanfile_content: + excluded.add(r) + continue + ret.append(RecipeReference.loads(ref)) + if excluded: + ConanOutput().warning(f"Excluding recipes not Conan 2.0 ready: {', '.join(excluded)}") return ret def get_recipe_folder(self, ref): @@ -164,4 +168,4 @@ def get_recipe_folder(self, ref): if str(ref.version) not in versions: raise RecipeNotFoundException(ref) subfolder = versions[str(ref.version)]["folder"] - return os.path.join(self.get_base_folder(ref.name), subfolder) + return os.path.join(self._get_base_folder(ref.name), subfolder) diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/local_tree_remote_test.py index 7b919041f9c..4fc37770fe7 100644 --- a/conans/test/integration/remote/local_tree_remote_test.py +++ b/conans/test/integration/remote/local_tree_remote_test.py @@ -133,11 +133,12 @@ def test_install(self, c3i_folder): assert "openssl/2.0: Already installed!" in c.out assert "libcurl/1.0: Already installed!" in c.out - # Can update + # Update doesn't fail, but doesn't update revision time c.run("install --requires libcurl/1.0 --update") - bins = {"libcurl/1.0": "Cache (Updated date) (ccifork)", - "openssl/2.0": "Cache (Updated date) (ccifork)", - "zlib/1.2.11": "Cache (Updated date) (ccifork)"} + bins = {"libcurl/1.0": "Cache (ccifork)", + "openssl/2.0": "Cache (ccifork)", + "zlib/1.2.11": "Cache (ccifork)"} + c.assert_listed_require(bins) assert "zlib/1.2.11: Already installed!" in c.out assert "openssl/2.0: Already installed!" in c.out @@ -148,4 +149,9 @@ def test_install(self, c3i_folder): save(os.path.join(c3i_folder, "recipes", "zlib", "all", "conanfile.py"), str(GenConanfile()) + "\n") c.run("install --requires=libcurl/1.0 --build missing --update") - assert "lib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (ccifork)" in c.out + # it is not updated + assert "zlib/1.2.11#6f5c31bb1219e9393743d1fbf2ee1b52 - Cache (ccifork)" in c.out + # We could remove the cached exported to force its re-creation + os.remove(os.path.join(c3i_folder, ".cache", "exported.json")) + c.run("install --requires=libcurl/1.0 --build missing --update") + assert "zlib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (ccifork)" in c.out From 6d9f80bb58ec7c121dd9acf3c170098385e06276 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 25 May 2023 00:47:19 +0200 Subject: [PATCH 04/29] raise on uploads --- conans/client/remote_manager_git.py | 11 +++++++++-- .../integration/remote/local_tree_remote_test.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py index add5cc75595..05993c3ea6a 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/remote_manager_git.py @@ -49,7 +49,7 @@ def authenticate(self, user, password): raise ConanException(f"Git remote '{self._remote.name}' doesn't support authentication") def check_credentials(self): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support check credentials") + raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") def search(self, pattern=None, ignorecase=True): return self.layout.get_recipes_references(pattern) @@ -133,7 +133,10 @@ def _get_base_folder(self, recipe_name): return os.path.join(self._base_folder, "recipes", recipe_name) def _load_config_yml(self, recipe_name): - content = load(os.path.join(self._get_base_folder(recipe_name), "config.yml")) + config = os.path.join(self._get_base_folder(recipe_name), "config.yml") + if not os.path.isfile(config): + return None + content = load(config) return yaml.safe_load(content) def get_recipes_references(self, pattern): @@ -144,6 +147,8 @@ def get_recipes_references(self, pattern): excluded = set() for r in recipes: config_yml = self._load_config_yml(r) + if config_yml is None: + raise ConanException(f"Corrupted repo, folder {r} without 'config.yml'") versions = config_yml["versions"] for v in versions: # TODO: Check the search pattern is the same as remotes and cache @@ -164,6 +169,8 @@ def get_recipes_references(self, pattern): def get_recipe_folder(self, ref): data = self._load_config_yml(ref.name) + if data is None: + raise RecipeNotFoundException(ref) versions = data["versions"] if str(ref.version) not in versions: raise RecipeNotFoundException(ref) diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/local_tree_remote_test.py index 4fc37770fe7..d5052873ed7 100644 --- a/conans/test/integration/remote/local_tree_remote_test.py +++ b/conans/test/integration/remote/local_tree_remote_test.py @@ -155,3 +155,15 @@ def test_install(self, c3i_folder): os.remove(os.path.join(c3i_folder, ".cache", "exported.json")) c.run("install --requires=libcurl/1.0 --build missing --update") assert "zlib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (ccifork)" in c.out + + +class TestRestrictedOperations: + def test_upload(self): + folder = temp_folder() + c3i_folder = os.path.join(folder, "recipes") + c = TestClient() + c.run("remote add ccifork 'file:///{}'".format(c3i_folder)) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .") + c.run("upload pkg/0.1 -r=ccifork", assert_error=True) + assert "ERROR: Git remote 'ccifork' doesn't support upload" in c.out From 8ff015400b0ac40102de8d5626bb0c2661bd117b Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 29 May 2023 13:42:32 +0200 Subject: [PATCH 05/29] remove file:/// protocol, use explicit --type=local --- conan/api/model.py | 5 ++++- conan/cli/commands/remote.py | 4 +++- conans/client/cache/remote_registry.py | 5 ++++- conans/client/remote_manager.py | 2 +- conans/client/remote_manager_git.py | 21 +++++++++++-------- .../remote/local_tree_remote_test.py | 12 +++++------ 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index cb9420425bb..87af934c50b 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -7,11 +7,12 @@ class Remote: - def __init__(self, name, url, verify_ssl=True, disabled=False): + def __init__(self, name, url, verify_ssl=True, disabled=False, remote_type=None): self._name = name # Read only, is the key self.url = url self.verify_ssl = verify_ssl self.disabled = disabled + self.remote_type = remote_type @property def name(self): @@ -26,6 +27,8 @@ def __eq__(self, other): self.disabled == other.disabled def __str__(self): + if self.remote_type == "local": + return "{}: {} [Local: True, Enabled: {}]".format(self.name, self.url, not self.disabled) return "{}: {} [Verify SSL: {}, Enabled: {}]".format(self.name, self.url, self.verify_ssl, not self.disabled) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 46ba43ee9e4..56c7b1eb042 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -72,9 +72,11 @@ def remote_add(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") + subparser.add_argument("-t", "--type", choices=["local"], + help="Define the remote type") subparser.set_defaults(secure=True) args = parser.parse_args(*args) - r = Remote(args.name, args.url, args.secure, disabled=False) + r = Remote(args.name, args.url, args.secure, disabled=False, remote_type=args.type) conan_api.remotes.add(r, force=args.force, index=args.index) diff --git a/conans/client/cache/remote_registry.py b/conans/client/cache/remote_registry.py index bcdcaa58eea..8eb51d1f73e 100644 --- a/conans/client/cache/remote_registry.py +++ b/conans/client/cache/remote_registry.py @@ -33,8 +33,9 @@ def load(filename): data = json.loads(text) for r in data.get("remotes", []): disabled = r.get("disabled", False) + remote_type = r.get("type", None) # TODO: Remote.serialize/deserialize - remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled) + remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled, remote_type) result._remotes[r["name"]] = remote return result @@ -44,6 +45,8 @@ def dumps(self): remote = {"name": r.name, "url": r.url, "verify_ssl": r.verify_ssl} if r.disabled: remote["disabled"] = True + if r.remote_type: + remote["type"] = r.remote_type remote_list.append(remote) ret = {"remotes": remote_list} return json.dumps(ret, indent=True) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 28654b8b886..308c84d628e 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -28,7 +28,7 @@ def __init__(self, cache, auth_manager): @staticmethod def _git_remote(remote): - if remote.url.startswith("file:///"): # FIXME: remote type? + if remote.remote_type == "local": return GitRemoteManager(remote) def check_credentials(self, remote): diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py index 05993c3ea6a..9174bfa2920 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/remote_manager_git.py @@ -18,7 +18,7 @@ class GitRemoteManager: def __init__(self, remote): self._remote = remote - cache_folder = self._remote.url.replace("file:///", "") + cache_folder = self._remote.url self._remote_cache_dir = "{}/.cache".format(cache_folder) self._exported_file = os.path.join(self._remote_cache_dir, "exported.json") from conan.internal.conan_app import ConanApp @@ -51,10 +51,11 @@ def authenticate(self, user, password): def check_credentials(self): raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") - def search(self, pattern=None, ignorecase=True): + def search(self, pattern=None): return self.layout.get_recipes_references(pattern) def search_packages(self, reference): + assert self and reference return {} def remove_recipe(self, ref): @@ -132,12 +133,12 @@ def __init__(self, base_folder): def _get_base_folder(self, recipe_name): return os.path.join(self._base_folder, "recipes", recipe_name) - def _load_config_yml(self, recipe_name): - config = os.path.join(self._get_base_folder(recipe_name), "config.yml") + @staticmethod + def _load_config_yml(folder): + config = os.path.join(folder, "config.yml") if not os.path.isfile(config): return None - content = load(config) - return yaml.safe_load(content) + return yaml.safe_load(load(config)) def get_recipes_references(self, pattern): recipes_dir = os.path.join(self._base_folder, "recipes") @@ -146,7 +147,8 @@ def get_recipes_references(self, pattern): ret = [] excluded = set() for r in recipes: - config_yml = self._load_config_yml(r) + folder = self._get_base_folder(r) + config_yml = self._load_config_yml(folder) if config_yml is None: raise ConanException(f"Corrupted repo, folder {r} without 'config.yml'") versions = config_yml["versions"] @@ -168,11 +170,12 @@ def get_recipes_references(self, pattern): return ret def get_recipe_folder(self, ref): - data = self._load_config_yml(ref.name) + folder = self._get_base_folder(ref.name) + data = self._load_config_yml(folder) if data is None: raise RecipeNotFoundException(ref) versions = data["versions"] if str(ref.version) not in versions: raise RecipeNotFoundException(ref) subfolder = versions[str(ref.version)]["folder"] - return os.path.join(self._get_base_folder(ref.name), subfolder) + return os.path.join(folder, subfolder) diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/local_tree_remote_test.py index d5052873ed7..f1dc7ff165c 100644 --- a/conans/test/integration/remote/local_tree_remote_test.py +++ b/conans/test/integration/remote/local_tree_remote_test.py @@ -65,7 +65,7 @@ def build(self): class TestSearchList: def test_basic_search(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) + client.run(f"remote add ccifork '{c3i_folder}' --type=local") client.run("search *") assert textwrap.dedent("""\ ccifork @@ -82,7 +82,7 @@ def test_basic_search(self, c3i_folder): def test_list_refs(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) + client.run(f"remote add ccifork '{c3i_folder}' --type=local") client.run("list *#* -r=ccifork --format=json") listjson = json.loads(client.stdout) revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] @@ -100,7 +100,7 @@ def test_list_refs(self, c3i_folder): def test_list_rrevs(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) + client.run(f"remote add ccifork '{c3i_folder}' --type=local") client.run("list libcurl/1.0#* -r=ccifork --format=json") listjson = json.loads(client.stdout) revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] @@ -108,7 +108,7 @@ def test_list_rrevs(self, c3i_folder): def test_list_binaries(self, c3i_folder): client = TestClient() - client.run("remote add ccifork 'file:///{}'".format(c3i_folder)) + client.run(f"remote add ccifork '{c3i_folder}' --type=local") client.run("list libcurl/1.0:* -r=ccifork --format=json") listjson = json.loads(client.stdout) rev = listjson["ccifork"]["libcurl/1.0"]["revisions"]["e468388f0e4e098d5b62ad68979aebd5"] @@ -118,7 +118,7 @@ def test_list_binaries(self, c3i_folder): class TestInstall: def test_install(self, c3i_folder): c = TestClient() - c.run("remote add ccifork 'file:///{}'".format(c3i_folder)) + c.run(f"remote add ccifork '{c3i_folder}' --type=local") c.run("install --requires=libcurl/1.0 --build missing") assert "zlib/1.2.11: CONANDATA: {}" in c.out assert "zlib/1.2.11: BUILDING: //myheader" in c.out @@ -162,7 +162,7 @@ def test_upload(self): folder = temp_folder() c3i_folder = os.path.join(folder, "recipes") c = TestClient() - c.run("remote add ccifork 'file:///{}'".format(c3i_folder)) + c.run(f"remote add ccifork '{c3i_folder}' --type=local") c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") c.run("upload pkg/0.1 -r=ccifork", assert_error=True) From 18e2a08d80b5dedde7fba101d3eb15613e8e7a82 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 21 Dec 2023 00:42:22 +0100 Subject: [PATCH 06/29] wip --- conans/client/remote_manager_git.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py index 9174bfa2920..8ce3d418989 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/remote_manager_git.py @@ -10,6 +10,7 @@ from conan.api.output import ConanOutput from conans.client.cmd.export import cmd_export from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException +from conans.model.conf import ConfDefinition from conans.model.recipe_ref import RecipeReference from conans.util.files import load, save @@ -22,21 +23,22 @@ def __init__(self, remote): self._remote_cache_dir = "{}/.cache".format(cache_folder) self._exported_file = os.path.join(self._remote_cache_dir, "exported.json") from conan.internal.conan_app import ConanApp - self._app = ConanApp(self._remote_cache_dir) + global_conf = ConfDefinition() + self._app = ConanApp(self._remote_cache_dir, global_conf) self.layout = _ConanCenterIndexLayout(cache_folder) def call_method(self, method_name, *args, **kwargs): return getattr(self, method_name)(*args, **kwargs) - def get_recipe(self, ref, dest_folder): - export_folder = self._app.cache.ref_layout(ref).export() + def get_recipe(self, ref, dest_folder, metadata, only_metadata): + export_folder = self._app.cache.recipe_layout(ref).export() return self._copy_files(export_folder, dest_folder) def get_recipe_sources(self, ref, dest_folder): - export_sources = self._app.cache.ref_layout(ref).export_sources() + export_sources = self._app.cache.recipe_layout(ref).export_sources() return self._copy_files(export_sources, dest_folder) - def get_package(self, pref, dest_folder): + def get_package(self, pref, dest_folder, metadata, only_metadata): raise ConanException(f"The remote '{self._remote.name}' doesn't support binary packages") def upload_recipe(self, ref, files_to_upload): @@ -104,7 +106,9 @@ def _export_recipe(self, ref): sys.stderr = StringIO() try: original_stderr.write(f"Git remote processing {ref}\n") - new_ref, _ = cmd_export(self._app, conanfile_path, ref.name, ref.version, None, None) + global_conf = ConfDefinition() + new_ref, _ = cmd_export(self._app, global_conf, conanfile_path, + ref.name, ref.version, None, None) finally: sys.stderr = original_stderr # Cache the result, so it is not constantly re-exported From 95eae24ace31077c40b543432089f1d401503431 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 21 Dec 2023 01:48:54 +0100 Subject: [PATCH 07/29] review --- conans/client/remote_manager_git.py | 23 +++++-------- .../remote/local_tree_remote_test.py | 34 ++++++++++++++----- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py index 8ce3d418989..010130560b6 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/remote_manager_git.py @@ -1,4 +1,3 @@ -import json import os import sys from distutils.dir_util import copy_tree @@ -12,7 +11,7 @@ from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException from conans.model.conf import ConfDefinition from conans.model.recipe_ref import RecipeReference -from conans.util.files import load, save +from conans.util.files import load class GitRemoteManager: @@ -21,7 +20,6 @@ def __init__(self, remote): self._remote = remote cache_folder = self._remote.url self._remote_cache_dir = "{}/.cache".format(cache_folder) - self._exported_file = os.path.join(self._remote_cache_dir, "exported.json") from conan.internal.conan_app import ConanApp global_conf = ConfDefinition() self._app = ConanApp(self._remote_cache_dir, global_conf) @@ -94,26 +92,20 @@ def get_package_revision_reference(self, pref): # Helper methods to implement the interface def _export_recipe(self, ref): - exported = load(self._exported_file) if os.path.isfile(self._exported_file) else "{}" - exported = json.loads(exported) - existing = exported.get(str(ref)) - if existing is not None: - return RecipeReference.loads(existing["ref"]) - folder = self.layout.get_recipe_folder(ref) conanfile_path = os.path.join(folder, "conanfile.py") original_stderr = sys.stderr sys.stderr = StringIO() try: - original_stderr.write(f"Git remote processing {ref}\n") global_conf = ConfDefinition() new_ref, _ = cmd_export(self._app, global_conf, conanfile_path, ref.name, ref.version, None, None) + except Exception as e: + raise ConanException(f"Error while exporting recipe from remote: {self._remote.name}\n" + f"{str(e)}") finally: sys.stderr = original_stderr - # Cache the result, so it is not constantly re-exported - exported[str(ref)] = {"ref": repr(new_ref)} - save(self._exported_file, json.dumps(exported)) + return new_ref @staticmethod @@ -145,12 +137,15 @@ def _load_config_yml(folder): return yaml.safe_load(load(config)) def get_recipes_references(self, pattern): + name_pattern = pattern.split("/", 1)[0] recipes_dir = os.path.join(self._base_folder, "recipes") recipes = os.listdir(recipes_dir) recipes.sort() ret = [] excluded = set() for r in recipes: + if not fnmatch(r, name_pattern): + continue folder = self._get_base_folder(r) config_yml = self._load_config_yml(folder) if config_yml is None: @@ -165,7 +160,7 @@ def get_recipes_references(self, pattern): # This check can be removed after compatibility with 2.0 conanfile = os.path.join(recipes_dir, r, subfolder, "conanfile.py") conanfile_content = load(conanfile) - if "from conans" in conanfile_content or "import conants" in conanfile_content: + if "from conans" in conanfile_content or "import conans" in conanfile_content: excluded.add(r) continue ret.append(RecipeReference.loads(ref)) diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/local_tree_remote_test.py index f1dc7ff165c..ce4c70a5c69 100644 --- a/conans/test/integration/remote/local_tree_remote_test.py +++ b/conans/test/integration/remote/local_tree_remote_test.py @@ -135,9 +135,9 @@ def test_install(self, c3i_folder): # Update doesn't fail, but doesn't update revision time c.run("install --requires libcurl/1.0 --update") - bins = {"libcurl/1.0": "Cache (ccifork)", - "openssl/2.0": "Cache (ccifork)", - "zlib/1.2.11": "Cache (ccifork)"} + bins = {"libcurl/1.0": "Cache (Updated date) (ccifork)", + "openssl/2.0": "Cache (Updated date) (ccifork)", + "zlib/1.2.11": "Cache (Updated date) (ccifork)"} c.assert_listed_require(bins) assert "zlib/1.2.11: Already installed!" in c.out @@ -149,11 +149,7 @@ def test_install(self, c3i_folder): save(os.path.join(c3i_folder, "recipes", "zlib", "all", "conanfile.py"), str(GenConanfile()) + "\n") c.run("install --requires=libcurl/1.0 --build missing --update") - # it is not updated - assert "zlib/1.2.11#6f5c31bb1219e9393743d1fbf2ee1b52 - Cache (ccifork)" in c.out - # We could remove the cached exported to force its re-creation - os.remove(os.path.join(c3i_folder, ".cache", "exported.json")) - c.run("install --requires=libcurl/1.0 --build missing --update") + # it is updated assert "zlib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (ccifork)" in c.out @@ -167,3 +163,25 @@ def test_upload(self): c.run("create .") c.run("upload pkg/0.1 -r=ccifork", assert_error=True) assert "ERROR: Git remote 'ccifork' doesn't support upload" in c.out + + +class TestErrorsUx: + def test_errors(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + zlib_config = textwrap.dedent(""" + versions: + "1.2.11": + folder: all + """) + zlib = textwrap.dedent(""" + class Zlib(ConanFile): + name = "zlib" + """) + save_files(recipes_folder, + {"zlib/config.yml": zlib_config, + "zlib/all/conanfile.py": zlib}) + c = TestClient() + c.run(f"remote add ccifork '{folder}' --type=local") + c.run("install --requires=zlib/[*] --build missing", assert_error=True) + assert "NameError: name 'ConanFile' is not defined" in c.out From 43d4d8cd78b6a635f7f219428264c4c86ad6fea4 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 21 Dec 2023 16:43:06 +0100 Subject: [PATCH 08/29] fix boost --- conans/client/remote_manager.py | 5 ++++ conans/client/remote_manager_git.py | 2 +- .../remote/local_tree_remote_test.py | 30 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index aec5635aa7f..942c1863155 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -51,6 +51,11 @@ def get_recipe(self, ref, remote, metadata=None): layout = self._cache.get_or_create_ref_layout(ref) layout.export_remove() + export_folder = layout.export() + git_remote = self._git_remote(remote) + if git_remote is not None: + return git_remote.get_recipe(ref, export_folder) + download_export = layout.download_export() try: zipped_files = self._call_remote(remote, "get_recipe", ref, download_export, metadata, diff --git a/conans/client/remote_manager_git.py b/conans/client/remote_manager_git.py index 010130560b6..52f34e063b0 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/remote_manager_git.py @@ -28,7 +28,7 @@ def __init__(self, remote): def call_method(self, method_name, *args, **kwargs): return getattr(self, method_name)(*args, **kwargs) - def get_recipe(self, ref, dest_folder, metadata, only_metadata): + def get_recipe(self, ref, dest_folder): export_folder = self._app.cache.recipe_layout(ref).export() return self._copy_files(export_folder, dest_folder) diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/local_tree_remote_test.py index ce4c70a5c69..e62f2bbb715 100644 --- a/conans/test/integration/remote/local_tree_remote_test.py +++ b/conans/test/integration/remote/local_tree_remote_test.py @@ -152,6 +152,36 @@ def test_install(self, c3i_folder): # it is updated assert "zlib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (ccifork)" in c.out + def test_install_with_exported_files(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + boost_config = textwrap.dedent(""" + versions: + "1.0": + folder: all + """) + boost = textwrap.dedent(""" + import os + from conan.tools.files import load + from conan import ConanFile + class Boost(ConanFile): + name = "boost" + version = "1.0" + exports = "*" + def source(self): + myfile = os.path.join(self.recipe_folder, "dependencies", "myfile.json") + self.output.info(load(self, myfile)) + """) + deps_json = '{"potato": 42}' + save_files(recipes_folder, + {"boost/config.yml": boost_config, + "boost/all/conanfile.py": boost, + "boost/all/dependencies/myfile.json": deps_json}) + c = TestClient() + c.run(f"remote add ccifork '{folder}' --type=local") + c.run("install --requires=boost/[*] --build missing") + assert 'boost/1.0: {"potato": 42}' in c.out + class TestRestrictedOperations: def test_upload(self): From aa8a0f438ad017eafd04e3a20fb591c0a8dd42e9 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 21 Dec 2023 16:53:11 +0100 Subject: [PATCH 09/29] wip --- conans/client/remote_manager.py | 26 ++++++++++--------- ...e_manager_git.py => rest_client_folder.py} | 6 ++++- 2 files changed, 19 insertions(+), 13 deletions(-) rename conans/client/{remote_manager_git.py => rest_client_folder.py} (97%) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 942c1863155..06c413be684 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -7,7 +7,7 @@ from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import METADATA from conans.client.cache.remote_registry import Remote -from conans.client.remote_manager_git import GitRemoteManager +from conans.client.rest_client_folder import RestApiClientFolder from conans.client.pkg_sign import PkgSignaturesPlugin from conans.errors import ConanConnectionError, ConanException, NotFoundException, \ PackageNotFoundException @@ -27,9 +27,9 @@ def __init__(self, cache, auth_manager): self._signer = PkgSignaturesPlugin(cache) @staticmethod - def _git_remote(remote): + def _local_folder_remote(remote): if remote.remote_type == "local": - return GitRemoteManager(remote) + return RestApiClientFolder(remote) def check_credentials(self, remote): self._call_remote(remote, "check_credentials") @@ -52,9 +52,10 @@ def get_recipe(self, ref, remote, metadata=None): layout.export_remove() export_folder = layout.export() - git_remote = self._git_remote(remote) - if git_remote is not None: - return git_remote.get_recipe(ref, export_folder) + local_folder_remote = self._local_folder_remote(remote) + if local_folder_remote is not None: + local_folder_remote.get_recipe(ref, export_folder) + return download_export = layout.download_export() try: @@ -109,9 +110,10 @@ def get_recipe_sources(self, ref, layout, remote): download_folder = layout.download_export() export_sources_folder = layout.export_sources() - git_remote = self._git_remote(remote) - if git_remote is not None: - return git_remote.get_recipe_sources(ref, export_sources_folder) + local_folder_remote = self._local_folder_remote(remote) + if local_folder_remote is not None: + local_folder_remote.get_recipe_sources(ref, export_sources_folder) + return zipped_files = self._call_remote(remote, "get_recipe_sources", ref, download_folder) if not zipped_files: @@ -247,9 +249,9 @@ def _call_remote(self, remote, method, *args, **kwargs): enforce_disabled = kwargs.pop("enforce_disabled", True) if remote.disabled and enforce_disabled: raise ConanException("Remote '%s' is disabled" % remote.name) - git_remote = self._git_remote(remote) - if git_remote is not None: - return git_remote.call_method(method, *args, **kwargs) + local_folder_remote = self._local_folder_remote(remote) + if local_folder_remote is not None: + return local_folder_remote.call_method(method, *args, **kwargs) try: return self._auth_manager.call_rest_api_method(remote, method, *args, **kwargs) except ConnectionError as exc: diff --git a/conans/client/remote_manager_git.py b/conans/client/rest_client_folder.py similarity index 97% rename from conans/client/remote_manager_git.py rename to conans/client/rest_client_folder.py index 52f34e063b0..0f280d52428 100644 --- a/conans/client/remote_manager_git.py +++ b/conans/client/rest_client_folder.py @@ -14,7 +14,11 @@ from conans.util.files import load -class GitRemoteManager: +class RestApiClientFolder: + """ + Implements the RestAPI but instead of over HTTP for a remote server, using just + a local folder assuming the conan-center-index repo layout + """ def __init__(self, remote): self._remote = remote From 46e0f07ef96d4d257b7dbc62ce6fde5261759f87 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 22 Dec 2023 00:34:54 +0100 Subject: [PATCH 10/29] add hook --- conans/client/rest_client_folder.py | 18 +++++++- ...te_test.py => test_local_folder_remote.py} | 42 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) rename conans/test/integration/remote/{local_tree_remote_test.py => test_local_folder_remote.py} (84%) diff --git a/conans/client/rest_client_folder.py b/conans/client/rest_client_folder.py index 0f280d52428..d97b22f3c0b 100644 --- a/conans/client/rest_client_folder.py +++ b/conans/client/rest_client_folder.py @@ -1,5 +1,6 @@ import os import sys +import textwrap from distutils.dir_util import copy_tree from fnmatch import fnmatch from io import StringIO @@ -7,11 +8,12 @@ import yaml from conan.api.output import ConanOutput +from conan.internal.cache.home_paths import HomePaths from conans.client.cmd.export import cmd_export from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException from conans.model.conf import ConfDefinition from conans.model.recipe_ref import RecipeReference -from conans.util.files import load +from conans.util.files import load, save class RestApiClientFolder: @@ -24,6 +26,17 @@ def __init__(self, remote): self._remote = remote cache_folder = self._remote.url self._remote_cache_dir = "{}/.cache".format(cache_folder) + hook_folder = HomePaths(self._remote_cache_dir).hooks_path + trim_hook = os.path.join(hook_folder, "hook_trim_conandata.py") + if not os.path.exists(trim_hook): + hook_content = textwrap.dedent("""\ + from conan.tools.files import trim_conandata + def post_export(conanfile): + if conanfile.conan_data: + trim_conandata(conanfile) + """) + save(trim_hook, hook_content) + from conan.internal.conan_app import ConanApp global_conf = ConfDefinition() self._app = ConanApp(self._remote_cache_dir, global_conf) @@ -108,7 +121,10 @@ def _export_recipe(self, ref): raise ConanException(f"Error while exporting recipe from remote: {self._remote.name}\n" f"{str(e)}") finally: + export_stderr = sys.stderr.getvalue() sys.stderr = original_stderr + ConanOutput().debug(f"Internal export for {ref}:\n" + f"{textwrap.indent(export_stderr, ' ')}") return new_ref diff --git a/conans/test/integration/remote/local_tree_remote_test.py b/conans/test/integration/remote/test_local_folder_remote.py similarity index 84% rename from conans/test/integration/remote/local_tree_remote_test.py rename to conans/test/integration/remote/test_local_folder_remote.py index e62f2bbb715..ffbf183b32d 100644 --- a/conans/test/integration/remote/local_tree_remote_test.py +++ b/conans/test/integration/remote/test_local_folder_remote.py @@ -182,6 +182,48 @@ def source(self): c.run("install --requires=boost/[*] --build missing") assert 'boost/1.0: {"potato": 42}' in c.out + def test_trim_conandata_yaml(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + config = textwrap.dedent(""" + versions: + "1.0": + folder: all + """) + conandata = textwrap.dedent("""\ + sources: + "1.0": + url: + sha256: "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e" + patches: + "1.0": + - patch_file: "patches/1.3/0001-fix-cmake.patch" + """) + save_files(recipes_folder, + {"pkg/config.yml": config, + "pkg/all/conanfile.py": str(GenConanfile("pkg")), + "pkg/all/conandata.yml": conandata}) + c = TestClient() + c.run(f"remote add ccifork '{folder}' --type=local") + c.run("install --requires=pkg/1.0 --build missing -vvv") + assert "pkg/1.0#86b609916bbdfe63c579f034ad0edfe7" in c.out + + # User modifies conandata.yml to add new version + new_conandata = textwrap.dedent("""\ + sources: + "1.0": + url: + sha256: "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e" + "1.1": + url: + patches: + "1.0": + - patch_file: "patches/1.3/0001-fix-cmake.patch" + """) + save_files(recipes_folder, {"pkg/all/conandata.yml": new_conandata}) + c.run("install --requires=pkg/1.0 --build missing --update -vvv") + assert "pkg/1.0#86b609916bbdfe63c579f034ad0edfe7" in c.out + class TestRestrictedOperations: def test_upload(self): From ff269e592daefd92e076aa48d914aa0873c549cf Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 12 Jan 2024 14:23:03 +0100 Subject: [PATCH 11/29] review --- conan/api/model.py | 7 +- conan/api/subapi/new.py | 7 +- conan/cli/commands/remote.py | 4 +- conan/internal/api/new/oss_recipe.py | 102 ++++++++++++++++++ conans/client/remote_manager.py | 9 +- ...t_folder.py => rest_client_oss_recipes.py} | 15 +-- ...der_remote.py => test_oss_recipes_repo.py} | 89 ++++++++++----- 7 files changed, 187 insertions(+), 46 deletions(-) create mode 100644 conan/internal/api/new/oss_recipe.py rename conans/client/{rest_client_folder.py => rest_client_oss_recipes.py} (95%) rename conans/test/integration/remote/{test_local_folder_remote.py => test_oss_recipes_repo.py} (72%) diff --git a/conan/api/model.py b/conan/api/model.py index b4738969c16..ad37d54f2c6 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -9,6 +9,8 @@ from conans.util.files import load from conans.model.version_range import VersionRange +OSS_RECIPES = "oss-recipes" + class Remote: @@ -30,8 +32,9 @@ def __eq__(self, other): self.verify_ssl == other.verify_ssl and self.disabled == other.disabled) def __str__(self): - if self.remote_type == "local": - return "{}: {} [Local: True, Enabled: {}]".format(self.name, self.url, not self.disabled) + if self.remote_type == OSS_RECIPES: + return "{}: {} [{}, Enabled: {}]".format(self.name, self.url, OSS_RECIPES, + not self.disabled) return "{}: {} [Verify SSL: {}, Enabled: {}]".format(self.name, self.url, self.verify_ssl, not self.disabled) diff --git a/conan/api/subapi/new.py b/conan/api/subapi/new.py index df08694b6f7..3cebe383620 100644 --- a/conan/api/subapi/new.py +++ b/conan/api/subapi/new.py @@ -14,7 +14,8 @@ class NewAPI: def __init__(self, conan_api): self.conan_api = conan_api - def get_builtin_template(self, template_name): + @staticmethod + def get_builtin_template(template_name): from conan.internal.api.new.basic import basic_file from conan.internal.api.new.alias_new import alias_file from conan.internal.api.new.cmake_exe import cmake_exe_files @@ -27,6 +28,7 @@ def get_builtin_template(self, template_name): from conan.internal.api.new.bazel_exe import bazel_exe_files from conan.internal.api.new.autotools_lib import autotools_lib_files from conan.internal.api.new.autoools_exe import autotools_exe_files + from conan.internal.api.new.oss_recipe import oss_recipe_files new_templates = {"basic": basic_file, "cmake_lib": cmake_lib_files, "cmake_exe": cmake_exe_files, @@ -38,7 +40,8 @@ def get_builtin_template(self, template_name): "bazel_exe": bazel_exe_files, "autotools_lib": autotools_lib_files, "autotools_exe": autotools_exe_files, - "alias": alias_file} + "alias": alias_file, + "oss_recipe": oss_recipe_files} template_files = new_templates.get(template_name) return template_files diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 0b2f57f94e3..dadf1a1a9a8 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -3,7 +3,7 @@ from conan.api.output import cli_out_write, Color from conan.api.conan_api import ConanAPI -from conan.api.model import Remote +from conan.api.model import Remote, OSS_RECIPES from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.commands.list import remote_color, error_color, recipe_color, \ reference_color @@ -72,7 +72,7 @@ def remote_add(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") - subparser.add_argument("-t", "--type", choices=["local"], + subparser.add_argument("-t", "--type", choices=[OSS_RECIPES], help="Define the remote type") subparser.set_defaults(secure=True) args = parser.parse_args(*args) diff --git a/conan/internal/api/new/oss_recipe.py b/conan/internal/api/new/oss_recipe.py new file mode 100644 index 00000000000..640ecc09651 --- /dev/null +++ b/conan/internal/api/new/oss_recipe.py @@ -0,0 +1,102 @@ +from conan.internal.api.new.cmake_lib import test_conanfile_v2, test_cmake_v2 + +config_yml = """\ +versions: + "{{version}}": + folder: all +""" + +conandata_yml = """\ +sources: + "{{version}}": + url: + {% if url is defined -%} + - "{{url}}" + {% else -%} + - "http://put/here/the/url/to/release.1.2.3.zip" + {% endif %} + {% if sha256 is defined -%} + sha256: "{{sha256}}" + {%- else -%} + sha256: "Put here your tarball sha256" + {% endif -%} +""" + + +conanfile = """\ +from conan import ConanFile +from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps +from conan.tools.files import apply_conandata_patches, export_conandata_patches, get, load, replace_in_file, save + + +class {{package_name}}Recipe(ConanFile): + name = "{{name}}" + version = "{{version}}" + package_type = "library" + + # Optional metadata + license = "" + author = " " + url = "" + description = "" + topics = ("", "", "") + + # Binary configuration + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def export_sources(self): + export_conandata_patches(self) + + def source(self): + get(self, **self.conan_data["sources"][self.version], destination=self.source_folder) + apply_conandata_patches(self) + + def layout(self): + cmake_layout(self, src_folder="src") + + def generate(self): + deps = CMakeDeps(self) + deps.generate() + tc = CMakeToolchain(self) + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.libs = ["{{name}}"] + + {% if requires is defined -%} + def requirements(self): + {% for require in requires -%} + self.requires("{{ require }}") + {% endfor %} + {%- endif %} + + {% if tool_requires is defined -%} + def build_requirements(self): + {% for require in tool_requires -%} + self.tool_requires("{{ require }}") + {% endfor %} + {%- endif %} +""" + +oss_recipe_files = {"recipes/{{name}}/config.yml": config_yml, + "recipes/{{name}}/all/conandata.yml": conandata_yml, + "recipes/{{name}}/all/conanfile.py": conanfile} diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index d0012da18d4..7e51ed07d50 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -4,10 +4,11 @@ from requests.exceptions import ConnectionError +from conan.api.model import OSS_RECIPES from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import METADATA from conans.client.cache.remote_registry import Remote -from conans.client.rest_client_folder import RestApiClientFolder +from conans.client.rest_client_oss_recipes import RestApiClientOSSRecipes from conans.client.pkg_sign import PkgSignaturesPlugin from conans.errors import ConanConnectionError, ConanException, NotFoundException, \ PackageNotFoundException @@ -28,8 +29,8 @@ def __init__(self, cache, auth_manager): @staticmethod def _local_folder_remote(remote): - if remote.remote_type == "local": - return RestApiClientFolder(remote) + if remote.remote_type == OSS_RECIPES: + return RestApiClientOSSRecipes(remote) def check_credentials(self, remote): self._call_remote(remote, "check_credentials") @@ -55,7 +56,7 @@ def get_recipe(self, ref, remote, metadata=None): local_folder_remote = self._local_folder_remote(remote) if local_folder_remote is not None: local_folder_remote.get_recipe(ref, export_folder) - return + return layout download_export = layout.download_export() try: diff --git a/conans/client/rest_client_folder.py b/conans/client/rest_client_oss_recipes.py similarity index 95% rename from conans/client/rest_client_folder.py rename to conans/client/rest_client_oss_recipes.py index d97b22f3c0b..dc1797ffb20 100644 --- a/conans/client/rest_client_folder.py +++ b/conans/client/rest_client_oss_recipes.py @@ -16,7 +16,7 @@ from conans.util.files import load, save -class RestApiClientFolder: +class RestApiClientOSSRecipes: """ Implements the RestAPI but instead of over HTTP for a remote server, using just a local folder assuming the conan-center-index repo layout @@ -38,9 +38,10 @@ def post_export(conanfile): save(trim_hook, hook_content) from conan.internal.conan_app import ConanApp - global_conf = ConfDefinition() - self._app = ConanApp(self._remote_cache_dir, global_conf) - self.layout = _ConanCenterIndexLayout(cache_folder) + from conan.api.conan_api import ConanAPI + conan_api = ConanAPI(self._remote_cache_dir) + self._app = ConanApp(conan_api) + self._layout = _OSSRecipesRepoLayout(cache_folder) def call_method(self, method_name, *args, **kwargs): return getattr(self, method_name)(*args, **kwargs) @@ -69,7 +70,7 @@ def check_credentials(self): raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") def search(self, pattern=None): - return self.layout.get_recipes_references(pattern) + return self._layout.get_recipes_references(pattern) def search_packages(self, reference): assert self and reference @@ -109,7 +110,7 @@ def get_package_revision_reference(self, pref): # Helper methods to implement the interface def _export_recipe(self, ref): - folder = self.layout.get_recipe_folder(ref) + folder = self._layout.get_recipe_folder(ref) conanfile_path = os.path.join(folder, "conanfile.py") original_stderr = sys.stderr sys.stderr = StringIO() @@ -141,7 +142,7 @@ def _copy_files(source_folder, dest_folder): return ret -class _ConanCenterIndexLayout: +class _OSSRecipesRepoLayout: def __init__(self, base_folder): self._base_folder = base_folder diff --git a/conans/test/integration/remote/test_local_folder_remote.py b/conans/test/integration/remote/test_oss_recipes_repo.py similarity index 72% rename from conans/test/integration/remote/test_local_folder_remote.py rename to conans/test/integration/remote/test_oss_recipes_repo.py index ffbf183b32d..511f4175800 100644 --- a/conans/test/integration/remote/test_local_folder_remote.py +++ b/conans/test/integration/remote/test_oss_recipes_repo.py @@ -4,10 +4,14 @@ import pytest +from conans.test.assets.cmake import gen_cmakelists from conans.test.assets.genconanfile import GenConanfile +from conans.test.assets.sources import gen_function_cpp, gen_function_h +from conans.test.utils.file_server import TestFileServer +from conans.test.utils.scm import create_local_git_repo from conans.test.utils.test_files import temp_folder -from conans.test.utils.tools import TestClient -from conans.util.files import mkdir, save, save_files +from conans.test.utils.tools import TestClient, zipdir +from conans.util.files import mkdir, save, save_files, sha256sum @pytest.fixture(scope="module") @@ -65,10 +69,10 @@ def build(self): class TestSearchList: def test_basic_search(self, c3i_folder): client = TestClient() - client.run(f"remote add ccifork '{c3i_folder}' --type=local") + client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") client.run("search *") assert textwrap.dedent("""\ - ccifork + local libcurl libcurl/1.0 openssl @@ -82,43 +86,43 @@ def test_basic_search(self, c3i_folder): def test_list_refs(self, c3i_folder): client = TestClient() - client.run(f"remote add ccifork '{c3i_folder}' --type=local") - client.run("list *#* -r=ccifork --format=json") + client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + client.run("list *#* -r=local --format=json") listjson = json.loads(client.stdout) - revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] + revs = listjson["local"]["libcurl/1.0"]["revisions"] assert len(revs) == 1 and "e468388f0e4e098d5b62ad68979aebd5" in revs - revs = listjson["ccifork"]["openssl/1.0"]["revisions"] + revs = listjson["local"]["openssl/1.0"]["revisions"] assert len(revs) == 1 and "b35ffb31b6d5a9d8af39f5de3cf4fd63" in revs - revs = listjson["ccifork"]["openssl/1.1"]["revisions"] + revs = listjson["local"]["openssl/1.1"]["revisions"] assert len(revs) == 1 and "b35ffb31b6d5a9d8af39f5de3cf4fd63" in revs - revs = listjson["ccifork"]["openssl/2.0"]["revisions"] + revs = listjson["local"]["openssl/2.0"]["revisions"] assert len(revs) == 1 and "e50e871efca149f160fa6354c8534449" in revs - revs = listjson["ccifork"]["zlib/1.2.8"]["revisions"] + revs = listjson["local"]["zlib/1.2.8"]["revisions"] assert len(revs) == 1 and "6f5c31bb1219e9393743d1fbf2ee1b52" in revs - revs = listjson["ccifork"]["zlib/1.2.11"]["revisions"] + revs = listjson["local"]["zlib/1.2.11"]["revisions"] assert len(revs) == 1 and "6f5c31bb1219e9393743d1fbf2ee1b52" in revs def test_list_rrevs(self, c3i_folder): client = TestClient() - client.run(f"remote add ccifork '{c3i_folder}' --type=local") - client.run("list libcurl/1.0#* -r=ccifork --format=json") + client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + client.run("list libcurl/1.0#* -r=local --format=json") listjson = json.loads(client.stdout) - revs = listjson["ccifork"]["libcurl/1.0"]["revisions"] + revs = listjson["local"]["libcurl/1.0"]["revisions"] assert len(revs) == 1 and "e468388f0e4e098d5b62ad68979aebd5" in revs def test_list_binaries(self, c3i_folder): client = TestClient() - client.run(f"remote add ccifork '{c3i_folder}' --type=local") - client.run("list libcurl/1.0:* -r=ccifork --format=json") + client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + client.run("list libcurl/1.0:* -r=local --format=json") listjson = json.loads(client.stdout) - rev = listjson["ccifork"]["libcurl/1.0"]["revisions"]["e468388f0e4e098d5b62ad68979aebd5"] + rev = listjson["local"]["libcurl/1.0"]["revisions"]["e468388f0e4e098d5b62ad68979aebd5"] assert rev["packages"] == {} class TestInstall: def test_install(self, c3i_folder): c = TestClient() - c.run(f"remote add ccifork '{c3i_folder}' --type=local") + c.run(f"remote add local '{c3i_folder}' --type=oss-recipes") c.run("install --requires=libcurl/1.0 --build missing") assert "zlib/1.2.11: CONANDATA: {}" in c.out assert "zlib/1.2.11: BUILDING: //myheader" in c.out @@ -135,9 +139,9 @@ def test_install(self, c3i_folder): # Update doesn't fail, but doesn't update revision time c.run("install --requires libcurl/1.0 --update") - bins = {"libcurl/1.0": "Cache (Updated date) (ccifork)", - "openssl/2.0": "Cache (Updated date) (ccifork)", - "zlib/1.2.11": "Cache (Updated date) (ccifork)"} + bins = {"libcurl/1.0": "Cache (Updated date) (local)", + "openssl/2.0": "Cache (Updated date) (local)", + "zlib/1.2.11": "Cache (Updated date) (local)"} c.assert_listed_require(bins) assert "zlib/1.2.11: Already installed!" in c.out @@ -150,7 +154,7 @@ def test_install(self, c3i_folder): str(GenConanfile()) + "\n") c.run("install --requires=libcurl/1.0 --build missing --update") # it is updated - assert "zlib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (ccifork)" in c.out + assert "zlib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (local)" in c.out def test_install_with_exported_files(self): folder = temp_folder() @@ -178,7 +182,7 @@ def source(self): "boost/all/conanfile.py": boost, "boost/all/dependencies/myfile.json": deps_json}) c = TestClient() - c.run(f"remote add ccifork '{folder}' --type=local") + c.run(f"remote add local '{folder}' --type=oss-recipes") c.run("install --requires=boost/[*] --build missing") assert 'boost/1.0: {"potato": 42}' in c.out @@ -204,7 +208,7 @@ def test_trim_conandata_yaml(self): "pkg/all/conanfile.py": str(GenConanfile("pkg")), "pkg/all/conandata.yml": conandata}) c = TestClient() - c.run(f"remote add ccifork '{folder}' --type=local") + c.run(f"remote add local '{folder}' --type=oss-recipes") c.run("install --requires=pkg/1.0 --build missing -vvv") assert "pkg/1.0#86b609916bbdfe63c579f034ad0edfe7" in c.out @@ -230,11 +234,11 @@ def test_upload(self): folder = temp_folder() c3i_folder = os.path.join(folder, "recipes") c = TestClient() - c.run(f"remote add ccifork '{c3i_folder}' --type=local") + c.run(f"remote add local '{c3i_folder}' --type=oss-recipes") c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") - c.run("upload pkg/0.1 -r=ccifork", assert_error=True) - assert "ERROR: Git remote 'ccifork' doesn't support upload" in c.out + c.run("upload pkg/0.1 -r=local", assert_error=True) + assert "ERROR: Git remote 'local' doesn't support upload" in c.out class TestErrorsUx: @@ -254,6 +258,33 @@ class Zlib(ConanFile): {"zlib/config.yml": zlib_config, "zlib/all/conanfile.py": zlib}) c = TestClient() - c.run(f"remote add ccifork '{folder}' --type=local") + c.run(f"remote add local '{folder}' --type=oss-recipes") c.run("install --requires=zlib/[*] --build missing", assert_error=True) assert "NameError: name 'ConanFile' is not defined" in c.out + + +class TestConanNewOssRecipe: + def test(self): + # Setup the release pkg0.1.zip http server + file_server = TestFileServer() + zippath = os.path.join(file_server.store, "pkg0.1.zip") + repo_folder = temp_folder() + cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, + public_header="pkg.h") + save_files(repo_folder, {"CMakeLists.txt": cmake, + "pkg.h": gen_function_h(name="pkg"), + "pkg.cpp": gen_function_cpp(name="pkg")}) + zipdir(repo_folder, zippath) + sha256 = sha256sum(zippath) + url = f"{file_server.fake_url}/pkg0.1.zip" + + c0 = TestClient() + c0.run(f"new oss_recipe -d name=pkg -d version=0.1 -d url={url} -d sha256={sha256}") + oss_recipe_repo = c0.current_folder + + c = TestClient() + c.servers["file_server"] = file_server + c.run(f"remote add local '{oss_recipe_repo}' --type=oss-recipes") + c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") + c.run("create . --build=missing") + assert "pkg: Release!" in c.out From 1b1cc2a39b1fced1cc2b029ceee46fc793a744c1 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 12 Jan 2024 14:58:33 +0100 Subject: [PATCH 12/29] fixes --- conan/internal/api/new/oss_recipe.py | 2 - .../test/functional/test_oss_recipes_repo.py | 112 ++++++++++++++++++ .../remote/test_oss_recipes_repo.py | 35 +----- 3 files changed, 114 insertions(+), 35 deletions(-) create mode 100644 conans/test/functional/test_oss_recipes_repo.py diff --git a/conan/internal/api/new/oss_recipe.py b/conan/internal/api/new/oss_recipe.py index 640ecc09651..e97875701d9 100644 --- a/conan/internal/api/new/oss_recipe.py +++ b/conan/internal/api/new/oss_recipe.py @@ -1,5 +1,3 @@ -from conan.internal.api.new.cmake_lib import test_conanfile_v2, test_cmake_v2 - config_yml = """\ versions: "{{version}}": diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py new file mode 100644 index 00000000000..3d33bebd00d --- /dev/null +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -0,0 +1,112 @@ +import os +import textwrap + +from conans.test.assets.cmake import gen_cmakelists +from conans.test.assets.sources import gen_function_cpp, gen_function_h +from conans.test.utils.file_server import TestFileServer +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient, zipdir +from conans.util.files import save_files, sha256sum + + +class TestConanNewOssRecipe: + def test_conan_new_oss_recipe(self): + # Setup the release pkg0.1.zip http server + file_server = TestFileServer() + zippath = os.path.join(file_server.store, "pkg0.1.zip") + repo_folder = temp_folder() + cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, + public_header="pkg.h") + save_files(repo_folder, {"CMakeLists.txt": cmake, + "pkg.h": gen_function_h(name="pkg"), + "pkg.cpp": gen_function_cpp(name="pkg")}) + zipdir(repo_folder, zippath) + sha256 = sha256sum(zippath) + url = f"{file_server.fake_url}/pkg0.1.zip" + + c0 = TestClient() + c0.run(f"new oss_recipe -d name=pkg -d version=0.1 -d url={url} -d sha256={sha256}") + oss_recipe_repo = c0.current_folder + + c = TestClient() + c.servers["file_server"] = file_server + c.run(f"remote add local '{oss_recipe_repo}' --type=oss-recipes") + c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") + c.run("create . --build=missing") + assert "pkg: Release!" in c.out + + +class TestInRepo: + def test_in_repo(self): + """testing that it is possible to put a "recipes" folder inside a source repo, and + use it as oss-recipes-repository, exporting the source from itself + """ + repo_folder = temp_folder() + cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, + public_header="pkg.h") + config_yml = textwrap.dedent("""\ + versions: + "0.1": + folder: all + """) + conanfile = textwrap.dedent("""\ + import os + from conan import ConanFile + from conan.tools.cmake import CMake, cmake_layout + from conan.tools.files import copy + + class PkgRecipe(ConanFile): + name = "pkg" + package_type = "library" + + # Binary configuration + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + generators = "CMakeToolchain" + + def export_sources(self): + src = os.path.dirname(os.path.dirname(os.path.dirname(self.recipe_folder))) + copy(self, "*", src=src, dst=self.export_sources_folder, excludes=["recipes*"]) + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.libs = [self.name] + """) + + save_files(repo_folder, {"recipes/pkg/config.yml": config_yml, + "recipes/pkg/all/conanfile.py": conanfile, + "CMakeLists.txt": cmake, + "pkg.h": gen_function_h(name="pkg"), + "pkg.cpp": gen_function_cpp(name="pkg")}) + + c = TestClient() + c.run(f"remote add local '{repo_folder}' --type=oss-recipes") + c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") + c.run("create . --build=missing") + assert "pkg: Release!" in c.out + + # Of course the recipe can also be created locally + path = os.path.join(repo_folder, "recipes/pkg/all") + c.run(f'create "{path}" --version=0.1') + print(c.out) diff --git a/conans/test/integration/remote/test_oss_recipes_repo.py b/conans/test/integration/remote/test_oss_recipes_repo.py index 511f4175800..aaadfa5ba50 100644 --- a/conans/test/integration/remote/test_oss_recipes_repo.py +++ b/conans/test/integration/remote/test_oss_recipes_repo.py @@ -4,14 +4,10 @@ import pytest -from conans.test.assets.cmake import gen_cmakelists from conans.test.assets.genconanfile import GenConanfile -from conans.test.assets.sources import gen_function_cpp, gen_function_h -from conans.test.utils.file_server import TestFileServer -from conans.test.utils.scm import create_local_git_repo from conans.test.utils.test_files import temp_folder -from conans.test.utils.tools import TestClient, zipdir -from conans.util.files import mkdir, save, save_files, sha256sum +from conans.test.utils.tools import TestClient +from conans.util.files import mkdir, save, save_files @pytest.fixture(scope="module") @@ -261,30 +257,3 @@ class Zlib(ConanFile): c.run(f"remote add local '{folder}' --type=oss-recipes") c.run("install --requires=zlib/[*] --build missing", assert_error=True) assert "NameError: name 'ConanFile' is not defined" in c.out - - -class TestConanNewOssRecipe: - def test(self): - # Setup the release pkg0.1.zip http server - file_server = TestFileServer() - zippath = os.path.join(file_server.store, "pkg0.1.zip") - repo_folder = temp_folder() - cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, - public_header="pkg.h") - save_files(repo_folder, {"CMakeLists.txt": cmake, - "pkg.h": gen_function_h(name="pkg"), - "pkg.cpp": gen_function_cpp(name="pkg")}) - zipdir(repo_folder, zippath) - sha256 = sha256sum(zippath) - url = f"{file_server.fake_url}/pkg0.1.zip" - - c0 = TestClient() - c0.run(f"new oss_recipe -d name=pkg -d version=0.1 -d url={url} -d sha256={sha256}") - oss_recipe_repo = c0.current_folder - - c = TestClient() - c.servers["file_server"] = file_server - c.run(f"remote add local '{oss_recipe_repo}' --type=oss-recipes") - c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") - c.run("create . --build=missing") - assert "pkg: Release!" in c.out From 73f0d9f43adf0b08c44364dd6d5070451927eb56 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 12 Jan 2024 17:20:50 +0100 Subject: [PATCH 13/29] wip --- conan/api/model.py | 1 + conan/cli/commands/remote.py | 29 ++++++++++++- conan/internal/cache/home_paths.py | 4 ++ conans/client/remote_manager.py | 9 ++-- conans/client/rest_client_oss_recipes.py | 30 ++++++++++---- .../test/functional/test_oss_recipes_repo.py | 2 +- .../remote/test_oss_recipes_repo.py | 41 +++++++++++++++++++ 7 files changed, 101 insertions(+), 15 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index ad37d54f2c6..b9c7870a27c 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -10,6 +10,7 @@ from conans.model.version_range import VersionRange OSS_RECIPES = "oss-recipes" +OSS_RECIPES_GIT = "oss-recipes-git" class Remote: diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index dadf1a1a9a8..55561e4f312 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -1,14 +1,18 @@ import json +import os from collections import OrderedDict from conan.api.output import cli_out_write, Color from conan.api.conan_api import ConanAPI -from conan.api.model import Remote, OSS_RECIPES +from conan.api.model import Remote, OSS_RECIPES, OSS_RECIPES_GIT from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.commands.list import remote_color, error_color, recipe_color, \ reference_color +from conan.internal.cache.home_paths import HomePaths from conans.client.rest.remote_credentials import RemoteCredentials from conan.errors import ConanException +from conans.util.files import chdir +from conans.util.runners import check_output_runner def formatter_remote_list_json(remotes): @@ -72,7 +76,7 @@ def remote_add(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") - subparser.add_argument("-t", "--type", choices=[OSS_RECIPES], + subparser.add_argument("-t", "--type", choices=[OSS_RECIPES, OSS_RECIPES_GIT], help="Define the remote type") subparser.set_defaults(secure=True) args = parser.parse_args(*args) @@ -283,3 +287,24 @@ def remote(conan_api, parser, *args): """ Manage the remote list and the users authenticated on them. """ + + +@conan_subcommand() +def remote_pull(conan_api: ConanAPI, parser, subparser, *args): + """ + Do a git pull for the given remotes + """ + subparser.add_argument("remote", help="Pattern or name of the oss-recipes remotes to pull") + args = parser.parse_args(*args) + + oss_recipes_path = HomePaths(conan_api.home_folder).oss_recipes_path + remotes = conan_api.remotes.list(pattern=args.remote) + if not remotes: + raise ConanException("There are no remotes matching the '{}' pattern".format(args.remote)) + + for r in remotes: + if r.remote_type != OSS_RECIPES_GIT: + continue + remote_folder = os.path.join(oss_recipes_path, r.name) + with chdir(remote_folder): + check_output_runner('git pull') diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index b192e64d76e..282a79e5798 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -12,6 +12,10 @@ class HomePaths: def __init__(self, home_folder): self._home = home_folder + @property + def oss_recipes_path(self): + return os.path.join(self._home, ".cache_oss_recipes") + @property def global_conf_path(self): return os.path.join(self._home, "global.conf") diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 7e51ed07d50..fd9277d5601 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -4,7 +4,7 @@ from requests.exceptions import ConnectionError -from conan.api.model import OSS_RECIPES +from conan.api.model import OSS_RECIPES, OSS_RECIPES_GIT from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import METADATA from conans.client.cache.remote_registry import Remote @@ -27,10 +27,9 @@ def __init__(self, cache, auth_manager): self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache) - @staticmethod - def _local_folder_remote(remote): - if remote.remote_type == OSS_RECIPES: - return RestApiClientOSSRecipes(remote) + def _local_folder_remote(self, remote): + if remote.remote_type in (OSS_RECIPES, OSS_RECIPES_GIT): + return RestApiClientOSSRecipes(remote, self._cache) def check_credentials(self, remote): self._call_remote(remote, "check_credentials") diff --git a/conans/client/rest_client_oss_recipes.py b/conans/client/rest_client_oss_recipes.py index dc1797ffb20..c064c68d38c 100644 --- a/conans/client/rest_client_oss_recipes.py +++ b/conans/client/rest_client_oss_recipes.py @@ -7,13 +7,15 @@ import yaml +from conan.api.model import OSS_RECIPES, OSS_RECIPES_GIT from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conans.client.cmd.export import cmd_export from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException from conans.model.conf import ConfDefinition from conans.model.recipe_ref import RecipeReference -from conans.util.files import load, save +from conans.util.files import load, save, mkdir, chdir +from conans.util.runners import check_output_runner class RestApiClientOSSRecipes: @@ -22,11 +24,25 @@ class RestApiClientOSSRecipes: a local folder assuming the conan-center-index repo layout """ - def __init__(self, remote): + def __init__(self, remote, cache): self._remote = remote - cache_folder = self._remote.url - self._remote_cache_dir = "{}/.cache".format(cache_folder) - hook_folder = HomePaths(self._remote_cache_dir).hooks_path + oss_recipes_path = HomePaths(cache.cache_folder).oss_recipes_path + # TODO: What when a remote name changes or is removed + oss_recipes_path = os.path.join(oss_recipes_path, remote.name) + cache_oss_recipes_path = os.path.join(oss_recipes_path, ".conan") + if remote.remote_type == OSS_RECIPES: + repo_folder = self._remote.url + else: + assert remote.remote_type == OSS_RECIPES_GIT + repo_folder = oss_recipes_path + if not os.path.exists(repo_folder): # clone it + output = ConanOutput() + output.info(f"Remote {remote.name} not cloned yet. Cloning {remote.url}") + mkdir(repo_folder) + with chdir(repo_folder): + check_output_runner(f'git clone "{remote.url}" .') + + hook_folder = HomePaths(cache_oss_recipes_path).hooks_path trim_hook = os.path.join(hook_folder, "hook_trim_conandata.py") if not os.path.exists(trim_hook): hook_content = textwrap.dedent("""\ @@ -39,9 +55,9 @@ def post_export(conanfile): from conan.internal.conan_app import ConanApp from conan.api.conan_api import ConanAPI - conan_api = ConanAPI(self._remote_cache_dir) + conan_api = ConanAPI(cache_oss_recipes_path) self._app = ConanApp(conan_api) - self._layout = _OSSRecipesRepoLayout(cache_folder) + self._layout = _OSSRecipesRepoLayout(repo_folder) def call_method(self, method_name, *args, **kwargs): return getattr(self, method_name)(*args, **kwargs) diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index 3d33bebd00d..48a96354b73 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -109,4 +109,4 @@ def package_info(self): # Of course the recipe can also be created locally path = os.path.join(repo_folder, "recipes/pkg/all") c.run(f'create "{path}" --version=0.1') - print(c.out) + assert "pkg/0.1: Created package" in c.out diff --git a/conans/test/integration/remote/test_oss_recipes_repo.py b/conans/test/integration/remote/test_oss_recipes_repo.py index aaadfa5ba50..3b86989f67f 100644 --- a/conans/test/integration/remote/test_oss_recipes_repo.py +++ b/conans/test/integration/remote/test_oss_recipes_repo.py @@ -5,6 +5,7 @@ import pytest from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.scm import create_local_git_repo, git_add_changes_commit from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient from conans.util.files import mkdir, save, save_files @@ -257,3 +258,43 @@ class Zlib(ConanFile): c.run(f"remote add local '{folder}' --type=oss-recipes") c.run("install --requires=zlib/[*] --build missing", assert_error=True) assert "NameError: name 'ConanFile' is not defined" in c.out + + +class TestOssRecipeGit: + def test_conan_new_oss_recipe(self): + # save repo + config_yml = textwrap.dedent("""\ + versions: + "0.1": + folder: all + """) + conanfile = GenConanfile("pkg") + files = {"recipes/pkg/config.yml": config_yml, + "recipes/pkg/all/conanfile.py": str(conanfile)} + url_folder, commit = create_local_git_repo(files) + + c = TestClient() + c.run(f'remote add local "{url_folder}" --type=oss-recipes-git') + c.run("list *:* -r=local") + assert "Remote local not cloned yet" in c.out + assert "pkg/0.1" in c.out + c.run("list *:* -r=local") + assert "Remote local not cloned yet" not in c.out + assert "pkg/0.1" in c.out + + config_yml = textwrap.dedent("""\ + versions: + "0.1": + folder: all + "0.2": + folder: all + """) + save_files(url_folder, {"recipes/pkg/config.yml": config_yml}) + git_add_changes_commit(url_folder) + c.run("list *:* -r=local") + assert "pkg/0.1" in c.out + assert "pkg/0.2" not in c.out + c.run("remote pull local") + c.run("list *:* -r=local") + assert "pkg/0.1" in c.out + assert "pkg/0.2" in c.out From 2ba740417e166dbc9dd674da890d60ca9139cea1 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 12 Jan 2024 18:31:32 +0100 Subject: [PATCH 14/29] wip --- .../test/functional/test_oss_recipes_repo.py | 42 +++++++++++++++++++ .../remote/test_oss_recipes_repo.py | 41 ------------------ 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index 48a96354b73..837845ed69d 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -2,8 +2,10 @@ import textwrap from conans.test.assets.cmake import gen_cmakelists +from conans.test.assets.genconanfile import GenConanfile from conans.test.assets.sources import gen_function_cpp, gen_function_h from conans.test.utils.file_server import TestFileServer +from conans.test.utils.scm import create_local_git_repo, git_add_changes_commit from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient, zipdir from conans.util.files import save_files, sha256sum @@ -110,3 +112,43 @@ def package_info(self): path = os.path.join(repo_folder, "recipes/pkg/all") c.run(f'create "{path}" --version=0.1') assert "pkg/0.1: Created package" in c.out + + +class TestOssRecipeGit: + def test_conan_new_oss_recipe(self): + # save repo + config_yml = textwrap.dedent("""\ + versions: + "0.1": + folder: all + """) + conanfile = GenConanfile("pkg") + files = {"recipes/pkg/config.yml": config_yml, + "recipes/pkg/all/conanfile.py": str(conanfile)} + url_folder, commit = create_local_git_repo(files) + + c = TestClient() + c.run(f'remote add local "{url_folder}" --type=oss-recipes-git') + c.run("list *:* -r=local") + assert "Remote local not cloned yet" in c.out + assert "pkg/0.1" in c.out + c.run("list *:* -r=local") + assert "Remote local not cloned yet" not in c.out + assert "pkg/0.1" in c.out + + config_yml = textwrap.dedent("""\ + versions: + "0.1": + folder: all + "0.2": + folder: all + """) + save_files(url_folder, {"recipes/pkg/config.yml": config_yml}) + git_add_changes_commit(url_folder) + c.run("list *:* -r=local") + assert "pkg/0.1" in c.out + assert "pkg/0.2" not in c.out + c.run("remote pull local") + c.run("list *:* -r=local") + assert "pkg/0.1" in c.out + assert "pkg/0.2" in c.out diff --git a/conans/test/integration/remote/test_oss_recipes_repo.py b/conans/test/integration/remote/test_oss_recipes_repo.py index 3b86989f67f..aaadfa5ba50 100644 --- a/conans/test/integration/remote/test_oss_recipes_repo.py +++ b/conans/test/integration/remote/test_oss_recipes_repo.py @@ -5,7 +5,6 @@ import pytest from conans.test.assets.genconanfile import GenConanfile -from conans.test.utils.scm import create_local_git_repo, git_add_changes_commit from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient from conans.util.files import mkdir, save, save_files @@ -258,43 +257,3 @@ class Zlib(ConanFile): c.run(f"remote add local '{folder}' --type=oss-recipes") c.run("install --requires=zlib/[*] --build missing", assert_error=True) assert "NameError: name 'ConanFile' is not defined" in c.out - - -class TestOssRecipeGit: - def test_conan_new_oss_recipe(self): - # save repo - config_yml = textwrap.dedent("""\ - versions: - "0.1": - folder: all - """) - conanfile = GenConanfile("pkg") - files = {"recipes/pkg/config.yml": config_yml, - "recipes/pkg/all/conanfile.py": str(conanfile)} - url_folder, commit = create_local_git_repo(files) - - c = TestClient() - c.run(f'remote add local "{url_folder}" --type=oss-recipes-git') - c.run("list *:* -r=local") - assert "Remote local not cloned yet" in c.out - assert "pkg/0.1" in c.out - c.run("list *:* -r=local") - assert "Remote local not cloned yet" not in c.out - assert "pkg/0.1" in c.out - - config_yml = textwrap.dedent("""\ - versions: - "0.1": - folder: all - "0.2": - folder: all - """) - save_files(url_folder, {"recipes/pkg/config.yml": config_yml}) - git_add_changes_commit(url_folder) - c.run("list *:* -r=local") - assert "pkg/0.1" in c.out - assert "pkg/0.2" not in c.out - c.run("remote pull local") - c.run("list *:* -r=local") - assert "pkg/0.1" in c.out - assert "pkg/0.2" in c.out From f4f03b3dad14ac7f75ce23a489c92cb86094b5fa Mon Sep 17 00:00:00 2001 From: James Date: Mon, 15 Jan 2024 16:16:58 +0100 Subject: [PATCH 15/29] Update conan/internal/api/new/oss_recipe.py Co-authored-by: Carlos Zoido --- conan/internal/api/new/oss_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/internal/api/new/oss_recipe.py b/conan/internal/api/new/oss_recipe.py index e97875701d9..bd651c28676 100644 --- a/conan/internal/api/new/oss_recipe.py +++ b/conan/internal/api/new/oss_recipe.py @@ -24,7 +24,7 @@ conanfile = """\ from conan import ConanFile from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps -from conan.tools.files import apply_conandata_patches, export_conandata_patches, get, load, replace_in_file, save +from conan.tools.files import apply_conandata_patches, export_conandata_patches, get class {{package_name}}Recipe(ConanFile): From 88433c7cb6978ab95d6a9277caf07610b75fdcf2 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 15 Jan 2024 16:18:42 +0100 Subject: [PATCH 16/29] Update conan/internal/api/new/oss_recipe.py Co-authored-by: Carlos Zoido --- conan/internal/api/new/oss_recipe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conan/internal/api/new/oss_recipe.py b/conan/internal/api/new/oss_recipe.py index bd651c28676..46607b24a4b 100644 --- a/conan/internal/api/new/oss_recipe.py +++ b/conan/internal/api/new/oss_recipe.py @@ -29,7 +29,6 @@ class {{package_name}}Recipe(ConanFile): name = "{{name}}" - version = "{{version}}" package_type = "library" # Optional metadata From 63d77b152f7154c881ddafb4d8f2ce45f693c860 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 15 Jan 2024 16:23:08 +0100 Subject: [PATCH 17/29] fix --- conans/test/functional/test_oss_recipes_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index 837845ed69d..4450c552402 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -34,7 +34,7 @@ def test_conan_new_oss_recipe(self): c.servers["file_server"] = file_server c.run(f"remote add local '{oss_recipe_repo}' --type=oss-recipes") c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") - c.run("create . --build=missing") + c.run("create . --version=0.1 --build=missing") assert "pkg: Release!" in c.out From 0abdde7b49c61c85aed8c3cc4bfd74dd61982367 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 15 Jan 2024 16:24:57 +0100 Subject: [PATCH 18/29] strip-root --- conan/internal/api/new/oss_recipe.py | 3 ++- conans/test/functional/test_oss_recipes_repo.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conan/internal/api/new/oss_recipe.py b/conan/internal/api/new/oss_recipe.py index 46607b24a4b..45e9ed6a41c 100644 --- a/conan/internal/api/new/oss_recipe.py +++ b/conan/internal/api/new/oss_recipe.py @@ -55,7 +55,8 @@ def export_sources(self): export_conandata_patches(self) def source(self): - get(self, **self.conan_data["sources"][self.version], destination=self.source_folder) + get(self, **self.conan_data["sources"][self.version], destination=self.source_folder, + strip_root=True) apply_conandata_patches(self) def layout(self): diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index 4450c552402..f3a981cd0aa 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -19,9 +19,9 @@ def test_conan_new_oss_recipe(self): repo_folder = temp_folder() cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, public_header="pkg.h") - save_files(repo_folder, {"CMakeLists.txt": cmake, - "pkg.h": gen_function_h(name="pkg"), - "pkg.cpp": gen_function_cpp(name="pkg")}) + save_files(repo_folder, {"pkg/CMakeLists.txt": cmake, + "pkg/pkg.h": gen_function_h(name="pkg"), + "pkg/pkg.cpp": gen_function_cpp(name="pkg")}) zipdir(repo_folder, zippath) sha256 = sha256sum(zippath) url = f"{file_server.fake_url}/pkg0.1.zip" From c555d7b0a938ec4bbc83663472ec824affcac212 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 15 Jan 2024 16:32:21 +0100 Subject: [PATCH 19/29] not warn remote-add --- conans/client/cache/remote_registry.py | 7 ++++--- conans/test/integration/remote/test_oss_recipes_repo.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conans/client/cache/remote_registry.py b/conans/client/cache/remote_registry.py index bdeeecc31b6..9d6625aa8d7 100644 --- a/conans/client/cache/remote_registry.py +++ b/conans/client/cache/remote_registry.py @@ -2,7 +2,7 @@ import os from urllib.parse import urlparse -from conan.api.model import Remote +from conan.api.model import Remote, OSS_RECIPES, OSS_RECIPES_GIT from conan.api.output import ConanOutput from conans.errors import ConanException from conans.util.files import load, save @@ -162,7 +162,8 @@ def read(self, remote_name): return ret def add(self, remote: Remote, force=False, index=None): - self._validate_url(remote.url) + if remote.remote_type not in (OSS_RECIPES, OSS_RECIPES_GIT): + self._validate_url(remote.url) remotes = self._load_remotes() remotes.add(remote, force=force, index=index) self.save_remotes(remotes) @@ -175,7 +176,7 @@ def remove(self, remote_name): def update(self, remote_name, url, secure, disabled, index): if url is not None: - self._validate_url(url) + self._validate_url(url) # TODO: Pending to not warn for oss-recipes repos remotes = self._load_remotes() remotes.update(remote_name, url, secure, disabled, index) self.save_remotes(remotes) diff --git a/conans/test/integration/remote/test_oss_recipes_repo.py b/conans/test/integration/remote/test_oss_recipes_repo.py index aaadfa5ba50..314e9941da9 100644 --- a/conans/test/integration/remote/test_oss_recipes_repo.py +++ b/conans/test/integration/remote/test_oss_recipes_repo.py @@ -66,6 +66,7 @@ class TestSearchList: def test_basic_search(self, c3i_folder): client = TestClient() client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + assert "WARN" not in client.out # Make sure it does not complain about url client.run("search *") assert textwrap.dedent("""\ local From c1c4b5339fd09e816d746c4250b7f0c9f6350fed Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 15 Jan 2024 17:55:05 +0100 Subject: [PATCH 20/29] test_package in template and remove clone in conan remote remove --- conan/api/subapi/remotes.py | 9 +++++++++ conan/internal/api/new/oss_recipe.py | 15 ++++++++++++++- conans/test/functional/test_oss_recipes_repo.py | 8 ++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index b5b1b23fb6c..6371053fd0a 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -1,11 +1,14 @@ import fnmatch import os +from conan.api.model import OSS_RECIPES_GIT +from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanApp from conans.client.cache.remote_registry import Remote, RemoteRegistry from conans.client.cmd.user import user_set, users_clean, users_list from conans.errors import ConanException +from conans.util.files import rmdir class RemotesAPI: @@ -69,6 +72,12 @@ def remove(self, pattern: str): app = ConanApp(self.conan_api) remotes = self.list(pattern, only_enabled=False) for remote in remotes: + if remote.remote_type == OSS_RECIPES_GIT: + oss_recipes_path = HomePaths(self.conan_api.cache_folder).oss_recipes_path + oss_recipes_path = os.path.join(oss_recipes_path, remote.name) + ConanOutput().info(f"Removing temporary clone for '{remote.name}' " + f"oss-recipes-git remote") + rmdir(oss_recipes_path) RemoteRegistry(self._remotes_file).remove(remote.name) users_clean(app.cache.localdb, remote.url) diff --git a/conan/internal/api/new/oss_recipe.py b/conan/internal/api/new/oss_recipe.py index 45e9ed6a41c..814b1e76f67 100644 --- a/conan/internal/api/new/oss_recipe.py +++ b/conan/internal/api/new/oss_recipe.py @@ -1,3 +1,5 @@ +from conan.internal.api.new.cmake_lib import test_conanfile_v2, test_cmake_v2 + config_yml = """\ versions: "{{version}}": @@ -95,6 +97,17 @@ def build_requirements(self): {%- endif %} """ + +test_main = """#include "{{name}}.h" + +int main() { + {{package_name}}(); +} +""" + oss_recipe_files = {"recipes/{{name}}/config.yml": config_yml, "recipes/{{name}}/all/conandata.yml": conandata_yml, - "recipes/{{name}}/all/conanfile.py": conanfile} + "recipes/{{name}}/all/conanfile.py": conanfile, + "recipes/{{name}}/all/test_package/conanfile.py": test_conanfile_v2, + "recipes/{{name}}/all/test_package/CMakeLists.txt": test_cmake_v2, + "recipes/{{name}}/all/test_package/src/example.cpp": test_main} diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index f3a981cd0aa..3fdf41f2396 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -27,7 +27,11 @@ def test_conan_new_oss_recipe(self): url = f"{file_server.fake_url}/pkg0.1.zip" c0 = TestClient() + c0.servers["file_server"] = file_server c0.run(f"new oss_recipe -d name=pkg -d version=0.1 -d url={url} -d sha256={sha256}") + # A local create is possible, and it includes a test_package + c0.run("create recipes/pkg/all --version=0.1") + assert "pkg: Release!" in c0.out oss_recipe_repo = c0.current_folder c = TestClient() @@ -152,3 +156,7 @@ def test_conan_new_oss_recipe(self): c.run("list *:* -r=local") assert "pkg/0.1" in c.out assert "pkg/0.2" in c.out + + # Finally lets remove the remote, check that the clone is cleared + c.run(f'remote remove local') + assert "Removing temporary clone for 'local' oss-recipes-git remote" in c.out From d8048d0b9faf960a031db9841c5f3c8458d6ed67 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 15 Jan 2024 18:27:59 +0100 Subject: [PATCH 21/29] fix export patches --- conans/client/rest_client_oss_recipes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/rest_client_oss_recipes.py b/conans/client/rest_client_oss_recipes.py index c064c68d38c..cfd4cc347ae 100644 --- a/conans/client/rest_client_oss_recipes.py +++ b/conans/client/rest_client_oss_recipes.py @@ -133,7 +133,7 @@ def _export_recipe(self, ref): try: global_conf = ConfDefinition() new_ref, _ = cmd_export(self._app, global_conf, conanfile_path, - ref.name, ref.version, None, None) + ref.name, str(ref.version), None, None) except Exception as e: raise ConanException(f"Error while exporting recipe from remote: {self._remote.name}\n" f"{str(e)}") From 70943cc585a676cef824458b1359382b63a9772a Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 15 Jan 2024 18:46:14 +0100 Subject: [PATCH 22/29] cover patches export too --- .../remote/test_oss_recipes_repo.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/conans/test/integration/remote/test_oss_recipes_repo.py b/conans/test/integration/remote/test_oss_recipes_repo.py index 314e9941da9..ec8a44a7f1c 100644 --- a/conans/test/integration/remote/test_oss_recipes_repo.py +++ b/conans/test/integration/remote/test_oss_recipes_repo.py @@ -225,6 +225,51 @@ def test_trim_conandata_yaml(self): c.run("install --requires=pkg/1.0 --build missing --update -vvv") assert "pkg/1.0#86b609916bbdfe63c579f034ad0edfe7" in c.out + def test_export_patches(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + zlib_config = textwrap.dedent(""" + versions: + "0.1": + folder: all + """) + zlib = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import export_conandata_patches, apply_conandata_patches + class Zlib(ConanFile): + name = "zlib" + exports_sources = "*.cpp" + def export_sources(self): + export_conandata_patches(self) + + def source(self): + apply_conandata_patches(self) + """) + conandata_yml = textwrap.dedent("""\ + versions: + "0.1": + patches: + "0.1": + - patch_file: "patches/patch1" + """) + patch = textwrap.dedent("""\ + --- a/main.cpp + +++ b/main.cpp + @@ -0,0 +1 @@ + +hello + """) + save_files(recipes_folder, + {"zlib/config.yml": zlib_config, + "zlib/all/conanfile.py": zlib, + "zlib/all/conandata.yml": conandata_yml, + "zlib/all/patches/patch1": patch, + "zlib/all/main.cpp": "\n"}) + client = TestClient() + client.run(f"remote add local '{folder}' --type=oss-recipes") + client.run("install --requires=zlib/0.1 --build=missing -vv") + assert "zlib/0.1: Copied 1 file: patch1" in client.out + assert "zlib/0.1: Apply patch (file): patches/patch1" in client.out + class TestRestrictedOperations: def test_upload(self): From 58892462c69ad20e6b9eee24aa6a01fc317d216e Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 16 Jan 2024 11:13:19 +0100 Subject: [PATCH 23/29] fix remove --- conan/api/subapi/remotes.py | 8 ++++---- conans/test/functional/test_oss_recipes_repo.py | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 6371053fd0a..46c26fdb9b7 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -1,7 +1,7 @@ import fnmatch import os -from conan.api.model import OSS_RECIPES_GIT +from conan.api.model import OSS_RECIPES_GIT, OSS_RECIPES from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanApp @@ -72,11 +72,11 @@ def remove(self, pattern: str): app = ConanApp(self.conan_api) remotes = self.list(pattern, only_enabled=False) for remote in remotes: - if remote.remote_type == OSS_RECIPES_GIT: + if remote.remote_type in (OSS_RECIPES, OSS_RECIPES_GIT): oss_recipes_path = HomePaths(self.conan_api.cache_folder).oss_recipes_path oss_recipes_path = os.path.join(oss_recipes_path, remote.name) - ConanOutput().info(f"Removing temporary clone for '{remote.name}' " - f"oss-recipes-git remote") + ConanOutput().info(f"Removing temporary files for '{remote.name}' " + f"oss-recipes remote") rmdir(oss_recipes_path) RemoteRegistry(self._remotes_file).remove(remote.name) users_clean(app.cache.localdb, remote.url) diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index 3fdf41f2396..0b67659c384 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -117,6 +117,10 @@ def package_info(self): c.run(f'create "{path}" --version=0.1') assert "pkg/0.1: Created package" in c.out + # Finally lets remove the remote, check that the clone is cleared + c.run('remote remove local') + assert "Removing temporary files for 'local' oss-recipes remote" in c.out + class TestOssRecipeGit: def test_conan_new_oss_recipe(self): @@ -158,5 +162,5 @@ def test_conan_new_oss_recipe(self): assert "pkg/0.2" in c.out # Finally lets remove the remote, check that the clone is cleared - c.run(f'remote remove local') - assert "Removing temporary clone for 'local' oss-recipes-git remote" in c.out + c.run('remote remove local') + assert "Removing temporary files for 'local' oss-recipes remote" in c.out From 90baa0a355cce7375b4ac59cade6015c366508d4 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 16 Jan 2024 11:55:39 +0100 Subject: [PATCH 24/29] refactor add/remove remote --- conan/api/model.py | 4 +- conan/api/subapi/remotes.py | 12 +--- conan/cli/commands/remote.py | 7 +- conans/client/rest_client_oss_recipes.py | 65 ++++++++++++------- .../test/functional/test_oss_recipes_repo.py | 7 +- 5 files changed, 55 insertions(+), 40 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index b9c7870a27c..47266fe1479 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -15,12 +15,14 @@ class Remote: - def __init__(self, name, url, verify_ssl=True, disabled=False, remote_type=None): + def __init__(self, name, url, verify_ssl=True, disabled=False, remote_type=None, + extra_args=None): self._name = name # Read only, is the key self.url = url self.verify_ssl = verify_ssl self.disabled = disabled self.remote_type = remote_type + self.extra_args = extra_args @property def name(self): diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 46c26fdb9b7..acefef65ba9 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -1,14 +1,12 @@ import fnmatch import os -from conan.api.model import OSS_RECIPES_GIT, OSS_RECIPES -from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanApp from conans.client.cache.remote_registry import Remote, RemoteRegistry from conans.client.cmd.user import user_set, users_clean, users_list +from conans.client.rest_client_oss_recipes import add_oss_recipes_remote, remove_oss_recipes_remote from conans.errors import ConanException -from conans.util.files import rmdir class RemotesAPI: @@ -66,18 +64,14 @@ def get(self, remote_name): return RemoteRegistry(self._remotes_file).read(remote_name) def add(self, remote: Remote, force=False, index=None): + add_oss_recipes_remote(self.conan_api, remote) return RemoteRegistry(self._remotes_file).add(remote, force=force, index=index) def remove(self, pattern: str): app = ConanApp(self.conan_api) remotes = self.list(pattern, only_enabled=False) for remote in remotes: - if remote.remote_type in (OSS_RECIPES, OSS_RECIPES_GIT): - oss_recipes_path = HomePaths(self.conan_api.cache_folder).oss_recipes_path - oss_recipes_path = os.path.join(oss_recipes_path, remote.name) - ConanOutput().info(f"Removing temporary files for '{remote.name}' " - f"oss-recipes remote") - rmdir(oss_recipes_path) + remove_oss_recipes_remote(self.conan_api, remote) RemoteRegistry(self._remotes_file).remove(remote.name) users_clean(app.cache.localdb, remote.url) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 55561e4f312..ff9caecc75d 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -78,9 +78,14 @@ def remote_add(conan_api, parser, subparser, *args): help="Force the definition of the remote even if duplicated") subparser.add_argument("-t", "--type", choices=[OSS_RECIPES, OSS_RECIPES_GIT], help="Define the remote type") + subparser.add_argument("-a", "--args", + help=f'"git clone" extra arguments string for --type={OSS_RECIPES_GIT}') subparser.set_defaults(secure=True) args = parser.parse_args(*args) - r = Remote(args.name, args.url, args.secure, disabled=False, remote_type=args.type) + if args.args and args.type != OSS_RECIPES_GIT: + raise ConanException(f"The --args argument is only valid for --type={OSS_RECIPES_GIT}") + r = Remote(args.name, args.url, args.secure, disabled=False, remote_type=args.type, + extra_args=args.args) conan_api.remotes.add(r, force=args.force, index=args.index) diff --git a/conans/client/rest_client_oss_recipes.py b/conans/client/rest_client_oss_recipes.py index cfd4cc347ae..5625bf30b0f 100644 --- a/conans/client/rest_client_oss_recipes.py +++ b/conans/client/rest_client_oss_recipes.py @@ -14,10 +14,49 @@ from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException from conans.model.conf import ConfDefinition from conans.model.recipe_ref import RecipeReference -from conans.util.files import load, save, mkdir, chdir +from conans.util.files import load, save, mkdir, chdir, rmdir from conans.util.runners import check_output_runner +def add_oss_recipes_remote(conan_api, remote): + if remote.remote_type not in (OSS_RECIPES, OSS_RECIPES_GIT): + return + oss_recipes_path = HomePaths(conan_api.cache_folder).oss_recipes_path + repo_folder = os.path.join(oss_recipes_path, remote.name) + + output = ConanOutput() + if os.path.exists(repo_folder): + output.warning(f"The cache folder for remote {remote.name} existed, removing it") + rmdir(repo_folder) + if remote.remote_type == OSS_RECIPES_GIT: + output.info(f"Remote {remote.name} cloning {remote.url}") + mkdir(repo_folder) + with chdir(repo_folder): + check_output_runner(f'git clone "{remote.url}" . {remote.extra_args or ""}') + # Finally, once it is cloned, change the remote URL to point to the local folder + remote.url = repo_folder + + cache_oss_recipes_path = os.path.join(oss_recipes_path, ".conan") + hook_folder = HomePaths(cache_oss_recipes_path).hooks_path + trim_hook = os.path.join(hook_folder, "hook_trim_conandata.py") + hook_content = textwrap.dedent("""\ + from conan.tools.files import trim_conandata + def post_export(conanfile): + if conanfile.conan_data: + trim_conandata(conanfile) + """) + save(trim_hook, hook_content) + + +def remove_oss_recipes_remote(conan_api, remote): + if remote.remote_type in (OSS_RECIPES, OSS_RECIPES_GIT): + oss_recipes_path = HomePaths(conan_api.cache_folder).oss_recipes_path + oss_recipes_path = os.path.join(oss_recipes_path, remote.name) + ConanOutput().info(f"Removing temporary files for '{remote.name}' " + f"oss-recipes remote") + rmdir(oss_recipes_path) + + class RestApiClientOSSRecipes: """ Implements the RestAPI but instead of over HTTP for a remote server, using just @@ -27,31 +66,9 @@ class RestApiClientOSSRecipes: def __init__(self, remote, cache): self._remote = remote oss_recipes_path = HomePaths(cache.cache_folder).oss_recipes_path - # TODO: What when a remote name changes or is removed oss_recipes_path = os.path.join(oss_recipes_path, remote.name) cache_oss_recipes_path = os.path.join(oss_recipes_path, ".conan") - if remote.remote_type == OSS_RECIPES: - repo_folder = self._remote.url - else: - assert remote.remote_type == OSS_RECIPES_GIT - repo_folder = oss_recipes_path - if not os.path.exists(repo_folder): # clone it - output = ConanOutput() - output.info(f"Remote {remote.name} not cloned yet. Cloning {remote.url}") - mkdir(repo_folder) - with chdir(repo_folder): - check_output_runner(f'git clone "{remote.url}" .') - - hook_folder = HomePaths(cache_oss_recipes_path).hooks_path - trim_hook = os.path.join(hook_folder, "hook_trim_conandata.py") - if not os.path.exists(trim_hook): - hook_content = textwrap.dedent("""\ - from conan.tools.files import trim_conandata - def post_export(conanfile): - if conanfile.conan_data: - trim_conandata(conanfile) - """) - save(trim_hook, hook_content) + repo_folder = self._remote.url from conan.internal.conan_app import ConanApp from conan.api.conan_api import ConanAPI diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index 0b67659c384..5b4f4d9a449 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -136,12 +136,9 @@ def test_conan_new_oss_recipe(self): url_folder, commit = create_local_git_repo(files) c = TestClient() - c.run(f'remote add local "{url_folder}" --type=oss-recipes-git') + c.run(f'remote add local "{url_folder}" --type=oss-recipes-git -a="-v"') + assert "Remote local cloning" in c.out c.run("list *:* -r=local") - assert "Remote local not cloned yet" in c.out - assert "pkg/0.1" in c.out - c.run("list *:* -r=local") - assert "Remote local not cloned yet" not in c.out assert "pkg/0.1" in c.out config_yml = textwrap.dedent("""\ From 09ba83ce9dcf6b41eaa4b3677c02572e6b8ce5cd Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 16 Jan 2024 12:35:36 +0100 Subject: [PATCH 25/29] fix --- conans/client/rest_client_oss_recipes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/rest_client_oss_recipes.py b/conans/client/rest_client_oss_recipes.py index 5625bf30b0f..eaf6d54b0bd 100644 --- a/conans/client/rest_client_oss_recipes.py +++ b/conans/client/rest_client_oss_recipes.py @@ -36,7 +36,7 @@ def add_oss_recipes_remote(conan_api, remote): # Finally, once it is cloned, change the remote URL to point to the local folder remote.url = repo_folder - cache_oss_recipes_path = os.path.join(oss_recipes_path, ".conan") + cache_oss_recipes_path = os.path.join(repo_folder, ".conan") hook_folder = HomePaths(cache_oss_recipes_path).hooks_path trim_hook = os.path.join(hook_folder, "hook_trim_conandata.py") hook_content = textwrap.dedent("""\ From c8f83238bd051c1b27375899c2e25eabe01b08a7 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 16 Jan 2024 16:46:37 +0100 Subject: [PATCH 26/29] removed git type --- conan/api/model.py | 5 +-- conan/cli/commands/remote.py | 34 ++------------- conans/client/cache/remote_registry.py | 4 +- conans/client/remote_manager.py | 4 +- conans/client/rest_client_oss_recipes.py | 16 ++------ .../test/functional/test_oss_recipes_repo.py | 41 ------------------- 6 files changed, 13 insertions(+), 91 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index 47266fe1479..ad37d54f2c6 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -10,19 +10,16 @@ from conans.model.version_range import VersionRange OSS_RECIPES = "oss-recipes" -OSS_RECIPES_GIT = "oss-recipes-git" class Remote: - def __init__(self, name, url, verify_ssl=True, disabled=False, remote_type=None, - extra_args=None): + def __init__(self, name, url, verify_ssl=True, disabled=False, remote_type=None): self._name = name # Read only, is the key self.url = url self.verify_ssl = verify_ssl self.disabled = disabled self.remote_type = remote_type - self.extra_args = extra_args @property def name(self): diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index ff9caecc75d..6f8d2b771c3 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -4,7 +4,7 @@ from conan.api.output import cli_out_write, Color from conan.api.conan_api import ConanAPI -from conan.api.model import Remote, OSS_RECIPES, OSS_RECIPES_GIT +from conan.api.model import Remote, OSS_RECIPES from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.commands.list import remote_color, error_color, recipe_color, \ reference_color @@ -76,16 +76,11 @@ def remote_add(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") - subparser.add_argument("-t", "--type", choices=[OSS_RECIPES, OSS_RECIPES_GIT], - help="Define the remote type") - subparser.add_argument("-a", "--args", - help=f'"git clone" extra arguments string for --type={OSS_RECIPES_GIT}') + subparser.add_argument("-t", "--type", choices=[OSS_RECIPES], help="Define the remote type") subparser.set_defaults(secure=True) args = parser.parse_args(*args) - if args.args and args.type != OSS_RECIPES_GIT: - raise ConanException(f"The --args argument is only valid for --type={OSS_RECIPES_GIT}") - r = Remote(args.name, args.url, args.secure, disabled=False, remote_type=args.type, - extra_args=args.args) + + r = Remote(args.name, args.url, args.secure, disabled=False, remote_type=args.type) conan_api.remotes.add(r, force=args.force, index=args.index) @@ -292,24 +287,3 @@ def remote(conan_api, parser, *args): """ Manage the remote list and the users authenticated on them. """ - - -@conan_subcommand() -def remote_pull(conan_api: ConanAPI, parser, subparser, *args): - """ - Do a git pull for the given remotes - """ - subparser.add_argument("remote", help="Pattern or name of the oss-recipes remotes to pull") - args = parser.parse_args(*args) - - oss_recipes_path = HomePaths(conan_api.home_folder).oss_recipes_path - remotes = conan_api.remotes.list(pattern=args.remote) - if not remotes: - raise ConanException("There are no remotes matching the '{}' pattern".format(args.remote)) - - for r in remotes: - if r.remote_type != OSS_RECIPES_GIT: - continue - remote_folder = os.path.join(oss_recipes_path, r.name) - with chdir(remote_folder): - check_output_runner('git pull') diff --git a/conans/client/cache/remote_registry.py b/conans/client/cache/remote_registry.py index 9d6625aa8d7..1336bbc4004 100644 --- a/conans/client/cache/remote_registry.py +++ b/conans/client/cache/remote_registry.py @@ -2,7 +2,7 @@ import os from urllib.parse import urlparse -from conan.api.model import Remote, OSS_RECIPES, OSS_RECIPES_GIT +from conan.api.model import Remote, OSS_RECIPES from conan.api.output import ConanOutput from conans.errors import ConanException from conans.util.files import load, save @@ -162,7 +162,7 @@ def read(self, remote_name): return ret def add(self, remote: Remote, force=False, index=None): - if remote.remote_type not in (OSS_RECIPES, OSS_RECIPES_GIT): + if remote.remote_type != OSS_RECIPES: self._validate_url(remote.url) remotes = self._load_remotes() remotes.add(remote, force=force, index=index) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index fd9277d5601..187fdb2817a 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -4,7 +4,7 @@ from requests.exceptions import ConnectionError -from conan.api.model import OSS_RECIPES, OSS_RECIPES_GIT +from conan.api.model import OSS_RECIPES from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import METADATA from conans.client.cache.remote_registry import Remote @@ -28,7 +28,7 @@ def __init__(self, cache, auth_manager): self._signer = PkgSignaturesPlugin(cache) def _local_folder_remote(self, remote): - if remote.remote_type in (OSS_RECIPES, OSS_RECIPES_GIT): + if remote.remote_type == OSS_RECIPES: return RestApiClientOSSRecipes(remote, self._cache) def check_credentials(self, remote): diff --git a/conans/client/rest_client_oss_recipes.py b/conans/client/rest_client_oss_recipes.py index eaf6d54b0bd..1bc8c3aae65 100644 --- a/conans/client/rest_client_oss_recipes.py +++ b/conans/client/rest_client_oss_recipes.py @@ -7,19 +7,18 @@ import yaml -from conan.api.model import OSS_RECIPES, OSS_RECIPES_GIT +from conan.api.model import OSS_RECIPES from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conans.client.cmd.export import cmd_export from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException from conans.model.conf import ConfDefinition from conans.model.recipe_ref import RecipeReference -from conans.util.files import load, save, mkdir, chdir, rmdir -from conans.util.runners import check_output_runner +from conans.util.files import load, save, rmdir def add_oss_recipes_remote(conan_api, remote): - if remote.remote_type not in (OSS_RECIPES, OSS_RECIPES_GIT): + if remote.remote_type != OSS_RECIPES: return oss_recipes_path = HomePaths(conan_api.cache_folder).oss_recipes_path repo_folder = os.path.join(oss_recipes_path, remote.name) @@ -28,13 +27,6 @@ def add_oss_recipes_remote(conan_api, remote): if os.path.exists(repo_folder): output.warning(f"The cache folder for remote {remote.name} existed, removing it") rmdir(repo_folder) - if remote.remote_type == OSS_RECIPES_GIT: - output.info(f"Remote {remote.name} cloning {remote.url}") - mkdir(repo_folder) - with chdir(repo_folder): - check_output_runner(f'git clone "{remote.url}" . {remote.extra_args or ""}') - # Finally, once it is cloned, change the remote URL to point to the local folder - remote.url = repo_folder cache_oss_recipes_path = os.path.join(repo_folder, ".conan") hook_folder = HomePaths(cache_oss_recipes_path).hooks_path @@ -49,7 +41,7 @@ def post_export(conanfile): def remove_oss_recipes_remote(conan_api, remote): - if remote.remote_type in (OSS_RECIPES, OSS_RECIPES_GIT): + if remote.remote_type == OSS_RECIPES: oss_recipes_path = HomePaths(conan_api.cache_folder).oss_recipes_path oss_recipes_path = os.path.join(oss_recipes_path, remote.name) ConanOutput().info(f"Removing temporary files for '{remote.name}' " diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index 5b4f4d9a449..e0f26037e2a 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -120,44 +120,3 @@ def package_info(self): # Finally lets remove the remote, check that the clone is cleared c.run('remote remove local') assert "Removing temporary files for 'local' oss-recipes remote" in c.out - - -class TestOssRecipeGit: - def test_conan_new_oss_recipe(self): - # save repo - config_yml = textwrap.dedent("""\ - versions: - "0.1": - folder: all - """) - conanfile = GenConanfile("pkg") - files = {"recipes/pkg/config.yml": config_yml, - "recipes/pkg/all/conanfile.py": str(conanfile)} - url_folder, commit = create_local_git_repo(files) - - c = TestClient() - c.run(f'remote add local "{url_folder}" --type=oss-recipes-git -a="-v"') - assert "Remote local cloning" in c.out - c.run("list *:* -r=local") - assert "pkg/0.1" in c.out - - config_yml = textwrap.dedent("""\ - versions: - "0.1": - folder: all - "0.2": - folder: all - """) - save_files(url_folder, {"recipes/pkg/config.yml": config_yml}) - git_add_changes_commit(url_folder) - c.run("list *:* -r=local") - assert "pkg/0.1" in c.out - assert "pkg/0.2" not in c.out - c.run("remote pull local") - c.run("list *:* -r=local") - assert "pkg/0.1" in c.out - assert "pkg/0.2" in c.out - - # Finally lets remove the remote, check that the clone is cleared - c.run('remote remove local') - assert "Removing temporary files for 'local' oss-recipes remote" in c.out From 3e3ff703b1dd1222ca855cf496b593395934d74d Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 16 Jan 2024 16:48:37 +0100 Subject: [PATCH 27/29] unused imports --- conans/test/functional/test_oss_recipes_repo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_oss_recipes_repo.py index e0f26037e2a..d5d5a82678d 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_oss_recipes_repo.py @@ -2,10 +2,8 @@ import textwrap from conans.test.assets.cmake import gen_cmakelists -from conans.test.assets.genconanfile import GenConanfile from conans.test.assets.sources import gen_function_cpp, gen_function_h from conans.test.utils.file_server import TestFileServer -from conans.test.utils.scm import create_local_git_repo, git_add_changes_commit from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient, zipdir from conans.util.files import save_files, sha256sum From bafcd9702c7feb5bce3220c42a577a34c42b8f90 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 17 Jan 2024 11:47:11 +0100 Subject: [PATCH 28/29] review and automatic deduction --- conan/cli/commands/remote.py | 13 ++++++----- conans/client/rest_client_oss_recipes.py | 18 ++++++++------- .../remote/test_oss_recipes_repo.py | 23 ++++++++++--------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 6f8d2b771c3..95a7fc969cc 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -2,17 +2,15 @@ import os from collections import OrderedDict -from conan.api.output import cli_out_write, Color from conan.api.conan_api import ConanAPI from conan.api.model import Remote, OSS_RECIPES +from conan.api.output import cli_out_write, Color +from conan.cli import make_abs_path from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.commands.list import remote_color, error_color, recipe_color, \ reference_color -from conan.internal.cache.home_paths import HomePaths -from conans.client.rest.remote_credentials import RemoteCredentials from conan.errors import ConanException -from conans.util.files import chdir -from conans.util.runners import check_output_runner +from conans.client.rest.remote_credentials import RemoteCredentials def formatter_remote_list_json(remotes): @@ -80,7 +78,10 @@ def remote_add(conan_api, parser, subparser, *args): subparser.set_defaults(secure=True) args = parser.parse_args(*args) - r = Remote(args.name, args.url, args.secure, disabled=False, remote_type=args.type) + url_folder = make_abs_path(args.url) + remote_type = args.type or (OSS_RECIPES if os.path.isdir(url_folder) else None) + url = url_folder if remote_type == OSS_RECIPES else args.url + r = Remote(args.name, url, args.secure, disabled=False, remote_type=remote_type) conan_api.remotes.add(r, force=args.force, index=args.index) diff --git a/conans/client/rest_client_oss_recipes.py b/conans/client/rest_client_oss_recipes.py index 1bc8c3aae65..7463da3a3df 100644 --- a/conans/client/rest_client_oss_recipes.py +++ b/conans/client/rest_client_oss_recipes.py @@ -80,19 +80,21 @@ def get_recipe_sources(self, ref, dest_folder): return self._copy_files(export_sources, dest_folder) def get_package(self, pref, dest_folder, metadata, only_metadata): - raise ConanException(f"The remote '{self._remote.name}' doesn't support binary packages") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support " + "binary packages") def upload_recipe(self, ref, files_to_upload): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support upload") def upload_package(self, pref, files_to_upload): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support upload") def authenticate(self, user, password): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support authentication") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support " + "authentication") def check_credentials(self): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support upload") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support upload") def search(self, pattern=None): return self._layout.get_recipes_references(pattern) @@ -102,13 +104,13 @@ def search_packages(self, reference): return {} def remove_recipe(self, ref): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support remove") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support remove") def remove_all_packages(self, ref): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support remove") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support remove") def remove_packages(self, prefs): - raise ConanException(f"Git remote '{self._remote.name}' doesn't support remove") + raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support remove") def get_recipe_revisions_references(self, ref): ref = self._export_recipe(ref) diff --git a/conans/test/integration/remote/test_oss_recipes_repo.py b/conans/test/integration/remote/test_oss_recipes_repo.py index ec8a44a7f1c..7882a355bff 100644 --- a/conans/test/integration/remote/test_oss_recipes_repo.py +++ b/conans/test/integration/remote/test_oss_recipes_repo.py @@ -65,7 +65,7 @@ def build(self): class TestSearchList: def test_basic_search(self, c3i_folder): client = TestClient() - client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") # Keep --type for test assert "WARN" not in client.out # Make sure it does not complain about url client.run("search *") assert textwrap.dedent("""\ @@ -83,7 +83,7 @@ def test_basic_search(self, c3i_folder): def test_list_refs(self, c3i_folder): client = TestClient() - client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + client.run(f"remote add local '{c3i_folder}'") client.run("list *#* -r=local --format=json") listjson = json.loads(client.stdout) revs = listjson["local"]["libcurl/1.0"]["revisions"] @@ -101,7 +101,7 @@ def test_list_refs(self, c3i_folder): def test_list_rrevs(self, c3i_folder): client = TestClient() - client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + client.run(f"remote add local '{c3i_folder}'") client.run("list libcurl/1.0#* -r=local --format=json") listjson = json.loads(client.stdout) revs = listjson["local"]["libcurl/1.0"]["revisions"] @@ -109,7 +109,7 @@ def test_list_rrevs(self, c3i_folder): def test_list_binaries(self, c3i_folder): client = TestClient() - client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + client.run(f"remote add local '{c3i_folder}'") client.run("list libcurl/1.0:* -r=local --format=json") listjson = json.loads(client.stdout) rev = listjson["local"]["libcurl/1.0"]["revisions"]["e468388f0e4e098d5b62ad68979aebd5"] @@ -119,7 +119,7 @@ def test_list_binaries(self, c3i_folder): class TestInstall: def test_install(self, c3i_folder): c = TestClient() - c.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + c.run(f"remote add local '{c3i_folder}'") c.run("install --requires=libcurl/1.0 --build missing") assert "zlib/1.2.11: CONANDATA: {}" in c.out assert "zlib/1.2.11: BUILDING: //myheader" in c.out @@ -179,7 +179,7 @@ def source(self): "boost/all/conanfile.py": boost, "boost/all/dependencies/myfile.json": deps_json}) c = TestClient() - c.run(f"remote add local '{folder}' --type=oss-recipes") + c.run(f"remote add local '{folder}'") c.run("install --requires=boost/[*] --build missing") assert 'boost/1.0: {"potato": 42}' in c.out @@ -205,7 +205,7 @@ def test_trim_conandata_yaml(self): "pkg/all/conanfile.py": str(GenConanfile("pkg")), "pkg/all/conandata.yml": conandata}) c = TestClient() - c.run(f"remote add local '{folder}' --type=oss-recipes") + c.run(f"remote add local '{folder}'") c.run("install --requires=pkg/1.0 --build missing -vvv") assert "pkg/1.0#86b609916bbdfe63c579f034ad0edfe7" in c.out @@ -265,7 +265,7 @@ def source(self): "zlib/all/patches/patch1": patch, "zlib/all/main.cpp": "\n"}) client = TestClient() - client.run(f"remote add local '{folder}' --type=oss-recipes") + client.run(f"remote add local '{folder}'") client.run("install --requires=zlib/0.1 --build=missing -vv") assert "zlib/0.1: Copied 1 file: patch1" in client.out assert "zlib/0.1: Apply patch (file): patches/patch1" in client.out @@ -275,12 +275,13 @@ class TestRestrictedOperations: def test_upload(self): folder = temp_folder() c3i_folder = os.path.join(folder, "recipes") + mkdir(c3i_folder) c = TestClient() - c.run(f"remote add local '{c3i_folder}' --type=oss-recipes") + c.run(f"remote add local '{c3i_folder}'") c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") c.run("upload pkg/0.1 -r=local", assert_error=True) - assert "ERROR: Git remote 'local' doesn't support upload" in c.out + assert "ERROR: Remote oss-recipes 'local' doesn't support upload" in c.out class TestErrorsUx: @@ -300,6 +301,6 @@ class Zlib(ConanFile): {"zlib/config.yml": zlib_config, "zlib/all/conanfile.py": zlib}) c = TestClient() - c.run(f"remote add local '{folder}' --type=oss-recipes") + c.run(f"remote add local '{folder}'") c.run("install --requires=zlib/[*] --build missing", assert_error=True) assert "NameError: name 'ConanFile' is not defined" in c.out From 74bf893183942c4737fc8efae2614aa9edf0655c Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 22 Jan 2024 13:53:22 +0100 Subject: [PATCH 29/29] renaming all --- conan/api/model.py | 6 +- conan/api/subapi/new.py | 4 +- conan/api/subapi/remotes.py | 13 +++-- conan/cli/commands/remote.py | 9 +-- .../{oss_recipe.py => local_recipes_index.py} | 12 ++-- conan/internal/cache/home_paths.py | 4 +- conans/client/remote_manager.py | 8 +-- ...s.py => rest_client_local_recipe_index.py} | 56 +++++++++---------- ...es_repo.py => test_local_recipes_index.py} | 16 +++--- ...es_repo.py => test_local_recipes_index.py} | 4 +- 10 files changed, 67 insertions(+), 65 deletions(-) rename conan/internal/api/new/{oss_recipe.py => local_recipes_index.py} (83%) rename conans/client/{rest_client_oss_recipes.py => rest_client_local_recipe_index.py} (77%) rename conans/test/functional/{test_oss_recipes_repo.py => test_local_recipes_index.py} (89%) rename conans/test/integration/remote/{test_oss_recipes_repo.py => test_local_recipes_index.py} (98%) diff --git a/conan/api/model.py b/conan/api/model.py index 1b031847850..d831d342901 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -9,7 +9,7 @@ from conans.util.files import load from conans.model.version_range import VersionRange -OSS_RECIPES = "oss-recipes" +LOCAL_RECIPES_INDEX = "local-recipes-index" class Remote: @@ -33,8 +33,8 @@ def __str__(self): allowed_msg = "" if self.allowed_packages: allowed_msg = ", Allowed packages: {}".format(", ".join(self.allowed_packages)) - if self.remote_type == OSS_RECIPES: - return "{}: {} [{}, Enabled: {}{}]".format(self.name, self.url, OSS_RECIPES, + if self.remote_type == LOCAL_RECIPES_INDEX: + return "{}: {} [{}, Enabled: {}{}]".format(self.name, self.url, LOCAL_RECIPES_INDEX, not self.disabled, allowed_msg) return "{}: {} [Verify SSL: {}, Enabled: {}{}]".format(self.name, self.url, self.verify_ssl, not self.disabled, allowed_msg) diff --git a/conan/api/subapi/new.py b/conan/api/subapi/new.py index 3cebe383620..573ce75d49d 100644 --- a/conan/api/subapi/new.py +++ b/conan/api/subapi/new.py @@ -28,7 +28,7 @@ def get_builtin_template(template_name): from conan.internal.api.new.bazel_exe import bazel_exe_files from conan.internal.api.new.autotools_lib import autotools_lib_files from conan.internal.api.new.autoools_exe import autotools_exe_files - from conan.internal.api.new.oss_recipe import oss_recipe_files + from conan.internal.api.new.local_recipes_index import local_recipes_index_files new_templates = {"basic": basic_file, "cmake_lib": cmake_lib_files, "cmake_exe": cmake_exe_files, @@ -41,7 +41,7 @@ def get_builtin_template(template_name): "autotools_lib": autotools_lib_files, "autotools_exe": autotools_exe_files, "alias": alias_file, - "oss_recipe": oss_recipe_files} + "local_recipes_index": local_recipes_index_files} template_files = new_templates.get(template_name) return template_files diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index b2d1806c033..95e6c5dee1d 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -3,11 +3,12 @@ import os from urllib.parse import urlparse -from conan.api.model import Remote, OSS_RECIPES +from conan.api.model import Remote, LOCAL_RECIPES_INDEX from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanApp -from conans.client.rest_client_oss_recipes import add_oss_recipes_remote, remove_oss_recipes_remote +from conans.client.rest_client_local_recipe_index import add_local_recipes_index_remote, \ + remove_local_recipes_index_remote from conans.errors import ConanException from conans.util.files import save, load @@ -105,9 +106,9 @@ def add(self, remote: Remote, force=False, index=None): :param index: if not defined, the new remote will be last one. Pass an integer to insert the remote in that position instead of the last one """ - add_oss_recipes_remote(self.conan_api, remote) + add_local_recipes_index_remote(self.conan_api, remote) remotes = _load(self._remotes_file) - if remote.remote_type != OSS_RECIPES: + if remote.remote_type != LOCAL_RECIPES_INDEX: _validate_url(remote.url) current = {r.name: r for r in remotes}.get(remote.name) if current: # same name remote existing! @@ -142,7 +143,7 @@ def remove(self, pattern): _save(self._remotes_file, remotes) app = ConanApp(self.conan_api) for remote in removed: - remove_oss_recipes_remote(self.conan_api, remote) + remove_local_recipes_index_remote(self.conan_api, remote) app.cache.localdb.clean(remote_url=remote.url) return removed @@ -164,7 +165,7 @@ def update(self, remote_name: str, url=None, secure=None, disabled=None, index=N except KeyError: raise ConanException(f"Remote '{remote_name}' doesn't exist") if url is not None: - if remote.remote_type != OSS_RECIPES: + if remote.remote_type != LOCAL_RECIPES_INDEX: _validate_url(url) _check_urls(remotes, url, force=False, current=remote) remote.url = url diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 61cc464af04..ed445dff330 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -3,7 +3,7 @@ from collections import OrderedDict from conan.api.conan_api import ConanAPI -from conan.api.model import Remote, OSS_RECIPES +from conan.api.model import Remote, LOCAL_RECIPES_INDEX from conan.api.output import cli_out_write, Color from conan.cli import make_abs_path from conan.cli.command import conan_command, conan_subcommand, OnceArgument @@ -78,13 +78,14 @@ def remote_add(conan_api, parser, subparser, *args): subparser.add_argument("-ap", "--allowed-packages", action="append", default=None, help="Add recipe reference pattern to list of allowed packages for " "this remote") - subparser.add_argument("-t", "--type", choices=[OSS_RECIPES], help="Define the remote type") + subparser.add_argument("-t", "--type", choices=[LOCAL_RECIPES_INDEX], + help="Define the remote type") subparser.set_defaults(secure=True) args = parser.parse_args(*args) url_folder = make_abs_path(args.url) - remote_type = args.type or (OSS_RECIPES if os.path.isdir(url_folder) else None) - url = url_folder if remote_type == OSS_RECIPES else args.url + remote_type = args.type or (LOCAL_RECIPES_INDEX if os.path.isdir(url_folder) else None) + url = url_folder if remote_type == LOCAL_RECIPES_INDEX else args.url r = Remote(args.name, url, args.secure, disabled=False, remote_type=remote_type, allowed_packages=args.allowed_packages) conan_api.remotes.add(r, force=args.force, index=args.index) diff --git a/conan/internal/api/new/oss_recipe.py b/conan/internal/api/new/local_recipes_index.py similarity index 83% rename from conan/internal/api/new/oss_recipe.py rename to conan/internal/api/new/local_recipes_index.py index 814b1e76f67..cea8a9c8491 100644 --- a/conan/internal/api/new/oss_recipe.py +++ b/conan/internal/api/new/local_recipes_index.py @@ -105,9 +105,9 @@ def build_requirements(self): } """ -oss_recipe_files = {"recipes/{{name}}/config.yml": config_yml, - "recipes/{{name}}/all/conandata.yml": conandata_yml, - "recipes/{{name}}/all/conanfile.py": conanfile, - "recipes/{{name}}/all/test_package/conanfile.py": test_conanfile_v2, - "recipes/{{name}}/all/test_package/CMakeLists.txt": test_cmake_v2, - "recipes/{{name}}/all/test_package/src/example.cpp": test_main} +local_recipes_index_files = {"recipes/{{name}}/config.yml": config_yml, + "recipes/{{name}}/all/conandata.yml": conandata_yml, + "recipes/{{name}}/all/conanfile.py": conanfile, + "recipes/{{name}}/all/test_package/conanfile.py": test_conanfile_v2, + "recipes/{{name}}/all/test_package/CMakeLists.txt": test_cmake_v2, + "recipes/{{name}}/all/test_package/src/example.cpp": test_main} diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index 282a79e5798..122e2ff4f44 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -13,8 +13,8 @@ def __init__(self, home_folder): self._home = home_folder @property - def oss_recipes_path(self): - return os.path.join(self._home, ".cache_oss_recipes") + def local_recipes_index_path(self): + return os.path.join(self._home, ".local_recipes_index") @property def global_conf_path(self): diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index af243ffb388..b6cbea4caf8 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -4,8 +4,8 @@ from requests.exceptions import ConnectionError -from conan.api.model import OSS_RECIPES -from conans.client.rest_client_oss_recipes import RestApiClientOSSRecipes +from conan.api.model import LOCAL_RECIPES_INDEX +from conans.client.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex from conan.api.model import Remote from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import METADATA @@ -28,8 +28,8 @@ def __init__(self, cache, auth_manager): self._signer = PkgSignaturesPlugin(cache) def _local_folder_remote(self, remote): - if remote.remote_type == OSS_RECIPES: - return RestApiClientOSSRecipes(remote, self._cache) + if remote.remote_type == LOCAL_RECIPES_INDEX: + return RestApiClientLocalRecipesIndex(remote, self._cache) def check_credentials(self, remote): self._call_remote(remote, "check_credentials") diff --git a/conans/client/rest_client_oss_recipes.py b/conans/client/rest_client_local_recipe_index.py similarity index 77% rename from conans/client/rest_client_oss_recipes.py rename to conans/client/rest_client_local_recipe_index.py index 7463da3a3df..a8a2ecb0cc8 100644 --- a/conans/client/rest_client_oss_recipes.py +++ b/conans/client/rest_client_local_recipe_index.py @@ -7,7 +7,7 @@ import yaml -from conan.api.model import OSS_RECIPES +from conan.api.model import LOCAL_RECIPES_INDEX from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conans.client.cmd.export import cmd_export @@ -17,19 +17,19 @@ from conans.util.files import load, save, rmdir -def add_oss_recipes_remote(conan_api, remote): - if remote.remote_type != OSS_RECIPES: +def add_local_recipes_index_remote(conan_api, remote): + if remote.remote_type != LOCAL_RECIPES_INDEX: return - oss_recipes_path = HomePaths(conan_api.cache_folder).oss_recipes_path - repo_folder = os.path.join(oss_recipes_path, remote.name) + local_recipes_index_path = HomePaths(conan_api.cache_folder).local_recipes_index_path + repo_folder = os.path.join(local_recipes_index_path, remote.name) output = ConanOutput() if os.path.exists(repo_folder): output.warning(f"The cache folder for remote {remote.name} existed, removing it") rmdir(repo_folder) - cache_oss_recipes_path = os.path.join(repo_folder, ".conan") - hook_folder = HomePaths(cache_oss_recipes_path).hooks_path + cache_path = os.path.join(repo_folder, ".conan") + hook_folder = HomePaths(cache_path).hooks_path trim_hook = os.path.join(hook_folder, "hook_trim_conandata.py") hook_content = textwrap.dedent("""\ from conan.tools.files import trim_conandata @@ -40,16 +40,16 @@ def post_export(conanfile): save(trim_hook, hook_content) -def remove_oss_recipes_remote(conan_api, remote): - if remote.remote_type == OSS_RECIPES: - oss_recipes_path = HomePaths(conan_api.cache_folder).oss_recipes_path - oss_recipes_path = os.path.join(oss_recipes_path, remote.name) +def remove_local_recipes_index_remote(conan_api, remote): + if remote.remote_type == LOCAL_RECIPES_INDEX: + local_recipes_index_path = HomePaths(conan_api.cache_folder).local_recipes_index_path + local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name) ConanOutput().info(f"Removing temporary files for '{remote.name}' " - f"oss-recipes remote") - rmdir(oss_recipes_path) + f"local-recipes-index remote") + rmdir(local_recipes_index_path) -class RestApiClientOSSRecipes: +class RestApiClientLocalRecipesIndex: """ Implements the RestAPI but instead of over HTTP for a remote server, using just a local folder assuming the conan-center-index repo layout @@ -57,16 +57,16 @@ class RestApiClientOSSRecipes: def __init__(self, remote, cache): self._remote = remote - oss_recipes_path = HomePaths(cache.cache_folder).oss_recipes_path - oss_recipes_path = os.path.join(oss_recipes_path, remote.name) - cache_oss_recipes_path = os.path.join(oss_recipes_path, ".conan") + local_recipes_index_path = HomePaths(cache.cache_folder).local_recipes_index_path + local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name) + local_recipes_index_path = os.path.join(local_recipes_index_path, ".conan") repo_folder = self._remote.url from conan.internal.conan_app import ConanApp from conan.api.conan_api import ConanAPI - conan_api = ConanAPI(cache_oss_recipes_path) + conan_api = ConanAPI(local_recipes_index_path) self._app = ConanApp(conan_api) - self._layout = _OSSRecipesRepoLayout(repo_folder) + self._layout = _LocalRecipesIndexLayout(repo_folder) def call_method(self, method_name, *args, **kwargs): return getattr(self, method_name)(*args, **kwargs) @@ -80,21 +80,21 @@ def get_recipe_sources(self, ref, dest_folder): return self._copy_files(export_sources, dest_folder) def get_package(self, pref, dest_folder, metadata, only_metadata): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support " + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support " "binary packages") def upload_recipe(self, ref, files_to_upload): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support upload") + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support upload") def upload_package(self, pref, files_to_upload): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support upload") + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support upload") def authenticate(self, user, password): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support " + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support " "authentication") def check_credentials(self): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support upload") + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support upload") def search(self, pattern=None): return self._layout.get_recipes_references(pattern) @@ -104,13 +104,13 @@ def search_packages(self, reference): return {} def remove_recipe(self, ref): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support remove") + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support remove") def remove_all_packages(self, ref): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support remove") + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support remove") def remove_packages(self, prefs): - raise ConanException(f"Remote oss-recipes '{self._remote.name}' doesn't support remove") + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support remove") def get_recipe_revisions_references(self, ref): ref = self._export_recipe(ref) @@ -169,7 +169,7 @@ def _copy_files(source_folder, dest_folder): return ret -class _OSSRecipesRepoLayout: +class _LocalRecipesIndexLayout: def __init__(self, base_folder): self._base_folder = base_folder diff --git a/conans/test/functional/test_oss_recipes_repo.py b/conans/test/functional/test_local_recipes_index.py similarity index 89% rename from conans/test/functional/test_oss_recipes_repo.py rename to conans/test/functional/test_local_recipes_index.py index d5d5a82678d..5c17dac6ed6 100644 --- a/conans/test/functional/test_oss_recipes_repo.py +++ b/conans/test/functional/test_local_recipes_index.py @@ -9,8 +9,8 @@ from conans.util.files import save_files, sha256sum -class TestConanNewOssRecipe: - def test_conan_new_oss_recipe(self): +class TestLocalRecipeIndexNew: + def test_conan_new_local_recipes_index(self): # Setup the release pkg0.1.zip http server file_server = TestFileServer() zippath = os.path.join(file_server.store, "pkg0.1.zip") @@ -26,15 +26,15 @@ def test_conan_new_oss_recipe(self): c0 = TestClient() c0.servers["file_server"] = file_server - c0.run(f"new oss_recipe -d name=pkg -d version=0.1 -d url={url} -d sha256={sha256}") + c0.run(f"new local_recipes_index -d name=pkg -d version=0.1 -d url={url} -d sha256={sha256}") # A local create is possible, and it includes a test_package c0.run("create recipes/pkg/all --version=0.1") assert "pkg: Release!" in c0.out - oss_recipe_repo = c0.current_folder + remote_folder = c0.current_folder c = TestClient() c.servers["file_server"] = file_server - c.run(f"remote add local '{oss_recipe_repo}' --type=oss-recipes") + c.run(f"remote add local '{remote_folder}'") c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") c.run("create . --version=0.1 --build=missing") assert "pkg: Release!" in c.out @@ -43,7 +43,7 @@ def test_conan_new_oss_recipe(self): class TestInRepo: def test_in_repo(self): """testing that it is possible to put a "recipes" folder inside a source repo, and - use it as oss-recipes-repository, exporting the source from itself + use it as local-recipes-index repository, exporting the source from itself """ repo_folder = temp_folder() cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, @@ -105,7 +105,7 @@ def package_info(self): "pkg.cpp": gen_function_cpp(name="pkg")}) c = TestClient() - c.run(f"remote add local '{repo_folder}' --type=oss-recipes") + c.run(f"remote add local '{repo_folder}'") c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") c.run("create . --build=missing") assert "pkg: Release!" in c.out @@ -117,4 +117,4 @@ def package_info(self): # Finally lets remove the remote, check that the clone is cleared c.run('remote remove local') - assert "Removing temporary files for 'local' oss-recipes remote" in c.out + assert "Removing temporary files for 'local' local-recipes-index remote" in c.out diff --git a/conans/test/integration/remote/test_oss_recipes_repo.py b/conans/test/integration/remote/test_local_recipes_index.py similarity index 98% rename from conans/test/integration/remote/test_oss_recipes_repo.py rename to conans/test/integration/remote/test_local_recipes_index.py index 7882a355bff..b53465f7028 100644 --- a/conans/test/integration/remote/test_oss_recipes_repo.py +++ b/conans/test/integration/remote/test_local_recipes_index.py @@ -65,7 +65,7 @@ def build(self): class TestSearchList: def test_basic_search(self, c3i_folder): client = TestClient() - client.run(f"remote add local '{c3i_folder}' --type=oss-recipes") # Keep --type for test + client.run(f"remote add local '{c3i_folder}' --type=local-recipes-index") # Keep --type test assert "WARN" not in client.out # Make sure it does not complain about url client.run("search *") assert textwrap.dedent("""\ @@ -281,7 +281,7 @@ def test_upload(self): c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") c.run("upload pkg/0.1 -r=local", assert_error=True) - assert "ERROR: Remote oss-recipes 'local' doesn't support upload" in c.out + assert "ERROR: Remote local-recipes-index 'local' doesn't support upload" in c.out class TestErrorsUx: