New local-recipes-index feature

conan/api/model.py

@@ -9,14 +9,17 @@
 from conans.util.files import load
 from conans.model.version_range import VersionRange
 
+OSS_RECIPES = "oss-recipes"
+
 
 class Remote:
 
-    def __init__(self, name, url, verify_ssl=True, disabled=False):
+    def __init__(self, name, url, verify_ssl=True, disabled=False, remote_type=None):
         self._name = name  # Read only, is the key
         self.url = url
         self.verify_ssl = verify_ssl
         self.disabled = disabled
+        self.remote_type = remote_type
 
     @property
     def name(self):
@@ -29,6 +32,9 @@ def __eq__(self, other):
                 self.verify_ssl == other.verify_ssl and self.disabled == other.disabled)
 
     def __str__(self):
+        if self.remote_type == OSS_RECIPES:
+            return "{}: {} [{}, Enabled: {}]".format(self.name, self.url, OSS_RECIPES,
+                                                     not self.disabled)
         return "{}: {} [Verify SSL: {}, Enabled: {}]".format(self.name, self.url, self.verify_ssl,
                                                              not self.disabled)
 

conan/api/subapi/new.py

@@ -14,7 +14,8 @@ class NewAPI:
     def __init__(self, conan_api):
         self.conan_api = conan_api
 
-    def get_builtin_template(self, template_name):
+    @staticmethod
+    def get_builtin_template(template_name):
         from conan.internal.api.new.basic import basic_file
         from conan.internal.api.new.alias_new import alias_file
         from conan.internal.api.new.cmake_exe import cmake_exe_files
@@ -27,6 +28,7 @@ def get_builtin_template(self, template_name):
         from conan.internal.api.new.bazel_exe import bazel_exe_files
         from conan.internal.api.new.autotools_lib import autotools_lib_files
         from conan.internal.api.new.autoools_exe import autotools_exe_files
+        from conan.internal.api.new.oss_recipe import oss_recipe_files
         new_templates = {"basic": basic_file,
                          "cmake_lib": cmake_lib_files,
                          "cmake_exe": cmake_exe_files,
@@ -38,7 +40,8 @@ def get_builtin_template(self, template_name):
                          "bazel_exe": bazel_exe_files,
                          "autotools_lib": autotools_lib_files,
                          "autotools_exe": autotools_exe_files,
-                         "alias": alias_file}
+                         "alias": alias_file,
+                         "oss_recipe": oss_recipe_files}
         template_files = new_templates.get(template_name)
         return template_files
 

conan/api/subapi/remotes.py

@@ -5,6 +5,7 @@
 from conan.internal.conan_app import ConanApp
 from conans.client.cache.remote_registry import Remote, RemoteRegistry
 from conans.client.cmd.user import user_set, users_clean, users_list
+from conans.client.rest_client_oss_recipes import add_oss_recipes_remote, remove_oss_recipes_remote
 from conans.errors import ConanException
 
 
@@ -63,12 +64,14 @@ def get(self, remote_name):
         return RemoteRegistry(self._remotes_file).read(remote_name)
 
     def add(self, remote: Remote, force=False, index=None):
+        add_oss_recipes_remote(self.conan_api, remote)
         return RemoteRegistry(self._remotes_file).add(remote, force=force, index=index)
 
     def remove(self, pattern: str):
         app = ConanApp(self.conan_api)
         remotes = self.list(pattern, only_enabled=False)
         for remote in remotes:
+            remove_oss_recipes_remote(self.conan_api, remote)
             RemoteRegistry(self._remotes_file).remove(remote.name)
             users_clean(app.cache.localdb, remote.url)
 

conan/cli/commands/remote.py

@@ -1,14 +1,18 @@
 import json
+import os
 from collections import OrderedDict
 
 from conan.api.output import cli_out_write, Color
 from conan.api.conan_api import ConanAPI
-from conan.api.model import Remote
+from conan.api.model import Remote, OSS_RECIPES
 from conan.cli.command import conan_command, conan_subcommand, OnceArgument
 from conan.cli.commands.list import remote_color, error_color, recipe_color, \
     reference_color
+from conan.internal.cache.home_paths import HomePaths
 from conans.client.rest.remote_credentials import RemoteCredentials
 from conan.errors import ConanException
+from conans.util.files import chdir
+from conans.util.runners import check_output_runner
 
 
 def formatter_remote_list_json(remotes):
@@ -72,9 +76,11 @@ def remote_add(conan_api, parser, subparser, *args):
                            help="Insert the remote at a specific position in the remote list")
     subparser.add_argument("-f", "--force", action='store_true',
                            help="Force the definition of the remote even if duplicated")
+    subparser.add_argument("-t", "--type", choices=[OSS_RECIPES], help="Define the remote type")
     subparser.set_defaults(secure=True)
     args = parser.parse_args(*args)
-    r = Remote(args.name, args.url, args.secure, disabled=False)
+
+    r = Remote(args.name, args.url, args.secure, disabled=False, remote_type=args.type)
     conan_api.remotes.add(r, force=args.force, index=args.index)
 
 

conan/internal/api/new/oss_recipe.py

@@ -0,0 +1,113 @@
+from conan.internal.api.new.cmake_lib import test_conanfile_v2, test_cmake_v2
+
+config_yml = """\
+versions:
+  "{{version}}":
+    folder: all
+"""
+
+conandata_yml = """\
+sources:
+  "{{version}}":
+    url:
+      {% if url is defined -%}
+      - "{{url}}"
+      {% else -%}
+      - "http://put/here/the/url/to/release.1.2.3.zip"
+      {% endif %}
+    {% if sha256 is defined -%}
+    sha256: "{{sha256}}"
+    {%- else -%}
+    sha256: "Put here your tarball sha256"
+    {% endif -%}
+"""
+
+
+conanfile = """\
+from conan import ConanFile
+from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps
+from conan.tools.files import apply_conandata_patches, export_conandata_patches, get
+
+
+class {{package_name}}Recipe(ConanFile):
+    name = "{{name}}"
+    package_type = "library"
+
+    # Optional metadata
+    license = "<Put the package license here>"
+    author = "<Put your name here> <And your email here>"
+    url = "<Package recipe repository url here, for issues about the package>"
+    description = "<Description of {{ name }} package here>"
+    topics = ("<Put some tag here>", "<here>", "<and here>")
+
+    # Binary configuration
+    settings = "os", "compiler", "build_type", "arch"
+    options = {"shared": [True, False], "fPIC": [True, False]}
+    default_options = {"shared": False, "fPIC": True}
+
+    def config_options(self):
+        if self.settings.os == "Windows":
+            self.options.rm_safe("fPIC")
+
+    def configure(self):
+        if self.options.shared:
+            self.options.rm_safe("fPIC")
+
+    def export_sources(self):
+        export_conandata_patches(self)
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version], destination=self.source_folder,
+            strip_root=True)
+        apply_conandata_patches(self)
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def generate(self):
+        deps = CMakeDeps(self)
+        deps.generate()
+        tc = CMakeToolchain(self)
+        tc.generate()
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def package(self):
+        cmake = CMake(self)
+        cmake.install()
+
+    def package_info(self):
+        self.cpp_info.libs = ["{{name}}"]
+
+    {% if requires is defined -%}
+    def requirements(self):
+        {% for require in requires -%}
+        self.requires("{{ require }}")
+        {% endfor %}
+    {%- endif %}
+
+    {% if tool_requires is defined -%}
+    def build_requirements(self):
+        {% for require in tool_requires -%}
+        self.tool_requires("{{ require }}")
+        {% endfor %}
+    {%- endif %}
+"""
+
+
+test_main = """#include "{{name}}.h"
+
+int main() {
+    {{package_name}}();
+}
+"""
+
+oss_recipe_files = {"recipes/{{name}}/config.yml": config_yml,
+                    "recipes/{{name}}/all/conandata.yml": conandata_yml,
+                    "recipes/{{name}}/all/conanfile.py": conanfile,
+                    "recipes/{{name}}/all/test_package/conanfile.py": test_conanfile_v2,
+                    "recipes/{{name}}/all/test_package/CMakeLists.txt": test_cmake_v2,
+                    "recipes/{{name}}/all/test_package/src/example.cpp": test_main}

conan/internal/cache/home_paths.py

@@ -12,6 +12,10 @@ class HomePaths:
     def __init__(self, home_folder):
         self._home = home_folder
 
+    @property
+    def oss_recipes_path(self):
+        return os.path.join(self._home, ".cache_oss_recipes")
+
     @property
     def global_conf_path(self):
         return os.path.join(self._home, "global.conf")

conans/client/cache/remote_registry.py

@@ -2,7 +2,7 @@
 import os
 from urllib.parse import urlparse
 
-from conan.api.model import Remote
+from conan.api.model import Remote, OSS_RECIPES
 from conan.api.output import ConanOutput
 from conans.errors import ConanException
 from conans.util.files import load, save
@@ -33,8 +33,9 @@ def load(filename):
         data = json.loads(text)
         for r in data.get("remotes", []):
             disabled = r.get("disabled", False)
+            remote_type = r.get("type", None)
             # TODO: Remote.serialize/deserialize
-            remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled)
+            remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled, remote_type)
             result._remotes[r["name"]] = remote
         return result
 
@@ -44,6 +45,8 @@ def dumps(self):
             remote = {"name": r.name, "url": r.url, "verify_ssl": r.verify_ssl}
             if r.disabled:
                 remote["disabled"] = True
+            if r.remote_type:
+                remote["type"] = r.remote_type
             remote_list.append(remote)
         ret = {"remotes": remote_list}
         return json.dumps(ret, indent=True)
@@ -159,7 +162,8 @@ def read(self, remote_name):
         return ret
 
     def add(self, remote: Remote, force=False, index=None):
-        self._validate_url(remote.url)
+        if remote.remote_type != OSS_RECIPES:
+            self._validate_url(remote.url)
         remotes = self._load_remotes()
         remotes.add(remote, force=force, index=index)
         self.save_remotes(remotes)
@@ -172,7 +176,7 @@ def remove(self, remote_name):
 
     def update(self, remote_name, url, secure, disabled, index):
         if url is not None:
-            self._validate_url(url)
+            self._validate_url(url)  # TODO: Pending to not warn for oss-recipes repos
         remotes = self._load_remotes()
         remotes.update(remote_name, url, secure, disabled, index)
         self.save_remotes(remotes)

conans/client/remote_manager.py

@@ -4,9 +4,11 @@
 
 from requests.exceptions import ConnectionError
 
+from conan.api.model import OSS_RECIPES
 from conan.api.output import ConanOutput
 from conan.internal.cache.conan_reference_layout import METADATA
 from conans.client.cache.remote_registry import Remote
+from conans.client.rest_client_oss_recipes import RestApiClientOSSRecipes
 from conans.client.pkg_sign import PkgSignaturesPlugin
 from conans.errors import ConanConnectionError, ConanException, NotFoundException, \
     PackageNotFoundException
@@ -18,14 +20,17 @@
 from conans.util.files import mkdir, tar_extract
 
 
-class RemoteManager(object):
+class RemoteManager:
     """ Will handle the remotes to get recipes, packages etc """
-
     def __init__(self, cache, auth_manager):
         self._cache = cache
         self._auth_manager = auth_manager
         self._signer = PkgSignaturesPlugin(cache)
 
+    def _local_folder_remote(self, remote):
+        if remote.remote_type == OSS_RECIPES:
+            return RestApiClientOSSRecipes(remote, self._cache)
+
     def check_credentials(self, remote):
         self._call_remote(remote, "check_credentials")
 
@@ -46,6 +51,12 @@ def get_recipe(self, ref, remote, metadata=None):
         layout = self._cache.get_or_create_ref_layout(ref)
         layout.export_remove()
 
+        export_folder = layout.export()
+        local_folder_remote = self._local_folder_remote(remote)
+        if local_folder_remote is not None:
+            local_folder_remote.get_recipe(ref, export_folder)
+            return layout
+
         download_export = layout.download_export()
         try:
             zipped_files = self._call_remote(remote, "get_recipe", ref, download_export, metadata,
@@ -100,6 +111,11 @@ def get_recipe_sources(self, ref, layout, remote):
 
         download_folder = layout.download_export()
         export_sources_folder = layout.export_sources()
+        local_folder_remote = self._local_folder_remote(remote)
+        if local_folder_remote is not None:
+            local_folder_remote.get_recipe_sources(ref, export_sources_folder)
+            return
+
         zipped_files = self._call_remote(remote, "get_recipe_sources", ref, download_folder)
         if not zipped_files:
             mkdir(export_sources_folder)  # create the folder even if no source files
@@ -234,6 +250,9 @@ def _call_remote(self, remote, method, *args, **kwargs):
         enforce_disabled = kwargs.pop("enforce_disabled", True)
         if remote.disabled and enforce_disabled:
             raise ConanException("Remote '%s' is disabled" % remote.name)
+        local_folder_remote = self._local_folder_remote(remote)
+        if local_folder_remote is not None:
+            return local_folder_remote.call_method(method, *args, **kwargs)
         try:
             return self._auth_manager.call_rest_api_method(remote, method, *args, **kwargs)
         except ConnectionError as exc: