Feature/workspace base (#17272)

* refactor: move editables to local api

* workspaces back

* wip

* Git.to_conandata and from_conandata

* Raise error if `check_type=int` and conf value is set to `bool` (#15378)

raise if bool on int check

* auto -FS in AutotoolsToolchain (#15375)

* auto -FS in AutotoolsToolchain

* moved test

* fix winsdk_version bug (#15373)

* allow to copy & paste from compact format into conan-lock-add (#15262)

* allow to copy & paste from compact format into conan-lock-add

* Revert, and just print the %timestamp too

* Remove float timestamp from conan list --format=text

* Fix test

---------

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

* wip

* wip

* open + add + remove proposal

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* initial minimal workspace

* wip

* new test

* minor change to make test fail

* making workspace decoupled from editables

* fix docstring

* new test for dynamic workspace

* UserWorkpaceAPI

* fix test

* review

---------

Co-authored-by: Carlos Zoido <mrgalleta@gmail.com>
Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

conan/api/conan_api.py

@@ -5,6 +5,7 @@
 from conan.api.subapi.command import CommandAPI
 from conan.api.subapi.local import LocalAPI
 from conan.api.subapi.lockfile import LockfileAPI
+from conan.api.subapi.workspace import WorkspaceAPI
 from conan import conan_version
 from conan.api.subapi.config import ConfigAPI
 from conan.api.subapi.download import DownloadAPI
@@ -32,7 +33,8 @@ def __init__(self, cache_folder=None):
             raise ConanException("Conan needs Python >= 3.6")
 
         init_colorama(sys.stderr)
-        self.cache_folder = cache_folder or get_conan_user_home()
+        self.workspace = WorkspaceAPI(self)
+        self.cache_folder = self.workspace.home_folder() or cache_folder or get_conan_user_home()
         self.home_folder = self.cache_folder  # Lets call it home, deprecate "cache"
 
         # Migration system

conan/api/subapi/local.py

@@ -17,6 +17,9 @@ class LocalAPI:
     def __init__(self, conan_api):
         self._conan_api = conan_api
         self.editable_packages = EditablePackages(conan_api.home_folder)
+        editables = conan_api.workspace.editables()
+        if editables:
+            self.editable_packages.edited_refs.update(editables)
 
     @staticmethod
     def get_conanfile_path(path, cwd, py):
@@ -55,17 +58,20 @@ def editable_add(self, path, name=None, version=None, user=None, channel=None, c
         target_path = self._conan_api.local.get_conanfile_path(path=path, cwd=cwd, py=True)
         output_folder = make_abs_path(output_folder) if output_folder else None
         # Check the conanfile is there, and name/version matches
-        self.editable_packages.add(ref, target_path, output_folder=output_folder)
+        editable_packages = EditablePackages(self._conan_api.home_folder)
+        editable_packages.add(ref, target_path, output_folder=output_folder)
         return ref
 
     def editable_remove(self, path=None, requires=None, cwd=None):
         if path:
             path = make_abs_path(path, cwd)
             path = os.path.join(path, "conanfile.py")
-        return self.editable_packages.remove(path, requires)
+        editable_packages = EditablePackages(self._conan_api.home_folder)
+        return editable_packages.remove(path, requires)
 
     def editable_list(self):
-        return self.editable_packages.edited_refs
+        editable_packages = EditablePackages(self._conan_api.home_folder)
+        return editable_packages.edited_refs
 
     def source(self, path, name=None, version=None, user=None, channel=None, remotes=None):
         """ calls the 'source()' method of the current (user folder) conanfile.py
@@ -116,4 +122,3 @@ def inspect(self, conanfile_path, remotes, lockfile, name=None, version=None, us
         conanfile = app.loader.load_named(conanfile_path, name=name, version=version, user=user,
                                           channel=channel, remotes=remotes, graph_lock=lockfile)
         return conanfile
-

conan/api/subapi/workspace.py

@@ -0,0 +1,82 @@
+import os
+import shutil
+
+from conan.cli import make_abs_path
+from conan.internal.conan_app import ConanApp
+from conan.internal.workspace import Workspace
+from conan.tools.scm import Git
+from conan.errors import ConanException
+from conans.client.graph.graph import RECIPE_EDITABLE
+from conans.client.source import retrieve_exports_sources
+from conans.model.recipe_ref import RecipeReference
+from conans.util.files import merge_directories
+
+
+class WorkspaceAPI:
+
+    def __init__(self, conan_api):
+        self._conan_api = conan_api
+        self._workspace = Workspace()
+
+    def home_folder(self):
+        return self._workspace.home_folder()
+
+    def folder(self):
+        return self._workspace.folder
+
+    def config_folder(self):
+        return self._workspace.config_folder()
+
+    def editables(self):
+        return self._workspace.editables()
+
+    def open(self, require, remotes, cwd=None):
+        app = ConanApp(self._conan_api)
+        ref = RecipeReference.loads(require)
+        recipe = app.proxy.get_recipe(ref, remotes, update=False, check_update=False)
+
+        layout, recipe_status, remote = recipe
+        if recipe_status == RECIPE_EDITABLE:
+            raise ConanException(f"Can't open a dependency that is already an editable: {ref}")
+        ref = layout.reference
+        conanfile_path = layout.conanfile()
+        conanfile, module = app.loader.load_basic_module(conanfile_path, remotes=remotes)
+
+        scm = conanfile.conan_data.get("scm") if conanfile.conan_data else None
+        dst_path = os.path.join(cwd or os.getcwd(), ref.name)
+        if scm is None:
+            conanfile.output.warning("conandata doesn't contain 'scm' information\n"
+                                     "doing a local copy!!!")
+            shutil.copytree(layout.export(), dst_path)
+            retrieve_exports_sources(app.remote_manager, layout, conanfile, ref, remotes)
+            export_sources = layout.export_sources()
+            if os.path.exists(export_sources):
+                conanfile.output.warning("There are export-sources, copying them, but the location"
+                                         " might be incorrect, use 'scm' approach")
+                merge_directories(export_sources, dst_path)
+        else:
+            git = Git(conanfile, folder=cwd)
+            git.clone(url=scm["url"], target=ref.name)
+            git.folder = ref.name  # change to the cloned folder
+            git.checkout(commit=scm["commit"])
+        return dst_path
+
+    def add(self, path, name=None, version=None, user=None, channel=None, cwd=None,
+            output_folder=None, remotes=None):
+        path = self._conan_api.local.get_conanfile_path(path, cwd, py=True)
+        app = ConanApp(self._conan_api)
+        conanfile = app.loader.load_named(path, name, version, user, channel, remotes=remotes)
+        if conanfile.name is None or conanfile.version is None:
+            raise ConanException("Editable package recipe should declare its name and version")
+        ref = RecipeReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel)
+        ref.validate_ref()
+        output_folder = make_abs_path(output_folder) if output_folder else None
+        # Check the conanfile is there, and name/version matches
+        self._workspace.add(ref, path, output_folder=output_folder)
+        return ref
+
+    def remove(self, path):
+        return self._workspace.remove(path)
+
+    def info(self):
+        return self._workspace.serialize()

conan/cli/commands/workspace.py

@@ -0,0 +1,108 @@
+import json
+import os
+
+from conan.api.conan_api import ConanAPI
+from conan.api.output import ConanOutput, cli_out_write
+from conan.cli import make_abs_path
+from conan.cli.args import add_reference_args
+from conan.cli.command import conan_command, conan_subcommand
+from conan.cli.commands.list import print_serial
+from conan.errors import ConanException
+
+
+@conan_subcommand(formatters={"text": cli_out_write})
+def workspace_root(conan_api: ConanAPI, parser, subparser, *args):
+    """
+    Return the folder containing the conanws.py/conanws.yml workspace file
+    """
+    ws = conan_api.workspace
+    if not ws.folder():
+        raise ConanException("No workspace defined, conanws.py file not found")
+    return ws.folder()
+
+
+@conan_subcommand()
+def workspace_open(conan_api: ConanAPI, parser, subparser, *args):
+    """
+    Open specific references
+    """
+    subparser.add_argument("reference",
+                           help="Open this package source repository")
+    group = subparser.add_mutually_exclusive_group()
+    group.add_argument("-r", "--remote", action="append", default=None,
+                       help='Look in the specified remote or remotes server')
+    group.add_argument("-nr", "--no-remote", action="store_true",
+                       help='Do not use remote, resolve exclusively in the cache')
+    args = parser.parse_args(*args)
+    remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    cwd = os.getcwd()
+    conan_api.workspace.open(args.reference, remotes=remotes, cwd=cwd)
+
+
+@conan_subcommand()
+def workspace_add(conan_api: ConanAPI, parser, subparser, *args):
+    """
+    Add packages to current workspace
+    """
+    subparser.add_argument('path',  nargs="?",
+                           help='Path to the package folder in the user workspace')
+    add_reference_args(subparser)
+    subparser.add_argument("--ref", nargs="?",
+                           help="Open and add this reference")
+    subparser.add_argument("-of", "--output-folder",
+                           help='The root output folder for generated and build files')
+    group = subparser.add_mutually_exclusive_group()
+    group.add_argument("-r", "--remote", action="append", default=None,
+                       help='Look in the specified remote or remotes server')
+    group.add_argument("-nr", "--no-remote", action="store_true",
+                       help='Do not use remote, resolve exclusively in the cache')
+    args = parser.parse_args(*args)
+    if args.path and args.ref:
+        raise ConanException("Do not use both 'path' and '--ref' argument")
+    remotes = conan_api.remotes.list(args.remote) if not args.no_remote else []
+    cwd = os.getcwd()
+    path = args.path
+    if args.ref:
+        # TODO: Use path here to open in this path
+        path = conan_api.workspace.open(args.ref, remotes, cwd=cwd)
+    ref = conan_api.workspace.add(path,
+                                  args.name, args.version, args.user, args.channel,
+                                  cwd, args.output_folder, remotes=remotes)
+    ConanOutput().success("Reference '{}' added to workspace".format(ref))
+
+
+@conan_subcommand()
+def workspace_remove(conan_api: ConanAPI, parser, subparser, *args):
+    """
+    Remove packages to current workspace
+    """
+    subparser.add_argument('path', help='Path to the package folder in the user workspace')
+    args = parser.parse_args(*args)
+    removed = conan_api.workspace.remove(make_abs_path(args.path))
+    ConanOutput().info(f"Removed from workspace: {removed}")
+
+
+def print_json(data):
+    results = data["info"]
+    myjson = json.dumps(results, indent=4)
+    cli_out_write(myjson)
+
+
+def _print_workspace_info(data):
+    print_serial(data["info"])
+
+
+@conan_subcommand(formatters={"text": _print_workspace_info, "json": print_json})
+def workspace_info(conan_api: ConanAPI, parser, subparser, *args):
+    """
+    Display info for current workspace
+    """
+    parser.parse_args(*args)
+    return {"info": conan_api.workspace.info()}
+
+
+@conan_command(group="Consumer")
+def workspace(conan_api, parser, *args):
+    """
+    Manage Conan workspaces (group of packages in editable mode)
+    """