diff --git a/dvc/cli/utils.py b/dvc/cli/utils.py index 6b794d7b27..1dfd835365 100644 --- a/dvc/cli/utils.py +++ b/dvc/cli/utils.py @@ -1,3 +1,27 @@ +import argparse + + +class DictAction(argparse.Action): + def __call__(self, parser, args, values, option_string=None): # noqa: ARG002 + d = getattr(args, self.dest) or {} + + if isinstance(values, list): + kvs = values + else: + kvs = [values] + + for kv in kvs: + key, value = kv.split("=") + if not value: + raise argparse.ArgumentError( + self, + f'Could not parse argument "{values}" as k1=v1 k2=v2 ... format', + ) + d[key] = value + + setattr(args, self.dest, d) + + def fix_subparsers(subparsers): """Workaround for bug in Python 3. See more info at: https://bugs.python.org/issue16308 diff --git a/dvc/commands/get.py b/dvc/commands/get.py index 016a234151..c1c4ad2d68 100644 --- a/dvc/commands/get.py +++ b/dvc/commands/get.py @@ -3,7 +3,7 @@ from dvc.cli import completion from dvc.cli.command import CmdBaseNoRepo -from dvc.cli.utils import append_doc_link +from dvc.cli.utils import DictAction, append_doc_link from dvc.exceptions import DvcException logger = logging.getLogger(__name__) @@ -38,6 +38,8 @@ def _get_file_from_repo(self): jobs=self.args.jobs, force=self.args.force, config=self.args.config, + remote=self.args.remote, + remote_config=self.args.remote_config, ) return 0 except CloneError: @@ -111,4 +113,19 @@ def add_parser(subparsers, parent_parser): "in the target repository." ), ) + get_parser.add_argument( + "--remote", + type=str, + help="Remote name to set as a default in the target repository.", + ) + get_parser.add_argument( + "--remote-config", + type=str, + nargs="*", + action=DictAction, + help=( + "Remote config options to merge with a remote's config (default or one " + "specified by '--remote') in the target repository." + ), + ) get_parser.set_defaults(func=CmdGet) diff --git a/dvc/commands/imp.py b/dvc/commands/imp.py index 217df87eb5..51874a76be 100644 --- a/dvc/commands/imp.py +++ b/dvc/commands/imp.py @@ -3,7 +3,7 @@ from dvc.cli import completion from dvc.cli.command import CmdBase -from dvc.cli.utils import append_doc_link +from dvc.cli.utils import DictAction, append_doc_link from dvc.exceptions import DvcException logger = logging.getLogger(__name__) @@ -23,6 +23,8 @@ def run(self): no_download=self.args.no_download, jobs=self.args.jobs, config=self.args.config, + remote=self.args.remote, + remote_config=self.args.remote_config, ) except CloneError: logger.exception("failed to import '%s'", self.args.path) @@ -103,4 +105,19 @@ def add_parser(subparsers, parent_parser): "in the target repository." ), ) + import_parser.add_argument( + "--remote", + type=str, + help="Remote name to set as a default in the target repository.", + ) + import_parser.add_argument( + "--remote-config", + type=str, + nargs="*", + action=DictAction, + help=( + "Remote config options to merge with a remote's config (default or one " + "specified by '--remote') in the target repository." + ), + ) import_parser.set_defaults(func=CmdImport) diff --git a/dvc/commands/ls/__init__.py b/dvc/commands/ls/__init__.py index ecb345064b..c044e3571a 100644 --- a/dvc/commands/ls/__init__.py +++ b/dvc/commands/ls/__init__.py @@ -3,7 +3,7 @@ from dvc.cli import completion from dvc.cli.command import CmdBaseNoRepo -from dvc.cli.utils import append_doc_link +from dvc.cli.utils import DictAction, append_doc_link from dvc.commands.ls.ls_colors import LsColors from dvc.exceptions import DvcException from dvc.ui import ui @@ -35,6 +35,8 @@ def run(self): recursive=self.args.recursive, dvc_only=self.args.dvc_only, config=self.args.config, + remote=self.args.remote, + remote_config=self.args.remote_config, ) if self.args.json: ui.write_json(entries) @@ -89,6 +91,21 @@ def add_parser(subparsers, parent_parser): "in the target repository." ), ) + list_parser.add_argument( + "--remote", + type=str, + help="Remote name to set as a default in the target repository.", + ) + list_parser.add_argument( + "--remote-config", + type=str, + nargs="*", + action=DictAction, + help=( + "Remote config options to merge with a remote's config (default or one " + "specified by '--remote') in the target repository." + ), + ) list_parser.add_argument( "path", nargs="?", diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 7c6fbb43b9..dfe6a8b72d 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Dict, Optional, Union -from voluptuous import Required +from voluptuous import Any, Required from dvc.utils import as_posix @@ -17,6 +17,7 @@ class RepoDependency(Dependency): PARAM_REV = "rev" PARAM_REV_LOCK = "rev_lock" PARAM_CONFIG = "config" + PARAM_REMOTE = "remote" REPO_SCHEMA = { PARAM_REPO: { @@ -24,6 +25,7 @@ class RepoDependency(Dependency): PARAM_REV: str, PARAM_REV_LOCK: str, PARAM_CONFIG: str, + PARAM_REMOTE: Any(str, dict), } } @@ -76,6 +78,10 @@ def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]: if config: repo[self.PARAM_CONFIG] = config + remote = self.def_repo.get(self.PARAM_REMOTE) + if remote: + repo[self.PARAM_REMOTE] = remote + return { self.PARAM_PATH: self.def_path, self.PARAM_REPO: repo, @@ -99,6 +105,14 @@ def _make_fs( from dvc.config import Config from dvc.fs import DVCFileSystem + rem = self.def_repo.get("remote") + if isinstance(rem, dict): + remote = None # type: ignore[unreachable] + remote_config = rem + else: + remote = rem + remote_config = None + conf = self.def_repo.get("config") if conf: config = Config.load_file(conf) @@ -113,6 +127,8 @@ def _make_fs( rev=rev or self._get_rev(locked=locked), subrepos=True, config=config, + remote=remote, + remote_config=remote_config, ) def _get_rev(self, locked: bool = True): diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 239a08318a..4617e0cd3b 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -19,7 +19,17 @@ def __init__(self): ) -def get(url, path, out=None, rev=None, jobs=None, force=False, config=None): +def get( + url, + path, + out=None, + rev=None, + jobs=None, + force=False, + config=None, + remote=None, + remote_config=None, +): from dvc.config import Config from dvc.dvcfile import is_valid_filename from dvc.repo import Repo @@ -38,6 +48,8 @@ def get(url, path, out=None, rev=None, jobs=None, force=False, config=None): subrepos=True, uninitialized=True, config=config, + remote=remote, + remote_config=remote_config, ) as repo: from dvc.fs import download from dvc.fs.data import DataFileSystem diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index 558fa56dae..8f4e2dcaf8 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -1,4 +1,14 @@ -def imp(self, url, path, out=None, rev=None, config=None, **kwargs): +def imp( + self, + url, + path, + out=None, + rev=None, + config=None, + remote=None, + remote_config=None, + **kwargs, +): erepo = {"url": url} if rev is not None: erepo["rev"] = rev @@ -6,4 +16,17 @@ def imp(self, url, path, out=None, rev=None, config=None, **kwargs): if config is not None: erepo["config"] = config + if remote is not None and remote_config is not None: + conf = erepo.get("config") or {} + remotes = conf.get("remote") or {} + remote_conf = remotes.get(remote) or {} + remote_conf.update(remote_config) + remotes[remote] = remote_conf + conf["remote"] = remotes + erepo["config"] = conf + elif remote is not None: + erepo["remote"] = remote + elif remote_config is not None: + erepo["remote"] = remote_config + return self.imp_url(path, out=out, erepo=erepo, frozen=True, **kwargs) diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index ce408bbb5e..5e6005ea38 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -14,6 +14,8 @@ def ls( recursive: Optional[bool] = None, dvc_only: bool = False, config: Optional[str] = None, + remote: Optional[str] = None, + remote_config: Optional[dict] = None, ): """Methods for getting files and outputs for the repo. @@ -23,7 +25,9 @@ def ls( rev (str, optional): SHA commit, branch or tag name recursive (bool, optional): recursively walk the repo dvc_only (bool, optional): show only DVC-artifacts - config (bool, optional): path to config file + config (str, optional): path to config file + remote (str, optional): remote name to set as a default remote in the repo + remote_config (str, dict): remote config to merge with a remote in the repo Returns: list of `entry` @@ -47,7 +51,13 @@ def ls( config_dict = None with Repo.open( - url, rev=rev, subrepos=True, uninitialized=True, config=config_dict + url, + rev=rev, + subrepos=True, + uninitialized=True, + config=config_dict, + remote=remote, + remote_config=remote_config, ) as repo: path = path or "" diff --git a/tests/unit/command/ls/test_ls.py b/tests/unit/command/ls/test_ls.py index 1d6702624d..884dae137b 100644 --- a/tests/unit/command/ls/test_ls.py +++ b/tests/unit/command/ls/test_ls.py @@ -19,7 +19,14 @@ def test_list(mocker): url = "local_dir" m = _test_cli(mocker, url) m.assert_called_once_with( - url, None, recursive=False, rev=None, dvc_only=False, config=None + url, + None, + recursive=False, + rev=None, + dvc_only=False, + config=None, + remote=None, + remote_config=None, ) @@ -27,7 +34,14 @@ def test_list_recursive(mocker): url = "local_dir" m = _test_cli(mocker, url, "-R") m.assert_called_once_with( - url, None, recursive=True, rev=None, dvc_only=False, config=None + url, + None, + recursive=True, + rev=None, + dvc_only=False, + config=None, + remote=None, + remote_config=None, ) @@ -35,7 +49,14 @@ def test_list_git_ssh_rev(mocker): url = "git@github.com:repo" m = _test_cli(mocker, url, "--rev", "123") m.assert_called_once_with( - url, None, recursive=False, rev="123", dvc_only=False, config=None + url, + None, + recursive=False, + rev="123", + dvc_only=False, + config=None, + remote=None, + remote_config=None, ) @@ -44,7 +65,14 @@ def test_list_targets(mocker): target = "subdir" m = _test_cli(mocker, url, target) m.assert_called_once_with( - url, target, recursive=False, rev=None, dvc_only=False, config=None + url, + target, + recursive=False, + rev=None, + dvc_only=False, + config=None, + remote=None, + remote_config=None, ) @@ -52,15 +80,40 @@ def test_list_outputs_only(mocker): url = "local_dir" m = _test_cli(mocker, url, None, "--dvc-only") m.assert_called_once_with( - url, None, recursive=False, rev=None, dvc_only=True, config=None + url, + None, + recursive=False, + rev=None, + dvc_only=True, + config=None, + remote=None, + remote_config=None, ) def test_list_config(mocker): url = "local_dir" - m = _test_cli(mocker, url, None, "--config", "myconfig") + m = _test_cli( + mocker, + url, + None, + "--config", + "myconfig", + "--remote", + "myremote", + "--remote-config", + "k1=v1", + "k2=v2", + ) m.assert_called_once_with( - url, None, recursive=False, rev=None, dvc_only=False, config="myconfig" + url, + None, + recursive=False, + rev=None, + dvc_only=False, + config="myconfig", + remote="myremote", + remote_config={"k1": "v1", "k2": "v2"}, ) diff --git a/tests/unit/command/test_get.py b/tests/unit/command/test_get.py index 3e2cfad507..05e77b7478 100644 --- a/tests/unit/command/test_get.py +++ b/tests/unit/command/test_get.py @@ -16,6 +16,11 @@ def test_get(mocker): "4", "--config", "myconfig", + "--remote", + "myremote", + "--remote-config", + "k1=v1", + "k2=v2", ] ) assert cli_args.func == CmdGet @@ -33,6 +38,8 @@ def test_get(mocker): jobs=4, config="myconfig", force=False, + remote="myremote", + remote_config={"k1": "v1", "k2": "v2"}, ) diff --git a/tests/unit/command/test_imp.py b/tests/unit/command/test_imp.py index 83b49d8bd8..b6aa9bc7b0 100644 --- a/tests/unit/command/test_imp.py +++ b/tests/unit/command/test_imp.py @@ -16,6 +16,11 @@ def test_import(mocker, dvc): "3", "--config", "myconfig", + "--remote", + "myremote", + "--remote-config", + "k1=v1", + "k2=v2", ] ) assert cli_args.func == CmdImport @@ -34,6 +39,8 @@ def test_import(mocker, dvc): no_download=False, jobs=3, config="myconfig", + remote="myremote", + remote_config={"k1": "v1", "k2": "v2"}, ) @@ -65,6 +72,8 @@ def test_import_no_exec(mocker, dvc): no_download=False, jobs=None, config=None, + remote=None, + remote_config=None, ) @@ -96,4 +105,6 @@ def test_import_no_download(mocker, dvc): no_download=True, jobs=None, config=None, + remote=None, + remote_config=None, )