diff --git a/.travis.yml b/.travis.yml index 9f4e90d..8ad75ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ install: # command to run tests script: + - git --version - flake8 . --exclude=__init__.py - coverage run --source git_aggregator setup.py test diff --git a/README.rst b/README.rst index e82ed56..9ef0682 100644 --- a/README.rst +++ b/README.rst @@ -64,6 +64,49 @@ You can specify that you want to fetch all references from all remotes you have fetch_all: true +Shallow repositories +-------------------- + +To save big amounts of bandwidth and disk space, you can use shallow clones. +These download only a restricted amount of commits depending on some criteria. +Available options are `depth`_, `shallow-since`_ and `shallow-exclude`_. + +.. warning:: + + Available options depend on server and client Git version, be sure to use + options available for your environment. + +.. _depth: https://git-scm.com/docs/git-fetch#git-fetch---depthltdepthgt +.. _shallow-since: https://git-scm.com/docs/git-fetch#git-fetch---shallow-sinceltdategt +.. _shallow-exclude: https://git-scm.com/docs/git-fetch#git-fetch---shallow-excludeltrevisiongt + +You can use those in the ``defaults`` sections to apply them everywhere, or +specifying them in the corresponding ``merges`` section, for which you must use +the ``dict`` alternate construction. If you need to disable a default in +``merges``, set it to ``false``: + +.. code-block:: yaml + + ./odoo: + defaults: + depth: 20 + remotes: + odoo: https://github.com/odoo/odoo.git + ocb: https://github.com/OCA/OCB.git + acsone: https://github.com/acsone/odoo.git + merges: + - + remote: ocb + ref: "9.0" + depth: 1000 + - + remote: odoo + ref: refs/pull/14859/head + target: acsone 9.0 + +Remember that you need to fetch at least the common ancestor of all merges for +it to succeed. + Triggers -------- diff --git a/git_aggregator/config.py b/git_aggregator/config.py index 1afe5de..b503088 100644 --- a/git_aggregator/config.py +++ b/git_aggregator/config.py @@ -25,6 +25,7 @@ def get_repos(config): directory = os.path.abspath(directory) repo_dict = { 'cwd': directory, + 'defaults': repo_data.get('defaults', dict()), } remote_names = set() if 'remotes' in repo_data: @@ -50,21 +51,32 @@ def get_repos(config): merges = [] merge_data = repo_data.get('merges') or [] for merge in merge_data: - parts = merge.split(' ') - if len(parts) != 2: - raise ConfigException( - '%s: Merge must be formatted as ' - '"remote_name ref".' % directory) - - remote_name, ref = merge.split(' ') - if remote_name not in remote_names: + try: + # Assume parts is a str + parts = merge.split(' ') + if len(parts) != 2: + raise ConfigException( + '%s: Merge must be formatted as ' + '"remote_name ref".' % directory) + merge = { + "remote": parts[0], + "ref": parts[1], + } + except AttributeError: + # Parts is a dict + try: + merge["remote"] = str(merge["remote"]) + merge["ref"] = str(merge["ref"]) + except KeyError: + raise ConfigException( + '%s: Merge lacks mandatory ' + '`remote` or `ref` keys.' % directory) + # Check remote is available + if merge["remote"] not in remote_names: raise ConfigException( '%s: Merge remote %s not defined in remotes.' % - (directory, remote_name)) - merges.append({ - 'remote': remote_name, - 'ref': ref, - }) + (directory, merge["remote"])) + merges.append(merge) repo_dict['merges'] = merges if not merges: raise ConfigException( diff --git a/git_aggregator/repo.py b/git_aggregator/repo.py index 5dde8ee..7cf1cc2 100644 --- a/git_aggregator/repo.py +++ b/git_aggregator/repo.py @@ -12,6 +12,7 @@ from .exception import GitAggregatorException from ._compat import console_to_str +FETCH_DEFAULTS = ("depth", "shallow-since", "shallow-exclude") logger = logging.getLogger(__name__) @@ -34,7 +35,7 @@ class Repo(object): _git_version = None def __init__(self, cwd, remotes, merges, target, - shell_command_after=None, fetch_all=False): + shell_command_after=None, fetch_all=False, defaults=None): """Initialize a git repository aggregator :param cwd: path to the directory where to initialize the repository @@ -43,12 +44,14 @@ def __init__(self, cwd, remotes, merges, target, :param: merges list of merge to apply to build the aggregated repository. A merge is a dict {'remote': '', 'ref': ''} :param target: + :param shell_command_after: an optional list of shell command to + execute after the aggregation :param fetch_all: Can be an iterable (recommended: ``frozenset``) that yields names of remotes where all refs should be fetched, or ``True`` to do it for every configured remote. - :param shell_command_after: an optional list of shell command to - execute after the aggregation + :param defaults: + Collection of default parameters to be passed to git. """ self.cwd = cwd self.remotes = remotes @@ -59,6 +62,7 @@ def __init__(self, cwd, remotes, merges, target, self.merges = merges self.target = target self.shell_command_after = shell_command_after or [] + self.defaults = defaults or dict() @property def git_version(self): @@ -174,9 +178,9 @@ def aggregate(self): # reset to the first merge origin = merges[0] merges = merges[1:] - self._reset_to(**origin) + self._reset_to(origin["remote"], origin["ref"]) for merge in merges: - self._merge(**merge) + self._merge(merge) self._execute_shell_command_after() logger.info('End aggregation of %s', self.cwd) @@ -188,7 +192,7 @@ def fetch(self): basecmd = ("git", "fetch") logger.info("Fetching required remotes") for merge in self.merges: - cmd = basecmd + (merge["remote"],) + cmd = basecmd + self._fetch_options(merge) + (merge["remote"],) if merge["remote"] not in self.fetch_all: cmd += (merge["ref"],) self.log_call(cmd, cwd=self.cwd) @@ -201,6 +205,15 @@ def push(self): os.chdir(self.cwd) self.log_call(['git', 'push', '-f', remote, branch]) + def _fetch_options(self, merge): + """Get the fetch options from the given merge dict.""" + cmd = tuple() + for option in FETCH_DEFAULTS: + value = merge.get(option, self.defaults.get(option)) + if value: + cmd += ("--%s" % option, str(value)) + return cmd + def _reset_to(self, remote, ref): logger.info('Reset branch to %s %s', remote, ref) rtype, sha = self.query_remote_ref(remote, ref) @@ -223,16 +236,17 @@ def _execute_shell_command_after(self): for cmd in self.shell_command_after: self.log_call(cmd, shell=True) - def _merge(self, remote, ref): - logger.info("Pull %s, %s", remote, ref) - cmd = ['git', 'pull', remote, ref] + def _merge(self, merge): + logger.info("Pull %s, %s", merge["remote"], merge["ref"]) + cmd = ("git", "pull") if self.git_version >= (1, 7, 10): # --edit and --no-edit appear with Git 1.7.10 # see Documentation/RelNotes/1.7.10.txt of Git # (https://git.kernel.org/cgit/git/git.git/tree) - cmd.insert(2, '--no-edit') + cmd += ('--no-edit',) if logger.getEffectiveLevel() != logging.DEBUG: - cmd.insert(2, '--quiet') + cmd += ('--quiet',) + cmd += self._fetch_options(merge) + (merge["remote"], merge["ref"]) self.log_call(cmd) def _get_remotes(self): diff --git a/tests/test_config.py b/tests/test_config.py index f794ee0..741dfac 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -41,6 +41,7 @@ def test_load(self): repos[0], {'cwd': '/product_attribute', 'fetch_all': False, + 'defaults': {}, 'merges': [{'ref': '8.0', 'remote': 'oca'}, {'ref': 'refs/pull/105/head', 'remote': 'oca'}, {'ref': 'refs/pull/106/head', 'remote': 'oca'}], @@ -57,6 +58,52 @@ def test_load(self): 'url': 'git+ssh://git@github.com/acsone/product-attribute.git'}]) + def test_load_defaults(self): + config_yaml = dedent(""" + /web: + defaults: + depth: 1 + remotes: + oca: https://github.com/OCA/web.git + acsone: git+ssh://git@github.com/acsone/web.git + merges: + - + remote: oca + ref: 8.0 + depth: 1000 + - oca refs/pull/105/head + - + remote: oca + ref: refs/pull/106/head + target: acsone aggregated_branch_name + """) + repos = config.get_repos(self._parse_config(config_yaml)) + self.assertEquals(len(repos), 1) + # remotes are configured as dict therefore the order is not preserved + # when parsed + remotes = repos[0]['remotes'] + repos[0]['remotes'] = [] + self.assertDictEqual( + repos[0], + {'cwd': '/web', + 'fetch_all': False, + 'defaults': {'depth': 1}, + 'merges': [{'ref': '8.0', 'remote': 'oca', 'depth': 1000}, + {'ref': 'refs/pull/105/head', 'remote': 'oca'}, + {'ref': 'refs/pull/106/head', 'remote': 'oca'}], + 'remotes': [], + 'shell_command_after': [], + 'target': {'branch': 'aggregated_branch_name', + 'remote': 'acsone'}}) + assertfn = self.assertItemsEqual if PY2 else self.assertCountEqual + assertfn( + remotes, + [{'name': 'oca', + 'url': 'https://github.com/OCA/web.git'}, + {'name': 'acsone', + 'url': + 'git+ssh://git@github.com/acsone/web.git'}]) + def test_load_shell_command_after(self): """Shell command after are alway parser as a list """ @@ -181,6 +228,21 @@ def test_load_merges_exception(self): ex.exception.args[0], '/product_attribute: Merge remote oba not defined in remotes.') + config_yaml = dedent(""" + /web: + remotes: + oca: https://github.com/OCA/web.git + merges: + - + depth: 1 + target: oca aggregated_branch + """) + with self.assertRaises(ConfigException) as ex: + config.get_repos(self._parse_config(config_yaml)) + self.assertEquals( + ex.exception.args[0], + '/web: Merge lacks mandatory `remote` or `ref` keys.') + def test_load_target_exception(self): config_yaml = """ /product_attribute: diff --git a/tests/test_repo.py b/tests/test_repo.py index c07029d..12e8daa 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -213,3 +213,70 @@ def test_update_aggregate_2(self): self.remote1, 'tracked_new', "last", msg="new file on remote1") repo.aggregate() self.assertFalse(os.path.isfile(os.path.join(self.cwd, 'tracked_new'))) + + def test_depth_1(self): + """Ensure a simple shallow clone with 1 commit works.""" + remotes = [{ + 'name': 'shallow', + 'url': self.url_remote1 + }] + merges = [{ + 'remote': 'shallow', + "ref": "master", + }] + target = { + 'remote': 'shallow', + 'branch': 'master' + } + defaults = { + "depth": 1, + } + repo = Repo(self.cwd, remotes, merges, target, defaults=defaults) + repo.aggregate() + self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked'))) + + with working_directory_keeper: + os.chdir(self.cwd) + log_shallow = subprocess.check_output( + ("git", "rev-list", "shallow/master")) + # Shallow fetch: just 1 commmit + self.assertEqual(len(log_shallow.splitlines()), 1) + + def test_depth(self): + """Ensure `depth` is used correctly.""" + remotes = [{ + 'name': 'r1', + 'url': self.url_remote1 + }, { + 'name': 'r2', + 'url': self.url_remote2 + }] + merges = [{ + 'remote': 'r1', + "ref": "master", + }, { + "remote": "r2", + 'ref': "b2", + }] + target = { + 'remote': 'r1', + 'branch': 'agg' + } + defaults = { + "depth": 2, + } + repo = Repo(self.cwd, remotes, merges, target, defaults=defaults) + repo.aggregate() + self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked'))) + self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked2'))) + + with working_directory_keeper: + os.chdir(self.cwd) + log_r1 = subprocess.check_output( + ("git", "rev-list", "r1/master")) + log_r2 = subprocess.check_output( + ("git", "rev-list", "r2/b2")) + # Shallow fetch: just 1 commmit + self.assertEqual(len(log_r1.splitlines()), 2) + # Full fetch: all 3 commits + self.assertEqual(len(log_r2.splitlines()), 2)