Merge pull request #269 from pelson/new_feedstocks_functions

New feedstocks functions

conda_smithy/feedstocks.py

@@ -1,9 +1,10 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 import argparse
 import glob
 import multiprocessing
 import os
 
+import git
 from git import Repo, GitCommandError
 from github import Github
 
@@ -127,6 +128,140 @@ def feedstocks_fetch_handle_args(args):
     return fetch_feedstocks(args.feedstocks_directory)
 
 
+def feedstocks_repos(organization, feedstocks_directory, pull_up_to_date=False, randomise=False, regexp=None):
+    """
+    Generator of (feedstock, yaml) for each feedstock in the given
+    feedstock_directory.
+
+    Parameters
+    ----------
+    pull_up_to_date : bool (default: False)
+        If True, clone all (missing) feedstocks before operation, and fetch
+        all feedstocks as each one is being yielded. 
+    randomise: bool (default: False)
+        If True, randomise the order of the generated feedstocks. This is
+        especially useful if no particular priority should be given to
+        the "aardvark" feedstock over the "zebra" feedstock ;) .
+    regepx: string (default: None)
+        If not None, a regular expression which can be used to filter the
+        feedstock based on its name. For example '^python' would yield
+        only feedstocks starting with the word "python".
+
+    """
+    # We can't do anything without having all of the feestocks cloned
+    # (the existing clones don't need to be fetched though).
+    if pull_up_to_date:
+        print('Cloning all missing repos...')
+        clone_all(organization, feedstocks_directory)
+
+    feedstocks = cloned_feedstocks(feedstocks_directory)
+
+    if regexp:
+        import re
+        regexp = re.compile(regexp)
+        feedstocks = [feedstock for feedstock in feedstocks
+                      if regexp.match(feedstock.package)]
+
+    if randomise:
+        import random
+        feedstocks = list(feedstocks)
+        random.shuffle(feedstocks)
+
+    for feedstock in feedstocks:
+        repo = git.Repo(feedstock.directory)
+        upstream = repo.remotes.upstream
+
+        if pull_up_to_date:
+            print('Fetching ', feedstock.package)
+            upstream.fetch()
+
+        yield repo, feedstock
+
+
+def yaml_meta(content):
+    """
+    Read the contents of meta.yaml into a ruamel.yaml document.
+
+    """
+    import jinja2
+    import ruamel.yaml
+    import six
+
+
+    class NullUndefined(jinja2.Undefined):
+        def __unicode__(self):
+            return six.text_type(self._undefined_name)
+
+        def __getattr__(self, name):
+            return six.text_type('{}.{}'.format(self, name))
+
+        def __getitem__(self, name):
+            return '{}["{}"]'.format(self, name)
+
+
+    class StrDict(dict):
+        def __getitem__(self, key, default=''):
+            # Unlike a normal dictionary, if the string doesn't exist, return an empty string. 
+            if key in self:
+                return self[key]
+            else:
+                return default
+
+
+    env = jinja2.Environment(undefined=NullUndefined)
+    parsable_content = env.from_string(content).render(os=os, environ=StrDict())
+    yaml = ruamel.yaml.load(parsable_content, ruamel.yaml.RoundTripLoader)
+    return yaml
+
+
+def feedstocks_yaml(organization, feedstocks_directory, use_local=False, **feedstocks_repo_kwargs):
+    """
+    Generator of (feedstock, ref, content, yaml) for each upstream git ref of each feedstock.
+
+    Parameters
+    ----------
+
+    use_local: bool
+        If True, the meta.yaml will be taken from the locally checked out branch, rather than from the commit
+        of the git ref.
+
+    """
+    for repo, feedstock in feedstocks_repos(organization, feedstocks_directory, **feedstocks_repo_kwargs):
+        upstream = repo.remotes.upstream
+        try:
+            refs = upstream.refs
+        except AssertionError:
+            # In early versions of gitpython and empty list of refs resulted in an
+            # assertion error (https://github.com/gitpython-developers/GitPython/pull/499).
+            refs = []
+
+        if not refs:
+            upstream.fetch()
+            refs = upstream.refs
+
+        for ref in refs:
+            remote_branch = ref.remote_head #.replace('{}/'.format(gh_me.login), '')
+
+            try:
+                if use_local:
+                    with open(os.path.join(feedstock.directory, 'recipe', 'meta.yaml'), 'r') as fh:
+                        content = ''.join(fh.readlines())
+                else:
+                    blob = ref.commit.tree['recipe']['meta.yaml']
+                    stream = blob.data_stream
+                    content = stream.read().decode('utf-8')
+                yaml = yaml_meta(content)
+            except:
+                # Add a helpful comment so we know what we are working with and reraise.
+                print('Failed on {}'.format(feedstock.package))
+                raise
+
+            yield (feedstock, ref, content, yaml)
+            if use_local:
+                # There is no point in us doing this for each branch - we will get the same result each time.
+                break
+
+
 def main():
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers()