Fixing source/path and GIT_* issues

conda_build/environ.py

@@ -2,7 +2,7 @@
 
 import os
 import sys
-from os.path import join, normpath, isabs
+from os.path import join, normpath
 from subprocess import STDOUT, check_output, CalledProcessError, Popen, PIPE
 import multiprocessing
 import warnings
@@ -12,6 +12,7 @@
 from conda_build.config import config
 
 from conda_build import source
+from conda_build import external
 from conda_build.scripts import prepend_bin_path
 
 
@@ -42,17 +43,11 @@ def get_sp_dir():
     return join(get_stdlib_dir(), 'site-packages')
 
 
-def get_git_build_info(src_dir, git_url, expected_rev):
-    expected_rev = expected_rev or 'HEAD'
+def verify_git_repo(git_dir, git_url, expected_rev='HEAD'):
     env = os.environ.copy()
-    d = {}
-    git_dir = join(src_dir, '.git')
-    if not isinstance(git_dir, str):
-        # On Windows, subprocess env can't handle unicode.
-        git_dir = git_dir.encode(sys.getfilesystemencoding() or 'utf-8')
 
-    if not os.path.exists(git_dir):
-        return d
+    if not expected_rev:
+        return False
 
     env['GIT_DIR'] = git_dir
     try:
@@ -65,59 +60,78 @@ def get_git_build_info(src_dir, git_url, expected_rev):
                                            env=env, stderr=STDOUT)
         expected_tag_commit = expected_tag_commit.decode('utf-8')
 
+        if current_commit != expected_tag_commit:
+            return False
+
         # Verify correct remote url. Need to find the git cache directory,
         # and check the remote from there.
         cache_details = check_output(["git", "remote", "-v"], env=env,
                                      stderr=STDOUT)
         cache_details = cache_details.decode('utf-8')
         cache_dir = cache_details.split('\n')[0].split()[1]
         assert "conda-bld/git_cache" in cache_dir
+
         if not isinstance(cache_dir, str):
             # On Windows, subprocess env can't handle unicode.
             cache_dir = cache_dir.encode(sys.getfilesystemencoding() or 'utf-8')
 
-        env['GIT_DIR'] = cache_dir
-        remote_details = check_output(["git", "remote", "-v"], env=env,
-                                      stderr=STDOUT)
+
+        remote_details = check_output(["git", "--git-dir", cache_dir, "remote", "-v"], env=env,
+                                                 stderr=STDOUT)
         remote_details = remote_details.decode('utf-8')
         remote_url = remote_details.split('\n')[0].split()[1]
-        if '://' not in remote_url:
+        if os.path.exists(remote_url):
             # Local filepaths are allowed, but make sure we normalize them
             remote_url = normpath(remote_url)
 
-        # If the current source directory in conda-bld/work doesn't match the
-        # user's metadata git_url or git_rev, then we aren't looking at the
-        # right source.
-        if remote_url != git_url or current_commit != expected_tag_commit:
-            return d
+        # If the current source directory in conda-bld/work doesn't match the user's
+        # metadata git_url or git_rev, then we aren't looking at the right source.
+        if remote_url != git_url:
+            return False
     except CalledProcessError:
-        return d
-
-    env['GIT_DIR'] = git_dir
+        return False
+    return True
+
+
+def get_git_info(repo):
+    """
+    Given a repo to a git repo, return a dictionary of:
+      GIT_DESCRIBE_TAG
+      GIT_DESCRIBE_NUMBER
+      GIT_DESCRIBE_HASH
+      GIT_FULL_HASH
+      GIT_BUILD_STR
+    from the output of git describe.
+    :return:
+    """
+    d = {}
 
     # grab information from describe
-    key_name = lambda a: "GIT_DESCRIBE_{}".format(a)
-    keys = [key_name("TAG"), key_name("NUMBER"), key_name("HASH")]
-    env = {str(key): str(value) for key, value in env.items()}
+    env = os.environ.copy()
+    env['GIT_DIR'] = repo
+    keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"]
+
     process = Popen(["git", "describe", "--tags", "--long", "HEAD"],
                     stdout=PIPE, stderr=PIPE,
                     env=env)
     output = process.communicate()[0].strip()
     output = output.decode('utf-8')
+
     parts = output.rsplit('-', 2)
-    parts_length = len(parts)
-    if parts_length == 3:
+    if len(parts) == 3:
         d.update(dict(zip(keys, parts)))
+
     # get the _full_ hash of the current HEAD
     process = Popen(["git", "rev-parse", "HEAD"],
                     stdout=PIPE, stderr=PIPE, env=env)
     output = process.communicate()[0].strip()
     output = output.decode('utf-8')
+
     d['GIT_FULL_HASH'] = output
     # set up the build string
-    if key_name('NUMBER') in d and key_name('HASH') in d:
-        d['GIT_BUILD_STR'] = '{}_{}'.format(d[key_name('NUMBER')],
-                                            d[key_name('HASH')])
+    if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d:
+        d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"],
+                                            d["GIT_DESCRIBE_HASH"])
 
     return d
 
@@ -160,6 +174,33 @@ def get_dict(m=None, prefix=None):
             else:
                 d[var_name] = value
 
+        git_dir = join(d['SRC_DIR'], '.git')
+        if not isinstance(git_dir, str):
+            # On Windows, subprocess env can't handle unicode.
+            git_dir = git_dir.encode(sys.getfilesystemencoding() or 'utf-8')
+
+        if external.find_executable('git') and os.path.exists(git_dir):
+            git_url = m.get_value('source/git_url')
+
+            if os.path.exists(git_url):
+                # If git_url is a relative path instead of a url, convert it to an abspath
+                git_url = normpath(join(m.path, git_url))
+
+            _x = False
+            if git_url:
+                _x = verify_git_repo(git_dir,
+                                     git_url,
+                                     m.get_value('source/git_rev', 'HEAD'))
+
+            if _x or m.get_value('source/path'):
+                d.update(get_git_info(git_dir))
+
+        d['PKG_NAME'] = m.name()
+        d['PKG_VERSION'] = m.version()
+        d['PKG_BUILDNUM'] = str(m.build_number())
+        d['PKG_BUILD_STRING'] = str(m.build_id())
+        d['RECIPE_DIR'] = m.path
+
     if sys.platform == "darwin":
         # multiprocessing.cpu_count() is not reliable on OSX
         # See issue #645 on github.com/conda/conda-build
@@ -172,18 +213,6 @@ def get_dict(m=None, prefix=None):
         except NotImplementedError:
             d['CPU_COUNT'] = "1"
 
-    if m and m.get_value('source/git_url'):
-        git_url = m.get_value('source/git_url')
-        if '://' not in git_url:
-            # If git_url is a relative path instead of a url, convert it to an
-            # abspath
-            if not isabs(git_url):
-                git_url = join(m.path, git_url)
-            git_url = normpath(join(m.path, git_url))
-        d.update(get_git_build_info(d['SRC_DIR'],
-                                    git_url,
-                                    m.get_value('source/git_rev')))
-
     d['PATH'] = dict(os.environ)['PATH']
     d = prepend_bin_path(d, prefix)
 
@@ -229,13 +258,6 @@ def get_dict(m=None, prefix=None):
             d['CFLAGS'] = cflags + ' -m32'
             d['CXXFLAGS'] = cxxflags + ' -m32'
 
-    if m:
-        d['PKG_NAME'] = m.name()
-        d['PKG_VERSION'] = m.version()
-        d['PKG_BUILDNUM'] = str(m.build_number())
-        d['PKG_BUILD_STRING'] = str(m.build_id())
-        d['RECIPE_DIR'] = m.path
-
     return d
 
 

conda_build/metadata.py

@@ -145,70 +145,59 @@ def parse(data):
     for field in FIELDS:
         if field not in res:
             continue
-        if not res[field]:
-            res[field] = {}
         if not isinstance(res[field], dict):
             raise RuntimeError("The %s field should be a dict, not %s" %
                                (field, res[field].__class__.__name__))
-    # ensure those are lists
-    for field in ('source/patches',
-                  'build/entry_points', 'build/script_env',
-                  'build/features', 'build/track_features',
-                  'requirements/build', 'requirements/run',
-                  'requirements/conflicts', 'test/requires',
-                  'test/files', 'test/commands', 'test/imports'):
-        section, key = field.split('/')
-        if res.get(section) is None:
-            res[section] = {}
-        if res[section].get(key, None) is None:
-            res[section][key] = []
-
-    # ensure those are strings
-    for field in ('package/version', 'build/string', 'build/pin_depends',
-                  'source/svn_rev', 'source/git_tag', 'source/git_branch',
-                  'source/md5', 'source/git_rev', 'source/path'):
-        section, key = field.split('/')
-        if res.get(section) is None:
-            res[section] = {}
-        val = res[section].get(key, '')
-        if val is None:
-            val = ''
-        res[section][key] = text_type(val)
-
-    # ensure these fields are booleans
-    trues = {'y', 'on', 'true', 'yes'}
-    falses = {'n', 'no', 'false', 'off'}
-    for field in ('build/osx_is_app', 'build/preserve_egg_dir',
-                  'build/binary_relocation', 'build/noarch_python',
-                  'build/detect_binary_files_with_prefix',
-                  'build/skip', 'app/own_environment'):
-        section, key = field.split('/')
-        if res.get(section) is None:
-            res[section] = {}
 
-        try:
-            val = res[section].get(key, '').lower()
-        except AttributeError:
-            # val wasn't a string
-            continue
 
-        if val in trues:
-            res[section][key] = True
-        elif val in falses:
-            res[section][key] = False
 
     ensure_valid_fields(res)
     ensure_valid_license_family(res)
     return sanitize(res)
 
 
+trues = {'y', 'on', 'true', 'yes'}
+falses = {'n', 'no', 'false', 'off'}
+
+default_stucts = {
+    'source/patches': list,
+    'build/entry_points': list,
+    'build/script_env': list,
+    'build/features': list,
+    'build/track_features': list,
+    'requirements/build': list,
+    'requirements/run': list,
+    'requirements/conflicts': list,
+    'test/requires': list,
+    'test/files': list,
+    'test/commands': list,
+    'test/imports': list,
+    'package/version': text_type,
+    'build/string': text_type,
+    'build/pin_depends': text_type,
+    'source/svn_rev': text_type,
+    'source/git_tag': text_type,
+    'source/git_branch': text_type,
+    'source/md5': text_type,
+    'source/git_rev': text_type,
+    'source/path': text_type,
+    'source/git_url': text_type,
+    'build/osx_is_app': bool,
+    'build/preserve_egg_dir': bool,
+    'build/binary_relocation': bool,
+    'build/noarch_python': bool,
+    'build/detect_binary_files_with_prefix': bool,
+    'build/skip': bool,
+    'app/own_environment': bool
+}
+
 def sanitize(meta):
     """
     Sanitize the meta-data to remove aliases/handle deprecation
 
     """
     # make a copy to avoid side-effects
-    meta = dict(meta)
+    meta = meta.copy()
     sanitize_funs = [('source', _git_clean), ]
     for section, func in sanitize_funs:
         if section in meta:
@@ -232,7 +221,7 @@ def _git_clean(source_meta):
 
     git_rev_tags = (git_rev,) + git_rev_tags_old
 
-    has_rev_tags = tuple(bool(source_meta[tag]) for
+    has_rev_tags = tuple(bool(source_meta.get(tag, text_type())) for
                           tag in git_rev_tags)
     if sum(has_rev_tags) > 1:
         msg = "Error: mulitple git_revs:"
@@ -241,14 +230,14 @@ def _git_clean(source_meta):
         sys.exit(msg)
 
     # make a copy of the input so we have no side-effects
-    ret_meta = dict(source_meta)
+    ret_meta = source_meta.copy()
     # loop over the old versions
     for key, has in zip(git_rev_tags[1:], has_rev_tags[1:]):
         # update if needed
         if has:
             ret_meta[git_rev_tags[0]] = ret_meta[key]
         # and remove
-        del ret_meta[key]
+        ret_meta.pop(key, None)
 
     return ret_meta
 
@@ -372,9 +361,30 @@ def fromdict(cls, metadata):
     def get_section(self, section):
         return self.meta.get(section, {})
 
-    def get_value(self, field, default=None):
+    def get_value(self, field, default=None, autotype=True):
+        """
+        Get a value from a meta.yaml.
+        :param field: Field to return
+        :param default: Default object to return if field doesn't exist
+        :param autotype: If True, return the default type of field if one exists.
+        False will return the default object.
+        :return:
+        """
         section, key = field.split('/')
+
+        # get correct default
+        if autotype and default is None and field in default_stucts:
+            default = default_stucts[field]()
+
         value = self.get_section(section).get(key, default)
+
+        # handle yaml 1.1 boolean values
+        if isinstance(value, text_type):
+            if value.lower() in trues:
+                value = True
+            elif value.lower() in falses:
+                value = False
+
         return value
 
     def check_fields(self):

tests/test-recipes/metadata/source_git/bld.bat

@@ -14,3 +14,13 @@ if errorlevel 1 exit 1
 set PYTHONPATH=.
 python -c "import conda_build; assert conda_build.__version__ == '1.8.1', conda_build.__version__"
 if errorlevel 1 exit 1
+
+
+rem check that GIT_* tags are present
+for %%i in (GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH) DO (
+  if defined %%i (
+      echo %%i
+  ) else (
+    exit 1
+  )
+)

tests/test-recipes/metadata/source_git/build.sh

@@ -5,3 +5,13 @@
 git describe
 [ "$(git describe)" = 1.8.1 ]
 PYTHONPATH=. python -c "import conda_build; assert conda_build.__version__ == '1.8.1', conda_build.__version__"
+
+# check if GIT_* variables are defined
+for i in GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH
+do
+  if [ -n "eval $i" ]; then
+    eval echo \$$i
+  else
+    exit 1
+  fi
+done