From 3e654ffc10cbe5332b72fb4a2edc5148ac656efb Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 1 Mar 2016 19:22:57 -0600 Subject: [PATCH 01/16] remove redundant checks --- conda_build/environ.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 64897975bd..b72ca7cbae 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -142,6 +142,24 @@ def get_dict(m=None, prefix=None): else: d[var_name] = value + if m.get_value('source/git_url'): + git_url = m.get_value('source/git_url') + if '://' not in git_url: + # If git_url is a relative path instead of a url, convert it to an abspath + if not isabs(git_url): + git_url = join(m.path, git_url) + git_url = normpath(join(m.path, git_url)) + d.update(**get_git_build_info(d['SRC_DIR'], + git_url, + m.get_value('source/git_rev'))) + + d['PKG_NAME'] = m.name() + d['PKG_VERSION'] = m.version() + d['PKG_BUILDNUM'] = str(m.build_number()) + d['PKG_BUILD_STRING'] = str(m.build_id()) + d['RECIPE_DIR'] = m.path + + if sys.platform == "darwin": # multiprocessing.cpu_count() is not reliable on OSX # See issue #645 on github.com/conda/conda-build @@ -153,17 +171,6 @@ def get_dict(m=None, prefix=None): except NotImplementedError: d['CPU_COUNT'] = "1" - if m and m.get_value('source/git_url'): - git_url = m.get_value('source/git_url') - if '://' not in git_url: - # If git_url is a relative path instead of a url, convert it to an abspath - if not isabs(git_url): - git_url = join(m.path, git_url) - git_url = normpath(join(m.path, git_url)) - d.update(**get_git_build_info(d['SRC_DIR'], - git_url, - m.get_value('source/git_rev'))) - d['PATH'] = dict(os.environ)['PATH'] d = prepend_bin_path(d, prefix) @@ -200,13 +207,6 @@ def get_dict(m=None, prefix=None): d['CFLAGS'] = cflags + ' -m32' d['CXXFLAGS'] = cxxflags + ' -m32' - if m: - d['PKG_NAME'] = m.name() - d['PKG_VERSION'] = m.version() - d['PKG_BUILDNUM'] = str(m.build_number()) - d['PKG_BUILD_STRING'] = str(m.build_id()) - d['RECIPE_DIR'] = m.path - return d From 46861a8e74cbcb32dd2a338d2cd4f332be6c877a Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 1 Mar 2016 19:26:18 -0600 Subject: [PATCH 02/16] If git executable can't be found, then don't try to execute git --- conda_build/environ.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index b72ca7cbae..4f873f5c54 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -12,6 +12,7 @@ from conda_build.config import config from conda_build import source +from conda_build import external from conda_build.scripts import prepend_bin_path @@ -42,7 +43,7 @@ def get_git_build_info(src_dir, git_url, expected_rev): env = os.environ.copy() d = {} git_dir = join(src_dir, '.git') - if not os.path.exists(git_dir): + if not (os.path.exists(git_dir) and external.find_executable('git')): return d env['GIT_DIR'] = git_dir From f1d33651dfa7b94da4f706c2505b10d720cab04b Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 1 Mar 2016 19:26:56 -0600 Subject: [PATCH 03/16] use keyword argument --- conda_build/environ.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 4f873f5c54..1bb36c12f5 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -38,8 +38,7 @@ def get_stdlib_dir(): def get_sp_dir(): return join(get_stdlib_dir(), 'site-packages') -def get_git_build_info(src_dir, git_url, expected_rev): - expected_rev = expected_rev or 'master' +def get_git_build_info(src_dir, git_url, expected_rev='master'): env = os.environ.copy() d = {} git_dir = join(src_dir, '.git') From 0af4e3508fd05d71e8162bb82bc473a41321bf9f Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Tue, 1 Mar 2016 19:30:20 -0600 Subject: [PATCH 04/16] remove redundant assignments --- conda_build/environ.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 1bb36c12f5..987f251eb9 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -41,11 +41,10 @@ def get_sp_dir(): def get_git_build_info(src_dir, git_url, expected_rev='master'): env = os.environ.copy() d = {} - git_dir = join(src_dir, '.git') - if not (os.path.exists(git_dir) and external.find_executable('git')): + env['GIT_DIR'] = join(src_dir, '.git') + if not (os.path.exists(env['GIT_DIR']) and external.find_executable('git')): return d - env['GIT_DIR'] = git_dir try: # Verify current commit matches expected commit current_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H"], env=env) @@ -60,8 +59,7 @@ def get_git_build_info(src_dir, git_url, expected_rev='master'): cache_dir = cache_details.split('\n')[0].split()[1] assert "conda-bld/git_cache" in cache_dir - env['GIT_DIR'] = cache_dir - remote_details = subprocess.check_output(["git", "remote", "-v"], env=env) + remote_details = subprocess.check_output(["git", "remote", "-v", "--git-dir", cache_dir], env=env) remote_details = remote_details.decode('utf-8') remote_url = remote_details.split('\n')[0].split()[1] if '://' not in remote_url: @@ -75,8 +73,6 @@ def get_git_build_info(src_dir, git_url, expected_rev='master'): except subprocess.CalledProcessError: return d - env['GIT_DIR'] = git_dir - # grab information from describe key_name = lambda a: "GIT_DESCRIBE_{}".format(a) keys = [key_name("TAG"), key_name("NUMBER"), key_name("HASH")] From 3bd490d6a280be6444781e885e07faf3edadba7a Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Wed, 2 Mar 2016 01:16:52 -0600 Subject: [PATCH 05/16] remove validation that was done incorrectly. By adding default values to the metadata during parsing, a very subtle bug in m.get_value() that rendered the method near useless. The default would never be returned if the field was validated during parsing (basically any field that required a certain type). For example, if m was a Metadata object, m.get_value('source/git_rev', 'master') would never return master, even if 'source/git_rev' was never defined in meta.yaml. Now, m.get_value will return the default type of a field, if defined, whenever default=None. Also, when accessing the metadata, never assume any field exists beyond package/name and package/version. --- conda_build/metadata.py | 101 ++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index ae2d20f8e8..ef5b59ccba 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -145,70 +145,58 @@ def parse(data): for field in FIELDS: if field not in res: continue - if not res[field]: - res[field] = {} if not isinstance(res[field], dict): raise RuntimeError("The %s field should be a dict, not %s" % (field, res[field].__class__.__name__)) - # ensure those are lists - for field in ('source/patches', - 'build/entry_points', 'build/script_env', - 'build/features', 'build/track_features', - 'requirements/build', 'requirements/run', - 'requirements/conflicts', 'test/requires', - 'test/files', 'test/commands', 'test/imports'): - section, key = field.split('/') - if res.get(section) is None: - res[section] = {} - if res[section].get(key, None) is None: - res[section][key] = [] - - # ensure those are strings - for field in ('package/version', 'build/string', 'build/pin_depends', - 'source/svn_rev', 'source/git_tag', 'source/git_branch', - 'source/md5', 'source/git_rev', 'source/path'): - section, key = field.split('/') - if res.get(section) is None: - res[section] = {} - val = res[section].get(key, '') - if val is None: - val = '' - res[section][key] = text_type(val) - - # ensure these fields are booleans - trues = {'y', 'on', 'true', 'yes'} - falses = {'n', 'no', 'false', 'off'} - for field in ('build/osx_is_app', 'build/preserve_egg_dir', - 'build/binary_relocation', 'build/noarch_python', - 'build/detect_binary_files_with_prefix', - 'build/skip', 'app/own_environment'): - section, key = field.split('/') - if res.get(section) is None: - res[section] = {} - try: - val = res[section].get(key, '').lower() - except AttributeError: - # val wasn't a string - continue - if val in trues: - res[section][key] = True - elif val in falses: - res[section][key] = False ensure_valid_fields(res) ensure_valid_license_family(res) return sanitize(res) +trues = {'y', 'on', 'true', 'yes'} +falses = {'n', 'no', 'false', 'off'} + +default_stucts = { + 'source/patches': list, + 'build/entry_points': list, + 'build/script_env': list, + 'build/features': list, + 'build/track_features': list, + 'requirements/build': list, + 'requirements/run': list, + 'requirements/conflicts': list, + 'test/requires': list, + 'test/files': list, + 'test/commands': list, + 'test/imports': list, + 'package/version': text_type, + 'build/string': text_type, + 'build/pin_depends': text_type, + 'source/svn_rev': text_type, + 'source/git_tag': text_type, + 'source/git_branch': text_type, + 'source/md5': text_type, + 'source/git_rev': text_type, + 'source/path': text_type, + 'build/osx_is_app': bool, + 'build/preserve_egg_dir': bool, + 'build/binary_relocation': bool, + 'build/noarch_python': bool, + 'build/detect_binary_files_with_prefix': bool, + 'build/skip': bool, + 'app/own_environment': bool +} + def sanitize(meta): """ Sanitize the meta-data to remove aliases/handle deprecation """ # make a copy to avoid side-effects - meta = dict(meta) + meta = meta.copy() sanitize_funs = [('source', _git_clean), ] for section, func in sanitize_funs: if section in meta: @@ -232,7 +220,7 @@ def _git_clean(source_meta): git_rev_tags = (git_rev,) + git_rev_tags_old - has_rev_tags = tuple(bool(source_meta[tag]) for + has_rev_tags = tuple(bool(source_meta.get(tag, text_type())) for tag in git_rev_tags) if sum(has_rev_tags) > 1: msg = "Error: mulitple git_revs:" @@ -241,14 +229,14 @@ def _git_clean(source_meta): sys.exit(msg) # make a copy of the input so we have no side-effects - ret_meta = dict(source_meta) + ret_meta = source_meta.copy() # loop over the old versions for key, has in zip(git_rev_tags[1:], has_rev_tags[1:]): # update if needed if has: ret_meta[git_rev_tags[0]] = ret_meta[key] # and remove - del ret_meta[key] + ret_meta.pop(key, None) return ret_meta @@ -372,7 +360,20 @@ def get_section(self, section): def get_value(self, field, default=None): section, key = field.split('/') + + # get correct default + if default is None and field in default_stucts: + default = default_stucts[field]() + value = self.get_section(section).get(key, default) + + # handle yaml 1.1 boolean values + if isinstance(value, text_type): + if value.lower() in trues: + value = True + elif value.lower() in falses: + value = False + return value def check_fields(self): From 85eb294e58f90872f58ba04f2552dec723b4a121 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Mon, 7 Mar 2016 20:42:41 -0600 Subject: [PATCH 06/16] initial rework of environ logic --- conda_build/environ.py | 46 ++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 987f251eb9..daee367ce4 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -38,44 +38,46 @@ def get_stdlib_dir(): def get_sp_dir(): return join(get_stdlib_dir(), 'site-packages') -def get_git_build_info(src_dir, git_url, expected_rev='master'): +def get_git_build_info(src_dir, git_url, expected_rev='HEAD'): env = os.environ.copy() d = {} env['GIT_DIR'] = join(src_dir, '.git') - if not (os.path.exists(env['GIT_DIR']) and external.find_executable('git')): + if not (os.path.exists(env['GIT_DIR']) and external.find_executable('git')) or not expected_rev: return d try: # Verify current commit matches expected commit current_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H"], env=env) current_commit = current_commit.decode('utf-8') + expected_tag_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H", expected_rev], env=env) expected_tag_commit = expected_tag_commit.decode('utf-8') + if current_commit != expected_tag_commit: + return d + # Verify correct remote url. # (Need to find the git cache directory, and check the remote from there.) cache_details = subprocess.check_output(["git", "remote", "-v"], env=env) cache_details = cache_details.decode('utf-8') cache_dir = cache_details.split('\n')[0].split()[1] - assert "conda-bld/git_cache" in cache_dir - remote_details = subprocess.check_output(["git", "remote", "-v", "--git-dir", cache_dir], env=env) + remote_details = subprocess.check_output(["git", "--git-dir", cache_dir, "remote", "-v"], env=env) remote_details = remote_details.decode('utf-8') remote_url = remote_details.split('\n')[0].split()[1] - if '://' not in remote_url: + if os.path.exists(remote_url): # Local filepaths are allowed, but make sure we normalize them remote_url = normpath(remote_url) # If the current source directory in conda-bld/work doesn't match the user's # metadata git_url or git_rev, then we aren't looking at the right source. - if remote_url != git_url or current_commit != expected_tag_commit: + if remote_url != git_url: return d except subprocess.CalledProcessError: return d # grab information from describe - key_name = lambda a: "GIT_DESCRIBE_{}".format(a) - keys = [key_name("TAG"), key_name("NUMBER"), key_name("HASH")] + keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] env = {str(key): str(value) for key, value in env.items()} process = subprocess.Popen(["git", "describe", "--tags", "--long", "HEAD"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -83,8 +85,7 @@ def get_git_build_info(src_dir, git_url, expected_rev='master'): output = process.communicate()[0].strip() output = output.decode('utf-8') parts = output.rsplit('-', 2) - parts_length = len(parts) - if parts_length == 3: + if len(parts) == 3: d.update(dict(zip(keys, parts))) # get the _full_ hash of the current HEAD process = subprocess.Popen(["git", "rev-parse", "HEAD"], @@ -94,9 +95,9 @@ def get_git_build_info(src_dir, git_url, expected_rev='master'): output = output.decode('utf-8') d['GIT_FULL_HASH'] = output # set up the build string - if key_name('NUMBER') in d and key_name('HASH') in d: - d['GIT_BUILD_STR'] = '{}_{}'.format(d[key_name('NUMBER')], - d[key_name('HASH')]) + if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d: + d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"], + d["GIT_DESCRIBE_HASH"]) return d @@ -138,16 +139,13 @@ def get_dict(m=None, prefix=None): else: d[var_name] = value - if m.get_value('source/git_url'): - git_url = m.get_value('source/git_url') - if '://' not in git_url: - # If git_url is a relative path instead of a url, convert it to an abspath - if not isabs(git_url): - git_url = join(m.path, git_url) - git_url = normpath(join(m.path, git_url)) - d.update(**get_git_build_info(d['SRC_DIR'], - git_url, - m.get_value('source/git_rev'))) + git_url = m.get_value('source/git_url', '') or m.get_value('source/path', '') + if os.path.exists(git_url): + # If git_url is a relative path instead of a url, convert it to an abspath + git_url = normpath(join(m.path, git_url)) + d.update(**get_git_build_info(d['SRC_DIR'], + git_url, + m.get_value('source/git_rev', 'HEAD'))) d['PKG_NAME'] = m.name() d['PKG_VERSION'] = m.version() @@ -155,7 +153,7 @@ def get_dict(m=None, prefix=None): d['PKG_BUILD_STRING'] = str(m.build_id()) d['RECIPE_DIR'] = m.path - + #print("D==", '\n'.join("{}: {}".format(k,v) for k, v in d.items())) if sys.platform == "darwin": # multiprocessing.cpu_count() is not reliable on OSX # See issue #645 on github.com/conda/conda-build From 31d7b394a2a05d94dd5264798c888bd4bc6261b2 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 10:40:34 -0500 Subject: [PATCH 07/16] Allow a way to bypass automatically returning the default type. If autotype is False and the field doesn't exist, always return the default object. Added docstring --- conda_build/metadata.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 727cdbd316..de71ed356a 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -360,11 +360,19 @@ def fromdict(cls, metadata): def get_section(self, section): return self.meta.get(section, {}) - def get_value(self, field, default=None): + def get_value(self, field, default=None, autotype=True): + """ + Get a value from a meta.yaml. + :param field: Field to return + :param default: Default object to return if field doesn't exist + :param autotype: If True, return the default type of field if one exists. + False will return the default object. + :return: + """ section, key = field.split('/') # get correct default - if default is None and field in default_stucts: + if autotype and default is None and field in default_stucts: default = default_stucts[field]() value = self.get_section(section).get(key, default) From f7d5e6883a6e4834ef012af3f001f56a6fbcffb0 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 10:42:57 -0500 Subject: [PATCH 08/16] Split verification of a git repo from git describe logic This is the cleanest way I see to allow source/git_url and source/path to behave as expected. --- conda_build/environ.py | 56 +++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 9636e74798..b7bec8b3d2 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -43,16 +43,11 @@ def get_sp_dir(): return join(get_stdlib_dir(), 'site-packages') -def get_git_build_info(src_dir, git_url, expected_rev='HEAD'): +def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): env = os.environ.copy() - d = {} - git_dir = join(src_dir, '.git') - if not isinstance(git_dir, str): - # On Windows, subprocess env can't handle unicode. - git_dir = git_dir.encode(sys.getfilesystemencoding() or 'utf-8') if not (os.path.exists(git_dir) and external.find_executable('git')) or not expected_rev: - return d + return False env['GIT_DIR'] = git_dir try: @@ -66,7 +61,8 @@ def get_git_build_info(src_dir, git_url, expected_rev='HEAD'): expected_tag_commit = expected_tag_commit.decode('utf-8') if current_commit != expected_tag_commit: - return d + raise ValueError("Commits don't match\n" + "Expected: {}\nGot: {}".format(current_commit, expected_tag_commit)) # Verify correct remote url. Need to find the git cache directory, # and check the remote from there. @@ -80,7 +76,7 @@ def get_git_build_info(src_dir, git_url, expected_rev='HEAD'): cache_dir = cache_dir.encode(sys.getfilesystemencoding() or 'utf-8') - remote_details = subprocess.check_output(["git", "--git-dir", cache_dir, "remote", "-v"], env=env, + remote_details = check_output(["git", "--git-dir", cache_dir, "remote", "-v"], env=env, stderr=STDOUT) remote_details = remote_details.decode('utf-8') remote_url = remote_details.split('\n')[0].split()[1] @@ -91,26 +87,44 @@ def get_git_build_info(src_dir, git_url, expected_rev='HEAD'): # If the current source directory in conda-bld/work doesn't match the user's # metadata git_url or git_rev, then we aren't looking at the right source. if remote_url != git_url: - return d + raise ValueError("Unexpected remote: {}".format(remote_url)) except CalledProcessError: - return d - + return False + return True + + +def get_git_info(repo): + """ + Given a repo to a git repo, return a dictionary of: + GIT_DESCRIBE_TAG + GIT_DESCRIBE_NUMBER + GIT_DESCRIBE_HASH + GIT_FULL_HASH + GIT_BUILD_STR + from the output of git describe. + :return: + """ # grab information from describe + env = os.environ.copy() + d = {} keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] - env = {str(key): str(value) for key, value in env.items()} + process = Popen(["git", "describe", "--tags", "--long", "HEAD"], stdout=PIPE, stderr=PIPE, env=env) output = process.communicate()[0].strip() output = output.decode('utf-8') + parts = output.rsplit('-', 2) if len(parts) == 3: d.update(dict(zip(keys, parts))) + # get the _full_ hash of the current HEAD process = Popen(["git", "rev-parse", "HEAD"], stdout=PIPE, stderr=PIPE, env=env) output = process.communicate()[0].strip() output = output.decode('utf-8') + d['GIT_FULL_HASH'] = output # set up the build string if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d: @@ -158,13 +172,21 @@ def get_dict(m=None, prefix=None): else: d[var_name] = value - git_url = m.get_value('source/git_url', '') or m.get_value('source/path', '') + git_url = m.get_value('source/git_url') + git_dir = join(d['SRC_DIR'], '.git') + if not isinstance(git_dir, str): + # On Windows, subprocess env can't handle unicode. + git_dir = git_dir.encode(sys.getfilesystemencoding() or 'utf-8') + if git_url: + # verify git repo + verify_git_repo(git_dir + git_url, + m.get_value('source/git_rev', 'HEAD')) + if os.path.exists(git_url): # If git_url is a relative path instead of a url, convert it to an abspath git_url = normpath(join(m.path, git_url)) - d.update(**get_git_build_info(d['SRC_DIR'], - git_url, - m.get_value('source/git_rev', 'HEAD'))) + d.update(get_git_info(git_dir)) d['PKG_NAME'] = m.name() d['PKG_VERSION'] = m.version() From eb53e19f2808e7df28024e4fab3aade8a8ffcc72 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 10:45:31 -0500 Subject: [PATCH 09/16] restore assertion for cache_dir --- conda_build/environ.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/environ.py b/conda_build/environ.py index b7bec8b3d2..de552cf400 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -70,6 +70,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): stderr=STDOUT) cache_details = cache_details.decode('utf-8') cache_dir = cache_details.split('\n')[0].split()[1] + assert "conda-bld/git_cache" in cache_dir if not isinstance(cache_dir, str): # On Windows, subprocess env can't handle unicode. From e6ac7b89390378c53efbd686c8e9327b7be5436b Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 10:50:51 -0500 Subject: [PATCH 10/16] add missing comma --- conda_build/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index de552cf400..83fdd76472 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -180,7 +180,7 @@ def get_dict(m=None, prefix=None): git_dir = git_dir.encode(sys.getfilesystemencoding() or 'utf-8') if git_url: # verify git repo - verify_git_repo(git_dir + verify_git_repo(git_dir, git_url, m.get_value('source/git_rev', 'HEAD')) From 5e19d9bacd89fd15bd2fca7c86768b7b4fc9212d Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 11:18:44 -0500 Subject: [PATCH 11/16] make text_type default for source/git_url --- conda_build/metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index de71ed356a..ea51591dec 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -181,6 +181,7 @@ def parse(data): 'source/md5': text_type, 'source/git_rev': text_type, 'source/path': text_type, + 'source/git_url': text_type, 'build/osx_is_app': bool, 'build/preserve_egg_dir': bool, 'build/binary_relocation': bool, From 1bf48fc25dbe3e58e1b9b4b84611bbec9290fb17 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 11:18:53 -0500 Subject: [PATCH 12/16] remove old comment --- conda_build/environ.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 83fdd76472..2c9881788e 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -195,7 +195,6 @@ def get_dict(m=None, prefix=None): d['PKG_BUILD_STRING'] = str(m.build_id()) d['RECIPE_DIR'] = m.path - #print("D==", '\n'.join("{}: {}".format(k,v) for k, v in d.items())) if sys.platform == "darwin": # multiprocessing.cpu_count() is not reliable on OSX # See issue #645 on github.com/conda/conda-build From 53b9fc150ba6dbb1123ab9fb96674f71f85b6d90 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 12:22:52 -0500 Subject: [PATCH 13/16] simplify/refactor logic --- conda_build/environ.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 2c9881788e..1d1944054d 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -46,7 +46,7 @@ def get_sp_dir(): def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): env = os.environ.copy() - if not (os.path.exists(git_dir) and external.find_executable('git')) or not expected_rev: + if not expected_rev: return False env['GIT_DIR'] = git_dir @@ -61,8 +61,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): expected_tag_commit = expected_tag_commit.decode('utf-8') if current_commit != expected_tag_commit: - raise ValueError("Commits don't match\n" - "Expected: {}\nGot: {}".format(current_commit, expected_tag_commit)) + return False # Verify correct remote url. Need to find the git cache directory, # and check the remote from there. @@ -88,7 +87,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): # If the current source directory in conda-bld/work doesn't match the user's # metadata git_url or git_rev, then we aren't looking at the right source. if remote_url != git_url: - raise ValueError("Unexpected remote: {}".format(remote_url)) + return False except CalledProcessError: return False return True @@ -105,9 +104,11 @@ def get_git_info(repo): from the output of git describe. :return: """ + d = {} + # grab information from describe env = os.environ.copy() - d = {} + env['GIT_DIR'] = repo keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] process = Popen(["git", "describe", "--tags", "--long", "HEAD"], @@ -173,21 +174,24 @@ def get_dict(m=None, prefix=None): else: d[var_name] = value - git_url = m.get_value('source/git_url') git_dir = join(d['SRC_DIR'], '.git') if not isinstance(git_dir, str): # On Windows, subprocess env can't handle unicode. git_dir = git_dir.encode(sys.getfilesystemencoding() or 'utf-8') - if git_url: - # verify git repo - verify_git_repo(git_dir, - git_url, - m.get_value('source/git_rev', 'HEAD')) - - if os.path.exists(git_url): - # If git_url is a relative path instead of a url, convert it to an abspath - git_url = normpath(join(m.path, git_url)) - d.update(get_git_info(git_dir)) + + if external.find_executable('git') and os.path.exists(git_dir): + git_url = m.get_value('source/git_url') + + if os.path.exists(git_url): + # If git_url is a relative path instead of a url, convert it to an abspath + git_url = normpath(join(m.path, git_url)) + + _x = verify_git_repo(git_dir, + git_url, + m.get_value('source/git_rev', 'HEAD')) + + if (git_url and _x) or m.get_value('source/path'): + d.update(get_git_info(git_dir)) d['PKG_NAME'] = m.name() d['PKG_VERSION'] = m.version() From 73d134e99ca0b27fec9225f3342d15e4726bdc33 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 12:38:32 -0500 Subject: [PATCH 14/16] keep verification from happening when source/path present --- conda_build/environ.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 1d1944054d..54a910cd0f 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -186,11 +186,13 @@ def get_dict(m=None, prefix=None): # If git_url is a relative path instead of a url, convert it to an abspath git_url = normpath(join(m.path, git_url)) - _x = verify_git_repo(git_dir, - git_url, - m.get_value('source/git_rev', 'HEAD')) + _x = False + if git_url: + _x = verify_git_repo(git_dir, + git_url, + m.get_value('source/git_rev', 'HEAD')) - if (git_url and _x) or m.get_value('source/path'): + if _x or m.get_value('source/path'): d.update(get_git_info(git_dir)) d['PKG_NAME'] = m.name() From f69af520faea819f24b11ba8ae45ea8c329f6108 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 12:41:06 -0500 Subject: [PATCH 15/16] remove unused import Fix pyflakes issue --- conda_build/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 54a910cd0f..f846884b50 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -2,7 +2,7 @@ import os import sys -from os.path import join, normpath, isabs +from os.path import join, normpath from subprocess import STDOUT, check_output, CalledProcessError, Popen, PIPE import multiprocessing import warnings From 2800838377ae96f7f404341ffb17559d0b50e7f6 Mon Sep 17 00:00:00 2001 From: Ryan Grout Date: Fri, 25 Mar 2016 17:27:54 -0500 Subject: [PATCH 16/16] check GIT_* variables to make sure they are defined for git repos. --- tests/test-recipes/metadata/source_git/bld.bat | 10 ++++++++++ tests/test-recipes/metadata/source_git/build.sh | 10 ++++++++++ tests/test-recipes/metadata/source_path/bld.bat | 10 ++++++++++ tests/test-recipes/metadata/source_path/build.sh | 10 ++++++++++ 4 files changed, 40 insertions(+) diff --git a/tests/test-recipes/metadata/source_git/bld.bat b/tests/test-recipes/metadata/source_git/bld.bat index bc002292ab..ca1489a42c 100644 --- a/tests/test-recipes/metadata/source_git/bld.bat +++ b/tests/test-recipes/metadata/source_git/bld.bat @@ -14,3 +14,13 @@ if errorlevel 1 exit 1 set PYTHONPATH=. python -c "import conda_build; assert conda_build.__version__ == '1.8.1', conda_build.__version__" if errorlevel 1 exit 1 + + +rem check that GIT_* tags are present +for %%i in (GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH) DO ( + if defined %%i ( + echo %%i + ) else ( + exit 1 + ) +) diff --git a/tests/test-recipes/metadata/source_git/build.sh b/tests/test-recipes/metadata/source_git/build.sh index efca936212..74571375c8 100644 --- a/tests/test-recipes/metadata/source_git/build.sh +++ b/tests/test-recipes/metadata/source_git/build.sh @@ -5,3 +5,13 @@ git describe [ "$(git describe)" = 1.8.1 ] PYTHONPATH=. python -c "import conda_build; assert conda_build.__version__ == '1.8.1', conda_build.__version__" + +# check if GIT_* variables are defined +for i in GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH +do + if [ -n "eval $i" ]; then + eval echo \$$i + else + exit 1 + fi +done diff --git a/tests/test-recipes/metadata/source_path/bld.bat b/tests/test-recipes/metadata/source_path/bld.bat index 3d9611cb95..28fa9812b0 100644 --- a/tests/test-recipes/metadata/source_path/bld.bat +++ b/tests/test-recipes/metadata/source_path/bld.bat @@ -1,3 +1,13 @@ rem If directory exists, we did it right cd tests/test-recipes/test-package if errorlevel 1 exit 1 + + +rem check that GIT_* tags are present +for %%i in (GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH) DO ( + if defined %%i ( + echo %%i + ) else ( + exit 1 + ) +) diff --git a/tests/test-recipes/metadata/source_path/build.sh b/tests/test-recipes/metadata/source_path/build.sh index 59287e39cb..004a333479 100644 --- a/tests/test-recipes/metadata/source_path/build.sh +++ b/tests/test-recipes/metadata/source_path/build.sh @@ -1,2 +1,12 @@ # If directory exists, we did it right cd tests/test-recipes/test-package + +# check if GIT_* variables are defined +for i in GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH +do + if [ -n "eval $i" ]; then + eval echo \$$i + else + exit 1 + fi +done