From e27aa17b286d50a44c36ede283775d27d573cef1 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 10:35:42 -0600 Subject: [PATCH 01/28] Some edits to files.json --- conda_build/build.py | 22 ++++++++++++---------- conda_build/conda_interface.py | 1 + 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index b9bcec7b01..d472f43bca 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -45,6 +45,7 @@ from .conda_interface import VersionOrder from .conda_interface import (PaddingError, LinkError, CondaValueError, NoPackagesFoundError, NoPackagesFound) +from .conda_interface import CrossPlatformStLink from conda_build import __version__ from conda_build import environ, source, tarcheck @@ -281,8 +282,9 @@ def get_files_with_prefix(m, files, prefix): ignore_files = [] if not m.get_value('build/detect_binary_files_with_prefix', True): ignore_types.update(('binary',)) + # files_with_prefix is a list of tuples containing (prefix_placeholder, file_mode) ignore_files.extend( - [f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files]) + f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files) files_with_prefix = [f for f in files_with_prefix if f[2] not in ignore_files] return files_with_prefix @@ -466,7 +468,7 @@ def create_info_files(m, files, config, prefix): fo.write(f + '\n') files_with_prefix = get_files_with_prefix(m, files, prefix) - create_info_files_json(m, config.info_dir, prefix, files, files_with_prefix) + create_info_files_json_v1(m, config.info_dir, prefix, files, files_with_prefix) detect_and_record_prefix_files(m, files, prefix, config) write_no_link(m, config, files) @@ -519,10 +521,9 @@ def is_no_link(no_link, short_path): def get_inode_paths(files, target_short_path, prefix): - ensure_list(files) - target_short_path_inode = os.stat(join(prefix, target_short_path)).st_ino + target_short_path_inode = os.lstat(join(prefix, target_short_path)).st_ino hardlinked_files = [sp for sp in files - if os.stat(join(prefix, sp)).st_ino == target_short_path_inode] + if os.lstat(join(prefix, sp)).st_ino == target_short_path_inode] return sorted(hardlinked_files) @@ -537,7 +538,7 @@ def file_type(path): def build_info_files_json(m, prefix, files, files_with_prefix): no_link = m.get_value('build/no_link') files_json = [] - for fi in files: + for fi in sorted(files): prefix_placeholder, file_mode = has_prefix(fi, files_with_prefix) path = os.path.join(prefix, fi) file_info = { @@ -552,7 +553,8 @@ def build_info_files_json(m, prefix, files, files_with_prefix): if prefix_placeholder and file_mode: file_info["prefix_placeholder"] = prefix_placeholder file_info["file_mode"] = file_mode - if file_info.get("file_type") == "hardlink" and os.stat(join(prefix, fi)).st_nlink > 1: + if file_info.get("file_type") == FileType.hardlink and CrossPlatformStLink.st_nlink( + join(prefix, fi)) > 1: inode_paths = get_inode_paths(files, fi, prefix) file_info["inode_paths"] = inode_paths files_json.append(file_info) @@ -563,17 +565,17 @@ def get_files_version(): return 1 -def create_info_files_json(m, info_dir, prefix, files, files_with_prefix): +def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix): files_json_fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "prefix_placeholder", "no_link", "inode_first_path"] files_json_files = build_info_files_json(m, prefix, files, files_with_prefix) files_json_info = { - "version": get_files_version(), + "version": 1, "fields": files_json_fields, "files": files_json_files, } with open(join(info_dir, 'files.json'), "w") as files_json: - json.dump(files_json_info, files_json) + json.dump(files_json_info, files_json, sort_keys=True, indent=2, separators=(',', ': ')) def get_build_index(config, clear_cache=True): diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index f12d92e675..1a796a188d 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -56,6 +56,7 @@ LinkError = conda.exceptions.LinkError NoPackagesFoundError = conda.exceptions.NoPackagesFoundError CondaValueError = conda.exceptions.CondaValueError + from conda.common.compat import CrossPlatformStLink # disallow softlinks. This avoids a lot of dumb issues, at the potential cost of disk space. conda.base.context.context.allow_softlinks = False From 451d571c4015a2e7047dc1322f9e85e0d09a2111 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 11:51:30 -0600 Subject: [PATCH 02/28] Create NodeType and FileMode Enums --- conda_build/build.py | 22 ++++++---------------- conda_build/conda_interface.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index d472f43bca..4f220e901d 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -46,6 +46,7 @@ from .conda_interface import (PaddingError, LinkError, CondaValueError, NoPackagesFoundError, NoPackagesFound) from .conda_interface import CrossPlatformStLink +from .conda_interface import NodeType, FileMode from conda_build import __version__ from conda_build import environ, source, tarcheck @@ -65,12 +66,7 @@ from conda_build.features import feature_list import conda_build.noarch_python as noarch_python - - -class FileType(Enum): - softlink = "softlink" - hardlink = "hardlink" - directory = "directory" +from conda_verify.verify import Verify if 'bsd' in sys.platform: @@ -528,11 +524,9 @@ def get_inode_paths(files, target_short_path, prefix): def file_type(path): - if isdir(path): - return FileType.directory - elif islink(path): - return FileType.softlink - return FileType.hardlink + if islink(path): + return NodeType.softlink + return NodeType.hardlink def build_info_files_json(m, prefix, files, files_with_prefix): @@ -553,7 +547,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): if prefix_placeholder and file_mode: file_info["prefix_placeholder"] = prefix_placeholder file_info["file_mode"] = file_mode - if file_info.get("file_type") == FileType.hardlink and CrossPlatformStLink.st_nlink( + if file_info.get("file_type") == NodeType.hardlink and CrossPlatformStLink.st_nlink( join(prefix, fi)) > 1: inode_paths = get_inode_paths(files, fi, prefix) file_info["inode_paths"] = inode_paths @@ -561,10 +555,6 @@ def build_info_files_json(m, prefix, files, files_with_prefix): return files_json -def get_files_version(): - return 1 - - def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix): files_json_fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "prefix_placeholder", "no_link", "inode_first_path"] diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 1a796a188d..e58cc12b87 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -24,6 +24,7 @@ import conda.config as cc # NOQA from conda.config import rc_path # NOQA from conda.version import VersionOrder # NOQA +from enum import Enum import os @@ -136,3 +137,35 @@ def which_prefix(path): # we cannot chop off any more directories, so we didn't find it return None prefix = dirname(prefix) + + +class NodeType(Enum): + hard_link = 1 + soft_link = 2 + + @classmethod + def from_string(cls, string): + return cls[string.replace('-', '_')] + + @classmethod + def make(cls, value): + if isinstance(value, string_types): + return cls.from_string(value) + elif isinstance(value, cls): + return value + else: + return cls(value) + + def __int__(self): + return self.value + + def __str__(self): + return self.name.replace('_', '-') + + +class FileMode(Enum): + text = 'text' + binary = 'binary' + + def __str__(self): + return "%s" % self.value From 1248ff4e54b6c190f5870890d0eb34baca469e1f Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 11:59:37 -0600 Subject: [PATCH 03/28] Rename file_type to node_type --- conda_build/build.py | 4 ++-- tests/test_api_build.py | 2 +- tests/test_build.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 4f220e901d..1544f7d0c4 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -539,7 +539,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): "path": get_short_path(m, fi), "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), - "file_type": getattr(file_type(path), "name"), + "node_type": getattr(file_type(path), "name"), } no_link = is_no_link(no_link, fi) if no_link: @@ -557,7 +557,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix): files_json_fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", - "prefix_placeholder", "no_link", "inode_first_path"] + "prefix_placeholder", "no_link", "inode_paths"] files_json_files = build_info_files_json(m, prefix, files, files_with_prefix) files_json_info = { "version": 1, diff --git a/tests/test_api_build.py b/tests/test_api_build.py index bbb2c76869..b46ea6467f 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -753,7 +753,7 @@ def test_info_files_json(test_config): with tarfile.open(fn) as tf: data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8')) fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link", - "prefix_placeholder", "inode_first_path"] + "prefix_placeholder", "inode_paths"] for key in data.keys(): assert key in ['files', 'fields', 'version'] for field in data.get('fields'): diff --git a/tests/test_build.py b/tests/test_build.py index 577e758d60..166a5dd87a 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -269,6 +269,7 @@ def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): "size_in_bytes": 0}], "fields": ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "prefix_placeholder", "no_link", "inode_first_path"], + "version": 1} with open(files_json_path, "r") as files_json: assert json.load(files_json) == expected_output From ba9e4fb161f7c74b913eda618b6dc7c2f3f51129 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 12:42:58 -0600 Subject: [PATCH 04/28] Add docstring for additional enums --- conda_build/conda_interface.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index e58cc12b87..e781dbefb0 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -140,6 +140,10 @@ def which_prefix(path): class NodeType(Enum): + """ + Refers to if the file in question is hard linked or soft linked. Originally designed to be used + in files.json + """ hard_link = 1 soft_link = 2 @@ -164,6 +168,10 @@ def __str__(self): class FileMode(Enum): + """ + Refers to the mode of the file. Originally referring to the has_prefix file, but adopted for + files.json + """ text = 'text' binary = 'binary' From 436f0348022bf732db4029b84cf690b9f9ece6b0 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 13:02:11 -0600 Subject: [PATCH 05/28] Refactor NodeType --- conda_build/build.py | 4 ++-- conda_build/conda_interface.py | 21 +++++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 1544f7d0c4..a1a3991d13 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -274,7 +274,7 @@ def get_files_with_prefix(m, files, prefix): ignore_types = set() if not hasattr(ignore_files, "__iter__"): if ignore_files is True: - ignore_types.update(('text', 'binary')) + ignore_types.update((FileMode.text, FileMode.binary)) ignore_files = [] if not m.get_value('build/detect_binary_files_with_prefix', True): ignore_types.update(('binary',)) @@ -539,7 +539,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): "path": get_short_path(m, fi), "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), - "node_type": getattr(file_type(path), "name"), + "node_type": file_type(path).name, } no_link = is_no_link(no_link, fi) if no_link: diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index e781dbefb0..c6dec1f17a 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -144,27 +144,24 @@ class NodeType(Enum): Refers to if the file in question is hard linked or soft linked. Originally designed to be used in files.json """ - hard_link = 1 - soft_link = 2 + hardlink = 1 + softlink = 2 @classmethod - def from_string(cls, string): - return cls[string.replace('-', '_')] + def __call__(cls, value, *args, **kwargs): + if isinstance(cls, value, *args, **kwargs): + return cls[value] + return super(NodeType, cls).__call__(value, *args, **kwargs) @classmethod - def make(cls, value): - if isinstance(value, string_types): - return cls.from_string(value) - elif isinstance(value, cls): - return value - else: - return cls(value) + def __getitem__(cls, name): + return cls._member_map_[name.replace('-', '').replace('_', '').lower()] def __int__(self): return self.value def __str__(self): - return self.name.replace('_', '-') + return self.name class FileMode(Enum): From c7f3e2fd5554e12b27052799f1ccfe7e2aeaf641 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 13:06:33 -0600 Subject: [PATCH 06/28] Rename file_type method to node_type --- conda_build/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index a1a3991d13..2aabf8aa30 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -523,7 +523,7 @@ def get_inode_paths(files, target_short_path, prefix): return sorted(hardlinked_files) -def file_type(path): +def node_type(path): if islink(path): return NodeType.softlink return NodeType.hardlink @@ -539,7 +539,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): "path": get_short_path(m, fi), "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), - "node_type": file_type(path).name, + "node_type": node_type(path).name, } no_link = is_no_link(no_link, fi) if no_link: From 48f37c70233590982e29cd1edac75cc24cf08762 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 15:01:31 -0600 Subject: [PATCH 07/28] Import NodeType and FileMode from conda --- conda_build/conda_interface.py | 75 ++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index c6dec1f17a..a0889d0672 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -138,39 +138,42 @@ def which_prefix(path): return None prefix = dirname(prefix) - -class NodeType(Enum): - """ - Refers to if the file in question is hard linked or soft linked. Originally designed to be used - in files.json - """ - hardlink = 1 - softlink = 2 - - @classmethod - def __call__(cls, value, *args, **kwargs): - if isinstance(cls, value, *args, **kwargs): - return cls[value] - return super(NodeType, cls).__call__(value, *args, **kwargs) - - @classmethod - def __getitem__(cls, name): - return cls._member_map_[name.replace('-', '').replace('_', '').lower()] - - def __int__(self): - return self.value - - def __str__(self): - return self.name - - -class FileMode(Enum): - """ - Refers to the mode of the file. Originally referring to the has_prefix file, but adopted for - files.json - """ - text = 'text' - binary = 'binary' - - def __str__(self): - return "%s" % self.value +if parse_version(conda.__version__) >= parse_version("4.3"): + from conda.exports import FileMode, NodeType + FileMode, NodeType = FileMode, NodeType +else: + class NodeType(Enum): + """ + Refers to if the file in question is hard linked or soft linked. Originally designed to be used + in files.json + """ + hardlink = 1 + softlink = 2 + + @classmethod + def __call__(cls, value, *args, **kwargs): + if isinstance(cls, value, *args, **kwargs): + return cls[value] + return super(NodeType, cls).__call__(value, *args, **kwargs) + + @classmethod + def __getitem__(cls, name): + return cls._member_map_[name.replace('-', '').replace('_', '').lower()] + + def __int__(self): + return self.value + + def __str__(self): + return self.name + + + class FileMode(Enum): + """ + Refers to the mode of the file. Originally referring to the has_prefix file, but adopted for + files.json + """ + text = 'text' + binary = 'binary' + + def __str__(self): + return "%s" % self.value From 59f3b78be5356a2c65d4461585a87ffb7f6ba53f Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 15:09:46 -0600 Subject: [PATCH 08/28] JSON encode node_type --- conda_build/build.py | 2 +- conda_build/conda_interface.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index 2aabf8aa30..b0b88bed0c 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -539,7 +539,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): "path": get_short_path(m, fi), "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), - "node_type": node_type(path).name, + "node_type": node_type(path), } no_link = is_no_link(no_link, fi) if no_link: diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index a0889d0672..892c320c42 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -141,7 +141,11 @@ def which_prefix(path): if parse_version(conda.__version__) >= parse_version("4.3"): from conda.exports import FileMode, NodeType FileMode, NodeType = FileMode, NodeType + from conda.export import EntityEncoder + EntityEncoder = EntityEncoder else: + from json import JSONEncoder + class NodeType(Enum): """ Refers to if the file in question is hard linked or soft linked. Originally designed to be used @@ -166,6 +170,9 @@ def __int__(self): def __str__(self): return self.name + def __json__(self): + return self.name + class FileMode(Enum): """ @@ -177,3 +184,17 @@ class FileMode(Enum): def __str__(self): return "%s" % self.value + + + class EntityEncoder(JSONEncoder): + # json.dumps(obj, cls=SetEncoder) + def default(self, obj): + if hasattr(obj, 'dump'): + return obj.dump() + elif hasattr(obj, '__json__'): + return obj.__json__() + elif hasattr(obj, 'to_json'): + return obj.to_json() + elif hasattr(obj, 'as_json'): + return obj.as_json() + return JSONEncoder.default(self, obj) From 341c5803107320a51b1f51820a7c6f7fd143d37c Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 15:22:32 -0600 Subject: [PATCH 09/28] Include CrossPlatformStLink --- conda_build/conda_interface.py | 97 +++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 892c320c42..e25ed75822 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -57,7 +57,6 @@ LinkError = conda.exceptions.LinkError NoPackagesFoundError = conda.exceptions.NoPackagesFoundError CondaValueError = conda.exceptions.CondaValueError - from conda.common.compat import CrossPlatformStLink # disallow softlinks. This avoids a lot of dumb issues, at the potential cost of disk space. conda.base.context.context.allow_softlinks = False @@ -143,8 +142,12 @@ def which_prefix(path): FileMode, NodeType = FileMode, NodeType from conda.export import EntityEncoder EntityEncoder = EntityEncoder + from conda.export import CrossPlatformStLink + CrossPlatformStLink = CrossPlatformStLink else: from json import JSONEncoder + from os import lstat + import os class NodeType(Enum): """ @@ -198,3 +201,95 @@ def default(self, obj): elif hasattr(obj, 'as_json'): return obj.as_json() return JSONEncoder.default(self, obj) + + + # work-around for python bug on Windows prior to python 3.2 + # https://bugs.python.org/issue10027 + # Adapted from the ntfsutils package, Copyright (c) 2012, the Mozilla Foundation + class CrossPlatformStLink(object): + _st_nlink = None + + def __call__(self, path): + return self.st_nlink(path) + + @classmethod + def st_nlink(cls, path): + if cls._st_nlink is None: + cls._initialize() + return cls._st_nlink(path) + + @classmethod + def _standard_st_nlink(cls, path): + return lstat(path).st_nlink + + @classmethod + def _windows_st_nlink(cls, path): + st_nlink = cls._standard_st_nlink(path) + if st_nlink != 0: + return st_nlink + else: + # cannot trust python on Windows when st_nlink == 0 + # get value using windows libraries to be sure of its true value + # Adapted from the ntfsutils package, Copyright (c) 2012, the Mozilla Foundation + GENERIC_READ = 0x80000000 + FILE_SHARE_READ = 0x00000001 + OPEN_EXISTING = 3 + hfile = cls.CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, + OPEN_EXISTING, 0, None) + if hfile is None: + from ctypes import WinError + raise WinError() + info = cls.BY_HANDLE_FILE_INFORMATION() + rv = cls.GetFileInformationByHandle(hfile, info) + cls.CloseHandle(hfile) + if rv == 0: + from ctypes import WinError + raise WinError() + return info.nNumberOfLinks + + @classmethod + def _initialize(cls): + if os.name != 'nt': + cls._st_nlink = cls._standard_st_nlink + else: + # http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858 + import ctypes + from ctypes import POINTER + from ctypes.wintypes import DWORD, HANDLE, BOOL + + cls.CreateFile = ctypes.windll.kernel32.CreateFileW + cls.CreateFile.argtypes = [ctypes.c_wchar_p, DWORD, DWORD, ctypes.c_void_p, + DWORD, DWORD, HANDLE] + cls.CreateFile.restype = HANDLE + + # http://msdn.microsoft.com/en-us/library/windows/desktop/ms724211 + cls.CloseHandle = ctypes.windll.kernel32.CloseHandle + cls.CloseHandle.argtypes = [HANDLE] + cls.CloseHandle.restype = BOOL + + class FILETIME(ctypes.Structure): + _fields_ = [("dwLowDateTime", DWORD), + ("dwHighDateTime", DWORD)] + + class BY_HANDLE_FILE_INFORMATION(ctypes.Structure): + _fields_ = [("dwFileAttributes", DWORD), + ("ftCreationTime", FILETIME), + ("ftLastAccessTime", FILETIME), + ("ftLastWriteTime", FILETIME), + ("dwVolumeSerialNumber", DWORD), + ("nFileSizeHigh", DWORD), + ("nFileSizeLow", DWORD), + ("nNumberOfLinks", DWORD), + ("nFileIndexHigh", DWORD), + ("nFileIndexLow", DWORD)] + + cls.BY_HANDLE_FILE_INFORMATION = BY_HANDLE_FILE_INFORMATION + + # http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952 + cls.GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle + cls.GetFileInformationByHandle.argtypes = [HANDLE, + POINTER(BY_HANDLE_FILE_INFORMATION)] + cls.GetFileInformationByHandle.restype = BOOL + + cls._st_nlink = cls._windows_st_nlink + From 511a5e712dc768ebe2d93b773ea82a456fe3478c Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 16:39:47 -0600 Subject: [PATCH 10/28] Json dump with EntityEncoder --- conda_build/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index b0b88bed0c..4d7ff78faa 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -47,6 +47,7 @@ NoPackagesFoundError, NoPackagesFound) from .conda_interface import CrossPlatformStLink from .conda_interface import NodeType, FileMode +from .conda_interface import EntityEncoder from conda_build import __version__ from conda_build import environ, source, tarcheck @@ -565,7 +566,8 @@ def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix): "files": files_json_files, } with open(join(info_dir, 'files.json'), "w") as files_json: - json.dump(files_json_info, files_json, sort_keys=True, indent=2, separators=(',', ': ')) + json.dump(files_json_info, files_json, sort_keys=True, indent=2, separators=(',', ': '), + cls=EntityEncoder) def get_build_index(config, clear_cache=True): From 2dad147df9fdc21f87cf8a9536e3de6c7d143026 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 16 Nov 2016 09:52:18 -0600 Subject: [PATCH 11/28] Fix tests --- conda_build/build.py | 2 +- tests/test_api_build.py | 2 +- tests/test_build.py | 50 +++++++++++++++++++++-------------------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 4d7ff78faa..c9b03e4db4 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -557,7 +557,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix): - files_json_fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", + files_json_fields = ["path", "sha256", "size_in_bytes", "node_type", "file_mode", "prefix_placeholder", "no_link", "inode_paths"] files_json_files = build_info_files_json(m, prefix, files, files_with_prefix) files_json_info = { diff --git a/tests/test_api_build.py b/tests/test_api_build.py index b46ea6467f..2f5433085a 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -752,7 +752,7 @@ def test_info_files_json(test_config): assert package_has_file(fn, "info/files.json") with tarfile.open(fn) as tf: data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8')) - fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link", + fields = ["path", "sha256", "size_in_bytes", "node_type", "file_mode", "no_link", "prefix_placeholder", "inode_paths"] for key in data.keys(): assert key in ['files', 'fields', 'version'] diff --git a/tests/test_build.py b/tests/test_build.py index 166a5dd87a..c570949853 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -14,6 +14,7 @@ from conda_build import build, api, __version__ from conda_build.metadata import MetaData from conda_build.utils import rm_rf, on_win +from conda_build.conda_interface import NodeType from .utils import (testing_workdir, test_config, test_metadata, metadata_dir, get_noarch_python_meta, put_bad_conda_on_path) @@ -215,20 +216,21 @@ def test_create_info_files_json(testing_workdir, test_metadata): files_with_prefix = [("prefix/path", "text", "foo")] files = ["one", "two", "foo"] - build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) + build.create_info_files_json_v1(test_metadata, info_dir, testing_workdir, files, files_with_prefix) files_json_path = os.path.join(info_dir, "files.json") expected_output = { - "files": [{"file_type": "hardlink", "path": "one", - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "size_in_bytes": 0}, - {"file_type": "hardlink", "path": "two", "size_in_bytes": 0, - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, - {"file_mode": "text", "file_type": "hardlink", - "path": "foo", "prefix_placeholder": "prefix/path", - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "size_in_bytes": 0}], - "fields": ["path", "sha256", "size_in_bytes", "file_type", "file_mode", - "prefix_placeholder", "no_link", "inode_first_path"], + "files": [{"file_mode": "text", "node_type": "hardlink", "path": "foo", + "prefix_placeholder": "prefix/path", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"node_type": "hardlink", "path": "one", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"node_type": "hardlink", "path": "two", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}], + "fields": ["path", "sha256", "size_in_bytes", "node_type", "file_mode", + "prefix_placeholder", "no_link", "inode_paths"], "version": 1} with open(files_json_path, "r") as files_json: output = json.load(files_json) @@ -251,25 +253,25 @@ def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): files_with_prefix = [("prefix/path", "text", "foo")] files = ["one", "two", "one_hl", "foo"] - build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) + build.create_info_files_json_v1(test_metadata, info_dir, testing_workdir, files, files_with_prefix) files_json_path = os.path.join(info_dir, "files.json") expected_output = { - "files": [{"inode_paths": ["one", "one_hl"], "file_type": "hardlink", "path": "one", + "files": [{"file_mode": "text", "node_type": "hardlink", "path": "foo", + "prefix_placeholder": "prefix/path", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_type": "hardlink", "path": "two", "size_in_bytes": 0, - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, - {"inode_paths": ["one", "one_hl"], "file_type": "hardlink", - "path": "one_hl", + {"node_type": "hardlink", "path": "one", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"file_mode": "text", "file_type": "hardlink", "path": "foo", - "prefix_placeholder": "prefix/path", + {"node_type": "hardlink", "path": "one_hl", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"node_type": "hardlink", "path": "two", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], - "fields": ["path", "sha256", "size_in_bytes", "file_type", "file_mode", - "prefix_placeholder", "no_link", "inode_first_path"], - + "fields": ["path", "sha256", "size_in_bytes", "node_type", "file_mode", + "prefix_placeholder", "no_link", "inode_paths"], "version": 1} with open(files_json_path, "r") as files_json: - assert json.load(files_json) == expected_output + output = json.load(files_json) + assert output == expected_output From 2e5cabf4cea31900af3c3e915d492d76f2dd397f Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 16 Nov 2016 13:22:47 -0600 Subject: [PATCH 12/28] Fix tests --- conda_build/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index c9b03e4db4..4d897c4775 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -275,10 +275,10 @@ def get_files_with_prefix(m, files, prefix): ignore_types = set() if not hasattr(ignore_files, "__iter__"): if ignore_files is True: - ignore_types.update((FileMode.text, FileMode.binary)) + ignore_types.update((FileMode.text.name, FileMode.binary.name)) ignore_files = [] if not m.get_value('build/detect_binary_files_with_prefix', True): - ignore_types.update(('binary',)) + ignore_types.update((FileMode.binary.name,)) # files_with_prefix is a list of tuples containing (prefix_placeholder, file_mode) ignore_files.extend( f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files) From 6b463cd80d273c524aa4dbe464a58afe231178bc Mon Sep 17 00:00:00 2001 From: Kale Franz Date: Thu, 17 Nov 2016 16:46:39 -0600 Subject: [PATCH 13/28] accurate names; pk ids sorted to top --- conda_build/build.py | 25 +++++++++++------------ conda_build/conda_interface.py | 14 ++++++------- tests/test_api_build.py | 16 +++++++-------- tests/test_build.py | 37 +++++++++++++++------------------- 4 files changed, 43 insertions(+), 49 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 4d897c4775..dd5330dc6b 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -46,7 +46,7 @@ from .conda_interface import (PaddingError, LinkError, CondaValueError, NoPackagesFoundError, NoPackagesFound) from .conda_interface import CrossPlatformStLink -from .conda_interface import NodeType, FileMode +from .conda_interface import PathType, FileMode from .conda_interface import EntityEncoder from conda_build import __version__ @@ -526,18 +526,18 @@ def get_inode_paths(files, target_short_path, prefix): def node_type(path): if islink(path): - return NodeType.softlink - return NodeType.hardlink + return PathType.softlink + return PathType.hardlink -def build_info_files_json(m, prefix, files, files_with_prefix): +def build_info_files_json_v1(m, prefix, files, files_with_prefix): no_link = m.get_value('build/no_link') files_json = [] for fi in sorted(files): prefix_placeholder, file_mode = has_prefix(fi, files_with_prefix) path = os.path.join(prefix, fi) file_info = { - "path": get_short_path(m, fi), + "_path": get_short_path(m, fi), "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), "node_type": node_type(path), @@ -548,7 +548,7 @@ def build_info_files_json(m, prefix, files, files_with_prefix): if prefix_placeholder and file_mode: file_info["prefix_placeholder"] = prefix_placeholder file_info["file_mode"] = file_mode - if file_info.get("file_type") == NodeType.hardlink and CrossPlatformStLink.st_nlink( + if file_info.get("path_type") == PathType.hardlink and CrossPlatformStLink.st_nlink( join(prefix, fi)) > 1: inode_paths = get_inode_paths(files, fi, prefix) file_info["inode_paths"] = inode_paths @@ -557,15 +557,14 @@ def build_info_files_json(m, prefix, files, files_with_prefix): def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix): - files_json_fields = ["path", "sha256", "size_in_bytes", "node_type", "file_mode", - "prefix_placeholder", "no_link", "inode_paths"] - files_json_files = build_info_files_json(m, prefix, files, files_with_prefix) + # fields: "_path", "sha256", "size_in_bytes", "path_type", "file_mode", + # "prefix_placeholder", "no_link", "inode_paths" + files_json_files = build_info_files_json_v1(m, prefix, files, files_with_prefix) files_json_info = { - "version": 1, - "fields": files_json_fields, - "files": files_json_files, + "paths_version": 1, + "paths": files_json_files, } - with open(join(info_dir, 'files.json'), "w") as files_json: + with open(join(info_dir, 'paths.json'), "w") as files_json: json.dump(files_json_info, files_json, sort_keys=True, indent=2, separators=(',', ': '), cls=EntityEncoder) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index e25ed75822..c55b2c9938 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -139,7 +139,7 @@ def which_prefix(path): if parse_version(conda.__version__) >= parse_version("4.3"): from conda.exports import FileMode, NodeType - FileMode, NodeType = FileMode, NodeType + FileMode, PathType = FileMode, NodeType from conda.export import EntityEncoder EntityEncoder = EntityEncoder from conda.export import CrossPlatformStLink @@ -149,19 +149,19 @@ def which_prefix(path): from os import lstat import os - class NodeType(Enum): + class PathType(Enum): """ Refers to if the file in question is hard linked or soft linked. Originally designed to be used - in files.json + in paths.json """ - hardlink = 1 - softlink = 2 + hardlink = "hardlink" + softlink = "softlink" @classmethod def __call__(cls, value, *args, **kwargs): if isinstance(cls, value, *args, **kwargs): return cls[value] - return super(NodeType, cls).__call__(value, *args, **kwargs) + return super(PathType, cls).__call__(value, *args, **kwargs) @classmethod def __getitem__(cls, name): @@ -180,7 +180,7 @@ def __json__(self): class FileMode(Enum): """ Refers to the mode of the file. Originally referring to the has_prefix file, but adopted for - files.json + paths.json """ text = 'text' binary = 'binary' diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 2f5433085a..d6c8da23ad 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -749,20 +749,20 @@ def test_info_files_json(test_config): recipe = os.path.join(metadata_dir, "ignore_some_prefix_files") fn = api.get_output_file_path(recipe, config=test_config) api.build(recipe, config=test_config) - assert package_has_file(fn, "info/files.json") + assert package_has_file(fn, "info/paths.json") with tarfile.open(fn) as tf: - data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8')) - fields = ["path", "sha256", "size_in_bytes", "node_type", "file_mode", "no_link", + data = json.loads(tf.extractfile('info/paths.json').read().decode('utf-8')) + fields = ["_path", "sha256", "size_in_bytes", "path_type", "file_mode", "no_link", "prefix_placeholder", "inode_paths"] for key in data.keys(): - assert key in ['files', 'fields', 'version'] - for field in data.get('fields'): + assert key in ['paths', 'paths_version'] + for field in data.get('paths'): assert field in fields - assert len(data.get('files')) == 2 - for file in data.get('files'): + assert len(data.get('paths')) == 2 + for file in data.get('paths'): for key in file.keys(): assert key in fields - short_path = file.get("path") + short_path = file.get("_path") if short_path == "test.sh" or short_path == "test.bat": assert file.get("prefix_placeholder") is not None assert file.get("file_mode") is not None diff --git a/tests/test_build.py b/tests/test_build.py index c570949853..da511455ba 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -14,7 +14,6 @@ from conda_build import build, api, __version__ from conda_build.metadata import MetaData from conda_build.utils import rm_rf, on_win -from conda_build.conda_interface import NodeType from .utils import (testing_workdir, test_config, test_metadata, metadata_dir, get_noarch_python_meta, put_bad_conda_on_path) @@ -217,20 +216,18 @@ def test_create_info_files_json(testing_workdir, test_metadata): files = ["one", "two", "foo"] build.create_info_files_json_v1(test_metadata, info_dir, testing_workdir, files, files_with_prefix) - files_json_path = os.path.join(info_dir, "files.json") + files_json_path = os.path.join(info_dir, "paths.json") expected_output = { - "files": [{"file_mode": "text", "node_type": "hardlink", "path": "foo", - "prefix_placeholder": "prefix/path", - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "size_in_bytes": 0}, - {"node_type": "hardlink", "path": "one", - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "size_in_bytes": 0}, - {"node_type": "hardlink", "path": "two", - "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "size_in_bytes": 0}], - "fields": ["path", "sha256", "size_in_bytes", "node_type", "file_mode", - "prefix_placeholder", "no_link", "inode_paths"], + "paths": [{"file_mode": "text", "path_type": "hardlink", "_path": "foo", + "prefix_placeholder": "prefix/path", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"path_type": "hardlink", "_path": "one", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"path_type": "hardlink", "_path": "two", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}], "version": 1} with open(files_json_path, "r") as files_json: output = json.load(files_json) @@ -254,23 +251,21 @@ def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): files = ["one", "two", "one_hl", "foo"] build.create_info_files_json_v1(test_metadata, info_dir, testing_workdir, files, files_with_prefix) - files_json_path = os.path.join(info_dir, "files.json") + files_json_path = os.path.join(info_dir, "paths.json") expected_output = { - "files": [{"file_mode": "text", "node_type": "hardlink", "path": "foo", + "paths": [{"file_mode": "text", "path_type": "hardlink", "_path": "foo", "prefix_placeholder": "prefix/path", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"node_type": "hardlink", "path": "one", + {"path_type": "hardlink", "_path": "one", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"node_type": "hardlink", "path": "one_hl", + {"path_type": "hardlink", "_path": "one_hl", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"node_type": "hardlink", "path": "two", + {"path_type": "hardlink", "_path": "two", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], - "fields": ["path", "sha256", "size_in_bytes", "node_type", "file_mode", - "prefix_placeholder", "no_link", "inode_paths"], "version": 1} with open(files_json_path, "r") as files_json: output = json.load(files_json) From 7442b6163e7c48cfda5ef3073308fd0cf2383328 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 10:35:42 -0600 Subject: [PATCH 14/28] Some edits to files.json --- conda_build/conda_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index c55b2c9938..33697fd1be 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -57,6 +57,7 @@ LinkError = conda.exceptions.LinkError NoPackagesFoundError = conda.exceptions.NoPackagesFoundError CondaValueError = conda.exceptions.CondaValueError + from conda.common.compat import CrossPlatformStLink # disallow softlinks. This avoids a lot of dumb issues, at the potential cost of disk space. conda.base.context.context.allow_softlinks = False From c4f473cfc6093796a80b431b5ccf9f77f5a8afec Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 11:51:30 -0600 Subject: [PATCH 15/28] Create NodeType and FileMode Enums --- conda_build/build.py | 2 -- conda_build/conda_interface.py | 1 - 2 files changed, 3 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index dd5330dc6b..65ff058c45 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -5,7 +5,6 @@ from collections import deque import copy -from enum import Enum import fnmatch from glob import glob import io @@ -28,7 +27,6 @@ # http://stackoverflow.com/a/13057751/1170370 import encodings.idna # NOQA -from conda_verify.verify import Verify # used to get version from .conda_interface import cc diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 33697fd1be..1cc8cc0ed2 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -293,4 +293,3 @@ class BY_HANDLE_FILE_INFORMATION(ctypes.Structure): cls.GetFileInformationByHandle.restype = BOOL cls._st_nlink = cls._windows_st_nlink - From 8b3140350bf340fdf861e28d3d257a993f44ac93 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 12:42:58 -0600 Subject: [PATCH 16/28] Add docstring for additional enums --- conda_build/conda_interface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 1cc8cc0ed2..89d12803d4 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -285,11 +285,9 @@ class BY_HANDLE_FILE_INFORMATION(ctypes.Structure): ("nFileIndexLow", DWORD)] cls.BY_HANDLE_FILE_INFORMATION = BY_HANDLE_FILE_INFORMATION - # http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952 cls.GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle cls.GetFileInformationByHandle.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)] cls.GetFileInformationByHandle.restype = BOOL - cls._st_nlink = cls._windows_st_nlink From 5876d8bb1b6cf3a98af14ea66a233501ffe2692d Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 15:01:31 -0600 Subject: [PATCH 17/28] Import NodeType and FileMode from conda --- conda_build/conda_interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 89d12803d4..e5f3c00dbd 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -177,7 +177,6 @@ def __str__(self): def __json__(self): return self.name - class FileMode(Enum): """ Refers to the mode of the file. Originally referring to the has_prefix file, but adopted for From 5125e15fa59e7858928c4c6902813976fb768cbe Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 15:09:46 -0600 Subject: [PATCH 18/28] JSON encode node_type --- conda_build/conda_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index e5f3c00dbd..89d12803d4 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -177,6 +177,7 @@ def __str__(self): def __json__(self): return self.name + class FileMode(Enum): """ Refers to the mode of the file. Originally referring to the has_prefix file, but adopted for From 44c83425ae25bff94024986964b4cf0e51a544e4 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 15 Nov 2016 15:22:32 -0600 Subject: [PATCH 19/28] Include CrossPlatformStLink --- conda_build/conda_interface.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 89d12803d4..80c138853c 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -57,7 +57,6 @@ LinkError = conda.exceptions.LinkError NoPackagesFoundError = conda.exceptions.NoPackagesFoundError CondaValueError = conda.exceptions.CondaValueError - from conda.common.compat import CrossPlatformStLink # disallow softlinks. This avoids a lot of dumb issues, at the potential cost of disk space. conda.base.context.context.allow_softlinks = False @@ -285,9 +284,12 @@ class BY_HANDLE_FILE_INFORMATION(ctypes.Structure): ("nFileIndexLow", DWORD)] cls.BY_HANDLE_FILE_INFORMATION = BY_HANDLE_FILE_INFORMATION + + # http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952 cls.GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle cls.GetFileInformationByHandle.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)] cls.GetFileInformationByHandle.restype = BOOL + cls._st_nlink = cls._windows_st_nlink From 17ac64a0ade91205e8cb8f6585e292b735bc49e2 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 16 Nov 2016 09:52:18 -0600 Subject: [PATCH 20/28] Fix tests --- tests/test_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_build.py b/tests/test_build.py index da511455ba..4deb86a273 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -14,6 +14,7 @@ from conda_build import build, api, __version__ from conda_build.metadata import MetaData from conda_build.utils import rm_rf, on_win +from conda_build.conda_interface import NodeType from .utils import (testing_workdir, test_config, test_metadata, metadata_dir, get_noarch_python_meta, put_bad_conda_on_path) From dbc044f5883bfc42229481fda3f8dd78288a7cdb Mon Sep 17 00:00:00 2001 From: sophia Date: Mon, 28 Nov 2016 13:53:12 -0600 Subject: [PATCH 21/28] Fix tests --- conda_build/build.py | 4 ++-- conda_build/conda_interface.py | 4 ++-- tests/test_api_build.py | 6 ++++-- tests/test_build.py | 9 ++++----- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 65ff058c45..caac50e24e 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -522,7 +522,7 @@ def get_inode_paths(files, target_short_path, prefix): return sorted(hardlinked_files) -def node_type(path): +def path_type(path): if islink(path): return PathType.softlink return PathType.hardlink @@ -538,7 +538,7 @@ def build_info_files_json_v1(m, prefix, files, files_with_prefix): "_path": get_short_path(m, fi), "sha256": sha256_checksum(path), "size_in_bytes": os.path.getsize(path), - "node_type": node_type(path), + "path_type": path_type(path), } no_link = is_no_link(no_link, fi) if no_link: diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 80c138853c..50f22cd698 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -179,8 +179,8 @@ def __json__(self): class FileMode(Enum): """ - Refers to the mode of the file. Originally referring to the has_prefix file, but adopted for - paths.json + Refers to the mode of the file. Originally referring to the has_prefix file, but adopted + for paths.json """ text = 'text' binary = 'binary' diff --git a/tests/test_api_build.py b/tests/test_api_build.py index d6c8da23ad..75f6ce98b8 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -754,10 +754,12 @@ def test_info_files_json(test_config): data = json.loads(tf.extractfile('info/paths.json').read().decode('utf-8')) fields = ["_path", "sha256", "size_in_bytes", "path_type", "file_mode", "no_link", "prefix_placeholder", "inode_paths"] + for key in data.keys(): assert key in ['paths', 'paths_version'] - for field in data.get('paths'): - assert field in fields + for paths in data.get('paths'): + for field in paths.keys(): + assert field in fields assert len(data.get('paths')) == 2 for file in data.get('paths'): for key in file.keys(): diff --git a/tests/test_build.py b/tests/test_build.py index 4deb86a273..6d4e379ed1 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -14,7 +14,6 @@ from conda_build import build, api, __version__ from conda_build.metadata import MetaData from conda_build.utils import rm_rf, on_win -from conda_build.conda_interface import NodeType from .utils import (testing_workdir, test_config, test_metadata, metadata_dir, get_noarch_python_meta, put_bad_conda_on_path) @@ -229,7 +228,7 @@ def test_create_info_files_json(testing_workdir, test_metadata): {"path_type": "hardlink", "_path": "two", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], - "version": 1} + "paths_version": 1} with open(files_json_path, "r") as files_json: output = json.load(files_json) assert output == expected_output @@ -258,16 +257,16 @@ def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): "prefix_placeholder": "prefix/path", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"path_type": "hardlink", "_path": "one", + {"path_type": "hardlink", "_path": "one", "inode_paths": ["one", "one_hl"], "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, - {"path_type": "hardlink", "_path": "one_hl", + {"path_type": "hardlink", "_path": "one_hl", "inode_paths": ["one", "one_hl"], "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}, {"path_type": "hardlink", "_path": "two", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size_in_bytes": 0}], - "version": 1} + "paths_version": 1} with open(files_json_path, "r") as files_json: output = json.load(files_json) assert output == expected_output From ab51199874160a2d6a210e5e619e579ee1fd6bc9 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 15 Dec 2016 12:55:13 -0600 Subject: [PATCH 22/28] Add test for conda master to travis --- .travis.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e19c3fc021..8032476679 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,9 @@ matrix: os: linux - python: '3.5' os: linux - # - python: '3.5' - # env: CANARY=true - # os: linux +# - python: '3.5' +# env: CANARY=true +# os: linux - python: '3.5' env: CONDA=4.1 os: linux @@ -15,6 +15,9 @@ matrix: env: - FLAKE8=true os: linux + - python: '3.5' + env: CONDA_MASTER=true + os: linux install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then @@ -35,6 +38,15 @@ install: if [ -n "$CONDA" ]; then conda install -y --no-deps conda=${CONDA}; fi + + if [[ "$CONDA_MASTER" == "true" ]]; then + git clone https://github.com/conda/conda.git; + pushd conda; + python setup.py install; + hash -r; + conda info; + popd; + fi fi - conda install -q anaconda-client requests=2.11.1 filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 conda-verify From 4af5fa3a944cb792105a98dfae377e4b5c531af5 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 15 Dec 2016 14:38:29 -0600 Subject: [PATCH 23/28] Better messages on win errors --- conda_build/conda_interface.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 50f22cd698..43c81c6e5b 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -157,21 +157,8 @@ class PathType(Enum): hardlink = "hardlink" softlink = "softlink" - @classmethod - def __call__(cls, value, *args, **kwargs): - if isinstance(cls, value, *args, **kwargs): - return cls[value] - return super(PathType, cls).__call__(value, *args, **kwargs) - - @classmethod - def __getitem__(cls, name): - return cls._member_map_[name.replace('-', '').replace('_', '').lower()] - - def __int__(self): - return self.value - def __str__(self): - return self.name + return self.value def __json__(self): return self.name @@ -238,13 +225,14 @@ def _windows_st_nlink(cls, path): OPEN_EXISTING, 0, None) if hfile is None: from ctypes import WinError - raise WinError() + raise WinError( + "Could not determine determine number of hardlinks for %s" % path) info = cls.BY_HANDLE_FILE_INFORMATION() rv = cls.GetFileInformationByHandle(hfile, info) cls.CloseHandle(hfile) if rv == 0: from ctypes import WinError - raise WinError() + raise WinError("Could not determine file information for %s" % path) return info.nNumberOfLinks @classmethod From de6b424135ba716bda897645a0993211fc468aac Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 15 Dec 2016 14:39:20 -0600 Subject: [PATCH 24/28] Test conda interface enums/classes --- tests/test_misc.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index 4291d3584d..b954185ab9 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -1,7 +1,10 @@ import unittest +import json +from os import link +from os.path import join import conda_build._link as _link - +from conda_build.conda_interface import PathType, EntityEncoder, CrossPlatformStLink class TestLink(unittest.TestCase): @@ -17,3 +20,38 @@ def test_pyc_f_3(self): 'sp/foo/__pycache__/utils.cpython-34.pyc'), ]: self.assertEqual(_link.pyc_f(f, (3, 4, 2)), r) + + +def test_pathtype(): + hardlink = PathType("hardlink") + assert str(hardlink) == "hardlink" + assert hardlink.__json__() == 'hardlink' + + softlink = PathType("softlink") + assert str(softlink) == "softlink" + assert softlink.__json__() == "softlink" + + +def test_entity_encoder(tmpdir): + test_file = join(str(tmpdir), "test-file") + test_json = {"a": PathType("hardlink"), "b": 1} + with open(test_file, "w") as f: + json.dump(test_json, f, cls=EntityEncoder) + + with open(test_file, "r") as f: + json_file = json.load(f) + assert json_file == {"a": "hardlink", "b": 1} + + +def test_crossplatform_st_link(tmpdir): + test_file = join(str(tmpdir), "test-file") + test_file_linked = join(str(tmpdir), "test-file-linked") + test_file_link = join(str(tmpdir), "test-file-link") + + open(test_file, "a").close() + open(test_file_link, "a").close() + link(test_file_link, test_file_linked) + assert 1 == CrossPlatformStLink.st_nlink(test_file) + assert 2 == CrossPlatformStLink.st_nlink(test_file_link) + assert 2 == CrossPlatformStLink.st_nlink(test_file_linked) + From b7d7ba3773796635bafe3a65277e4ab91fa7aa21 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 15 Dec 2016 14:41:48 -0600 Subject: [PATCH 25/28] Update travis to test against conda canary --- .travis.yml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8032476679..1f3761d38f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,9 @@ matrix: os: linux - python: '3.5' os: linux -# - python: '3.5' -# env: CANARY=true -# os: linux + - python: '3.5' + env: CANARY=true + os: linux - python: '3.5' env: CONDA=4.1 os: linux @@ -15,9 +15,6 @@ matrix: env: - FLAKE8=true os: linux - - python: '3.5' - env: CONDA_MASTER=true - os: linux install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then @@ -38,17 +35,7 @@ install: if [ -n "$CONDA" ]; then conda install -y --no-deps conda=${CONDA}; fi - - if [[ "$CONDA_MASTER" == "true" ]]; then - git clone https://github.com/conda/conda.git; - pushd conda; - python setup.py install; - hash -r; - conda info; - popd; - fi fi - - conda install -q anaconda-client requests=2.11.1 filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 conda-verify - pip install pkginfo - if [[ "$FLAKE8" == "true" ]]; then From e67d419fcfdf5cead901a8fe53945204a3e6c72f Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 15 Dec 2016 15:08:52 -0600 Subject: [PATCH 26/28] Import PathType for conda instead of NodeType --- conda_build/conda_interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 43c81c6e5b..bdc6111e2f 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -138,8 +138,8 @@ def which_prefix(path): prefix = dirname(prefix) if parse_version(conda.__version__) >= parse_version("4.3"): - from conda.exports import FileMode, NodeType - FileMode, PathType = FileMode, NodeType + from conda.exports import FileMode, PathType + FileMode, PathType = FileMode, PathType from conda.export import EntityEncoder EntityEncoder = EntityEncoder from conda.export import CrossPlatformStLink From 47b9136ae138d4403be3f3d3949c56f0d2dd5b86 Mon Sep 17 00:00:00 2001 From: sophia Date: Thu, 15 Dec 2016 15:15:23 -0600 Subject: [PATCH 27/28] Add test for st_nlink on win --- .travis.yml | 1 + tests/test_misc.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1f3761d38f..f2cb739e69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,7 @@ install: conda install -y --no-deps conda=${CONDA}; fi fi + - conda install -q anaconda-client requests=2.11.1 filelock contextlib2 jinja2 patchelf python=$TRAVIS_PYTHON_VERSION pyflakes=1.1 conda-verify - pip install pkginfo - if [[ "$FLAKE8" == "true" ]]; then diff --git a/tests/test_misc.py b/tests/test_misc.py index b954185ab9..547b640e6e 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -3,9 +3,12 @@ from os import link from os.path import join +import pytest +from conda_build.utils import on_win import conda_build._link as _link from conda_build.conda_interface import PathType, EntityEncoder, CrossPlatformStLink + class TestLink(unittest.TestCase): def test_pyc_f_2(self): @@ -43,6 +46,7 @@ def test_entity_encoder(tmpdir): assert json_file == {"a": "hardlink", "b": 1} +@pytest.mark.skipif(on_win, reason="link not available on win/py2.7") def test_crossplatform_st_link(tmpdir): test_file = join(str(tmpdir), "test-file") test_file_linked = join(str(tmpdir), "test-file-linked") @@ -55,3 +59,9 @@ def test_crossplatform_st_link(tmpdir): assert 2 == CrossPlatformStLink.st_nlink(test_file_link) assert 2 == CrossPlatformStLink.st_nlink(test_file_linked) + +def test_crossplatform_st_link_on_win(tmpdir): + test_file = join(str(tmpdir), "test-file") + open(test_file, "a").close() + assert 1 == CrossPlatformStLink.st_nlink(test_file) + From e9885a120caf33bca76aa80325bdf3a938c8f75a Mon Sep 17 00:00:00 2001 From: sophia Date: Fri, 16 Dec 2016 10:30:27 -0600 Subject: [PATCH 28/28] Skip crossplatform_st_link_on_win if not on win --- tests/test_misc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index 547b640e6e..6d05b3332d 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -1,6 +1,5 @@ import unittest import json -from os import link from os.path import join import pytest @@ -48,6 +47,7 @@ def test_entity_encoder(tmpdir): @pytest.mark.skipif(on_win, reason="link not available on win/py2.7") def test_crossplatform_st_link(tmpdir): + from os import link test_file = join(str(tmpdir), "test-file") test_file_linked = join(str(tmpdir), "test-file-linked") test_file_link = join(str(tmpdir), "test-file-link") @@ -60,6 +60,7 @@ def test_crossplatform_st_link(tmpdir): assert 2 == CrossPlatformStLink.st_nlink(test_file_linked) +@pytest.mark.skipif(not on_win, reason="already tested") def test_crossplatform_st_link_on_win(tmpdir): test_file = join(str(tmpdir), "test-file") open(test_file, "a").close()