Merge pull request #1535 from soapy1/files-json-review

finalizing paths.json spec

.travis.yml

@@ -5,9 +5,9 @@ matrix:
       os: linux
     - python: '3.5'
       os: linux
-    # - python: '3.5'
-    #   env: CANARY=true
-    #   os: linux
+    - python: '3.5'
+      env: CANARY=true
+      os: linux
     - python: '3.5'
       env: CONDA=4.1
       os: linux

conda_build/build.py

@@ -5,7 +5,6 @@
 
 from collections import deque
 import copy
-from enum import Enum
 import fnmatch
 from glob import glob
 import io
@@ -28,7 +27,6 @@
 #    http://stackoverflow.com/a/13057751/1170370
 import encodings.idna  # NOQA
 
-from conda_verify.verify import Verify
 
 # used to get version
 from .conda_interface import cc
@@ -45,6 +43,9 @@
 from .conda_interface import VersionOrder
 from .conda_interface import (PaddingError, LinkError, CondaValueError,
                               NoPackagesFoundError, NoPackagesFound)
+from .conda_interface import CrossPlatformStLink
+from .conda_interface import PathType, FileMode
+from .conda_interface import EntityEncoder
 
 from conda_build import __version__
 from conda_build import environ, source, tarcheck
@@ -65,12 +66,7 @@
 from conda_build.features import feature_list
 
 import conda_build.noarch_python as noarch_python
-
-
-class FileType(Enum):
-    softlink = "softlink"
-    hardlink = "hardlink"
-    directory = "directory"
+from conda_verify.verify import Verify
 
 
 if 'bsd' in sys.platform:
@@ -283,12 +279,13 @@ def get_files_with_prefix(m, files, prefix):
     ignore_types = set()
     if not hasattr(ignore_files, "__iter__"):
         if ignore_files is True:
-            ignore_types.update(('text', 'binary'))
+            ignore_types.update((FileMode.text.name,  FileMode.binary.name))
         ignore_files = []
     if not m.get_value('build/detect_binary_files_with_prefix', True):
-        ignore_types.update(('binary',))
+        ignore_types.update((FileMode.binary.name,))
+    # files_with_prefix is a list of tuples containing (prefix_placeholder, file_mode)
     ignore_files.extend(
-        [f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files])
+        f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files)
     files_with_prefix = [f for f in files_with_prefix if f[2] not in ignore_files]
     return files_with_prefix
 
@@ -473,7 +470,7 @@ def create_info_files(m, files, config, prefix):
                 fo.write(f + '\n')
 
     files_with_prefix = get_files_with_prefix(m, files, prefix)
-    create_info_files_json(m, config.info_dir, prefix, files, files_with_prefix)
+    create_info_files_json_v1(m, config.info_dir, prefix, files, files_with_prefix)
 
     detect_and_record_prefix_files(m, files, prefix, config)
     write_no_link(m, config, files)
@@ -528,61 +525,55 @@ def is_no_link(no_link, short_path):
 
 
 def get_inode_paths(files, target_short_path, prefix):
-    ensure_list(files)
-    target_short_path_inode = os.stat(join(prefix, target_short_path)).st_ino
+    target_short_path_inode = os.lstat(join(prefix, target_short_path)).st_ino
     hardlinked_files = [sp for sp in files
-                        if os.stat(join(prefix, sp)).st_ino == target_short_path_inode]
+                        if os.lstat(join(prefix, sp)).st_ino == target_short_path_inode]
     return sorted(hardlinked_files)
 
 
-def file_type(path):
-    if isdir(path):
-        return FileType.directory
-    elif islink(path):
-        return FileType.softlink
-    return FileType.hardlink
+def path_type(path):
+    if islink(path):
+        return PathType.softlink
+    return PathType.hardlink
 
 
-def build_info_files_json(m, prefix, files, files_with_prefix):
+def build_info_files_json_v1(m, prefix, files, files_with_prefix):
     no_link = m.get_value('build/no_link')
     files_json = []
-    for fi in files:
+    for fi in sorted(files):
         prefix_placeholder, file_mode = has_prefix(fi, files_with_prefix)
         path = os.path.join(prefix, fi)
         file_info = {
-            "path": get_short_path(m, fi),
+            "_path": get_short_path(m, fi),
             "sha256": sha256_checksum(path),
             "size_in_bytes": os.path.getsize(path),
-            "file_type": getattr(file_type(path), "name"),
+            "path_type": path_type(path),
         }
         no_link = is_no_link(no_link, fi)
         if no_link:
             file_info["no_link"] = no_link
         if prefix_placeholder and file_mode:
             file_info["prefix_placeholder"] = prefix_placeholder
             file_info["file_mode"] = file_mode
-        if file_info.get("file_type") == "hardlink" and os.stat(join(prefix, fi)).st_nlink > 1:
+        if file_info.get("path_type") == PathType.hardlink and CrossPlatformStLink.st_nlink(
+                join(prefix, fi)) > 1:
             inode_paths = get_inode_paths(files, fi, prefix)
             file_info["inode_paths"] = inode_paths
         files_json.append(file_info)
     return files_json
 
 
-def get_files_version():
-    return 1
-
-
-def create_info_files_json(m, info_dir, prefix, files, files_with_prefix):
-    files_json_fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode",
-                         "prefix_placeholder", "no_link", "inode_first_path"]
-    files_json_files = build_info_files_json(m, prefix, files, files_with_prefix)
+def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix):
+    # fields: "_path", "sha256", "size_in_bytes", "path_type", "file_mode",
+    #         "prefix_placeholder", "no_link", "inode_paths"
+    files_json_files = build_info_files_json_v1(m, prefix, files, files_with_prefix)
     files_json_info = {
-        "version": get_files_version(),
-        "fields": files_json_fields,
-        "files": files_json_files,
+        "paths_version": 1,
+        "paths": files_json_files,
     }
-    with open(join(info_dir, 'files.json'), "w") as files_json:
-        json.dump(files_json_info, files_json)
+    with open(join(info_dir, 'paths.json'), "w") as files_json:
+        json.dump(files_json_info, files_json, sort_keys=True, indent=2, separators=(',', ': '),
+                  cls=EntityEncoder)
 
 
 def get_build_index(config, clear_cache=True):

conda_build/conda_interface.py

@@ -24,6 +24,7 @@
 import conda.config as cc  # NOQA
 from conda.config import rc_path  # NOQA
 from conda.version import VersionOrder  # NOQA
+from enum import Enum
 
 import os
 
@@ -136,3 +137,148 @@ def which_prefix(path):
             # we cannot chop off any more directories, so we didn't find it
             return None
         prefix = dirname(prefix)
+
+if parse_version(conda.__version__) >= parse_version("4.3"):
+    from conda.exports import FileMode, PathType
+    FileMode, PathType = FileMode, PathType
+    from conda.export import EntityEncoder
+    EntityEncoder = EntityEncoder
+    from conda.export import CrossPlatformStLink
+    CrossPlatformStLink = CrossPlatformStLink
+else:
+    from json import JSONEncoder
+    from os import lstat
+    import os
+
+    class PathType(Enum):
+        """
+        Refers to if the file in question is hard linked or soft linked. Originally designed to be used
+        in paths.json
+        """
+        hardlink = "hardlink"
+        softlink = "softlink"
+
+        def __str__(self):
+            return self.value
+
+        def __json__(self):
+            return self.name
+
+
+    class FileMode(Enum):
+        """
+        Refers to the mode of the file. Originally referring to the has_prefix file, but adopted
+        for paths.json
+        """
+        text = 'text'
+        binary = 'binary'
+
+        def __str__(self):
+            return "%s" % self.value
+
+
+    class EntityEncoder(JSONEncoder):
+        # json.dumps(obj, cls=SetEncoder)
+        def default(self, obj):
+            if hasattr(obj, 'dump'):
+                return obj.dump()
+            elif hasattr(obj, '__json__'):
+                return obj.__json__()
+            elif hasattr(obj, 'to_json'):
+                return obj.to_json()
+            elif hasattr(obj, 'as_json'):
+                return obj.as_json()
+            return JSONEncoder.default(self, obj)
+
+
+    # work-around for python bug on Windows prior to python 3.2
+    # https://bugs.python.org/issue10027
+    # Adapted from the ntfsutils package, Copyright (c) 2012, the Mozilla Foundation
+    class CrossPlatformStLink(object):
+        _st_nlink = None
+
+        def __call__(self, path):
+            return self.st_nlink(path)
+
+        @classmethod
+        def st_nlink(cls, path):
+            if cls._st_nlink is None:
+                cls._initialize()
+            return cls._st_nlink(path)
+
+        @classmethod
+        def _standard_st_nlink(cls, path):
+            return lstat(path).st_nlink
+
+        @classmethod
+        def _windows_st_nlink(cls, path):
+            st_nlink = cls._standard_st_nlink(path)
+            if st_nlink != 0:
+                return st_nlink
+            else:
+                # cannot trust python on Windows when st_nlink == 0
+                # get value using windows libraries to be sure of its true value
+                # Adapted from the ntfsutils package, Copyright (c) 2012, the Mozilla Foundation
+                GENERIC_READ = 0x80000000
+                FILE_SHARE_READ = 0x00000001
+                OPEN_EXISTING = 3
+                hfile = cls.CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None,
+                                       OPEN_EXISTING, 0, None)
+                if hfile is None:
+                    from ctypes import WinError
+                    raise WinError(
+                        "Could not determine determine number of hardlinks for %s" % path)
+                info = cls.BY_HANDLE_FILE_INFORMATION()
+                rv = cls.GetFileInformationByHandle(hfile, info)
+                cls.CloseHandle(hfile)
+                if rv == 0:
+                    from ctypes import WinError
+                    raise WinError("Could not determine file information for %s" % path)
+                return info.nNumberOfLinks
+
+        @classmethod
+        def _initialize(cls):
+            if os.name != 'nt':
+                cls._st_nlink = cls._standard_st_nlink
+            else:
+                # http://msdn.microsoft.com/en-us/library/windows/desktop/aa363858
+                import ctypes
+                from ctypes import POINTER
+                from ctypes.wintypes import DWORD, HANDLE, BOOL
+
+                cls.CreateFile = ctypes.windll.kernel32.CreateFileW
+                cls.CreateFile.argtypes = [ctypes.c_wchar_p, DWORD, DWORD, ctypes.c_void_p,
+                                           DWORD, DWORD, HANDLE]
+                cls.CreateFile.restype = HANDLE
+
+                # http://msdn.microsoft.com/en-us/library/windows/desktop/ms724211
+                cls.CloseHandle = ctypes.windll.kernel32.CloseHandle
+                cls.CloseHandle.argtypes = [HANDLE]
+                cls.CloseHandle.restype = BOOL
+
+                class FILETIME(ctypes.Structure):
+                    _fields_ = [("dwLowDateTime", DWORD),
+                                ("dwHighDateTime", DWORD)]
+
+                class BY_HANDLE_FILE_INFORMATION(ctypes.Structure):
+                    _fields_ = [("dwFileAttributes", DWORD),
+                                ("ftCreationTime", FILETIME),
+                                ("ftLastAccessTime", FILETIME),
+                                ("ftLastWriteTime", FILETIME),
+                                ("dwVolumeSerialNumber", DWORD),
+                                ("nFileSizeHigh", DWORD),
+                                ("nFileSizeLow", DWORD),
+                                ("nNumberOfLinks", DWORD),
+                                ("nFileIndexHigh", DWORD),
+                                ("nFileIndexLow", DWORD)]
+
+                cls.BY_HANDLE_FILE_INFORMATION = BY_HANDLE_FILE_INFORMATION
+
+
+                # http://msdn.microsoft.com/en-us/library/windows/desktop/aa364952
+                cls.GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle
+                cls.GetFileInformationByHandle.argtypes = [HANDLE,
+                                                           POINTER(BY_HANDLE_FILE_INFORMATION)]
+                cls.GetFileInformationByHandle.restype = BOOL
+
+                cls._st_nlink = cls._windows_st_nlink

tests/test_api_build.py

@@ -724,21 +724,23 @@ def test_output_folder_moves_file(test_metadata, testing_workdir):
 
 def test_info_files_json(test_config):
     recipe = os.path.join(metadata_dir, "ignore_some_prefix_files")
-    outputs = api.build(recipe, config=test_config)
-    assert package_has_file(outputs[0], "info/files.json")
-    with tarfile.open(outputs[0]) as tf:
-        data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8'))
-    fields = ["path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link",
-              "prefix_placeholder", "inode_first_path"]
+    fn = api.get_output_file_path(recipe, config=test_config)
+    api.build(recipe, config=test_config)
+    assert package_has_file(fn, "info/paths.json")
+    with tarfile.open(fn) as tf:
+        data = json.loads(tf.extractfile('info/paths.json').read().decode('utf-8'))
+    fields = ["_path", "sha256", "size_in_bytes", "path_type", "file_mode", "no_link",
+              "prefix_placeholder", "inode_paths"]
     for key in data.keys():
-        assert key in ['files', 'fields', 'version']
-    for field in data.get('fields'):
-        assert field in fields
-    assert len(data.get('files')) == 2
-    for file in data.get('files'):
+        assert key in ['paths', 'paths_version']
+    for paths in data.get('paths'):
+        for field in paths.keys():
+            assert field in fields
+    assert len(data.get('paths')) == 2
+    for file in data.get('paths'):
         for key in file.keys():
             assert key in fields
-        short_path = file.get("path")
+        short_path = file.get("_path")
         if short_path == "test.sh" or short_path == "test.bat":
             assert file.get("prefix_placeholder") is not None
             assert file.get("file_mode") is not None