diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 8105b7bf10..205b3b644e 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -32,6 +32,7 @@ requirements: - python - pyyaml - pkginfo + - enum34 [py<34] test: requires: diff --git a/conda_build/build.py b/conda_build/build.py index 61c3ed2bf4..25af32b89f 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -18,6 +18,7 @@ import subprocess import sys import tarfile +import hashlib # this is to compensate for a requests idna encoding error. Conda is a better place to fix, # eventually @@ -60,6 +61,13 @@ import conda_build.noarch_python as noarch_python from conda_verify.verify import Verify +from enum import Enum + + +class FileType(Enum): + softlink = "softlink" + hardlink = "hardlink" + directory = "directory" if 'bsd' in sys.platform: @@ -235,22 +243,27 @@ def copy_license(m, config): join(config.info_dir, 'LICENSE.txt'), config.timeout) -def detect_and_record_prefix_files(m, files, prefix, config): +def get_files_with_prefix(m, files, prefix): files_with_prefix = sorted(have_prefix_files(files, prefix)) - binary_has_prefix_files = m.binary_has_prefix_files() - text_has_prefix_files = m.has_prefix_files() ignore_files = m.ignore_prefix_files() ignore_types = set() if not hasattr(ignore_files, "__iter__"): - if ignore_files == True: + if ignore_files is True: ignore_types.update(('text', 'binary')) ignore_files = [] if not m.get_value('build/detect_binary_files_with_prefix', True): ignore_types.update(('binary',)) - ignore_files.extend([f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files]) + ignore_files.extend( + [f[2] for f in files_with_prefix if f[1] in ignore_types and f[2] not in ignore_files]) files_with_prefix = [f for f in files_with_prefix if f[2] not in ignore_files] + return files_with_prefix + +def detect_and_record_prefix_files(m, files, prefix, config): + files_with_prefix = get_files_with_prefix(m, files, prefix) + binary_has_prefix_files = m.binary_has_prefix_files() + text_has_prefix_files = m.has_prefix_files() is_noarch = m.get_value('build/noarch_python') or is_noarch_python(m) or m.get_value('build/noarch') if files_with_prefix and not is_noarch: @@ -425,6 +438,9 @@ def create_info_files(m, files, config, prefix): for f in files: fo.write(f + '\n') + files_with_prefix = get_files_with_prefix(m, files, prefix) + create_info_files_json(m, config.info_dir, prefix, files, files_with_prefix) + detect_and_record_prefix_files(m, files, prefix, config) write_no_link(m, config, files) @@ -438,6 +454,99 @@ def create_info_files(m, files, config, prefix): config.timeout) +def get_short_path(m, target_file): + entry_point_script_names = get_entry_point_script_names(m.get_value('build/entry_points')) + if is_noarch_python(m): + if target_file.find("site-packages") > 0: + return target_file[target_file.find("site-packages"):] + elif target_file.startswith("bin") and (target_file not in entry_point_script_names): + return target_file.replace("bin", "python-scripts") + elif target_file.startswith("Scripts") and (target_file not in entry_point_script_names): + return target_file.replace("Scripts", "python-scripts") + elif m.get_value('build/noarch_python'): + return None + else: + return target_file + + +def sha256_checksum(filename, buffersize=65536): + sha256 = hashlib.sha256() + with open(filename, 'rb') as f: + for block in iter(lambda: f.read(buffersize), b''): + sha256.update(block) + return sha256.hexdigest() + + +def has_prefix(short_path, files_with_prefix): + for prefix, mode, filename in files_with_prefix: + if short_path == filename: + return prefix, mode + return None, None + + +def is_no_link(no_link, short_path): + if no_link is not None and short_path in no_link: + return True + + +def get_inode_paths(files, target_short_path, prefix): + ensure_list(files) + target_short_path_inode = os.stat(join(prefix, target_short_path)).st_ino + hardlinked_files = [sp for sp in files + if os.stat(join(prefix, sp)).st_ino == target_short_path_inode] + return sorted(hardlinked_files) + + +def file_type(path): + if isdir(path): + return FileType.directory + elif islink(path): + return FileType.softlink + return FileType.hardlink + + +def build_info_files_json(m, prefix, files, files_with_prefix): + no_link = m.get_value('build/no_link') + files_json = [] + for fi in files: + prefix_placeholder, file_mode = has_prefix(fi, files_with_prefix) + path = os.path.join(prefix, fi) + file_info = { + "short_path": get_short_path(m, fi), + "sha256": sha256_checksum(path), + "size_in_bytes": os.path.getsize(path), + "file_type": getattr(file_type(path), "name"), + } + no_link = is_no_link(no_link, fi) + if no_link: + file_info["no_link"] = no_link + if prefix_placeholder and file_mode: + file_info["prefix_placeholder"] = prefix_placeholder + file_info["file_mode"] = file_mode + if file_info.get("file_type") == "hardlink" and os.stat(join(prefix, fi)).st_nlink > 1: + inode_paths = get_inode_paths(files, fi, prefix) + file_info["inode_paths"] = inode_paths + files_json.append(file_info) + return files_json + + +def get_files_version(): + return 1 + + +def create_info_files_json(m, info_dir, prefix, files, files_with_prefix): + files_json_fields = ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "prefix_placeholder", "no_link", "inode_first_path"] + files_json_files = build_info_files_json(m, prefix, files, files_with_prefix) + files_json_info = { + "version": get_files_version(), + "fields": files_json_fields, + "files": files_json_files, + } + with open(join(info_dir, 'files.json'), "w") as files_json: + json.dump(files_json_info, files_json) + + def get_build_index(config, clear_cache=True): # priority: local by croot (can vary), then channels passed as args, # then channels from config. diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 8ec98539f4..00c1f530d6 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -727,3 +727,29 @@ def test_script_win_creates_exe(test_config): api.build(recipe, config=test_config) assert package_has_file(fn, 'Scripts/test-script.exe') assert package_has_file(fn, 'Scripts/test-script-script.py') + + +def test_info_files_json(test_config): + recipe = os.path.join(metadata_dir, "ignore_some_prefix_files") + fn = api.get_output_file_path(recipe, config=test_config) + api.build(recipe, config=test_config) + assert package_has_file(fn, "info/files.json") + with tarfile.open(fn) as tf: + data = json.loads(tf.extractfile('info/files.json').read().decode('utf-8')) + fields = ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", "no_link", + "prefix_placeholder", "inode_first_path"] + for key in data.keys(): + assert key in ['files', 'fields', 'version'] + for field in data.get('fields'): + assert field in fields + assert len(data.get('files')) == 2 + for file in data.get('files'): + for key in file.keys(): + assert key in fields + short_path = file.get("short_path") + if short_path == "test.sh" or short_path == "test.bat": + assert file.get("prefix_placeholder") is not None + assert file.get("file_mode") is not None + else: + assert file.get("prefix_placeholder") is None + assert file.get("file_mode") is None diff --git a/tests/test_build.py b/tests/test_build.py index 3301755bb8..ac112f0064 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -15,7 +15,8 @@ from conda_build.metadata import MetaData from conda_build.utils import rm_rf, on_win -from .utils import testing_workdir, test_config, test_metadata, metadata_dir, put_bad_conda_on_path +from .utils import (testing_workdir, test_config, test_metadata, metadata_dir, + get_noarch_python_meta, put_bad_conda_on_path) prefix_tests = {"normal": os.path.sep} if sys.platform == "win32": @@ -159,3 +160,115 @@ def test_write_about_json_without_conda_on_path(testing_workdir, test_metadata): about = json.load(f) assert 'conda_version' in about assert 'conda_build_version' in about + + +def test_get_short_path(test_metadata): + # Test for regular package + assert build.get_short_path(test_metadata, "test/file") == "test/file" + + # Test for noarch: python + meta = get_noarch_python_meta(test_metadata) + assert build.get_short_path(meta, "lib/site-packages/test") == "site-packages/test" + assert build.get_short_path(meta, "bin/test") == "python-scripts/test" + assert build.get_short_path(meta, "Scripts/test") == "python-scripts/test" + + +def test_has_prefix(): + files_with_prefix = [("prefix/path", "text", "short/path/1"), + ("prefix/path", "text", "short/path/2")] + assert build.has_prefix("short/path/1", files_with_prefix) == ("prefix/path", "text") + assert build.has_prefix("short/path/nope", files_with_prefix) == (None, None) + + +def test_is_no_link(): + no_link = ["path/1", "path/2"] + assert build.is_no_link(no_link, "path/1") is True + assert build.is_no_link(no_link, "path/nope") is None + + +@pytest.mark.skipif(on_win and sys.version[:3] == "2.7", + reason="os.link is not available so can't setup test") +def test_sorted_inode_first_path(testing_workdir): + path_one = os.path.join(testing_workdir, "one") + path_two = os.path.join(testing_workdir, "two") + path_one_hardlink = os.path.join(testing_workdir, "one_hl") + open(path_one, "a").close() + open(path_two, "a").close() + + os.link(path_one, path_one_hardlink) + + files = ["one", "two", "one_hl"] + assert build.get_inode_paths(files, "one", testing_workdir) == ["one", "one_hl"] + assert build.get_inode_paths(files, "one_hl", testing_workdir) == ["one", "one_hl"] + assert build.get_inode_paths(files, "two", testing_workdir) == ["two"] + + +def test_create_info_files_json(testing_workdir, test_metadata): + info_dir = os.path.join(testing_workdir, "info") + os.mkdir(info_dir) + path_one = os.path.join(testing_workdir, "one") + path_two = os.path.join(testing_workdir, "two") + path_foo = os.path.join(testing_workdir, "foo") + open(path_one, "a").close() + open(path_two, "a").close() + open(path_foo, "a").close() + files_with_prefix = [("prefix/path", "text", "foo")] + files = ["one", "two", "foo"] + + build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) + files_json_path = os.path.join(info_dir, "files.json") + expected_output = { + "files": [{"file_type": "hardlink", "short_path": "one", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"file_type": "hardlink", "short_path": "two", "size_in_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {"file_mode": "text", "file_type": "hardlink", + "short_path": "foo", "prefix_placeholder": "prefix/path", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}], + "fields": ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "prefix_placeholder", "no_link", "inode_first_path"], + "version": 1} + with open(files_json_path, "r") as files_json: + output = json.load(files_json) + assert output == expected_output + + +@pytest.mark.skipif(on_win and sys.version[:3] == "2.7", + reason="os.link is not available so can't setup test") +def test_create_info_files_json_no_inodes(testing_workdir, test_metadata): + info_dir = os.path.join(testing_workdir, "info") + os.mkdir(info_dir) + path_one = os.path.join(testing_workdir, "one") + path_two = os.path.join(testing_workdir, "two") + path_foo = os.path.join(testing_workdir, "foo") + path_one_hardlink = os.path.join(testing_workdir, "one_hl") + open(path_one, "a").close() + open(path_two, "a").close() + open(path_foo, "a").close() + os.link(path_one, path_one_hardlink) + files_with_prefix = [("prefix/path", "text", "foo")] + files = ["one", "two", "one_hl", "foo"] + + build.create_info_files_json(test_metadata, info_dir, testing_workdir, files, files_with_prefix) + files_json_path = os.path.join(info_dir, "files.json") + expected_output = { + "files": [{"inode_paths": ["one", "one_hl"], "file_type": "hardlink", "short_path": "one", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"file_type": "hardlink", "short_path": "two", "size_in_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {"inode_paths": ["one", "one_hl"], "file_type": "hardlink", + "short_path": "one_hl", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}, + {"file_mode": "text", "file_type": "hardlink", "short_path": "foo", + "prefix_placeholder": "prefix/path", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size_in_bytes": 0}], + "fields": ["short_path", "sha256", "size_in_bytes", "file_type", "file_mode", + "prefix_placeholder", "no_link", "inode_first_path"], + "version": 1} + with open(files_json_path, "r") as files_json: + assert json.load(files_json) == expected_output diff --git a/tests/utils.py b/tests/utils.py index 11353d69f9..496b5af900 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -122,3 +122,9 @@ def put_bad_conda_on_path(testing_workdir): raise finally: os.environ['PATH'] = path_backup + + +def get_noarch_python_meta(meta): + d = meta.meta + d['build']['noarch'] = "python" + return MetaData.fromdict(d, config=meta.config)