diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 9b150bdce0..e1904912fd 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -105,7 +105,6 @@ def _create_whl_repos( # containers to aggregate outputs from this function whl_map = {} - exposed_packages = {} extra_aliases = { whl_name: {alias: True for alias in aliases} for whl_name, aliases in pip_attr.extra_hub_aliases.items() @@ -219,8 +218,6 @@ def _create_whl_repos( ) for whl_name, requirements in requirements_by_platform.items(): - whl_name = normalize_name(whl_name) - group_name = whl_group_mapping.get(whl_name) group_deps = requirement_cycles.get(group_name, []) @@ -261,68 +258,55 @@ def _create_whl_repos( if v != default }) - is_exposed = False - if get_index_urls: - # TODO @aignas 2024-05-26: move to a separate function - found_something = False - for requirement in requirements: - is_exposed = is_exposed or requirement.is_exposed - dists = requirement.whls - if not pip_attr.download_only and requirement.sdist: - dists = dists + [requirement.sdist] - - for distribution in dists: - found_something = True - is_reproducible = False - - args = dict(whl_library_args) - if pip_attr.netrc: - args["netrc"] = pip_attr.netrc - if pip_attr.auth_patterns: - args["auth_patterns"] = pip_attr.auth_patterns - - if not distribution.filename.endswith(".whl"): - # pip is not used to download wheels and the python - # `whl_library` helpers are only extracting things, however - # for sdists, they will be built by `pip`, so we still - # need to pass the extra args there. - args["extra_pip_args"] = requirement.extra_pip_args - - # This is no-op because pip is not used to download the wheel. - args.pop("download_only", None) - - repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) - args["requirement"] = requirement.srcs.requirement - args["urls"] = [distribution.url] - args["sha256"] = distribution.sha256 - args["filename"] = distribution.filename - args["experimental_target_platforms"] = requirement.target_platforms - - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): - if len(requirements) > 1: - target_platforms = requirement.target_platforms - - whl_libraries[repo_name] = args - - whl_map.setdefault(whl_name, {})[whl_config_setting( - version = major_minor, - filename = distribution.filename, - target_platforms = target_platforms, - )] = repo_name - - if found_something: - if is_exposed: - exposed_packages[whl_name] = None - continue - - is_exposed = False + # TODO @aignas 2024-05-26: move to a separate function for requirement in requirements: - is_exposed = is_exposed or requirement.is_exposed - if get_index_urls: - logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.srcs.requirement_line)) + dists = requirement.whls + if not pip_attr.download_only and requirement.sdist: + dists = dists + [requirement.sdist] + + for distribution in dists: + args = dict(whl_library_args) + if pip_attr.netrc: + args["netrc"] = pip_attr.netrc + if pip_attr.auth_patterns: + args["auth_patterns"] = pip_attr.auth_patterns + + if not distribution.filename.endswith(".whl"): + # pip is not used to download wheels and the python + # `whl_library` helpers are only extracting things, however + # for sdists, they will be built by `pip`, so we still + # need to pass the extra args there. + args["extra_pip_args"] = requirement.extra_pip_args + + # This is no-op because pip is not used to download the wheel. + args.pop("download_only", None) + + repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) + args["requirement"] = requirement.srcs.requirement + args["urls"] = [distribution.url] + args["sha256"] = distribution.sha256 + args["filename"] = distribution.filename + args["experimental_target_platforms"] = requirement.target_platforms + + # Pure python wheels or sdists may need to have a platform here + target_platforms = None + if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): + if len(requirements) > 1: + target_platforms = requirement.target_platforms + + whl_libraries[repo_name] = args + + whl_map.setdefault(whl_name, {})[whl_config_setting( + version = major_minor, + filename = distribution.filename, + target_platforms = target_platforms, + )] = repo_name + + if dists: + is_reproducible = False + continue + # Fallback to a pip-installed wheel args = dict(whl_library_args) # make a copy args["requirement"] = requirement.srcs.requirement_line if requirement.extra_pip_args: @@ -343,13 +327,14 @@ def _create_whl_repos( target_platforms = target_platforms or None, )] = repo_name - if is_exposed: - exposed_packages[whl_name] = None - return struct( is_reproducible = is_reproducible, whl_map = whl_map, - exposed_packages = exposed_packages, + exposed_packages = { + whl_name: None + for whl_name, requirements in requirements_by_platform.items() + if len([r for r in requirements if r.is_exposed]) > 0 + }, extra_aliases = extra_aliases, whl_libraries = whl_libraries, ) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 821913d6de..d7ee285c0e 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -203,6 +203,9 @@ def parse_requirements( sorted(requirements), )) + # Return normalized names + ret_requirements = ret.setdefault(normalize_name(whl_name), []) + for r in sorted(reqs.values(), key = lambda r: r.requirement_line): whls, sdist = _add_dists( requirement = r, @@ -211,7 +214,7 @@ def parse_requirements( ) target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - ret.setdefault(whl_name, []).append( + ret_requirements.append( struct( distribution = r.distribution, srcs = r.srcs,