Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[internal] Introduce OptionalPex/OptionalPexRequest. #13712

Merged
merged 1 commit into from
Nov 25, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion src/python/pants/backend/python/util_rules/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@
from pants.util.ordered_set import FrozenOrderedSet
from pants.util.strutil import pluralize

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class Lockfile:
Expand Down Expand Up @@ -255,6 +257,11 @@ def debug_hint(self) -> str:
return self.output_filename


@dataclass(frozen=True)
class OptionalPexRequest:
maybe_pex_request: PexRequest | None


@dataclass(frozen=True)
class Pex:
"""Wrapper for a digest containing a pex file created with some filename."""
Expand All @@ -264,7 +271,9 @@ class Pex:
python: PythonExecutable | None


logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class OptionalPex:
maybe_pex: Pex | None


@rule(desc="Find Python interpreter for constraints", level=LogLevel.DEBUG)
Expand Down Expand Up @@ -690,6 +699,14 @@ async def create_pex(request: PexRequest) -> Pex:
return result.create_pex()


@rule
async def create_optional_pex(request: OptionalPexRequest) -> OptionalPex:
if request.maybe_pex_request is None:
return OptionalPex(None)
result = await Get(Pex, PexRequest, request.maybe_pex_request)
return OptionalPex(result)


@dataclass(frozen=True)
class Script:
path: PurePath
Expand Down
122 changes: 54 additions & 68 deletions src/python/pants/backend/python/util_rules/pex_from_targets.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
from pants.backend.python.util_rules.local_dists import rules as local_dists_rules
from pants.backend.python.util_rules.pex import (
Lockfile,
Pex,
OptionalPex,
OptionalPexRequest,
PexPlatforms,
PexRequest,
PexRequirements,
Expand Down Expand Up @@ -230,11 +231,6 @@ async def interpreter_constraints_for_targets(
return interpreter_constraints


@dataclass(frozen=True)
class _RepositoryPex:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now we use a standard OptionalPexRequest instead

maybe_pex: Pex | None


@dataclass(frozen=True)
class _RepositoryPexRequest:
requirements: PexRequirements
Expand Down Expand Up @@ -322,7 +318,7 @@ async def pex_from_targets(request: PexFromTargetsRequest) -> PexRequest:

if requirements:
repository_pex = await Get(
_RepositoryPex,
OptionalPex,
_RepositoryPexRequest(
requirements,
request.platforms,
Expand Down Expand Up @@ -351,15 +347,14 @@ async def pex_from_targets(request: PexFromTargetsRequest) -> PexRequest:
@rule
async def get_repository_pex(
request: _RepositoryPexRequest, python_setup: PythonSetup
) -> _RepositoryPex:
repository_pex: Pex | None = None
) -> OptionalPexRequest:
repository_pex_request: PexRequest | None = None
if python_setup.requirement_constraints:
maybe_constraints_repository_pex = await Get(
_RepositoryPex,
constraints_repository_pex_request = await Get(
OptionalPexRequest,
_ConstraintsRepositoryPexRequest(request),
)
if maybe_constraints_repository_pex.maybe_pex:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There was no need for this check even before!

repository_pex = maybe_constraints_repository_pex.maybe_pex
repository_pex_request = constraints_repository_pex_request.maybe_pex_request
elif (
python_setup.resolve_all_constraints
and python_setup.resolve_all_constraints_was_set_explicitly()
Expand All @@ -370,59 +365,53 @@ async def get_repository_pex(
)
elif request.resolve_and_lockfile:
resolve, lockfile = request.resolve_and_lockfile
repository_pex = await Get(
Pex,
PexRequest(
description=f"Installing {lockfile} for the resolve `{resolve}`",
output_filename=f"{path_safe(resolve)}_lockfile.pex",
internal_only=request.internal_only,
requirements=Lockfile(
file_path=lockfile,
file_path_description_of_origin=(
f"the resolve `{resolve}` (from "
"`[python].experimental_resolves_to_lockfiles`)"
),
# TODO(#12314): Hook up lockfile staleness check.
lockfile_hex_digest=None,
req_strings=None,
repository_pex_request = PexRequest(
description=f"Installing {lockfile} for the resolve `{resolve}`",
output_filename=f"{path_safe(resolve)}_lockfile.pex",
internal_only=request.internal_only,
requirements=Lockfile(
file_path=lockfile,
file_path_description_of_origin=(
f"the resolve `{resolve}` (from "
"`[python].experimental_resolves_to_lockfiles`)"
),
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
# TODO(#12314): Hook up lockfile staleness check.
lockfile_hex_digest=None,
req_strings=None,
),
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
)
elif python_setup.lockfile:
repository_pex = await Get(
Pex,
PexRequest(
description=f"Installing {python_setup.lockfile}",
output_filename="lockfile.pex",
internal_only=request.internal_only,
requirements=Lockfile(
file_path=python_setup.lockfile,
file_path_description_of_origin=("the option `[python].experimental_lockfile`"),
# TODO(#12314): Hook up lockfile staleness check once multiple lockfiles
# are supported.
lockfile_hex_digest=None,
req_strings=None,
),
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
repository_pex_request = PexRequest(
description=f"Installing {python_setup.lockfile}",
output_filename="lockfile.pex",
internal_only=request.internal_only,
requirements=Lockfile(
file_path=python_setup.lockfile,
file_path_description_of_origin=("the option `[python].experimental_lockfile`"),
# TODO(#12314): Hook up lockfile staleness check once multiple lockfiles
# are supported.
lockfile_hex_digest=None,
req_strings=None,
),
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
)
return _RepositoryPex(repository_pex)
return OptionalPexRequest(repository_pex_request)


@rule
async def _setup_constraints_repository_pex(
constraints_request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup
) -> _RepositoryPex:
) -> OptionalPexRequest:
request = constraints_request.repository_pex_request
# NB: it isn't safe to resolve against the whole constraints file if
# platforms are in use. See https://github.com/pantsbuild/pants/issues/12222.
if not python_setup.resolve_all_constraints or request.platforms:
return _RepositoryPex(None)
return OptionalPexRequest(None)

constraints_path = python_setup.requirement_constraints
assert constraints_path is not None
Expand Down Expand Up @@ -468,7 +457,7 @@ async def _setup_constraints_repository_pex(
f"entries for the following requirements: {', '.join(unconstrained_projects)}.\n\n"
f"Ignoring `[python_setup].resolve_all_constraints` option."
)
return _RepositoryPex(None)
return OptionalPexRequest(None)

# To get a full set of requirements we must add the URL requirements to the
# constraints file, since the latter cannot contain URL requirements.
Expand All @@ -479,24 +468,21 @@ async def _setup_constraints_repository_pex(
# all these repository pexes will have identical pinned versions of everything,
# this is not a correctness issue, only a performance one.
all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)}
repository_pex = await Get(
Pex,
PexRequest(
description=f"Resolving {constraints_path}",
output_filename="repository.pex",
internal_only=request.internal_only,
requirements=PexRequirements(
all_constraints,
apply_constraints=True,
# TODO: See PexRequirements docs.
is_all_constraints_resolve=True,
),
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
repository_pex = PexRequest(
description=f"Resolving {constraints_path}",
output_filename="repository.pex",
internal_only=request.internal_only,
requirements=PexRequirements(
all_constraints,
apply_constraints=True,
# TODO: See PexRequirements docs.
is_all_constraints_resolve=True,
),
interpreter_constraints=request.interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
)
return _RepositoryPex(repository_pex)
return OptionalPexRequest(repository_pex)


@frozen_after_init
Expand Down