From 5b87df6b181ba15b83f37c42e20877bb88222742 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Wed, 24 Nov 2021 16:00:28 -0800 Subject: [PATCH] Introduce OptionalPex/OptionalPexRequest. So we can operate monadically in situations where there may or may not be an underlying Pex. Utilizes this in one such case we currently have - requesting a repository pex. This brings repository pexes more inline with other pexes - you request them via a PexRequest, except you must also account for the possibility that no repository pex can be used. [ci skip-rust] [ci skip-build-wheels] --- .../pants/backend/python/util_rules/pex.py | 19 ++- .../python/util_rules/pex_from_targets.py | 122 ++++++++---------- 2 files changed, 72 insertions(+), 69 deletions(-) diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index ee813d5f684..f3c3c44624b 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -66,6 +66,8 @@ from pants.util.ordered_set import FrozenOrderedSet from pants.util.strutil import pluralize +logger = logging.getLogger(__name__) + @dataclass(frozen=True) class Lockfile: @@ -255,6 +257,11 @@ def debug_hint(self) -> str: return self.output_filename +@dataclass(frozen=True) +class OptionalPexRequest: + maybe_pex_request: PexRequest | None + + @dataclass(frozen=True) class Pex: """Wrapper for a digest containing a pex file created with some filename.""" @@ -264,7 +271,9 @@ class Pex: python: PythonExecutable | None -logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class OptionalPex: + maybe_pex: Pex | None @rule(desc="Find Python interpreter for constraints", level=LogLevel.DEBUG) @@ -690,6 +699,14 @@ async def create_pex(request: PexRequest) -> Pex: return result.create_pex() +@rule +async def create_optional_pex(request: OptionalPexRequest) -> OptionalPex: + if request.maybe_pex_request is None: + return OptionalPex(None) + result = await Get(Pex, PexRequest, request.maybe_pex_request) + return OptionalPex(result) + + @dataclass(frozen=True) class Script: path: PurePath diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets.py b/src/python/pants/backend/python/util_rules/pex_from_targets.py index 5d0592b2d20..15eb3204d57 100644 --- a/src/python/pants/backend/python/util_rules/pex_from_targets.py +++ b/src/python/pants/backend/python/util_rules/pex_from_targets.py @@ -23,7 +23,8 @@ from pants.backend.python.util_rules.local_dists import rules as local_dists_rules from pants.backend.python.util_rules.pex import ( Lockfile, - Pex, + OptionalPex, + OptionalPexRequest, PexPlatforms, PexRequest, PexRequirements, @@ -230,11 +231,6 @@ async def interpreter_constraints_for_targets( return interpreter_constraints -@dataclass(frozen=True) -class _RepositoryPex: - maybe_pex: Pex | None - - @dataclass(frozen=True) class _RepositoryPexRequest: requirements: PexRequirements @@ -322,7 +318,7 @@ async def pex_from_targets(request: PexFromTargetsRequest) -> PexRequest: if requirements: repository_pex = await Get( - _RepositoryPex, + OptionalPex, _RepositoryPexRequest( requirements, request.platforms, @@ -351,15 +347,14 @@ async def pex_from_targets(request: PexFromTargetsRequest) -> PexRequest: @rule async def get_repository_pex( request: _RepositoryPexRequest, python_setup: PythonSetup -) -> _RepositoryPex: - repository_pex: Pex | None = None +) -> OptionalPexRequest: + repository_pex_request: PexRequest | None = None if python_setup.requirement_constraints: - maybe_constraints_repository_pex = await Get( - _RepositoryPex, + constraints_repository_pex_request = await Get( + OptionalPexRequest, _ConstraintsRepositoryPexRequest(request), ) - if maybe_constraints_repository_pex.maybe_pex: - repository_pex = maybe_constraints_repository_pex.maybe_pex + repository_pex_request = constraints_repository_pex_request.maybe_pex_request elif ( python_setup.resolve_all_constraints and python_setup.resolve_all_constraints_was_set_explicitly() @@ -370,59 +365,53 @@ async def get_repository_pex( ) elif request.resolve_and_lockfile: resolve, lockfile = request.resolve_and_lockfile - repository_pex = await Get( - Pex, - PexRequest( - description=f"Installing {lockfile} for the resolve `{resolve}`", - output_filename=f"{path_safe(resolve)}_lockfile.pex", - internal_only=request.internal_only, - requirements=Lockfile( - file_path=lockfile, - file_path_description_of_origin=( - f"the resolve `{resolve}` (from " - "`[python].experimental_resolves_to_lockfiles`)" - ), - # TODO(#12314): Hook up lockfile staleness check. - lockfile_hex_digest=None, - req_strings=None, + repository_pex_request = PexRequest( + description=f"Installing {lockfile} for the resolve `{resolve}`", + output_filename=f"{path_safe(resolve)}_lockfile.pex", + internal_only=request.internal_only, + requirements=Lockfile( + file_path=lockfile, + file_path_description_of_origin=( + f"the resolve `{resolve}` (from " + "`[python].experimental_resolves_to_lockfiles`)" ), - interpreter_constraints=request.interpreter_constraints, - platforms=request.platforms, - additional_args=request.additional_lockfile_args, + # TODO(#12314): Hook up lockfile staleness check. + lockfile_hex_digest=None, + req_strings=None, ), + interpreter_constraints=request.interpreter_constraints, + platforms=request.platforms, + additional_args=request.additional_lockfile_args, ) elif python_setup.lockfile: - repository_pex = await Get( - Pex, - PexRequest( - description=f"Installing {python_setup.lockfile}", - output_filename="lockfile.pex", - internal_only=request.internal_only, - requirements=Lockfile( - file_path=python_setup.lockfile, - file_path_description_of_origin=("the option `[python].experimental_lockfile`"), - # TODO(#12314): Hook up lockfile staleness check once multiple lockfiles - # are supported. - lockfile_hex_digest=None, - req_strings=None, - ), - interpreter_constraints=request.interpreter_constraints, - platforms=request.platforms, - additional_args=request.additional_lockfile_args, + repository_pex_request = PexRequest( + description=f"Installing {python_setup.lockfile}", + output_filename="lockfile.pex", + internal_only=request.internal_only, + requirements=Lockfile( + file_path=python_setup.lockfile, + file_path_description_of_origin=("the option `[python].experimental_lockfile`"), + # TODO(#12314): Hook up lockfile staleness check once multiple lockfiles + # are supported. + lockfile_hex_digest=None, + req_strings=None, ), + interpreter_constraints=request.interpreter_constraints, + platforms=request.platforms, + additional_args=request.additional_lockfile_args, ) - return _RepositoryPex(repository_pex) + return OptionalPexRequest(repository_pex_request) @rule async def _setup_constraints_repository_pex( constraints_request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup -) -> _RepositoryPex: +) -> OptionalPexRequest: request = constraints_request.repository_pex_request # NB: it isn't safe to resolve against the whole constraints file if # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222. if not python_setup.resolve_all_constraints or request.platforms: - return _RepositoryPex(None) + return OptionalPexRequest(None) constraints_path = python_setup.requirement_constraints assert constraints_path is not None @@ -468,7 +457,7 @@ async def _setup_constraints_repository_pex( f"entries for the following requirements: {', '.join(unconstrained_projects)}.\n\n" f"Ignoring `[python_setup].resolve_all_constraints` option." ) - return _RepositoryPex(None) + return OptionalPexRequest(None) # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. @@ -479,24 +468,21 @@ async def _setup_constraints_repository_pex( # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} - repository_pex = await Get( - Pex, - PexRequest( - description=f"Resolving {constraints_path}", - output_filename="repository.pex", - internal_only=request.internal_only, - requirements=PexRequirements( - all_constraints, - apply_constraints=True, - # TODO: See PexRequirements docs. - is_all_constraints_resolve=True, - ), - interpreter_constraints=request.interpreter_constraints, - platforms=request.platforms, - additional_args=request.additional_lockfile_args, + repository_pex = PexRequest( + description=f"Resolving {constraints_path}", + output_filename="repository.pex", + internal_only=request.internal_only, + requirements=PexRequirements( + all_constraints, + apply_constraints=True, + # TODO: See PexRequirements docs. + is_all_constraints_resolve=True, ), + interpreter_constraints=request.interpreter_constraints, + platforms=request.platforms, + additional_args=request.additional_lockfile_args, ) - return _RepositoryPex(repository_pex) + return OptionalPexRequest(repository_pex) @frozen_after_init