Add overrides field to python_sources and python_tests target

build-support/bin/BUILD

@@ -1,20 +1,31 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-shell_sources(name="sh_scripts")
-
-python_sources(name="py_scripts")
-python_tests(name="py_tests", timeout=90)  # reversion_test.py times out occasionally.
 resources(name="docs_templates", sources=["docs_templates/*.mustache"])
 resources(name="user_list_templates", sources=["user_list_templates/*.mustache"])
 
+shell_sources(name="sh_scripts")
+
+python_sources(
+    name="py_scripts",
+    overrides={
+        "generate_docs.py": {"dependencies": [":docs_templates"]},
+        "generate_user_list.py": {"dependencies": [":user_list_templates"]},
+    }
+)
+
+python_tests(
+    name="py_tests",
+    overrides={"reversion_test.py": {"timeout": 90}},
+)
+
 pex_binary(name="changelog", entry_point="changelog.py")
 pex_binary(name="check_banned_imports", entry_point="check_banned_imports.py")
 pex_binary(name="check_inits", entry_point="check_inits.py")
 pex_binary(name="deploy_to_s3", entry_point="deploy_to_s3.py")
 pex_binary(name="generate_all_lockfiles_helper", entry_point="_generate_all_lockfiles_helper.py")
-pex_binary(name="generate_docs", entry_point="generate_docs.py", dependencies=[":docs_templates"])
+pex_binary(name="generate_docs", entry_point="generate_docs.py")
 pex_binary(name="generate_github_workflows", entry_point="generate_github_workflows.py")
-pex_binary(name="generate_user_list", entry_point="generate_user_list.py", dependencies=[":user_list_templates"])
+pex_binary(name="generate_user_list", entry_point="generate_user_list.py")
 pex_binary(name="release_helper", entry_point="_release_helper.py")
 pex_binary(name="reversion", entry_point="reversion.py")

src/python/pants/backend/project_info/peek.py

@@ -8,9 +8,7 @@
 import os
 from dataclasses import asdict, dataclass, is_dataclass
 from enum import Enum
-from typing import Iterable, cast
-
-from pkg_resources import Requirement
+from typing import Any, Iterable, cast
 
 from pants.engine.addresses import Address, BuildFileAddress
 from pants.engine.collection import Collection
@@ -111,12 +109,19 @@ class TargetDatas(Collection[TargetData]):
     pass
 
 
-def _render_json(tds: Iterable[TargetData], exclude_defaults: bool = False) -> str:
+def render_json(tds: Iterable[TargetData], exclude_defaults: bool = False) -> str:
     nothing = object()
 
+    def normalize_value(val: Any) -> Any:
+        if isinstance(val, collections.abc.Mapping):
+            return {str(k): normalize_value(v) for k, v in val.items()}
+        return val
+
     def to_json(td: TargetData) -> dict:
         fields = {
-            (f"{k.alias}_raw" if issubclass(k, (SourcesField, Dependencies)) else k.alias): v.value
+            (
+                f"{k.alias}_raw" if issubclass(k, (SourcesField, Dependencies)) else k.alias
+            ): normalize_value(v.value)
             for k, v in td.target.field_values.items()
             if not (exclude_defaults and getattr(k, "default", nothing) == v.value)
         }
@@ -138,8 +143,6 @@ def to_json(td: TargetData) -> dict:
 class _PeekJsonEncoder(json.JSONEncoder):
     """Allow us to serialize some commmonly found types in BUILD files."""
 
-    safe_to_str_types = (Requirement,)
-
     def default(self, o):
         """Return a serializable object for o."""
         if is_dataclass(o):
@@ -220,7 +223,7 @@ async def peek(
         return Peek(exit_code=0)
 
     tds = await Get(TargetDatas, UnexpandedTargets, targets)
-    output = _render_json(tds, subsys.exclude_defaults)
+    output = render_json(tds, subsys.exclude_defaults)
     with subsys.output(console) as write_stdout:
         write_stdout(output)
     return Peek(exit_code=0)

src/python/pants/backend/project_info/peek_test.py

@@ -11,9 +11,15 @@
 from pants.core.target_types import ArchiveTarget, FilesGeneratorTarget, GenericTarget
 from pants.engine.addresses import Address
 from pants.engine.rules import QueryRule
+from pants.engine.target import OverridesField
 from pants.testutil.rule_runner import RuleRunner
 
 
+# TODO: Remove this if/when we add an overrides field to `files`.
+class FilesGeneratorTargetWithOverrides(FilesGeneratorTarget):
+    core_fields = (*FilesGeneratorTarget.core_fields, OverridesField)  # type: ignore[assignment]
+
+
 @pytest.mark.parametrize(
     "expanded_target_infos, exclude_defaults, expected_output",
     [
@@ -26,8 +32,14 @@
         pytest.param(
             [
                 TargetData(
-                    FilesGeneratorTarget(
-                        {"sources": ["*.txt"]}, Address("example", target_name="files_target")
+                    FilesGeneratorTargetWithOverrides(
+                        {
+                            "sources": ["*.txt"],
+                            # Regression test that we can handle a dict with `tuple[str, ...]` as
+                            # key.
+                            "overrides": {("foo.txt",): {"tags": ["overridden"]}},
+                        },
+                        Address("example", target_name="files_target"),
                     ),
                     ("foo.txt", "bar.txt"),
                     tuple(),
@@ -41,6 +53,13 @@
                     "address": "example:files_target",
                     "target_type": "files",
                     "dependencies": [],
+                    "overrides": {
+                      "('foo.txt',)": {
+                        "tags": [
+                          "overridden"
+                        ]
+                      }
+                    },
                     "sources": [
                       "foo.txt",
                       "bar.txt"
@@ -143,7 +162,7 @@
     ],
 )
 def test_render_targets_as_json(expanded_target_infos, exclude_defaults, expected_output):
-    actual_output = peek._render_json(expanded_target_infos, exclude_defaults)
+    actual_output = peek.render_json(expanded_target_infos, exclude_defaults)
     assert actual_output == expected_output
 
 

src/python/pants/backend/python/target_types.py

@@ -46,6 +46,7 @@
     InvalidTargetException,
     MultipleSourcesField,
     NestedDictStringToStringField,
+    OverridesField,
     ProvidesField,
     ScalarField,
     SecondaryOwnerMixin,
@@ -642,9 +643,36 @@ class PythonTestsGeneratingSourcesField(PythonGeneratingSourcesBase):
     )
 
 
+class PythonTestsOverrideField(OverridesField):
+    help = (
+        "Override the field values for generated `python_test` targets.\n\n"
+        "Expects a dictionary of relative file paths and globs to a dictionary for the "
+        "overrides. You may either use a string for a single path / glob, "
+        "or a string tuple for multiple paths / globs. Each override is a dictionary of "
+        "field names to the overridden value.\n\n"
+        "For example:\n\n"
+        "  overrides={\n"
+        '    "foo_test.py": {"timeout": 120]},\n'
+        '    "bar_test.py": {"timeout": 200]},\n'
+        '    ("foo_test.py", "bar_test.py"): {"tags": ["slow_tests"]},\n'
+        "  }\n\n"
+        "File paths and globs are relative to the BUILD file's directory. Every overridden file is "
+        "validated to belong to this target's `sources` field.\n\n"
+        "If you'd like to override a field's value for every `python_test` target generated by "
+        "this target, change the field directly on this target rather than using the "
+        "`overrides` field.\n\n"
+        "You can specify the same file name in multiple keys, so long as you don't override the "
+        "same field more than one time for the file."
+    )
+
+
 class PythonTestsGeneratorTarget(Target):
     alias = "python_tests"
-    core_fields = (*_PYTHON_TEST_COMMON_FIELDS, PythonTestsGeneratingSourcesField)
+    core_fields = (
+        *_PYTHON_TEST_COMMON_FIELDS,
+        PythonTestsGeneratingSourcesField,
+        PythonTestsOverrideField,
+    )
     help = "Generate a `python_test` target for each file in the `sources` field."
 
 
@@ -670,13 +698,37 @@ class PythonSourcesGeneratingSourcesField(PythonGeneratingSourcesBase):
     )
 
 
+class PythonSourcesOverridesField(OverridesField):
+    help = (
+        "Override the field values for generated `python_source` targets.\n\n"
+        "Expects a dictionary of relative file paths and globs to a dictionary for the "
+        "overrides. You may either use a string for a single path / glob, "
+        "or a string tuple for multiple paths / globs. Each override is a dictionary of "
+        "field names to the overridden value.\n\n"
+        "For example:\n\n"
+        "  overrides={\n"
+        '    "foo.py": {"skip_pylint": True]},\n'
+        '    "bar.py": {"skip_flake8": True]},\n'
+        '    ("foo.py", "bar.py"): {"tags": ["linter_disabled"]},\n'
+        "  }\n\n"
+        "File paths and globs are relative to the BUILD file's directory. Every overridden file is "
+        "validated to belong to this target's `sources` field.\n\n"
+        "If you'd like to override a field's value for every `python_source` target generated by "
+        "this target, change the field directly on this target rather than using the "
+        "`overrides` field.\n\n"
+        "You can specify the same file name in multiple keys, so long as you don't override the "
+        "same field more than one time for the file."
+    )
+
+
 class PythonSourcesGeneratorTarget(Target):
     alias = "python_sources"
     core_fields = (
         *COMMON_TARGET_FIELDS,
         InterpreterConstraintsField,
         Dependencies,
         PythonSourcesGeneratingSourcesField,
+        PythonSourcesOverridesField,
     )
     help = "Generate a `python_source` target for each file in the `sources` field."
 

src/python/pants/backend/python/target_types_rules.py

@@ -49,13 +49,15 @@
     InjectDependenciesRequest,
     InjectedDependencies,
     InvalidFieldException,
+    OverridesField,
     SourcesPaths,
     SourcesPathsRequest,
     Targets,
     WrappedTarget,
     generate_file_level_targets,
 )
 from pants.engine.unions import UnionMembership, UnionRule
+from pants.option.global_options import FilesNotFoundBehavior
 from pants.source.source_root import SourceRoot, SourceRootRequest
 from pants.util.docutil import doc_url
 from pants.util.frozendict import FrozenDict
@@ -77,18 +79,32 @@ class GenerateTargetsFromPythonTests(GenerateTargetsRequest):
 @rule
 async def generate_targets_from_python_tests(
     request: GenerateTargetsFromPythonTests,
+    files_not_found_behavior: FilesNotFoundBehavior,
     python_infer: PythonInferSubsystem,
     union_membership: UnionMembership,
 ) -> GeneratedTargets:
-    paths = await Get(
+    sources_paths = await Get(
         SourcesPaths, SourcesPathsRequest(request.generator[PythonTestsGeneratingSourcesField])
     )
+
+    all_overrides = {}
+    overrides_field = request.generator[OverridesField]
+    if overrides_field.value:
+        _all_override_paths = await MultiGet(
+            Get(Paths, PathGlobs, path_globs)
+            for path_globs in overrides_field.to_path_globs(files_not_found_behavior)
+        )
+        all_overrides = overrides_field.flatten_paths(
+            dict(zip(_all_override_paths, overrides_field.value.values()))
+        )
+
     return generate_file_level_targets(
         PythonTestTarget,
         request.generator,
-        paths.files,
+        sources_paths.files,
         union_membership,
         add_dependencies_on_all_siblings=not python_infer.imports,
+        overrides=all_overrides,
     )
 
 
@@ -100,17 +116,31 @@ class GenerateTargetsFromPythonSources(GenerateTargetsRequest):
 async def generate_targets_from_python_sources(
     request: GenerateTargetsFromPythonSources,
     python_infer: PythonInferSubsystem,
+    files_not_found_behavior: FilesNotFoundBehavior,
     union_membership: UnionMembership,
 ) -> GeneratedTargets:
-    paths = await Get(
+    sources_paths = await Get(
         SourcesPaths, SourcesPathsRequest(request.generator[PythonSourcesGeneratingSourcesField])
     )
+
+    all_overrides = {}
+    overrides_field = request.generator[OverridesField]
+    if overrides_field.value:
+        _all_override_paths = await MultiGet(
+            Get(Paths, PathGlobs, path_globs)
+            for path_globs in overrides_field.to_path_globs(files_not_found_behavior)
+        )
+        all_overrides = overrides_field.flatten_paths(
+            dict(zip(_all_override_paths, overrides_field.value.values()))
+        )
+
     return generate_file_level_targets(
         PythonSourceTarget,
         request.generator,
-        paths.files,
+        sources_paths.files,
         union_membership,
         add_dependencies_on_all_siblings=not python_infer.imports,
+        overrides=all_overrides,
     )