From 4e49e28ec329573906cd23251c61ba47132094c2 Mon Sep 17 00:00:00 2001 From: brandjon Date: Fri, 22 Feb 2019 11:18:16 -0800 Subject: [PATCH] Expose PyRuntimeProvider to Starlark as PyRuntimeInfo This creates a top-level symbol `PyRuntimeInfo` representing the provider created by the `py_runtime` rule and consumed by `py_binary` and `py_test`. PyRuntimeProvider is refactored to not implement TransitiveInfoProvider, and to now be split into separate classes for the provider instance vs provider type. Fakes are created for Stardoc and friends. Tests are added for the API of `PyRuntimeInfo` and for the ability to sandwich a Starlark rule between a py_runtime and its consuming py_binary. As drive-by cleanups, the provider is now instantiated in Java code via factory methods instead of its constructor, and the isHermetic method is renamed to isInBuild to be consistent with the terminology in the design doc for #7375. Rule documentation is also updated. Work toward #7375. RELNOTES: (Python rules) PyRuntimeInfo is exposed to Starlark, making it possible for Starlark rules to depend on or imitate `py_runtime`. The `files` attribute of `py_runtime` is no longer mandatory. PiperOrigin-RevId: 235224774 --- .../devtools/build/docgen/SymbolFamilies.java | 4 +- .../java/com/google/devtools/build/lib/BUILD | 1 - .../bazel/rules/BazelRuleClassProvider.java | 3 +- .../rules/python/BazelPyRuleClasses.java | 4 +- .../rules/python/BazelPythonSemantics.java | 14 +- .../build/lib/rules/python/PyRuntime.java | 8 +- .../build/lib/rules/python/PyRuntimeInfo.java | 190 ++++++++++++++++++ .../lib/rules/python/PyRuntimeProvider.java | 60 ------ .../build/lib/skylarkbuildapi/python/BUILD | 1 + .../skylarkbuildapi/python/PyBootstrap.java | 7 +- .../python/PyRuntimeInfoApi.java | 130 ++++++++++++ .../devtools/build/skydoc/SkydocMain.java | 4 +- .../python/FakePyRuntimeInfo.java | 58 ++++++ .../packages/util/BazelMockPythonSupport.java | 7 + .../lib/packages/util/MockPythonSupport.java | 8 + .../devtools/build/lib/rules/python/BUILD | 17 +- .../lib/rules/python/PyRuntimeInfoTest.java | 151 ++++++++++++++ .../build/lib/rules/python/PyRuntimeTest.java | 10 +- .../rules/python/PythonStarlarkApiTest.java | 44 ++++ 19 files changed, 636 insertions(+), 85 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfo.java delete mode 100644 src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyRuntimeInfoApi.java create mode 100644 src/main/java/com/google/devtools/build/skydoc/fakebuildapi/python/FakePyRuntimeInfo.java create mode 100644 src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfoTest.java diff --git a/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java b/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java index db7af731a1681b..e3ef9ea2547e24 100644 --- a/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java +++ b/src/main/java/com/google/devtools/build/docgen/SymbolFamilies.java @@ -58,6 +58,7 @@ import com.google.devtools.build.skydoc.fakebuildapi.java.FakeJavaProtoCommon; import com.google.devtools.build.skydoc.fakebuildapi.platform.FakePlatformCommon; import com.google.devtools.build.skydoc.fakebuildapi.python.FakePyInfo.FakePyInfoProvider; +import com.google.devtools.build.skydoc.fakebuildapi.python.FakePyRuntimeInfo.FakePyRuntimeInfoProvider; import com.google.devtools.build.skydoc.fakebuildapi.repository.FakeRepositoryModule; import com.google.devtools.build.skydoc.fakebuildapi.test.FakeAnalysisFailureInfoProvider; import com.google.devtools.build.skydoc.fakebuildapi.test.FakeAnalysisTestResultInfoProvider; @@ -194,7 +195,8 @@ private Map collectBzlGlobals() { new FakeJavaProtoCommon(), new FakeJavaCcLinkParamsProvider.Provider()); PlatformBootstrap platformBootstrap = new PlatformBootstrap(new FakePlatformCommon()); - PyBootstrap pyBootstrap = new PyBootstrap(new FakePyInfoProvider()); + PyBootstrap pyBootstrap = + new PyBootstrap(new FakePyInfoProvider(), new FakePyRuntimeInfoProvider()); RepositoryBootstrap repositoryBootstrap = new RepositoryBootstrap(new FakeRepositoryModule()); TestingBootstrap testingBootstrap = new TestingBootstrap( diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index 7eeea00e01b6e0..a05ecf9dd28d05 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -1143,7 +1143,6 @@ java_library( "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:crosstool_config_java_proto", "//src/main/protobuf:extra_actions_base_java_proto", - "//third_party:auto_value", "//third_party:guava", "//third_party:jsr305", "//third_party/protobuf:protobuf_java", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java index fa3a7037c9aa11..e081dd51333873 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java @@ -83,6 +83,7 @@ import com.google.devtools.build.lib.rules.proto.ProtoInfo; import com.google.devtools.build.lib.rules.proto.ProtoLangToolchainRule; import com.google.devtools.build.lib.rules.python.PyInfo; +import com.google.devtools.build.lib.rules.python.PyRuntimeInfo; import com.google.devtools.build.lib.rules.python.PythonConfigurationLoader; import com.google.devtools.build.lib.rules.repository.CoreWorkspaceRules; import com.google.devtools.build.lib.rules.repository.NewLocalRepositoryRule; @@ -348,7 +349,7 @@ public void init(ConfiguredRuleClassProvider.Builder builder) { builder.addRuleDefinition(new BazelPyTestRule()); builder.addRuleDefinition(new BazelPyRuntimeRule()); - builder.addSkylarkBootstrap(new PyBootstrap(PyInfo.PROVIDER)); + builder.addSkylarkBootstrap(new PyBootstrap(PyInfo.PROVIDER, PyRuntimeInfo.PROVIDER)); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java index 48a766e3523e7e..361042327a39c4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java @@ -130,8 +130,8 @@ executable Python rule (py_binary or py_test). .value(PythonVersion.DEFAULT_SRCS_VALUE.toString()) .allowedValues(new AllowedValueSet(PythonVersion.SRCS_STRINGS))) // TODO(brandjon): Consider adding to py_interpreter a .mandatoryNativeProviders() of - // BazelPyRuntimeProvider. (Add a test case to PythonConfigurationTest for violations - // of this requirement.) + // PyRuntimeInfoProvider. (Add a test case to PythonConfigurationTest for violations of + // this requirement.) Probably moot now that this is going to be replaced by toolchains. .add(attr(":py_interpreter", LABEL).value(PY_INTERPRETER)) // do not depend on lib2to3:2to3 rule, because it creates circular dependencies // 2to3 is itself written in Python and depends on many libraries. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java index a7e0a28e96bfa5..e20438ac35ba53 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java @@ -44,7 +44,7 @@ import com.google.devtools.build.lib.rules.cpp.CcInfo; import com.google.devtools.build.lib.rules.python.PyCcLinkParamsProvider; import com.google.devtools.build.lib.rules.python.PyCommon; -import com.google.devtools.build.lib.rules.python.PyRuntimeProvider; +import com.google.devtools.build.lib.rules.python.PyRuntimeInfo; import com.google.devtools.build.lib.rules.python.PythonConfiguration; import com.google.devtools.build.lib.rules.python.PythonSemantics; import com.google.devtools.build.lib.util.FileTypeSet; @@ -314,9 +314,9 @@ private static void createPythonZipAction( } private static void addRuntime(RuleContext ruleContext, Runfiles.Builder builder) { - PyRuntimeProvider provider = - ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeProvider.class); - if (provider != null && provider.isHermetic()) { + PyRuntimeInfo provider = + ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeInfo.PROVIDER); + if (provider != null && provider.isInBuild()) { builder.addArtifact(provider.getInterpreter()); // WARNING: we are adding the all Python runtime files here, // and it would fail if the filenames of them contain spaces. @@ -334,12 +334,12 @@ private static String getPythonBinary( String pythonBinary; - PyRuntimeProvider provider = - ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeProvider.class); + PyRuntimeInfo provider = + ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeInfo.PROVIDER); if (provider != null) { // make use of py_runtime defined by --python_top - if (!provider.isHermetic()) { + if (!provider.isInBuild()) { // absolute Python path in py_runtime pythonBinary = provider.getInterpreterPath().getPathString(); } else { diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntime.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntime.java index 74627e1357bab4..c5a904ef210d87 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntime.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntime.java @@ -57,15 +57,15 @@ public ConfiguredTarget create(RuleContext ruleContext) throws ActionConflictExc return null; } - PyRuntimeProvider provider = + PyRuntimeInfo provider = hermetic - ? PyRuntimeProvider.create(files, interpreter, /*interpreterPath=*/ null) - : PyRuntimeProvider.create(/*files=*/ null, /*interpreter=*/ null, interpreterPath); + ? PyRuntimeInfo.createForInBuildRuntime(interpreter, files) + : PyRuntimeInfo.createForPlatformRuntime(interpreterPath); return new RuleConfiguredTargetBuilder(ruleContext) .setFilesToBuild(files) .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY) - .addProvider(PyRuntimeProvider.class, provider) + .addNativeDeclaredProvider(provider) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfo.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfo.java new file mode 100644 index 00000000000000..4dab2b311f050c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfo.java @@ -0,0 +1,190 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.python; + +import static com.google.devtools.build.lib.syntax.Runtime.NONE; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.BuiltinProvider; +import com.google.devtools.build.lib.packages.Info; +import com.google.devtools.build.lib.skylarkbuildapi.python.PyRuntimeInfoApi; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; +import com.google.devtools.build.lib.vfs.PathFragment; +import java.util.Objects; +import javax.annotation.Nullable; + +/** + * Instance of the provider type that describes Python runtimes. + * + *

Invariant: Exactly one of {@link #interpreterPath} and {@link #interpreter} is non-null. The + * former corresponds to a platform runtime, and the latter to an in-build runtime; these two cases + * are mutually exclusive. In addition, {@link #files} is non-null if and only if {@link + * #interpreter} is non-null; in other words files are only used for in-build runtimes. These + * invariants mirror the user-visible API on {@link PyRuntimeInfoApi} except that {@code None} is + * replaced by null. + */ +public class PyRuntimeInfo extends Info implements PyRuntimeInfoApi { + + /** The Starlark-accessible top-level builtin name for this provider type. */ + public static final String STARLARK_NAME = "PyRuntimeInfo"; + + /** The singular {@code PyRuntimeInfo} provider type object. */ + public static final PyRuntimeInfoProvider PROVIDER = new PyRuntimeInfoProvider(); + + @Nullable private final PathFragment interpreterPath; + @Nullable private final Artifact interpreter; + @Nullable private final SkylarkNestedSet files; + + private PyRuntimeInfo( + @Nullable Location location, + @Nullable PathFragment interpreterPath, + @Nullable Artifact interpreter, + @Nullable SkylarkNestedSet files) { + super(PROVIDER, location); + Preconditions.checkArgument((interpreterPath == null) != (interpreter == null)); + Preconditions.checkArgument((interpreter == null) == (files == null)); + if (files != null) { + // Work around #7266 by special-casing the empty set in the type check. + Preconditions.checkArgument( + files.isEmpty() || files.getContentType().canBeCastTo(Artifact.class)); + } + this.interpreterPath = interpreterPath; + this.interpreter = interpreter; + this.files = files; + } + + /** Constructs an instance from native rule logic (built-in location) for an in-build runtime. */ + public static PyRuntimeInfo createForInBuildRuntime( + Artifact interpreter, NestedSet files) { + return new PyRuntimeInfo( + /*location=*/ null, + /*interpreterPath=*/ null, + interpreter, + SkylarkNestedSet.of(Artifact.class, files)); + } + + /** Constructs an instance from native rule logic (built-in location) for a platform runtime. */ + public static PyRuntimeInfo createForPlatformRuntime(PathFragment interpreterPath) { + return new PyRuntimeInfo( + /*location=*/ null, interpreterPath, /*interpreter=*/ null, /*files=*/ null); + } + + @Override + public boolean equals(Object other) { + // PyRuntimeInfo implements value equality, but note that it contains identity-equality fields + // (depsets), so you generally shouldn't rely on equality comparisons. + if (!(other instanceof PyRuntimeInfo)) { + return false; + } + PyRuntimeInfo otherInfo = (PyRuntimeInfo) other; + return (this.interpreterPath.equals(otherInfo.interpreterPath) + && this.interpreter.equals(otherInfo.interpreter) + && this.files.equals(otherInfo.files)); + } + + @Override + public int hashCode() { + return Objects.hash(PyRuntimeInfo.class, interpreterPath, interpreter, files); + } + + /** + * Returns true if this is an in-build runtime as opposed to a platform runtime -- that is, if + * this refers to a target within the build as opposed to a path to a system interpreter. + * + *

{@link #getInterpreter} and {@link #getFiles} are non-null if and only if this is an + * in-build runtime, whereas {@link #getInterpreterPath} is non-null if and only if this is a + * platform runtime. + * + *

Note: It is still possible for an in-build runtime to reference the system interpreter, as + * in the case where it is a wrapper script. + */ + public boolean isInBuild() { + return getInterpreter() != null; + } + + @Nullable + public PathFragment getInterpreterPath() { + return interpreterPath; + } + + @Override + @Nullable + public String getInterpreterPathString() { + return interpreterPath == null ? null : interpreterPath.getPathString(); + } + + @Override + @Nullable + public Artifact getInterpreter() { + return interpreter; + } + + @Nullable + public NestedSet getFiles() { + return files == null ? null : files.getSet(Artifact.class); + } + + @Override + @Nullable + public SkylarkNestedSet getFilesForStarlark() { + return files; + } + + /** The class of the {@code PyRuntimeInfo} provider type. */ + public static class PyRuntimeInfoProvider extends BuiltinProvider + implements PyRuntimeInfoApi.PyRuntimeInfoProviderApi { + + private PyRuntimeInfoProvider() { + super(STARLARK_NAME, PyRuntimeInfo.class); + } + + @Override + public PyRuntimeInfo constructor( + Object interpreterPathUncast, Object interpreterUncast, Object filesUncast, Location loc) + throws EvalException { + String interpreterPath = + interpreterPathUncast == NONE ? null : (String) interpreterPathUncast; + Artifact interpreter = interpreterUncast == NONE ? null : (Artifact) interpreterUncast; + SkylarkNestedSet files = filesUncast == NONE ? null : (SkylarkNestedSet) filesUncast; + + if ((interpreter == null) == (interpreterPath == null)) { + throw new EvalException( + loc, + "exactly one of the 'interpreter' or 'interpreter_path' arguments must be specified"); + } + boolean isInBuildRuntime = interpreter != null; + if (!isInBuildRuntime && files != null) { + throw new EvalException(loc, "cannot specify 'files' if 'interpreter_path' is given"); + } + + if (isInBuildRuntime) { + if (files == null) { + files = + SkylarkNestedSet.of(Artifact.class, NestedSetBuilder.emptySet(Order.STABLE_ORDER)); + } + return new PyRuntimeInfo(loc, /*interpreterPath=*/ null, interpreter, files); + } else { + return new PyRuntimeInfo( + loc, PathFragment.create(interpreterPath), /*interpreter=*/ null, /*files=*/ null); + } + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeProvider.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeProvider.java deleted file mode 100644 index c3230c0ee29a59..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PyRuntimeProvider.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.python; - -import com.google.auto.value.AutoValue; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; -import com.google.devtools.build.lib.vfs.PathFragment; -import javax.annotation.Nullable; - -/** Information about the Python runtime used by the py_* rules. */ -@AutoValue -@Immutable -public abstract class PyRuntimeProvider implements TransitiveInfoProvider { - - public static PyRuntimeProvider create( - @Nullable NestedSet files, - @Nullable Artifact interpreter, - @Nullable PathFragment interpreterPath) { - return new AutoValue_PyRuntimeProvider(files, interpreter, interpreterPath); - } - - /** - * Returns whether this runtime is hermetic, i.e. represents an in-build interpreter as opposed to - * a system interpreter. - * - *

Hermetic runtimes have non-null values for {@link #getInterpreter} and {@link #getFiles}, - * while non-hermetic runtimes have non-null {@link #getInterpreterPath}. - * - *

Note: Despite the name, it is still possible for a hermetic runtime to reference in-build - * files that have non-hermetic behavior. For example, {@link #getInterpreter} could reference a - * checked-in wrapper script that calls the system interpreter at execution time. - */ - public boolean isHermetic() { - return getInterpreter() != null; - } - - @Nullable - public abstract NestedSet getFiles(); - - @Nullable - public abstract Artifact getInterpreter(); - - @Nullable - public abstract PathFragment getInterpreterPath(); -} diff --git a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/BUILD b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/BUILD index 220dcb18839686..c49c44d83e56c3 100644 --- a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/BUILD @@ -24,5 +24,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib:syntax", "//src/main/java/com/google/devtools/build/lib/skylarkbuildapi", "//third_party:guava", + "//third_party:jsr305", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyBootstrap.java b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyBootstrap.java index 5c182de58b4e53..dd7fb13ad84a50 100644 --- a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyBootstrap.java +++ b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyBootstrap.java @@ -17,18 +17,23 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.skylarkbuildapi.Bootstrap; import com.google.devtools.build.lib.skylarkbuildapi.python.PyInfoApi.PyInfoProviderApi; +import com.google.devtools.build.lib.skylarkbuildapi.python.PyRuntimeInfoApi.PyRuntimeInfoProviderApi; /** {@link Bootstrap} for Starlark objects related to the Python rules. */ public class PyBootstrap implements Bootstrap { private final PyInfoProviderApi pyInfoProviderApi; + private final PyRuntimeInfoProviderApi pyRuntimeInfoProviderApi; - public PyBootstrap(PyInfoProviderApi pyInfoProviderApi) { + public PyBootstrap( + PyInfoProviderApi pyInfoProviderApi, PyRuntimeInfoProviderApi pyRuntimeInfoProviderApi) { this.pyInfoProviderApi = pyInfoProviderApi; + this.pyRuntimeInfoProviderApi = pyRuntimeInfoProviderApi; } @Override public void addBindingsToBuilder(ImmutableMap.Builder builder) { builder.put("PyInfo", pyInfoProviderApi); + builder.put("PyRuntimeInfo", pyRuntimeInfoProviderApi); } } diff --git a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyRuntimeInfoApi.java b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyRuntimeInfoApi.java new file mode 100644 index 00000000000000..ebd79d1911cdf9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/python/PyRuntimeInfoApi.java @@ -0,0 +1,130 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skylarkbuildapi.python; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.skylarkbuildapi.FileApi; +import com.google.devtools.build.lib.skylarkbuildapi.ProviderApi; +import com.google.devtools.build.lib.skylarkinterface.Param; +import com.google.devtools.build.lib.skylarkinterface.SkylarkCallable; +import com.google.devtools.build.lib.skylarkinterface.SkylarkConstructor; +import com.google.devtools.build.lib.skylarkinterface.SkylarkModule; +import com.google.devtools.build.lib.skylarkinterface.SkylarkModuleCategory; +import com.google.devtools.build.lib.skylarkinterface.SkylarkValue; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; +import javax.annotation.Nullable; + +/** Provider instance for {@code py_runtime}. */ +@SkylarkModule( + name = "PyRuntimeInfo", + doc = + "Contains information about a Python runtime, as returned by the py_runtime" + + "rule." + + "" + + "

A Python runtime describes either a platform runtime or an in-build " + + "runtime. A platform runtime accesses a system-installed interpreter at a known " + + "path, whereas an in-build runtime points to a File that acts as the " + + "interpreter. In both cases, an \"interpreter\" is really any executable binary or " + + "wrapper script that is capable of running a Python script passed on the command " + + "line, following the same conventions as the standard CPython interpreter.", + category = SkylarkModuleCategory.PROVIDER) +public interface PyRuntimeInfoApi extends SkylarkValue { + + @SkylarkCallable( + name = "interpreter_path", + structField = true, + allowReturnNones = true, + doc = + "If this is a platform runtime, this field is the absolute filesystem path to the " + + "interpreter on the target platform. Otherwise, this is None.") + @Nullable + String getInterpreterPathString(); + + @SkylarkCallable( + name = "interpreter", + structField = true, + allowReturnNones = true, + doc = + "If this is an in-build runtime, this field is a File representing the " + + "interpreter. Otherwise, this is None. Note that an in-build runtime " + + "can use either a prebuilt, checked-in interpreter or an interpreter built from " + + "source.") + @Nullable + FileT getInterpreter(); + + @SkylarkCallable( + name = "files", + structField = true, + allowReturnNones = true, + doc = + "If this is an in-build runtime, this field is a depset of File" + + "s that need to be added to the runfiles of an executable target that uses " + + "this runtime (in particular, files needed by interpreter). The value " + + "of interpreter need not be included in this " + + "field. If this is a platform runtime then this field is None.") + @Nullable + SkylarkNestedSet getFilesForStarlark(); + + /** Provider type for {@link PyRuntimeInfoApi} objects. */ + @SkylarkModule(name = "Provider", documented = false, doc = "") + interface PyRuntimeInfoProviderApi extends ProviderApi { + + @SkylarkCallable( + name = "PyRuntimeInfo", + doc = "The PyRuntimeInfo constructor.", + parameters = { + @Param( + name = "interpreter_path", + type = String.class, + noneable = true, + positional = false, + named = true, + defaultValue = "None", + doc = + "The value for the new object's interpreter_path field. Do not give " + + "a value for this argument if you pass in interpreter."), + @Param( + name = "interpreter", + type = FileApi.class, + noneable = true, + positional = false, + named = true, + defaultValue = "None", + doc = + "The value for the new object's interpreter field. Do not give " + + "a value for this argument if you pass in interpreter_path."), + @Param( + name = "files", + type = SkylarkNestedSet.class, + generic1 = FileApi.class, + noneable = true, + positional = false, + named = true, + defaultValue = "None", + doc = + "The value for the new object's files field. Do not give a value " + + "for this argument if you pass in interpreter_path. If " + + "interpreter is given and this argument is None, " + + "files becomes an empty depset instead."), + }, + selfCall = true, + useLocation = true) + @SkylarkConstructor(objectType = PyRuntimeInfoApi.class, receiverNameForDoc = "PyRuntimeInfo") + PyRuntimeInfoApi constructor( + Object interpreterPathUncast, Object interpreterUncast, Object filesUncast, Location loc) + throws EvalException; + } +} diff --git a/src/main/java/com/google/devtools/build/skydoc/SkydocMain.java b/src/main/java/com/google/devtools/build/skydoc/SkydocMain.java index 2ba2c799fcb29d..bf7a026ea4f0a7 100644 --- a/src/main/java/com/google/devtools/build/skydoc/SkydocMain.java +++ b/src/main/java/com/google/devtools/build/skydoc/SkydocMain.java @@ -92,6 +92,7 @@ import com.google.devtools.build.skydoc.fakebuildapi.platform.FakePlatformCommon; import com.google.devtools.build.skydoc.fakebuildapi.proto.FakeProtoInfoApiProvider; import com.google.devtools.build.skydoc.fakebuildapi.python.FakePyInfo.FakePyInfoProvider; +import com.google.devtools.build.skydoc.fakebuildapi.python.FakePyRuntimeInfo.FakePyRuntimeInfoProvider; import com.google.devtools.build.skydoc.fakebuildapi.repository.FakeRepositoryModule; import com.google.devtools.build.skydoc.fakebuildapi.test.FakeAnalysisFailureInfoProvider; import com.google.devtools.build.skydoc.fakebuildapi.test.FakeAnalysisTestResultInfoProvider; @@ -510,7 +511,8 @@ private static GlobalFrame globalFrame(List ruleInfoList, new FakeJavaCcLinkParamsProvider.Provider()); PlatformBootstrap platformBootstrap = new PlatformBootstrap(new FakePlatformCommon()); ProtoBootstrap protoBootstrap = new ProtoBootstrap(new FakeProtoInfoApiProvider()); - PyBootstrap pyBootstrap = new PyBootstrap(new FakePyInfoProvider()); + PyBootstrap pyBootstrap = + new PyBootstrap(new FakePyInfoProvider(), new FakePyRuntimeInfoProvider()); RepositoryBootstrap repositoryBootstrap = new RepositoryBootstrap(new FakeRepositoryModule()); TestingBootstrap testingBootstrap = new TestingBootstrap( diff --git a/src/main/java/com/google/devtools/build/skydoc/fakebuildapi/python/FakePyRuntimeInfo.java b/src/main/java/com/google/devtools/build/skydoc/fakebuildapi/python/FakePyRuntimeInfo.java new file mode 100644 index 00000000000000..8ab56ab97fd8e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skydoc/fakebuildapi/python/FakePyRuntimeInfo.java @@ -0,0 +1,58 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.skydoc.fakebuildapi.python; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.skylarkbuildapi.FileApi; +import com.google.devtools.build.lib.skylarkbuildapi.python.PyRuntimeInfoApi; +import com.google.devtools.build.lib.skylarkinterface.SkylarkPrinter; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +/** Fake implementation of {@link PyRuntimeInfoApi}. */ +public class FakePyRuntimeInfo implements PyRuntimeInfoApi { + + @Override + public String getInterpreterPathString() { + return null; + } + + @Override + public FileApi getInterpreter() { + return null; + } + + @Override + public SkylarkNestedSet getFilesForStarlark() { + return null; + } + + @Override + public void repr(SkylarkPrinter printer) {} + + /** Fake implementation of {@link PyRuntimeInfoProviderApi}. */ + public static class FakePyRuntimeInfoProvider implements PyRuntimeInfoProviderApi { + + @Override + public PyRuntimeInfoApi constructor( + Object interpreterPathUncast, Object interpreterUncast, Object filesUncast, Location loc) + throws EvalException { + return new FakePyRuntimeInfo(); + } + + @Override + public void repr(SkylarkPrinter printer) {} + } +} diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java index a4cab68135baa0..698a9bad7c3c2c 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java @@ -51,4 +51,11 @@ public void setup(MockToolsConfig config) throws IOException { "exports_files(['precompile.py'])", "sh_binary(name='2to3', srcs=['2to3.sh'])"); } + + @Override + public String createPythonTopEntryPoint(MockToolsConfig config, String pyRuntimeLabel) + throws IOException { + // Under BazelPythonSemantics, we can simply set --python_top to be the py_runtime target. + return pyRuntimeLabel; + } } diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/MockPythonSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/MockPythonSupport.java index 4852df34bdad88..b08677847fdfd3 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/MockPythonSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/MockPythonSupport.java @@ -23,4 +23,12 @@ public abstract class MockPythonSupport { /** Setup the support for building Python. */ public abstract void setup(MockToolsConfig config) throws IOException; + + /** + * Setup support for, and return the string label of, a target that can be passed to {@code + * --python_top} that causes the {@code py_runtime} consumed by Python rules to be the given + * {@code pyRuntimeLabel}. + */ + public abstract String createPythonTopEntryPoint(MockToolsConfig config, String pyRuntimeLabel) + throws IOException; } diff --git a/src/test/java/com/google/devtools/build/lib/rules/python/BUILD b/src/test/java/com/google/devtools/build/lib/rules/python/BUILD index 3df2ae76c4b48d..bd56e2eb12c644 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/python/BUILD +++ b/src/test/java/com/google/devtools/build/lib/rules/python/BUILD @@ -16,6 +16,7 @@ test_suite( ":PyInfoTest", ":PyLibraryConfiguredTargetTest", ":PyProviderUtilsTest", + ":PyRuntimeInfoTest", ":PyRuntimeTest", ":PyStructUtilsTest", ":PyTestConfiguredTargetTest", @@ -214,7 +215,21 @@ java_test( "//src/main/java/com/google/devtools/build/lib/collect/nestedset", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/test/java/com/google/devtools/build/lib:analysis_testutil", - "//src/test/java/com/google/devtools/build/lib:testutil", + "//src/test/java/com/google/devtools/build/lib/skylark:testutil", + "//third_party:junit4", + "//third_party:truth", + ], +) + +java_test( + name = "PyRuntimeInfoTest", + srcs = ["PyRuntimeInfoTest.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib:events", + "//src/main/java/com/google/devtools/build/lib:python-rules", + "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/collect/nestedset", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/test/java/com/google/devtools/build/lib/skylark:testutil", "//third_party:junit4", "//third_party:truth", diff --git a/src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfoTest.java b/src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfoTest.java new file mode 100644 index 00000000000000..7da0c00d77ddfe --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeInfoTest.java @@ -0,0 +1,151 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.python; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.skylark.util.SkylarkTestCase; +import com.google.devtools.build.lib.vfs.PathFragment; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link PyRuntimeInfo}. */ +@RunWith(JUnit4.class) +public class PyRuntimeInfoTest extends SkylarkTestCase { + + private Artifact dummyInterpreter; + private Artifact dummyFile; + + @Before + public void setUp() throws Exception { + dummyInterpreter = getSourceArtifact("dummy_interpreter"); + dummyFile = getSourceArtifact("dummy_file"); + update("PyRuntimeInfo", PyRuntimeInfo.PROVIDER); + update("dummy_interpreter", dummyInterpreter); + update("dummy_file", dummyFile); + } + + /** We need this because {@code NestedSet}s don't have value equality. */ + private static void assertHasOrderAndContainsExactly( + NestedSet set, Order order, Object... values) { + assertThat(set.getOrder()).isEqualTo(order); + assertThat(set).containsExactly(values); + } + + @Test + public void factoryMethod_InBuildRuntime() { + NestedSet files = NestedSetBuilder.create(Order.STABLE_ORDER, dummyFile); + PyRuntimeInfo inBuildRuntime = PyRuntimeInfo.createForInBuildRuntime(dummyInterpreter, files); + + assertThat(inBuildRuntime.getCreationLoc()).isEqualTo(Location.BUILTIN); + assertThat(inBuildRuntime.getInterpreterPath()).isNull(); + assertThat(inBuildRuntime.getInterpreterPathString()).isNull(); + assertThat(inBuildRuntime.getInterpreter()).isEqualTo(dummyInterpreter); + assertThat(inBuildRuntime.getFiles()).isEqualTo(files); + assertThat(inBuildRuntime.getFilesForStarlark().getSet(Artifact.class)).isEqualTo(files); + } + + @Test + public void factoryMethod_PlatformRuntime() { + PathFragment path = PathFragment.create("/system/interpreter"); + PyRuntimeInfo platformRuntime = PyRuntimeInfo.createForPlatformRuntime(path); + + assertThat(platformRuntime.getCreationLoc()).isEqualTo(Location.BUILTIN); + assertThat(platformRuntime.getInterpreterPath()).isEqualTo(path); + assertThat(platformRuntime.getInterpreterPathString()).isEqualTo("/system/interpreter"); + assertThat(platformRuntime.getInterpreter()).isNull(); + assertThat(platformRuntime.getFiles()).isNull(); + assertThat(platformRuntime.getFilesForStarlark()).isNull(); + } + + @Test + public void starlarkConstructor_InBuildRuntime() throws Exception { + eval( + "info = PyRuntimeInfo(", + " interpreter = dummy_interpreter,", + " files = depset([dummy_file]),", + ")"); + PyRuntimeInfo info = (PyRuntimeInfo) lookup("info"); + assertThat(info.getCreationLoc().getStartOffset()).isEqualTo(7); + assertThat(info.getInterpreterPath()).isNull(); + assertThat(info.getInterpreter()).isEqualTo(dummyInterpreter); + assertHasOrderAndContainsExactly(info.getFiles(), Order.STABLE_ORDER, dummyFile); + } + + @Test + public void starlarkConstructor_PlatformRuntime() throws Exception { + eval( + "info = PyRuntimeInfo(", // + " interpreter_path = '/system/interpreter',", + ")"); + PyRuntimeInfo info = (PyRuntimeInfo) lookup("info"); + assertThat(info.getCreationLoc().getStartOffset()).isEqualTo(7); + assertThat(info.getInterpreterPath()).isEqualTo(PathFragment.create("/system/interpreter")); + assertThat(info.getInterpreter()).isNull(); + assertThat(info.getFiles()).isNull(); + } + + @Test + public void starlarkConstructor_FilesDefaultsToEmpty() throws Exception { + eval( + "info = PyRuntimeInfo(", // + " interpreter = dummy_interpreter,", + ")"); + PyRuntimeInfo info = (PyRuntimeInfo) lookup("info"); + assertHasOrderAndContainsExactly(info.getFiles(), Order.STABLE_ORDER); + } + + @Test + public void starlarkConstructorErrors_InBuildXorPlatform() throws Exception { + checkEvalErrorContains( + "exactly one of the 'interpreter' or 'interpreter_path' arguments must be specified", + "PyRuntimeInfo()"); + checkEvalErrorContains( + "exactly one of the 'interpreter' or 'interpreter_path' arguments must be specified", + "PyRuntimeInfo(", + " interpreter_path = '/system/interpreter',", + " interpreter = dummy_interpreter,", + ")"); + } + + @Test + public void starlarkConstructorErrors_Files() throws Exception { + checkEvalErrorContains( + "expected value of type 'depset of Files or NoneType' for parameter 'files'", + "PyRuntimeInfo(", + " interpreter = dummy_interpreter,", + " files = 'abc',", + ")"); + checkEvalErrorContains( + "expected value of type 'depset of Files or NoneType' for parameter 'files'", + "PyRuntimeInfo(", + " interpreter = dummy_interpreter,", + " files = depset(['abc']),", + ")"); + checkEvalErrorContains( + "cannot specify 'files' if 'interpreter_path' is given", + "PyRuntimeInfo(", + " interpreter_path = '/system/interpreter',", + " files = depset([dummy_file]),", + ")"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeTest.java b/src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeTest.java index bf3b51526e03fb..3c528f509a14a0 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/python/PyRuntimeTest.java @@ -41,10 +41,9 @@ public void hermeticRuntime() throws Exception { " files = [':myfile'],", " interpreter = ':myinterpreter',", ")"); - PyRuntimeProvider info = - getConfiguredTarget("//pkg:myruntime").getProvider(PyRuntimeProvider.class); + PyRuntimeInfo info = getConfiguredTarget("//pkg:myruntime").get(PyRuntimeInfo.PROVIDER); - assertThat(info.isHermetic()).isTrue(); + assertThat(info.isInBuild()).isTrue(); assertThat(info.getInterpreterPath()).isNull(); assertThat(info.getInterpreter().getExecPathString()).isEqualTo("pkg/myinterpreter"); assertThat(ActionsTestUtil.baseArtifactNames(info.getFiles())).containsExactly("myfile"); @@ -58,10 +57,9 @@ public void nonhermeticRuntime() throws Exception { " name = 'myruntime',", " interpreter_path = '/system/interpreter',", ")"); - PyRuntimeProvider info = - getConfiguredTarget("//pkg:myruntime").getProvider(PyRuntimeProvider.class); + PyRuntimeInfo info = getConfiguredTarget("//pkg:myruntime").get(PyRuntimeInfo.PROVIDER); - assertThat(info.isHermetic()).isFalse(); + assertThat(info.isInBuild()).isFalse(); assertThat(info.getInterpreterPath().getPathString()).isEqualTo("/system/interpreter"); assertThat(info.getInterpreter()).isNull(); assertThat(info.getFiles()).isNull(); diff --git a/src/test/java/com/google/devtools/build/lib/rules/python/PythonStarlarkApiTest.java b/src/test/java/com/google/devtools/build/lib/rules/python/PythonStarlarkApiTest.java index 32fa86b293a3c4..c9cf589ca02aac 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/python/PythonStarlarkApiTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/python/PythonStarlarkApiTest.java @@ -131,4 +131,48 @@ public void librarySandwich() throws Exception { assertThat(modernInfo.getHasPy2OnlySources()).isTrue(); assertThat(modernInfo.getHasPy3OnlySources()).isTrue(); } + + @Test + public void runtimeSandwich() throws Exception { + scratch.file( + "pkg/rules.bzl", + "def _userruntime_impl(ctx):", + " info = ctx.attr.runtime[PyRuntimeInfo]", + " return [PyRuntimeInfo(", + " interpreter = ctx.file.interpreter,", + " files = depset(direct = ctx.files.files, transitive=[info.files]))]", + "", + "userruntime = rule(", + " implementation = _userruntime_impl,", + " attrs = {", + " 'runtime': attr.label(),", + " 'interpreter': attr.label(allow_single_file=True),", + " 'files': attr.label_list(allow_files=True),", + " },", + ")"); + scratch.file( + "pkg/BUILD", + "load(':rules.bzl', 'userruntime')", + "py_runtime(", + " name = 'pyruntime',", + " interpreter = ':intr',", + " files = ['data.txt'],", + ")", + "userruntime(", + " name = 'userruntime',", + " runtime = ':pyruntime',", + " interpreter = ':userintr',", + " files = ['userdata.txt'],", + ")", + "py_binary(", + " name = 'pybin',", + " srcs = ['pybin.py'],", + ")"); + String pythonTopLabel = + analysisMock.pySupport().createPythonTopEntryPoint(mockToolsConfig, "//pkg:userruntime"); + useConfiguration("--python_top=" + pythonTopLabel); + ConfiguredTarget target = getConfiguredTarget("//pkg:pybin"); + assertThat(collectRunfiles(target)) + .containsAllOf(getSourceArtifact("pkg/data.txt"), getSourceArtifact("pkg/userdata.txt")); + } }