Skip to content

Commit

Permalink
Expose PyRuntimeProvider to Starlark as PyRuntimeInfo
Browse files Browse the repository at this point in the history
This creates a top-level symbol `PyRuntimeInfo` representing the provider created by the `py_runtime` rule and consumed by `py_binary` and `py_test`.

PyRuntimeProvider is refactored to not implement TransitiveInfoProvider, and to now be split into separate classes for the provider instance vs provider type. Fakes are created for Stardoc and friends.

Tests are added for the API of `PyRuntimeInfo` and for the ability to sandwich a Starlark rule between a py_runtime and its consuming py_binary.

As drive-by cleanups, the provider is now instantiated in Java code via factory methods instead of its constructor, and the isHermetic method is renamed to isInBuild to be consistent with the terminology in the design doc for #7375. Rule documentation is also updated.

Work toward #7375.

RELNOTES: (Python rules) PyRuntimeInfo is exposed to Starlark, making it possible for Starlark rules to depend on or imitate `py_runtime`. The `files` attribute of `py_runtime` is no longer mandatory.
PiperOrigin-RevId: 235224774
  • Loading branch information
brandjon authored and copybara-github committed Feb 22, 2019
1 parent 64e833c commit 4e49e28
Show file tree
Hide file tree
Showing 19 changed files with 636 additions and 85 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import com.google.devtools.build.skydoc.fakebuildapi.java.FakeJavaProtoCommon;
import com.google.devtools.build.skydoc.fakebuildapi.platform.FakePlatformCommon;
import com.google.devtools.build.skydoc.fakebuildapi.python.FakePyInfo.FakePyInfoProvider;
import com.google.devtools.build.skydoc.fakebuildapi.python.FakePyRuntimeInfo.FakePyRuntimeInfoProvider;
import com.google.devtools.build.skydoc.fakebuildapi.repository.FakeRepositoryModule;
import com.google.devtools.build.skydoc.fakebuildapi.test.FakeAnalysisFailureInfoProvider;
import com.google.devtools.build.skydoc.fakebuildapi.test.FakeAnalysisTestResultInfoProvider;
Expand Down Expand Up @@ -194,7 +195,8 @@ private Map<String, Object> collectBzlGlobals() {
new FakeJavaProtoCommon(),
new FakeJavaCcLinkParamsProvider.Provider());
PlatformBootstrap platformBootstrap = new PlatformBootstrap(new FakePlatformCommon());
PyBootstrap pyBootstrap = new PyBootstrap(new FakePyInfoProvider());
PyBootstrap pyBootstrap =
new PyBootstrap(new FakePyInfoProvider(), new FakePyRuntimeInfoProvider());
RepositoryBootstrap repositoryBootstrap = new RepositoryBootstrap(new FakeRepositoryModule());
TestingBootstrap testingBootstrap =
new TestingBootstrap(
Expand Down
1 change: 0 additions & 1 deletion src/main/java/com/google/devtools/build/lib/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -1143,7 +1143,6 @@ java_library(
"//src/main/java/com/google/devtools/common/options",
"//src/main/protobuf:crosstool_config_java_proto",
"//src/main/protobuf:extra_actions_base_java_proto",
"//third_party:auto_value",
"//third_party:guava",
"//third_party:jsr305",
"//third_party/protobuf:protobuf_java",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@
import com.google.devtools.build.lib.rules.proto.ProtoInfo;
import com.google.devtools.build.lib.rules.proto.ProtoLangToolchainRule;
import com.google.devtools.build.lib.rules.python.PyInfo;
import com.google.devtools.build.lib.rules.python.PyRuntimeInfo;
import com.google.devtools.build.lib.rules.python.PythonConfigurationLoader;
import com.google.devtools.build.lib.rules.repository.CoreWorkspaceRules;
import com.google.devtools.build.lib.rules.repository.NewLocalRepositoryRule;
Expand Down Expand Up @@ -348,7 +349,7 @@ public void init(ConfiguredRuleClassProvider.Builder builder) {
builder.addRuleDefinition(new BazelPyTestRule());
builder.addRuleDefinition(new BazelPyRuntimeRule());

builder.addSkylarkBootstrap(new PyBootstrap(PyInfo.PROVIDER));
builder.addSkylarkBootstrap(new PyBootstrap(PyInfo.PROVIDER, PyRuntimeInfo.PROVIDER));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@ executable Python rule (<code>py_binary</code> or <code>py_test</code>).
.value(PythonVersion.DEFAULT_SRCS_VALUE.toString())
.allowedValues(new AllowedValueSet(PythonVersion.SRCS_STRINGS)))
// TODO(brandjon): Consider adding to py_interpreter a .mandatoryNativeProviders() of
// BazelPyRuntimeProvider. (Add a test case to PythonConfigurationTest for violations
// of this requirement.)
// PyRuntimeInfoProvider. (Add a test case to PythonConfigurationTest for violations of
// this requirement.) Probably moot now that this is going to be replaced by toolchains.
.add(attr(":py_interpreter", LABEL).value(PY_INTERPRETER))
// do not depend on lib2to3:2to3 rule, because it creates circular dependencies
// 2to3 is itself written in Python and depends on many libraries.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import com.google.devtools.build.lib.rules.cpp.CcInfo;
import com.google.devtools.build.lib.rules.python.PyCcLinkParamsProvider;
import com.google.devtools.build.lib.rules.python.PyCommon;
import com.google.devtools.build.lib.rules.python.PyRuntimeProvider;
import com.google.devtools.build.lib.rules.python.PyRuntimeInfo;
import com.google.devtools.build.lib.rules.python.PythonConfiguration;
import com.google.devtools.build.lib.rules.python.PythonSemantics;
import com.google.devtools.build.lib.util.FileTypeSet;
Expand Down Expand Up @@ -314,9 +314,9 @@ private static void createPythonZipAction(
}

private static void addRuntime(RuleContext ruleContext, Runfiles.Builder builder) {
PyRuntimeProvider provider =
ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeProvider.class);
if (provider != null && provider.isHermetic()) {
PyRuntimeInfo provider =
ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeInfo.PROVIDER);
if (provider != null && provider.isInBuild()) {
builder.addArtifact(provider.getInterpreter());
// WARNING: we are adding the all Python runtime files here,
// and it would fail if the filenames of them contain spaces.
Expand All @@ -334,12 +334,12 @@ private static String getPythonBinary(

String pythonBinary;

PyRuntimeProvider provider =
ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeProvider.class);
PyRuntimeInfo provider =
ruleContext.getPrerequisite(":py_interpreter", Mode.TARGET, PyRuntimeInfo.PROVIDER);

if (provider != null) {
// make use of py_runtime defined by --python_top
if (!provider.isHermetic()) {
if (!provider.isInBuild()) {
// absolute Python path in py_runtime
pythonBinary = provider.getInterpreterPath().getPathString();
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,15 @@ public ConfiguredTarget create(RuleContext ruleContext) throws ActionConflictExc
return null;
}

PyRuntimeProvider provider =
PyRuntimeInfo provider =
hermetic
? PyRuntimeProvider.create(files, interpreter, /*interpreterPath=*/ null)
: PyRuntimeProvider.create(/*files=*/ null, /*interpreter=*/ null, interpreterPath);
? PyRuntimeInfo.createForInBuildRuntime(interpreter, files)
: PyRuntimeInfo.createForPlatformRuntime(interpreterPath);

return new RuleConfiguredTargetBuilder(ruleContext)
.setFilesToBuild(files)
.addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY)
.addProvider(PyRuntimeProvider.class, provider)
.addNativeDeclaredProvider(provider)
.build();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.rules.python;

import static com.google.devtools.build.lib.syntax.Runtime.NONE;

import com.google.common.base.Preconditions;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.packages.BuiltinProvider;
import com.google.devtools.build.lib.packages.Info;
import com.google.devtools.build.lib.skylarkbuildapi.python.PyRuntimeInfoApi;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.Objects;
import javax.annotation.Nullable;

/**
* Instance of the provider type that describes Python runtimes.
*
* <p>Invariant: Exactly one of {@link #interpreterPath} and {@link #interpreter} is non-null. The
* former corresponds to a platform runtime, and the latter to an in-build runtime; these two cases
* are mutually exclusive. In addition, {@link #files} is non-null if and only if {@link
* #interpreter} is non-null; in other words files are only used for in-build runtimes. These
* invariants mirror the user-visible API on {@link PyRuntimeInfoApi} except that {@code None} is
* replaced by null.
*/
public class PyRuntimeInfo extends Info implements PyRuntimeInfoApi<Artifact> {

/** The Starlark-accessible top-level builtin name for this provider type. */
public static final String STARLARK_NAME = "PyRuntimeInfo";

/** The singular {@code PyRuntimeInfo} provider type object. */
public static final PyRuntimeInfoProvider PROVIDER = new PyRuntimeInfoProvider();

@Nullable private final PathFragment interpreterPath;
@Nullable private final Artifact interpreter;
@Nullable private final SkylarkNestedSet files;

private PyRuntimeInfo(
@Nullable Location location,
@Nullable PathFragment interpreterPath,
@Nullable Artifact interpreter,
@Nullable SkylarkNestedSet files) {
super(PROVIDER, location);
Preconditions.checkArgument((interpreterPath == null) != (interpreter == null));
Preconditions.checkArgument((interpreter == null) == (files == null));
if (files != null) {
// Work around #7266 by special-casing the empty set in the type check.
Preconditions.checkArgument(
files.isEmpty() || files.getContentType().canBeCastTo(Artifact.class));
}
this.interpreterPath = interpreterPath;
this.interpreter = interpreter;
this.files = files;
}

/** Constructs an instance from native rule logic (built-in location) for an in-build runtime. */
public static PyRuntimeInfo createForInBuildRuntime(
Artifact interpreter, NestedSet<Artifact> files) {
return new PyRuntimeInfo(
/*location=*/ null,
/*interpreterPath=*/ null,
interpreter,
SkylarkNestedSet.of(Artifact.class, files));
}

/** Constructs an instance from native rule logic (built-in location) for a platform runtime. */
public static PyRuntimeInfo createForPlatformRuntime(PathFragment interpreterPath) {
return new PyRuntimeInfo(
/*location=*/ null, interpreterPath, /*interpreter=*/ null, /*files=*/ null);
}

@Override
public boolean equals(Object other) {
// PyRuntimeInfo implements value equality, but note that it contains identity-equality fields
// (depsets), so you generally shouldn't rely on equality comparisons.
if (!(other instanceof PyRuntimeInfo)) {
return false;
}
PyRuntimeInfo otherInfo = (PyRuntimeInfo) other;
return (this.interpreterPath.equals(otherInfo.interpreterPath)
&& this.interpreter.equals(otherInfo.interpreter)
&& this.files.equals(otherInfo.files));
}

@Override
public int hashCode() {
return Objects.hash(PyRuntimeInfo.class, interpreterPath, interpreter, files);
}

/**
* Returns true if this is an in-build runtime as opposed to a platform runtime -- that is, if
* this refers to a target within the build as opposed to a path to a system interpreter.
*
* <p>{@link #getInterpreter} and {@link #getFiles} are non-null if and only if this is an
* in-build runtime, whereas {@link #getInterpreterPath} is non-null if and only if this is a
* platform runtime.
*
* <p>Note: It is still possible for an in-build runtime to reference the system interpreter, as
* in the case where it is a wrapper script.
*/
public boolean isInBuild() {
return getInterpreter() != null;
}

@Nullable
public PathFragment getInterpreterPath() {
return interpreterPath;
}

@Override
@Nullable
public String getInterpreterPathString() {
return interpreterPath == null ? null : interpreterPath.getPathString();
}

@Override
@Nullable
public Artifact getInterpreter() {
return interpreter;
}

@Nullable
public NestedSet<Artifact> getFiles() {
return files == null ? null : files.getSet(Artifact.class);
}

@Override
@Nullable
public SkylarkNestedSet getFilesForStarlark() {
return files;
}

/** The class of the {@code PyRuntimeInfo} provider type. */
public static class PyRuntimeInfoProvider extends BuiltinProvider<PyRuntimeInfo>
implements PyRuntimeInfoApi.PyRuntimeInfoProviderApi {

private PyRuntimeInfoProvider() {
super(STARLARK_NAME, PyRuntimeInfo.class);
}

@Override
public PyRuntimeInfo constructor(
Object interpreterPathUncast, Object interpreterUncast, Object filesUncast, Location loc)
throws EvalException {
String interpreterPath =
interpreterPathUncast == NONE ? null : (String) interpreterPathUncast;
Artifact interpreter = interpreterUncast == NONE ? null : (Artifact) interpreterUncast;
SkylarkNestedSet files = filesUncast == NONE ? null : (SkylarkNestedSet) filesUncast;

if ((interpreter == null) == (interpreterPath == null)) {
throw new EvalException(
loc,
"exactly one of the 'interpreter' or 'interpreter_path' arguments must be specified");
}
boolean isInBuildRuntime = interpreter != null;
if (!isInBuildRuntime && files != null) {
throw new EvalException(loc, "cannot specify 'files' if 'interpreter_path' is given");
}

if (isInBuildRuntime) {
if (files == null) {
files =
SkylarkNestedSet.of(Artifact.class, NestedSetBuilder.emptySet(Order.STABLE_ORDER));
}
return new PyRuntimeInfo(loc, /*interpreterPath=*/ null, interpreter, files);
} else {
return new PyRuntimeInfo(
loc, PathFragment.create(interpreterPath), /*interpreter=*/ null, /*files=*/ null);
}
}
}
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,6 @@ java_library(
"//src/main/java/com/google/devtools/build/lib:syntax",
"//src/main/java/com/google/devtools/build/lib/skylarkbuildapi",
"//third_party:guava",
"//third_party:jsr305",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,23 @@
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.skylarkbuildapi.Bootstrap;
import com.google.devtools.build.lib.skylarkbuildapi.python.PyInfoApi.PyInfoProviderApi;
import com.google.devtools.build.lib.skylarkbuildapi.python.PyRuntimeInfoApi.PyRuntimeInfoProviderApi;

/** {@link Bootstrap} for Starlark objects related to the Python rules. */
public class PyBootstrap implements Bootstrap {

private final PyInfoProviderApi pyInfoProviderApi;
private final PyRuntimeInfoProviderApi pyRuntimeInfoProviderApi;

public PyBootstrap(PyInfoProviderApi pyInfoProviderApi) {
public PyBootstrap(
PyInfoProviderApi pyInfoProviderApi, PyRuntimeInfoProviderApi pyRuntimeInfoProviderApi) {
this.pyInfoProviderApi = pyInfoProviderApi;
this.pyRuntimeInfoProviderApi = pyRuntimeInfoProviderApi;
}

@Override
public void addBindingsToBuilder(ImmutableMap.Builder<String, Object> builder) {
builder.put("PyInfo", pyInfoProviderApi);
builder.put("PyRuntimeInfo", pyRuntimeInfoProviderApi);
}
}
Loading

0 comments on commit 4e49e28

Please sign in to comment.