Skip to content

Commit

Permalink
Add helper for abstracting over Python provider representation and de…
Browse files Browse the repository at this point in the history
…faults

This creates PyProviderUtils, a collection of helpers that take in a TransitiveInfoCollection and return fields obtained from the Python provider. Currently this just dispatches to PyStructUtils to retrieve from the legacy "py" struct provider. In a follow-up CL, we'll add a modern PyInfo type, and PyProviderUtils will access either one as appropriate.

PyProviderUtils also takes over the default logic from PyCommon, for when no Python provider information is available.

Work toward #7010.

RELNOTES: None
PiperOrigin-RevId: 230791581
  • Loading branch information
brandjon authored and Copybara-Service committed Jan 24, 2019
1 parent 14c1793 commit 4b3725e
Show file tree
Hide file tree
Showing 8 changed files with 388 additions and 213 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import com.google.devtools.build.lib.packages.BuildType;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException;
import com.google.devtools.build.lib.packages.StructImpl;
import com.google.devtools.build.lib.rules.cpp.CcInfo;
import com.google.devtools.build.lib.rules.cpp.CppFileTypes;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
Expand Down Expand Up @@ -230,20 +229,11 @@ private static NestedSet<Artifact> initTransitivePythonSources(RuleContext ruleC
private static void collectTransitivePythonSourcesFromDeps(
RuleContext ruleContext, NestedSetBuilder<Artifact> builder) {
for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (PyStructUtils.hasProvider(dep)) {
try {
StructImpl info = PyStructUtils.getProvider(dep);
NestedSet<Artifact> sources = PyStructUtils.getTransitiveSources(info);
builder.addTransitive(sources);
} catch (EvalException e) {
// Either the provider type or field type is bad.
ruleContext.ruleError(e.getMessage());
}
} else {
// TODO(bazel-team): We also collect .py source files from deps (e.g. for proto_library
// rules). We should have rules implement a "PythonSourcesProvider" instead.
FileProvider provider = dep.getProvider(FileProvider.class);
builder.addAll(FileType.filter(provider.getFilesToBuild(), PyRuleClasses.PYTHON_SOURCE));
try {
builder.addTransitive(PyProviderUtils.getTransitiveSources(dep));
} catch (EvalException e) {
// Either the provider type or field type is bad.
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}
}
Expand All @@ -268,48 +258,36 @@ private static boolean initUsesSharedLibraries(RuleContext ruleContext) {
} else {
targets = ruleContext.getPrerequisites("deps", Mode.TARGET);
}
try {
for (TransitiveInfoCollection target : targets) {
if (checkForSharedLibraries(target)) {
for (TransitiveInfoCollection target : targets) {
try {
if (PyProviderUtils.getUsesSharedLibraries(target)) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", target.getLabel(), e.getMessage()));
}
return false;
} catch (EvalException e) {
ruleContext.ruleError(e.getMessage());
return false;
}
}

private static boolean checkForSharedLibraries(TransitiveInfoCollection target)
throws EvalException {
if (PyStructUtils.hasProvider(target)) {
return PyStructUtils.getUsesSharedLibraries(PyStructUtils.getProvider(target));
} else {
NestedSet<Artifact> files = target.getProvider(FileProvider.class).getFilesToBuild();
return FileType.contains(files, CppFileTypes.SHARED_LIBRARY);
}
return false;
}

private static NestedSet<String> initImports(RuleContext ruleContext, PythonSemantics semantics) {
NestedSetBuilder<String> builder = NestedSetBuilder.compileOrder();
builder.addAll(semantics.getImports(ruleContext));

for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (dep.getProvider(PythonImportsProvider.class) != null) {
PythonImportsProvider provider = dep.getProvider(PythonImportsProvider.class);
NestedSet<String> imports = provider.getTransitivePythonImports();
try {
NestedSet<String> imports = PyProviderUtils.getImports(dep);
if (!builder.getOrder().isCompatible(imports.getOrder())) {
// TODO(brandjon): Add test case for this error, once we replace PythonImportsProvider
// with the normal Python provider and once we clean up our provider merge logic.
// TODO(brandjon): We should make order an invariant of the Python provider. Then once we
/// remove PythonImportsProvider we can move this check into PyProvider/PyStructUtils.
ruleContext.ruleError(
getOrderErrorMessage(PyStructUtils.IMPORTS, builder.getOrder(), imports.getOrder()));
} else {
builder.addTransitive(imports);
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}

return builder.build();
}

Expand All @@ -324,14 +302,12 @@ private static boolean initHasPy2OnlySources(
return true;
}
for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (PyStructUtils.hasProvider(dep)) {
try {
if (PyStructUtils.getHasPy2OnlySources(PyStructUtils.getProvider(dep))) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
try {
if (PyProviderUtils.getHasPy2OnlySources(dep)) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}
return false;
Expand All @@ -347,14 +323,12 @@ private static boolean initHasPy3OnlySources(
return true;
}
for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (PyStructUtils.hasProvider(dep)) {
try {
if (PyStructUtils.getHasPy3OnlySources(PyStructUtils.getProvider(dep))) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
try {
if (PyProviderUtils.getHasPy3OnlySources(dep)) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}
return false;
Expand Down Expand Up @@ -617,10 +591,9 @@ public void addCommonTransitiveInfoProviders(
*/
public List<Artifact> validateSrcs() {
List<Artifact> sourceFiles = new ArrayList<>();
// TODO(bazel-team): Need to get the transitive deps closure, not just the
// sources of the rule.
for (TransitiveInfoCollection src : ruleContext
.getPrerequisitesIf("srcs", Mode.TARGET, FileProvider.class)) {
// TODO(bazel-team): Need to get the transitive deps closure, not just the sources of the rule.
for (TransitiveInfoCollection src :
ruleContext.getPrerequisitesIf("srcs", Mode.TARGET, FileProvider.class)) {
// Make sure that none of the sources contain hyphens.
if (Util.containsHyphen(src.getLabel().getPackageFragment())) {
ruleContext.attributeError("srcs",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.rules.python;

import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.FileProvider;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.packages.StructImpl;
import com.google.devtools.build.lib.rules.cpp.CppFileTypes;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.SkylarkType;
import com.google.devtools.build.lib.util.FileType;

/**
* Static helper class for creating and accessing Python provider information.
*
* <p>This class exposes a unified view over both the legacy and modern Python providers.
*/
public class PyProviderUtils {

// Disable construction.
private PyProviderUtils() {}

/** Returns whether a given target has the py provider. */
public static boolean hasProvider(TransitiveInfoCollection target) {
return target.get(PyStructUtils.PROVIDER_NAME) != null;
}

/**
* Returns the struct representing the py provider, from the given target info.
*
* @throws EvalException if the provider does not exist or has the wrong type.
*/
public static StructImpl getProvider(TransitiveInfoCollection target) throws EvalException {
Object info = target.get(PyStructUtils.PROVIDER_NAME);
if (info == null) {
throw new EvalException(/*location=*/ null, "Target does not have 'py' provider");
}
return SkylarkType.cast(
info,
StructImpl.class,
null,
"'%s' provider should be a struct",
PyStructUtils.PROVIDER_NAME);
}

/**
* Returns the transitive sources of a given target.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we fall
* back on collecting .py source files from the target's filesToBuild.
*
* @throws EvalException if the legacy struct provider is present but malformed
*/
// TODO(bazel-team): Eliminate the fallback behavior by returning an appropriate py provider from
// the relevant rules.
public static NestedSet<Artifact> getTransitiveSources(TransitiveInfoCollection target)
throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getTransitiveSources(getProvider(target));
} else {
NestedSet<Artifact> files = target.getProvider(FileProvider.class).getFilesToBuild();
return NestedSetBuilder.<Artifact>compileOrder()
.addAll(FileType.filter(files, PyRuleClasses.PYTHON_SOURCE))
.build();
}
}

/**
* Returns whether a target uses shared libraries.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we fall
* back on checking whether the target's filesToBuild contains a shared library file type (e.g., a
* .so file).
*
* @throws EvalException if the legacy struct provider is present but malformed
*/
public static boolean getUsesSharedLibraries(TransitiveInfoCollection target)
throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getUsesSharedLibraries(getProvider(target));
} else {
NestedSet<Artifact> files = target.getProvider(FileProvider.class).getFilesToBuild();
return FileType.contains(files, CppFileTypes.SHARED_LIBRARY);
}
}

/**
* Returns the transitive import paths of a target.
*
* <p>Imports are not currently propagated correctly (#7054). Currently the behavior is to return
* the imports contained in the target's {@link PythonImportsProvider}, ignoring the py provider,
* or no imports if there is no {@code PythonImportsProvider}. When #7054 is fixed, we'll instead
* return the imports specified by the py provider, or those from {@code PythonImportsProvider} if
* the py provider is not present, with an eventual goal of removing {@code
* PythonImportsProvider}.
*/
// TODO(#7054) Implement the above change.
public static NestedSet<String> getImports(TransitiveInfoCollection target) throws EvalException {
PythonImportsProvider importsProvider = target.getProvider(PythonImportsProvider.class);
if (importsProvider != null) {
return importsProvider.getTransitivePythonImports();
} else {
return NestedSetBuilder.emptySet(Order.COMPILE_ORDER);
}
}

/**
* Returns whether the target has transitive sources requiring Python 2.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we default
* to false.
*/
public static boolean getHasPy2OnlySources(TransitiveInfoCollection target) throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getHasPy2OnlySources(getProvider(target));
} else {
return false;
}
}

/**
* Returns whether the target has transitive sources requiring Python 3.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we default
* to false.
*/
public static boolean getHasPy3OnlySources(TransitiveInfoCollection target) throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getHasPy3OnlySources(getProvider(target));
} else {
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
Expand All @@ -28,7 +27,7 @@
import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
import com.google.devtools.build.lib.syntax.SkylarkType;

/** Static helper class for creating and accessing instances of the "py" legacy struct provider. */
/** Static helper class for creating and accessing instances of the legacy "py" struct provider. */
// TODO(#7010): Replace this with a real provider.
public class PyStructUtils {

Expand All @@ -54,9 +53,7 @@ private PyStructUtils() {}
* target).
*/
// TODO(brandjon): Make this a pre-order depset, since higher-level targets should get precedence
// on PYTHONPATH.
// TODO(brandjon): Add assertions that this depset and transitive_sources have an order compatible
// with the one expected by the rules.
// on PYTHONPATH. Add assertions on its order compatibility.
public static final String IMPORTS = "imports";

/**
Expand Down Expand Up @@ -85,25 +82,6 @@ private PyStructUtils() {}
DEFAULTS = builder.build();
}

/** Returns whether a given dependency has the py provider. */
public static boolean hasProvider(TransitiveInfoCollection dep) {
return dep.get(PROVIDER_NAME) != null;
}

/**
* Returns the struct representing the py provider, from the given target info.
*
* @throws EvalException if the provider does not exist or has the wrong type.
*/
public static StructImpl getProvider(TransitiveInfoCollection dep) throws EvalException {
Object info = dep.get(PROVIDER_NAME);
if (info == null) {
throw new EvalException(/*location=*/ null, "Target does not have 'py' provider");
}
return SkylarkType.cast(
info, StructImpl.class, null, "'%s' provider should be a struct", PROVIDER_NAME);
}

private static Object getValue(StructImpl info, String fieldName) throws EvalException {
Object fieldValue = info.getValue(fieldName);
if (fieldValue == null) {
Expand Down Expand Up @@ -220,7 +198,7 @@ public static Builder builder() {
return new Builder();
}

/** Builder for a py provider struct. */
/** Builder for a legacy py provider struct. */
public static class Builder {
SkylarkNestedSet transitiveSources = null;
Boolean usesSharedLibraries = null;
Expand Down
Loading

0 comments on commit 4b3725e

Please sign in to comment.