Skip to content

Commit

Permalink
[#7024] Allow chaining of the same function type in aquery.
Browse files Browse the repository at this point in the history
For example:

inputs('a', inputs('b', expr)) would now filter all actions which results from
evaluating expr and whose inputs matches both patterns 'a' and 'b'.

RELNOTES: [#7024] Allow chaining of the same function type in aquery.
PiperOrigin-RevId: 228322146
  • Loading branch information
joeleba authored and Copybara-Service committed Jan 8, 2019
1 parent 1ed98e7 commit c8d9be5
Show file tree
Hide file tree
Showing 13 changed files with 365 additions and 84 deletions.
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -1259,6 +1259,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/profiler/memory:allocationtracker",
"//src/main/java/com/google/devtools/build/lib/query2",
"//src/main/java/com/google/devtools/build/lib/query2:abstract-blaze-query-env",
"//src/main/java/com/google/devtools/build/lib/query2:aquery-utils",
"//src/main/java/com/google/devtools/build/lib/query2:query-engine",
"//src/main/java/com/google/devtools/build/lib/query2:query-output",
"//src/main/java/com/google/devtools/build/lib/shell",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
package com.google.devtools.build.lib.buildtool;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.query2.ActionGraphQueryEnvironment;
import com.google.devtools.build.lib.query2.AqueryActionFilter;
import com.google.devtools.build.lib.query2.PostAnalysisQueryEnvironment;
import com.google.devtools.build.lib.query2.PostAnalysisQueryEnvironment.TopLevelConfigurations;
import com.google.devtools.build.lib.query2.engine.ActionFilterFunction;
Expand All @@ -34,12 +34,12 @@

/** A version of {@link BuildTool} that handles all aquery work. */
public class AqueryBuildTool extends PostAnalysisQueryBuildTool<ConfiguredTargetValue> {
private final ImmutableMap<String, Pattern> actionFilters;
private final AqueryActionFilter actionFilters;

public AqueryBuildTool(CommandEnvironment env, QueryExpression queryExpression)
throws AqueryActionFilterException {
super(env, queryExpression);
actionFilters = getActionFilters(queryExpression);
actionFilters = buildActionFilters(queryExpression);
}

@Override
Expand Down Expand Up @@ -79,9 +79,9 @@ protected PostAnalysisQueryEnvironment<ConfiguredTargetValue> getQueryEnvironmen
* @throws AqueryActionFilterException if an aquery filter function is preceded by any other
* function types
*/
private ImmutableMap<String, Pattern> getActionFilters(QueryExpression queryExpression)
private AqueryActionFilter buildActionFilters(QueryExpression queryExpression)
throws AqueryActionFilterException {
ImmutableMap.Builder<String, Pattern> actionFiltersBuilder = ImmutableMap.builder();
AqueryActionFilter.Builder actionFiltersBuilder = AqueryActionFilter.builder();

if (!(queryExpression instanceof FunctionExpression)) {
return actionFiltersBuilder.build();
Expand All @@ -108,7 +108,6 @@ private ImmutableMap<String, Pattern> getActionFilters(QueryExpression queryExpr
ActionFilterFunction actionFilterFunction =
(ActionFilterFunction) functionExpression.getFunction();

// TODO(leba) support multiple patterns for 1 function type
String patternString = functionExpression.getArgs().get(0).getWord();
actionFiltersBuilder.put(actionFilterFunction.getName(), Pattern.compile(patternString));
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
package com.google.devtools.build.lib.query2;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.analysis.AnalysisProtos;
import com.google.devtools.build.lib.analysis.AnalysisProtos.ActionGraphContainer;
Expand All @@ -29,7 +28,6 @@
import com.google.protobuf.TextFormat;
import java.io.IOException;
import java.io.OutputStream;
import java.util.regex.Pattern;

/** Default output callback for aquery, prints proto output. */
public class ActionGraphProtoOutputFormatterCallback extends AqueryThreadsafeCallback {
Expand All @@ -52,7 +50,7 @@ public String formatName() {

private final OutputType outputType;
private final ActionGraphDump actionGraphDump;
private final ImmutableMap<String, Pattern> actionFilters;
private final AqueryActionFilter actionFilters;

ActionGraphProtoOutputFormatterCallback(
ExtendedEventHandler eventHandler,
Expand All @@ -61,7 +59,7 @@ public String formatName() {
SkyframeExecutor skyframeExecutor,
TargetAccessor<ConfiguredTargetValue> accessor,
OutputType outputType,
ImmutableMap<String, Pattern> actionFilters) {
AqueryActionFilter actionFilters) {
super(eventHandler, options, out, skyframeExecutor, accessor);
this.outputType = outputType;
this.actionFilters = actionFilters;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
package com.google.devtools.build.lib.query2;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.Futures;
Expand Down Expand Up @@ -53,7 +52,6 @@
import java.util.List;
import java.util.Set;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import javax.annotation.Nullable;

/**
Expand All @@ -67,7 +65,7 @@ public class ActionGraphQueryEnvironment
public static final ImmutableList<QueryFunction> FUNCTIONS = populateFunctions();
AqueryOptions aqueryOptions;

private ImmutableMap<String, Pattern> actionFilters;
private AqueryActionFilter actionFilters;
private final KeyExtractor<ConfiguredTargetValue, ConfiguredTargetKey>
configuredTargetKeyExtractor;
private final ConfiguredTargetValueAccessor accessor;
Expand Down Expand Up @@ -336,7 +334,7 @@ public ThreadSafeMutableSet<ConfiguredTargetValue> createThreadSafeMutableSet()
SkyQueryEnvironment.DEFAULT_THREAD_COUNT);
}

public void setActionFilters(ImmutableMap<String, Pattern> actionFilters) {
public void setActionFilters(AqueryActionFilter actionFilters) {
this.actionFilters = actionFilters;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
Expand Down Expand Up @@ -46,22 +45,21 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/** Output callback for aquery, prints human readable output. */
public class ActionGraphTextOutputFormatterCallback extends AqueryThreadsafeCallback {

private final ActionKeyContext actionKeyContext = new ActionKeyContext();
private final ImmutableMap<String, Pattern> actionFilters;
private final AqueryActionFilter actionFilters;

ActionGraphTextOutputFormatterCallback(
ExtendedEventHandler eventHandler,
AqueryOptions options,
OutputStream out,
SkyframeExecutor skyframeExecutor,
TargetAccessor<ConfiguredTargetValue> accessor,
ImmutableMap<String, Pattern> actionFilters) {
AqueryActionFilter actionFilters) {
super(eventHandler, options, out, skyframeExecutor, accessor);
this.actionFilters = actionFilters;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.query2;

import com.google.common.collect.ImmutableMap;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

/** Encapsulate the action filters parsed from aquery command. */
public class AqueryActionFilter {
// TODO(leba): Use Enum for list of filters.
private final ImmutableMap<String, List<Pattern>> filterMap;

private AqueryActionFilter(Builder builder) {
filterMap = ImmutableMap.copyOf(builder.filterMap);
}

public static AqueryActionFilter emptyInstance() {
return builder().build();
}

public static Builder builder() {
return new Builder();
}

public boolean hasFilterForFunction(String function) {
return filterMap.containsKey(function);
}

/**
* Returns whether the input string matches ALL the filter patterns of a specific type parsed from
* aquery command.
*
* @param function the name of the aquery function (inputs, outputs, mnemonic)
* @param input the string to be matched against
*/
public boolean matchesAllPatternsForFunction(String function, String input) {
if (!hasFilterForFunction(function)) {
return false;
}

return filterMap.get(function).stream()
.map(pattern -> pattern.matcher(input).matches())
.reduce(true, Boolean::logicalAnd);
}

/** Builder class for {@code AqueryActionFilter} */
public static class Builder {
private final Map<String, List<Pattern>> filterMap;

public Builder() {
filterMap = new HashMap<>();
}

public Builder put(String key, Pattern value) {
filterMap.putIfAbsent(key, new ArrayList<>());
filterMap.get(key).add(value);

return this;
}

public AqueryActionFilter build() {
return new AqueryActionFilter(this);
}
}
}
24 changes: 13 additions & 11 deletions src/main/java/com/google/devtools/build/lib/query2/AqueryUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,9 @@
import static com.google.devtools.build.lib.query2.engine.MnemonicFunction.MNEMONIC;
import static com.google.devtools.build.lib.query2.engine.OutputsFunction.OUTPUTS;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.Artifact;
import java.util.regex.Pattern;

/** Utility class for Aquery */
public class AqueryUtils {
Expand All @@ -34,34 +32,38 @@ public class AqueryUtils {
* @return whether the action matches the filtering patterns
*/
public static boolean matchesAqueryFilters(
ActionAnalysisMetadata action, ImmutableMap<String, Pattern> actionFilters) {
ActionAnalysisMetadata action, AqueryActionFilter actionFilters) {
Iterable<Artifact> inputs = action.getInputs();
Iterable<Artifact> outputs = action.getOutputs();
String mnemonic = action.getMnemonic();

if (actionFilters.containsKey(MNEMONIC)) {
if (!actionFilters.get(MNEMONIC).matcher(mnemonic).matches()) {
if (actionFilters.hasFilterForFunction(MNEMONIC)) {
if (!actionFilters.matchesAllPatternsForFunction(MNEMONIC, mnemonic)) {
return false;
}
}

if (actionFilters.containsKey(INPUTS)) {
Pattern inputsPattern = actionFilters.get(INPUTS);
if (actionFilters.hasFilterForFunction(INPUTS)) {
Boolean containsFile =
Streams.stream(inputs)
.map(artifact -> inputsPattern.matcher(artifact.getExecPathString()).matches())
.map(
artifact ->
actionFilters.matchesAllPatternsForFunction(
INPUTS, artifact.getExecPathString()))
.reduce(false, Boolean::logicalOr);

if (!containsFile) {
return false;
}
}

if (actionFilters.containsKey(OUTPUTS)) {
Pattern outputsPattern = actionFilters.get(OUTPUTS);
if (actionFilters.hasFilterForFunction(OUTPUTS)) {
Boolean containsFile =
Streams.stream(outputs)
.map(artifact -> outputsPattern.matcher(artifact.getExecPathString()).matches())
.map(
artifact ->
actionFilters.matchesAllPatternsForFunction(
OUTPUTS, artifact.getExecPathString()))
.reduce(false, Boolean::logicalOr);

return containsFile;
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/google/devtools/build/lib/query2/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ java_library(
"AbstractBlazeQueryEnvironment.java",
"FakeLoadTarget.java",
"AqueryUtils.java",
"AqueryActionFilter.java",
],
),
deps = [
Expand Down Expand Up @@ -149,6 +150,7 @@ java_library(
java_library(
name = "aquery-utils",
srcs = [
"AqueryActionFilter.java",
"AqueryUtils.java",
],
deps = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
*
* <pre>expr ::= MNEMONIC '(' WORD ',' expr ')'</pre>
*
* Example patterns: TODO(leba) example patterns
* Example patterns:
*
* <pre>
* 'CppCompile' Match all actions whose mnemonic is CppComile
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.Action;
Expand All @@ -37,13 +36,13 @@
import com.google.devtools.build.lib.collect.nestedset.NestedSetView;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.packages.AspectDescriptor;
import com.google.devtools.build.lib.query2.AqueryActionFilter;
import com.google.devtools.build.lib.query2.AqueryUtils;
import com.google.devtools.build.lib.skyframe.AspectValue;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;

/**
* Encapsulates necessary functionality to dump the current skyframe state of the action graph to
Expand All @@ -62,21 +61,23 @@ public class ActionGraphDump {
private final KnownAspectDescriptors knownAspectDescriptors;
private final KnownRuleConfiguredTargets knownRuleConfiguredTargets;
private final boolean includeActionCmdLine;
private final ImmutableMap<String, Pattern> actionFilters;
private final AqueryActionFilter actionFilters;

public ActionGraphDump(
boolean includeActionCmdLine, ImmutableMap<String, Pattern> actionFilters) {
public ActionGraphDump(boolean includeActionCmdLine, AqueryActionFilter actionFilters) {
this(/* actionGraphTargets= */ ImmutableList.of("..."), includeActionCmdLine, actionFilters);
}

public ActionGraphDump(List<String> actionGraphTargets, boolean includeActionCmdLine) {
this(actionGraphTargets, includeActionCmdLine, /* actionFilters= */ ImmutableMap.of());
this(
actionGraphTargets,
includeActionCmdLine,
/* actionFilters= */ AqueryActionFilter.emptyInstance());
}

public ActionGraphDump(
List<String> actionGraphTargets,
boolean includeActionCmdLine,
ImmutableMap<String, Pattern> actionFilters) {
AqueryActionFilter actionFilters) {
this.actionGraphTargets = ImmutableSet.copyOf(actionGraphTargets);
this.includeActionCmdLine = includeActionCmdLine;
this.actionFilters = actionFilters;
Expand Down
Loading

0 comments on commit c8d9be5

Please sign in to comment.