Skip to content

Commit

Permalink
Use a CommandLine to represent executable arguments in SpawnAction
Browse files Browse the repository at this point in the history 
This reduced analysis memory usage for actions that construct complex
argument lists.

PiperOrigin-RevId: 218388661
  • Loading branch information
@cushon
cushon authored and Copybara-Service committed Oct 23, 2018
1 parent b7747f3 commit 9fe597d
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 47 deletions.
16 changes: 11 additions & 5 deletions src/main/java/com/google/devtools/build/lib/actions/CommandLine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.collect.CollectionUtils;
import com.google.devtools.build.lib.collect.IterablesChain;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec.VisibleForSerialization;
import com.google.devtools.build.lib.util.Fingerprint;
Expand Down Expand Up @@ -94,13 +94,13 @@ static class PrefixedCommandLine extends CommandLine {

@Override
public Iterable<String> arguments() throws CommandLineExpansionException {
return Iterables.concat(executableArgs, commandLine.arguments());
return IterablesChain.concat(executableArgs, commandLine.arguments());
}

@Override
public Iterable<String> arguments(ArtifactExpander artifactExpander)
throws CommandLineExpansionException {
return Iterables.concat(executableArgs, commandLine.arguments(artifactExpander));
return IterablesChain.concat(executableArgs, commandLine.arguments(artifactExpander));
}
}

Expand All @@ -118,13 +118,13 @@ static class SuffixedCommandLine extends CommandLine {

@Override
public Iterable<String> arguments() throws CommandLineExpansionException {
return Iterables.concat(commandLine.arguments(), executableArgs);
return IterablesChain.concat(commandLine.arguments(), executableArgs);
}

@Override
public Iterable<String> arguments(ArtifactExpander artifactExpander)
throws CommandLineExpansionException {
return Iterables.concat(commandLine.arguments(artifactExpander), executableArgs);
return IterablesChain.concat(commandLine.arguments(artifactExpander), executableArgs);
}
}

Expand All @@ -137,6 +137,9 @@ public static CommandLine concat(
if (executableArgs.isEmpty()) {
return commandLine;
}
if (commandLine == EMPTY) {
return CommandLine.of(executableArgs);
}
return new PrefixedCommandLine(executableArgs, commandLine);
}

Expand All @@ -149,6 +152,9 @@ public static CommandLine concat(
if (args.isEmpty()) {
return commandLine;
}
if (commandLine == EMPTY) {
return CommandLine.of(args);
}
return new SuffixedCommandLine(args, commandLine);
}

Expand Down
67 changes: 25 additions & 42 deletions src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.devtools.build.lib.actions.AbstractAction;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionEnvironment;
Expand Down Expand Up @@ -76,7 +75,6 @@
import com.google.errorprone.annotations.FormatMethod;
import com.google.errorprone.annotations.FormatString;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -569,9 +567,7 @@ public static class Builder {
private boolean isShellCommand = false;
private boolean useDefaultShellEnvironment = false;
protected boolean executeUnconditionally;
private PathFragment executable;
// executableArgs does not include the executable itself.
private List<String> executableArgs;
private CustomCommandLine.Builder executableArgs;
private List<CommandLineAndParamFileInfo> commandLines = new ArrayList<>();

private CharSequence progressMessage;
Expand All @@ -598,10 +594,7 @@ public Builder(Builder other) {
this.executionInfo = other.executionInfo;
this.isShellCommand = other.isShellCommand;
this.useDefaultShellEnvironment = other.useDefaultShellEnvironment;
this.executable = other.executable;
this.executableArgs = (other.executableArgs != null)
? Lists.newArrayList(other.executableArgs)
: null;
this.executableArgs = other.executableArgs;
this.commandLines = new ArrayList<>(other.commandLines);
this.progressMessage = other.progressMessage;
this.mnemonic = other.mnemonic;
Expand Down Expand Up @@ -631,10 +624,9 @@ public Action[] build(ActionConstructionContext context) {

@VisibleForTesting @CheckReturnValue
public Action[] build(ActionOwner owner, BuildConfiguration configuration) {
ImmutableList<String> executableArgs = buildExecutableArgs();
Action[] actions = new Action[1];
CommandLines.Builder result = CommandLines.builder();
result.addCommandLine(CommandLine.of(executableArgs));
result.addCommandLine(executableArgs.build());
for (CommandLineAndParamFileInfo pair : this.commandLines) {
result.addCommandLine(pair);
}
Expand All @@ -653,8 +645,7 @@ public Action[] build(ActionOwner owner, BuildConfiguration configuration) {
@CheckReturnValue
SpawnAction buildForActionTemplate(ActionOwner owner) {
CommandLines.Builder result = CommandLines.builder();
ImmutableList<String> executableArgs = buildExecutableArgs();
result.addCommandLine(CommandLine.of(executableArgs));
result.addCommandLine(executableArgs.build());
for (CommandLineAndParamFileInfo pair : commandLines) {
result.addCommandLine(pair.commandLine);
}
Expand Down Expand Up @@ -749,16 +740,6 @@ protected SpawnAction createSpawnAction(
extraActionInfoSupplier);
}

private ImmutableList<String> buildExecutableArgs() {
Preconditions.checkNotNull(executable);
Preconditions.checkNotNull(executableArgs);

return ImmutableList.<String>builder()
.add(executable.getPathString())
.addAll(executableArgs)
.build();
}

/**
* Adds an artifact that is necessary for executing the spawn itself (e.g. a compiler), in
* contrast to an artifact that is necessary for the spawn to do its work (e.g. source code).
Expand Down Expand Up @@ -934,8 +915,7 @@ public Builder executeUnconditionally() {
* {@link #setShellCommand(String)}.
*/
public Builder setExecutable(PathFragment executable) {
this.executable = executable;
this.executableArgs = Lists.newArrayList();
this.executableArgs = CustomCommandLine.builder().addPath(executable);
this.isShellCommand = false;
return this;
}
Expand Down Expand Up @@ -982,11 +962,12 @@ public Builder setExecutable(FilesToRunProvider executableProvider) {

private Builder setJavaExecutable(PathFragment javaExecutable, Artifact deployJar,
List<String> jvmArgs, String... launchArgs) {
this.executable = javaExecutable;
this.executableArgs = Lists.newArrayList();
executableArgs.add("-Xverify:none");
executableArgs.addAll(jvmArgs);
Collections.addAll(executableArgs, launchArgs);
this.executableArgs =
CustomCommandLine.builder()
.addPath(javaExecutable)
.add("-Xverify:none")
.addAll(ImmutableList.copyOf(jvmArgs))
.addAll(ImmutableList.copyOf(launchArgs));
toolsBuilder.add(deployJar);
this.isShellCommand = false;
return this;
Expand Down Expand Up @@ -1032,9 +1013,9 @@ public Builder setJarExecutable(PathFragment javaExecutable,
* #setExecutable(Artifact)}, {@link #setJavaExecutable}, or {@link #setShellCommand(String)}.
*/
public Builder setShellCommand(PathFragment shExecutable, String command) {
this.executable = shExecutable;
// 0=shell command switch, 1=command
this.executableArgs = Lists.newArrayList("-c", command);
this.executableArgs =
CustomCommandLine.builder().addPath(shExecutable).add("-c").addDynamicString(command);
this.isShellCommand = true;
return this;
}
Expand All @@ -1044,9 +1025,7 @@ public Builder setShellCommand(PathFragment shExecutable, String command) {
* commands to be executed.
*/
public Builder setShellCommand(Iterable<String> command) {
this.executable = PathFragment.create(Iterables.getFirst(command, null));
// The first item of the commands is the shell executable that should be used.
this.executableArgs = ImmutableList.copyOf(Iterables.skip(command, 1));
this.executableArgs = CustomCommandLine.builder().addAll(ImmutableList.copyOf(command));
this.isShellCommand = true;
return this;
}
Expand All @@ -1062,22 +1041,26 @@ public Builder addTool(FilesToRunProvider tool) {
return this;
}

/**
* Appends the arguments to the list of executable arguments.
*/
/** Returns a {@link CustomCommandLine.Builder} for executable arguments. */
public CustomCommandLine.Builder executableArguments() {
Preconditions.checkState(executableArgs != null);
return this.executableArgs;
}

/** Appends the arguments to the list of executable arguments. */
public Builder addExecutableArguments(String... arguments) {
Preconditions.checkState(executableArgs != null);
Collections.addAll(executableArgs, arguments);
this.executableArgs.addAll(ImmutableList.copyOf(arguments));
return this;
}

/**
* Add multiple arguments in the order they are returned by the collection
* to the list of executable arguments.
* Add multiple arguments in the order they are returned by the collection to the list of
* executable arguments.
*/
public Builder addExecutableArguments(Iterable<String> arguments) {
Preconditions.checkState(executableArgs != null);
Iterables.addAll(executableArgs, arguments);
this.executableArgs.addAll(ImmutableList.copyOf(arguments));
return this;
}

Expand Down

0 comments on commit 9fe597d

Please sign in to comment.