From de780307c7e360be1a4b29d6e2386f1a8480b3a6 Mon Sep 17 00:00:00 2001 From: Ulf Adams Date: Thu, 26 Mar 2020 20:03:18 +0100 Subject: [PATCH] Experimental support for remote persistent workers Add a new --experimental_remote_mark_tool_inputs flag, which makes Bazel tag tool inputs when executing actions remotely, and also adds a tools input key to the platform proto sent as part of the remote execution request. This allows a remote execution system to implement persistent workers, i.e., to keep worker processes around and reuse them for subsequent actions. In a trivial example, this improves build performance by ~3x. We use "persistentWorkerKey" for the platform property, with the value being a hash of the tool inputs, and "bazel_tool_input" as the node property name, with an empty string as value (this is just a boolean tag). Implements #10091. Change-Id: Iccb36081fee399855be7c487c2d4091cb36f8df3 --- .../lib/analysis/platform/PlatformUtils.java | 49 +++++++++++--- .../lib/exec/local/LocalEnvProvider.java | 1 + .../google/devtools/build/lib/remote/BUILD | 3 + .../build/lib/remote/RemoteSpawnRunner.java | 66 ++++++++++++++++++- .../lib/remote/merkletree/DirectoryTree.java | 30 +++++++-- .../merkletree/DirectoryTreeBuilder.java | 27 ++++++-- .../lib/remote/merkletree/MerkleTree.java | 46 +++++++++++-- .../lib/remote/options/RemoteOptions.java | 9 +++ .../devtools/build/lib/worker/WorkerKey.java | 2 +- .../build/lib/worker/WorkerParser.java | 2 +- .../ActionInputDirectoryTreeTest.java | 2 +- 11 files changed, 206 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/platform/PlatformUtils.java b/src/main/java/com/google/devtools/build/lib/analysis/platform/PlatformUtils.java index 621a598b25aea4..292a19d32508f1 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/platform/PlatformUtils.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/platform/PlatformUtils.java @@ -17,6 +17,8 @@ import build.bazel.remote.execution.v2.Platform; import build.bazel.remote.execution.v2.Platform.Property; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Ordering; import com.google.devtools.build.lib.actions.Spawn; @@ -28,8 +30,11 @@ import com.google.protobuf.TextFormat; import com.google.protobuf.TextFormat.ParseException; import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import javax.annotation.Nullable; @@ -63,6 +68,15 @@ public static Platform buildPlatformProto(Map executionPropertie @Nullable public static Platform getPlatformProto(Spawn spawn, @Nullable RemoteOptions remoteOptions) throws UserExecException { + return getPlatformProto(spawn, remoteOptions, ImmutableMap.of()); + } + + @Nullable + public static Platform getPlatformProto( + Spawn spawn, + @Nullable RemoteOptions remoteOptions, + Map additionalProperties) + throws UserExecException { SortedMap defaultExecProperties = remoteOptions != null ? remoteOptions.getRemoteDefaultExecProperties() @@ -70,22 +84,26 @@ public static Platform getPlatformProto(Spawn spawn, @Nullable RemoteOptions rem if (spawn.getExecutionPlatform() == null && spawn.getCombinedExecProperties().isEmpty() - && defaultExecProperties.isEmpty()) { + && defaultExecProperties.isEmpty() + && additionalProperties.isEmpty()) { return null; } - Platform.Builder platformBuilder = Platform.newBuilder(); - + Map properties; if (!spawn.getCombinedExecProperties().isEmpty()) { - for (Map.Entry entry : spawn.getCombinedExecProperties().entrySet()) { - platformBuilder.addPropertiesBuilder().setName(entry.getKey()).setValue(entry.getValue()); - } + properties = spawn.getCombinedExecProperties(); } else if (spawn.getExecutionPlatform() != null && !Strings.isNullOrEmpty(spawn.getExecutionPlatform().remoteExecutionProperties())) { - // Try and get the platform info from the execution properties. + properties = new HashMap<>(); + // Try and get the platform info from the execution properties. This is pretty inefficient; it + // would be better to store the parsed properties instead of the String text proto. try { + Platform.Builder platformBuilder = Platform.newBuilder(); TextFormat.getParser() .merge(spawn.getExecutionPlatform().remoteExecutionProperties(), platformBuilder); + for (Property property : platformBuilder.getPropertiesList()) { + properties.put(property.getName(), property.getValue()); + } } catch (ParseException e) { String message = String.format( @@ -95,12 +113,23 @@ public static Platform getPlatformProto(Spawn spawn, @Nullable RemoteOptions rem e, createFailureDetail(message, Code.INVALID_REMOTE_EXECUTION_PROPERTIES)); } } else { - for (Map.Entry property : defaultExecProperties.entrySet()) { - platformBuilder.addProperties( - Property.newBuilder().setName(property.getKey()).setValue(property.getValue()).build()); + properties = defaultExecProperties; + } + + if (!additionalProperties.isEmpty()) { + if (properties.isEmpty()) { + properties = additionalProperties; + } else { + // Merge the two maps. + properties = new HashMap<>(properties); + properties.putAll(additionalProperties); } } + Platform.Builder platformBuilder = Platform.newBuilder(); + for (Map.Entry entry : properties.entrySet()) { + platformBuilder.addPropertiesBuilder().setName(entry.getKey()).setValue(entry.getValue()); + } sortPlatformProperties(platformBuilder); return platformBuilder.build(); } diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java index c93008ca175830..614e652ec5faca 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java +++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java @@ -24,6 +24,7 @@ * probably should not exist, but is currently necessary for our local MacOS support. */ public interface LocalEnvProvider { + LocalEnvProvider NOOP = (env, binTools, fallbackTmpDir) -> ImmutableMap.copyOf(env); /** * Creates a local environment provider for the current OS. diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index 741a246f7edf19..9f58c8a5d06954 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -69,6 +69,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/exec:spawn_cache", "//src/main/java/com/google/devtools/build/lib/exec:spawn_runner", "//src/main/java/com/google/devtools/build/lib/exec:spawn_strategy_registry", + "//src/main/java/com/google/devtools/build/lib/exec/local", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/remote/common", @@ -83,6 +84,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/sandbox", "//src/main/java/com/google/devtools/build/lib/skyframe:mutable_supplier", "//src/main/java/com/google/devtools/build/lib/skyframe:tree_artifact_value", + "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", "//src/main/java/com/google/devtools/build/lib/util:exit_code", @@ -91,6 +93,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:output_service", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/build/lib/worker", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", "//third_party:auth", diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index 88d4de8e55532c..11a53e321debba 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java @@ -47,9 +47,11 @@ import com.google.common.collect.Maps; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput; import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnMetrics; import com.google.devtools.build.lib.actions.SpawnResult; @@ -57,6 +59,7 @@ import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.actions.cache.VirtualActionInput; import com.google.devtools.build.lib.analysis.platform.PlatformUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; @@ -64,6 +67,7 @@ import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.RemoteLocalFallbackRegistry; import com.google.devtools.build.lib.exec.SpawnRunner; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; @@ -82,15 +86,22 @@ import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.worker.WorkerKey; +import com.google.devtools.build.lib.worker.WorkerOptions; +import com.google.devtools.build.lib.worker.WorkerParser; +import com.google.devtools.common.options.Options; import com.google.longrunning.Operation; import com.google.protobuf.Message; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; import io.grpc.Status.Code; +import io.grpc.protobuf.StatusProto; +import javax.annotation.Nullable; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -99,11 +110,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; -import javax.annotation.Nullable; +import java.util.stream.Collectors; /** A client for the remote execution service. */ @ThreadSafe @@ -214,8 +226,19 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) context.report(ProgressStatus.SCHEDULING, getName()); RemoteOutputsMode remoteOutputsMode = remoteOptions.remoteOutputsMode; SortedMap inputMap = context.getInputMapping(); + ToolSignature toolSignature = + remoteOptions.markToolInputs + && Spawns.supportsWorkers(spawn) + && !spawn.getToolFiles().isEmpty() + ? computePersistentWorkerSignature(spawn, context) + : null; final MerkleTree merkleTree = - MerkleTree.build(inputMap, context.getMetadataProvider(), execRoot, digestUtil); + MerkleTree.build( + inputMap, + toolSignature == null ? ImmutableSet.of() : toolSignature.toolInputs, + context.getMetadataProvider(), + execRoot, + digestUtil); SpawnMetrics.Builder spawnMetrics = SpawnMetrics.Builder.forRemoteExec() .setInputBytes(merkleTree.getInputBytes()) @@ -223,7 +246,14 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) maybeWriteParamFilesLocally(spawn); // Get the remote platform properties. - Platform platform = PlatformUtils.getPlatformProto(spawn, remoteOptions); + Platform platform; + if (toolSignature != null) { + platform = + PlatformUtils.getPlatformProto( + spawn, remoteOptions, ImmutableMap.of("persistentWorkerKey", toolSignature.key)); + } else { + platform = PlatformUtils.getPlatformProto(spawn, remoteOptions); + } Command command = buildCommand( @@ -453,6 +483,22 @@ static void spawnMetricsAccounting( } } + @Nullable + private ToolSignature computePersistentWorkerSignature(Spawn spawn, SpawnExecutionContext context) + throws IOException, ExecException, InterruptedException { + WorkerParser workerParser = + new WorkerParser( + execRoot, false, Options.getDefaults(WorkerOptions.class), LocalEnvProvider.NOOP, null); + WorkerKey workerKey = workerParser.compute(spawn, context).getWorkerKey(); + Fingerprint fingerprint = new Fingerprint(); + fingerprint.addBytes(workerKey.getWorkerFilesCombinedHash().asBytes()); + fingerprint.addIterableStrings(workerKey.getArgs()); + fingerprint.addStringMap(workerKey.getEnv()); + return new ToolSignature( + fingerprint.hexDigestAndReset(), + workerKey.getWorkerFilesWithHashes().keySet()); + } + private SpawnResult downloadAndFinalizeSpawnResult( RemoteActionExecutionContext remoteActionExecutionContext, String actionId, @@ -848,4 +894,18 @@ private static RemoteRetrier createExecuteRetrier( return new ExecuteRetrier( options.remoteMaxRetryAttempts, retryService, Retrier.ALLOW_ALL_CALLS); } + + /** + * A simple value class combining a hash of the tool inputs (and their digests) as well as a set + * of the relative paths of all tool inputs. + */ + private static final class ToolSignature { + private final String key; + private final Set toolInputs; + + private ToolSignature(String key, Set toolInputs) { + this.key = key; + this.toolInputs = toolInputs; + } + } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTree.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTree.java index c31e9b40470c42..ac37ef800cb8b4 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTree.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTree.java @@ -73,6 +73,7 @@ static class FileNode extends Node { private final ByteString data; private final Digest digest; private final boolean isExecutable; + private final boolean toolInput; /** * Create a FileNode with its executable bit set. @@ -83,28 +84,40 @@ static class FileNode extends Node { * https://github.com/bazelbuild/bazel/issues/13262 for more details. */ static FileNode createExecutable(String pathSegment, Path path, Digest digest) { - return new FileNode(pathSegment, path, digest, /* isExecutable= */ true); + return new FileNode(pathSegment, path, digest, /* isExecutable= */ true, false); } - static FileNode createExecutable(String pathSegment, ByteString data, Digest digest) { - return new FileNode(pathSegment, data, digest, /* isExecutable= */ true); + static FileNode createExecutable(String pathSegment, Path path, Digest digest, boolean toolInput) { + return new FileNode(pathSegment, path, digest, /* isExecutable= */ true, toolInput); } - private FileNode(String pathSegment, Path path, Digest digest, boolean isExecutable) { + static FileNode createExecutable(String pathSegment, ByteString data, Digest digest, boolean toolInput) { + return new FileNode(pathSegment, data, digest, /* isExecutable= */ true, toolInput); + } + + private FileNode( + String pathSegment, Path path, Digest digest, boolean isExecutable, boolean toolInput) { super(pathSegment); this.path = Preconditions.checkNotNull(path, "path"); this.data = null; this.digest = Preconditions.checkNotNull(digest, "digest"); this.isExecutable = isExecutable; + this.toolInput = toolInput; } - private FileNode(String pathSegment, ByteString data, Digest digest, boolean isExecutable) { + private FileNode( + String pathSegment, + ByteString data, + Digest digest, + boolean isExecutable, + boolean toolInput) { super(pathSegment); this.path = null; this.data = Preconditions.checkNotNull(data, "data"); ; this.digest = Preconditions.checkNotNull(digest, "digest"); this.isExecutable = isExecutable; + this.toolInput = toolInput; } Digest getDigest() { @@ -123,9 +136,13 @@ public boolean isExecutable() { return isExecutable; } + boolean isToolInput() { + return toolInput; + } + @Override public int hashCode() { - return Objects.hash(super.hashCode(), path, data, digest, isExecutable); + return Objects.hash(super.hashCode(), path, data, digest, toolInput, isExecutable); } @Override @@ -136,6 +153,7 @@ public boolean equals(Object o) { && Objects.equals(path, other.path) && Objects.equals(data, other.data) && Objects.equals(digest, other.digest) + && toolInput == other.toolInput && isExecutable == other.isExecutable; } return false; diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java index 2522bd3fb90d50..fc232e0946b339 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java @@ -15,6 +15,7 @@ import build.bazel.remote.execution.v2.Digest; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.FileArtifactValue; @@ -31,6 +32,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; @@ -58,8 +60,18 @@ static DirectoryTree fromActionInputs( Path execRoot, DigestUtil digestUtil) throws IOException { + return fromActionInputs(inputs, ImmutableSet.of(), metadataProvider, execRoot, digestUtil); + } + + static DirectoryTree fromActionInputs( + SortedMap inputs, + Set toolInputs, + MetadataProvider metadataProvider, + Path execRoot, + DigestUtil digestUtil) + throws IOException { Map tree = new HashMap<>(); - int numFiles = buildFromActionInputs(inputs, metadataProvider, execRoot, digestUtil, tree); + int numFiles = buildFromActionInputs(inputs, toolInputs, metadataProvider, execRoot, digestUtil, tree); return new DirectoryTree(tree, numFiles); } @@ -115,6 +127,7 @@ private static int buildFromPaths( */ private static int buildFromActionInputs( SortedMap inputs, + Set toolInputs, MetadataProvider metadataProvider, Path execRoot, DigestUtil digestUtil, @@ -128,7 +141,11 @@ private static int buildFromActionInputs( VirtualActionInput virtualActionInput = (VirtualActionInput) input; Digest d = digestUtil.compute(virtualActionInput); currDir.addChild( - FileNode.createExecutable(path.getBaseName(), virtualActionInput.getBytes(), d)); + FileNode.createExecutable( + path.getBaseName(), + virtualActionInput.getBytes(), + d, + toolInputs.contains(path))); return 1; } @@ -141,14 +158,16 @@ private static int buildFromActionInputs( case REGULAR_FILE: Digest d = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize()); Path inputPath = ActionInputHelper.toInputPath(input, execRoot); - currDir.addChild(FileNode.createExecutable(path.getBaseName(), inputPath, d)); + currDir.addChild( + FileNode.createExecutable( + path.getBaseName(), inputPath, d, toolInputs.contains(path))); return 1; case DIRECTORY: SortedMap directoryInputs = explodeDirectory(path, execRoot); return buildFromActionInputs( - directoryInputs, metadataProvider, execRoot, digestUtil, tree); + directoryInputs, toolInputs, metadataProvider, execRoot, digestUtil, tree); case SYMLINK: throw new IllegalStateException( diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java index 5b33c57e53cc18..4a16c407c2c47b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java @@ -32,12 +32,14 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; /** A merkle tree representation as defined by the remote execution api. */ public class MerkleTree { + private static final String BAZEL_TOOL_INPUT_MARKER = "bazel_tool_input"; /** A path or contents */ public static class PathOrBytes { @@ -142,6 +144,32 @@ public static MerkleTree build( } } + /** + * Constructs a merkle tree from a lexicographically sorted map of inputs (files). + * + * @param inputs a map of path to input. The map is required to be sorted lexicographically by + * paths. Inputs of type tree artifacts are not supported and are expected to have been + * expanded before. + * @param metadataProvider provides metadata for all {@link ActionInput}s in {@code inputs}, as + * well as any {@link ActionInput}s being discovered via directory expansion. + * @param execRoot all paths in {@code inputs} need to be relative to this {@code execRoot}. + * @param digestUtil a hashing utility + */ + public static MerkleTree build( + SortedMap inputs, + Set toolInputs, + MetadataProvider metadataProvider, + Path execRoot, + DigestUtil digestUtil) + throws IOException { + try (SilentCloseable c = Profiler.instance().profile("MerkleTree.build(ActionInput)")) { + DirectoryTree tree = + DirectoryTreeBuilder.fromActionInputs( + inputs, toolInputs, metadataProvider, execRoot, digestUtil); + return build(tree, digestUtil); + } + } + /** * Constructs a merkle tree from a lexicographically sorted map of files. * @@ -195,11 +223,19 @@ private static MerkleTree build(DirectoryTree tree, DigestUtil digestUtil) { } private static FileNode buildProto(DirectoryTree.FileNode file) { - return FileNode.newBuilder() - .setName(file.getPathSegment()) - .setDigest(file.getDigest()) - .setIsExecutable(file.isExecutable()) - .build(); + FileNode.Builder builder = + FileNode.newBuilder() + .setName(file.getPathSegment()) + .setDigest(file.getDigest()) + .setIsExecutable(file.isExecutable()); + if (file.isToolInput()) { + builder + .getNodePropertiesBuilder() + .addPropertiesBuilder() + .setName(BAZEL_TOOL_INPUT_MARKER) + .setValue(""); + } + return builder.build(); } private static DirectoryNode buildProto(DirectoryTree.DirectoryNode dir, Digest protoDirDigest) { diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java index 74790cea6dd6ef..11659fc3b9e574 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java @@ -507,6 +507,15 @@ public RemoteOutputsStrategyConverter() { + "files and also deduplicates uploads.") public boolean experimentalRemoteDeduplicateUploads; + @Option( + name = "experimental_remote_mark_tool_inputs", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "If set to true, Bazel will mark inputs as tool inputs for the remote executor. This " + + "can be used to implement remote persistent workers.") + public boolean markToolInputs; + // The below options are not configurable by users, only tests. // This is part of the effort to reduce the overall number of flags. diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java index d04ecaba4f2f0a..3ac086745c980c 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerKey.java @@ -31,7 +31,7 @@ *

We expect a small number of WorkerKeys per mnemonic. Unbounded creation of WorkerKeys will * break various things as well as render the workers less useful. */ -final class WorkerKey { +public final class WorkerKey { /** Build options. */ private final ImmutableList args; /** Environment variables. */ diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerParser.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerParser.java index 1cbae1079742a2..4fc6c16b9374b8 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerParser.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerParser.java @@ -21,7 +21,7 @@ import java.util.SortedMap; import java.util.regex.Pattern; -class WorkerParser { +public class WorkerParser { public static final String ERROR_MESSAGE_PREFIX = "Worker strategy cannot execute this %s action, "; public static final String REASON_NO_FLAGFILE = diff --git a/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java b/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java index fcca592bb49bc1..6f871b73fc91a6 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/merkletree/ActionInputDirectoryTreeTest.java @@ -72,7 +72,7 @@ public void virtualActionInputShouldWork() throws Exception { FileNode expectedFooNode = FileNode.createExecutable("foo.cc", foo.getPath(), digestUtil.computeAsUtf8("foo")); FileNode expectedBarNode = - FileNode.createExecutable("bar.cc", bar.getBytes(), digestUtil.computeAsUtf8("bar")); + FileNode.createExecutable("bar.cc", bar.getBytes(), digestUtil.computeAsUtf8("bar"), false); assertThat(fileNodesAtDepth(tree, 0)).isEmpty(); assertThat(fileNodesAtDepth(tree, 1)).containsExactly(expectedFooNode, expectedBarNode); }