Skip to content

Commit

Permalink
Implement lease service
Browse files Browse the repository at this point in the history
  • Loading branch information
coeuvre committed Jan 27, 2023
1 parent 850b334 commit b58f1b3
Show file tree
Hide file tree
Showing 36 changed files with 927 additions and 124 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,12 @@
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
import com.google.devtools.build.lib.skyframe.TreeArtifactValue.ArchivedRepresentation;
import com.google.devtools.build.lib.vfs.OutputPermissions;
import com.google.devtools.build.lib.vfs.LeaseService;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
Expand Down Expand Up @@ -199,11 +201,18 @@ private static boolean validateArtifacts(
NestedSet<Artifact> actionInputs,
MetadataHandler metadataHandler,
boolean checkOutput,
@Nullable CachedOutputMetadata cachedOutputMetadata) {
@Nullable CachedOutputMetadata cachedOutputMetadata,
@Nullable LeaseService leaseService) {
Map<String, FileArtifactValue> mdMap = new HashMap<>();
if (checkOutput) {
for (Artifact artifact : action.getOutputs()) {
FileArtifactValue metadata = getCachedMetadata(cachedOutputMetadata, artifact);
if (leaseService != null && metadata != null && metadata.isRemote()) {
if (!leaseService.isAlive(
metadata.getDigest(), metadata.getSize(), metadata.getLocationIndex())) {
metadata = null;
}
}
if (metadata == null) {
metadata = getMetadataMaybe(metadataHandler, artifact);
}
Expand Down Expand Up @@ -434,7 +443,8 @@ public Token getTokenIfNeedToExecute(
MetadataHandler metadataHandler,
ArtifactExpander artifactExpander,
Map<String, String> remoteDefaultPlatformProperties,
boolean loadCachedOutputMetadata)
boolean loadCachedOutputMetadata,
@Nullable LeaseService leaseService)
throws InterruptedException {
// TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that
// produce only symlinks we should not check whether inputs are valid at all - all that matters
Expand Down Expand Up @@ -494,7 +504,8 @@ public Token getTokenIfNeedToExecute(
clientEnv,
outputPermissions,
remoteDefaultPlatformProperties,
cachedOutputMetadata)) {
cachedOutputMetadata,
leaseService)) {
if (entry != null) {
removeCacheEntry(action);
}
Expand Down Expand Up @@ -524,7 +535,8 @@ private boolean mustExecute(
Map<String, String> clientEnv,
OutputPermissions outputPermissions,
Map<String, String> remoteDefaultPlatformProperties,
@Nullable CachedOutputMetadata cachedOutputMetadata)
@Nullable CachedOutputMetadata cachedOutputMetadata,
@Nullable LeaseService leaseService)
throws InterruptedException {
// Unconditional execution can be applied only for actions that are allowed to be executed.
if (unconditionalExecution(action)) {
Expand All @@ -544,7 +556,7 @@ private boolean mustExecute(
actionCache.accountMiss(MissReason.CORRUPTED_CACHE_ENTRY);
return true;
} else if (validateArtifacts(
entry, action, actionInputs, metadataHandler, true, cachedOutputMetadata)) {
entry, action, actionInputs, metadataHandler, true, cachedOutputMetadata, leaseService)) {
reportChanged(handler, action);
actionCache.accountMiss(MissReason.DIFFERENT_FILES);
return true;
Expand Down Expand Up @@ -747,7 +759,8 @@ private void checkMiddlemanAction(
action.getInputs(),
metadataHandler,
false,
/*cachedOutputMetadata=*/ null)) {
/*cachedOutputMetadata=*/ null,
/*leaseService=*/ null)) {
reportChanged(handler, action);
actionCache.accountMiss(MissReason.DIFFERENT_FILES);
changed = true;
Expand Down Expand Up @@ -788,7 +801,8 @@ public Token getTokenUnconditionallyAfterFailureToRecordActionCacheHit(
MetadataHandler metadataHandler,
ArtifactExpander artifactExpander,
Map<String, String> remoteDefaultPlatformProperties,
boolean loadCachedOutputMetadata)
boolean loadCachedOutputMetadata,
@Nullable LeaseService leaseService)
throws InterruptedException {
if (action != null) {
removeCacheEntry(action);
Expand All @@ -802,7 +816,8 @@ public Token getTokenUnconditionallyAfterFailureToRecordActionCacheHit(
metadataHandler,
artifactExpander,
remoteDefaultPlatformProperties,
loadCachedOutputMetadata);
loadCachedOutputMetadata,
leaseService);
}

/** Returns an action key. It is always set to the first output exec path string. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,8 @@ private static RemoteFileArtifactValue decodeRemoteMetadata(
PathFragment.create(getStringForIndex(indexer, VarInt.getVarInt(source)));
}

// TODO(chiwang): setting expiredAtEpochMilli to 0 for now. Save/Load it once the implementation
// details in other part are clear.
return RemoteFileArtifactValue.create(
digest, size, locationIndex, actionId, materializationExecPath);
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/remote/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ java_library(
"//src/main/java/com/google/devtools/build/skyframe",
"//src/main/java/com/google/devtools/common/options",
"//src/main/protobuf:failure_details_java_proto",
"//src/main/protobuf:remote_lease_java_proto",
"//src/main/protobuf:spawn_java_proto",
"//third_party:auth",
"//third_party:caffeine",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import com.google.devtools.build.lib.buildtool.buildevent.ProfilerStartedEvent;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.remote.common.MissingDigestsFinder.Intention;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext.CachePolicy;
import com.google.devtools.build.lib.remote.options.RemoteBuildEventUploadMode;
Expand Down Expand Up @@ -253,7 +254,9 @@ private Single<List<PathMetadata>> queryRemoteCache(
if (digestsToQuery.isEmpty()) {
return Single.just(knownRemotePaths);
}
return toSingle(() -> remoteCache.findMissingDigests(context, digestsToQuery), executor)
return toSingle(
() -> remoteCache.findMissingDigests(context, Intention.WRITE, digestsToQuery),
executor)
.onErrorResumeNext(
error -> {
reporterUploadError(error);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ public static boolean isRemoteCacheOptions(RemoteOptions options) {

@Override
public ListenableFuture<ImmutableSet<Digest>> findMissingDigests(
RemoteActionExecutionContext context, Iterable<Digest> digests) {
RemoteActionExecutionContext context, Intention intention, Iterable<Digest> digests) {
if (Iterables.isEmpty(digests)) {
return Futures.immediateFuture(ImmutableSet.of());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.ReadableByteChannel;
import java.time.Instant;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -74,6 +75,8 @@ public class RemoteActionFileSystem extends DelegateFileSystem {
private final ActionInputMap inputArtifactData;
private final ImmutableMap<PathFragment, Artifact> outputMapping;
private final RemoteActionInputFetcher inputFetcher;
private final RemoteLeaseService remoteLeaseService;

private final RemoteInMemoryFileSystem remoteOutputTree;

@Nullable private MetadataInjector metadataInjector = null;
Expand All @@ -84,14 +87,17 @@ public class RemoteActionFileSystem extends DelegateFileSystem {
String relativeOutputPath,
ActionInputMap inputArtifactData,
Iterable<Artifact> outputArtifacts,
RemoteActionInputFetcher inputFetcher) {
RemoteActionInputFetcher inputFetcher,
RemoteLeaseService remoteLeaseService) {
super(localDelegate);
this.execRoot = checkNotNull(execRootFragment, "execRootFragment");
this.outputBase = execRoot.getRelative(checkNotNull(relativeOutputPath, "relativeOutputPath"));
this.inputArtifactData = checkNotNull(inputArtifactData, "inputArtifactData");
this.outputMapping =
stream(outputArtifacts).collect(toImmutableMap(Artifact::getExecPath, a -> a));
this.inputFetcher = checkNotNull(inputFetcher, "inputFetcher");
this.remoteLeaseService = remoteLeaseService;

this.remoteOutputTree = new RemoteInMemoryFileSystem(getDigestFunction());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.exec.SpawnProgressEvent;
import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
import com.google.devtools.build.lib.remote.common.MissingDigestsFinder.Intention;
import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException;
import com.google.devtools.build.lib.remote.common.ProgressStatusListener;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
Expand Down Expand Up @@ -115,11 +116,11 @@ public CachedActionResult downloadActionResult(
* guaranteed to be a subset of {@code digests}.
*/
public ListenableFuture<ImmutableSet<Digest>> findMissingDigests(
RemoteActionExecutionContext context, Iterable<Digest> digests) {
RemoteActionExecutionContext context, Intention intention, Iterable<Digest> digests) {
if (Iterables.isEmpty(digests)) {
return immediateFuture(ImmutableSet.of());
}
return cacheProtocol.findMissingDigests(context, digests);
return cacheProtocol.findMissingDigests(context, intention, digests);
}

/** Returns whether the action cache supports updating action results. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.remote.common.MissingDigestsFinder.Intention;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
import com.google.devtools.build.lib.remote.merkletree.MerkleTree;
Expand Down Expand Up @@ -252,7 +253,7 @@ private Single<List<UploadTask>> findMissingBlobs(
if (digestsToQuery.isEmpty()) {
return immediateFuture(ImmutableSet.of());
}
return findMissingDigests(context, digestsToQuery);
return findMissingDigests(context, Intention.WRITE, digestsToQuery);
},
directExecutor())
.map(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -788,8 +788,7 @@ private void createSymlinks(Iterable<SymlinkMetadata> symlinks) throws IOExcepti
}
}

private void injectRemoteArtifacts(RemoteAction action, ActionResultMetadata metadata)
throws IOException {
private void injectRemoteArtifacts(RemoteAction action, ActionResultMetadata metadata) throws IOException {
FileSystem actionFileSystem = action.getSpawnExecutionContext().getActionFileSystem();
checkState(actionFileSystem instanceof RemoteActionFileSystem);

Expand Down
Loading

0 comments on commit b58f1b3

Please sign in to comment.