Skip to content

Commit

Permalink
Save remote output metadata to ActionCache and load them before check…
Browse files Browse the repository at this point in the history 
…ing action cache

This PR extends `ActionCache.Entry` to store output metadata by having a map of <Path, Metadata>. This map is updated after action execution when we update action cache so that metadata of all outputs of the action are saved. Before checking the action cache (when executing actions), we will load the output metadata into output store if it is remote and the correspondingly local one is missing.

With this change, remote output metadata is saved to disk so build without bytes can use them among server restarts.
 We can also download outputs after action execution since remote output metadata can be accessed outside.

Part of #12665.

Fixes #8248.

Closes #13604.

PiperOrigin-RevId: 388586691
  • Loading branch information
@coeuvre @copybara-github
coeuvre authored and copybara-github committed Aug 4, 2021
1 parent f913ef1 commit 4e29042
Show file tree
Hide file tree
Showing 15 changed files with 1,587 additions and 185 deletions.
276 changes: 237 additions & 39 deletions src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
View file

Large diffs are not rendered by default.

165 changes: 153 additions & 12 deletions src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,23 @@

package com.google.devtools.build.lib.actions.cache;

import com.google.common.base.Preconditions;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableMap.toImmutableMap;

import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.io.BaseEncoding;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics;
import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.IOException;
import java.io.PrintStream;
Expand All @@ -32,6 +40,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;

/**
Expand Down Expand Up @@ -84,31 +93,148 @@ final class Entry {
private Map<String, FileArtifactValue> mdMap;
private byte[] digest;
private final byte[] usedClientEnvDigest;
private final Map<String, RemoteFileArtifactValue> outputFileMetadata;
private final Map<String, SerializableTreeArtifactValue> outputTreeMetadata;

/**
* The metadata for output tree that can be serialized.
*
* <p>We can't serialize {@link TreeArtifactValue} directly as it contains some objects that we
* don't want to serialize, e.g. {@link SpecialArtifact}.
*/
@AutoValue
public abstract static class SerializableTreeArtifactValue {
public static SerializableTreeArtifactValue create(
ImmutableMap<String, RemoteFileArtifactValue> childValues,
Optional<RemoteFileArtifactValue> archivedFileValue) {
return new AutoValue_ActionCache_Entry_SerializableTreeArtifactValue(
childValues, archivedFileValue);
}

/**
* Creates {@link SerializableTreeArtifactValue} from {@link TreeArtifactValue} by collecting
* children and archived artifact which are remote.
*
* <p>If no remote value, {@link Optional#empty} is returned.
*/
public static Optional<SerializableTreeArtifactValue> createSerializable(
TreeArtifactValue treeMetadata) {
ImmutableMap<String, RemoteFileArtifactValue> childValues =
treeMetadata.getChildValues().entrySet().stream()
// Only save remote tree file
.filter(e -> e.getValue().isRemote())
.collect(
toImmutableMap(
e -> e.getKey().getTreeRelativePathString(),
e -> (RemoteFileArtifactValue) e.getValue()));

// Only save remote archived artifact
Optional<RemoteFileArtifactValue> archivedFileValue =
treeMetadata
.getArchivedRepresentation()
.filter(ar -> ar.archivedFileValue().isRemote())
.map(ar -> (RemoteFileArtifactValue) ar.archivedFileValue());

if (childValues.isEmpty() && !archivedFileValue.isPresent()) {
return Optional.empty();
}

return Optional.of(SerializableTreeArtifactValue.create(childValues, archivedFileValue));
}

// A map from parentRelativePath to the file metadata
public abstract ImmutableMap<String, RemoteFileArtifactValue> childValues();

public abstract Optional<RemoteFileArtifactValue> archivedFileValue();
}

public Entry(String key, Map<String, String> usedClientEnv, boolean discoversInputs) {
actionKey = key;
this.usedClientEnvDigest = MetadataDigestUtils.fromEnv(usedClientEnv);
files = discoversInputs ? new ArrayList<String>() : null;
mdMap = new HashMap<>();
outputFileMetadata = new HashMap<>();
outputTreeMetadata = new HashMap<>();
}

public Entry(
String key, byte[] usedClientEnvDigest, @Nullable List<String> files, byte[] digest) {
String key,
byte[] usedClientEnvDigest,
@Nullable List<String> files,
byte[] digest,
Map<String, RemoteFileArtifactValue> outputFileMetadata,
Map<String, SerializableTreeArtifactValue> outputTreeMetadata) {
actionKey = key;
this.usedClientEnvDigest = usedClientEnvDigest;
this.files = files;
this.digest = digest;
mdMap = null;
this.outputFileMetadata = outputFileMetadata;
this.outputTreeMetadata = outputTreeMetadata;
}

/**
* Adds the artifact, specified by the executable relative path and its metadata into the cache
* entry.
*/
public void addFile(PathFragment relativePath, FileArtifactValue md, boolean saveExecPath) {
Preconditions.checkState(mdMap != null);
Preconditions.checkState(!isCorrupted());
Preconditions.checkState(digest == null);
/** Adds metadata of an output file */
public void addOutputFile(Artifact output, FileArtifactValue value, boolean saveFileMetadata) {
checkArgument(
!output.isTreeArtifact() && !output.isChildOfDeclaredDirectory(),
"Must use addOutputTree to save tree artifacts and their children: %s",
output);
checkState(mdMap != null);
checkState(!isCorrupted());
checkState(digest == null);

String execPath = output.getExecPathString();
// Only save remote file metadata
if (saveFileMetadata && value.isRemote()) {
outputFileMetadata.put(execPath, (RemoteFileArtifactValue) value);
}
mdMap.put(execPath, value);
}

/** Gets metadata of an output file */
@Nullable
public RemoteFileArtifactValue getOutputFile(Artifact output) {
checkState(!isCorrupted());
return outputFileMetadata.get(output.getExecPathString());
}

Map<String, RemoteFileArtifactValue> getOutputFiles() {
return outputFileMetadata;
}

/** Adds metadata of an output tree */
public void addOutputTree(
SpecialArtifact output, TreeArtifactValue metadata, boolean saveTreeMetadata) {
checkArgument(output.isTreeArtifact(), "artifact must be a tree artifact: %s", output);
checkState(mdMap != null);
checkState(!isCorrupted());
checkState(digest == null);

String execPath = output.getExecPathString();
if (saveTreeMetadata) {
SerializableTreeArtifactValue.createSerializable(metadata)
.ifPresent(value -> outputTreeMetadata.put(execPath, value));
}
mdMap.put(execPath, metadata.getMetadata());
}

/** Gets metadata of an output tree */
@Nullable
public SerializableTreeArtifactValue getOutputTree(SpecialArtifact output) {
checkState(!isCorrupted());
return outputTreeMetadata.get(output.getExecPathString());
}

Map<String, SerializableTreeArtifactValue> getOutputTrees() {
return outputTreeMetadata;
}

/** Adds metadata of an input file */
public void addInputFile(
PathFragment relativePath, FileArtifactValue md, boolean saveExecPath) {
checkState(mdMap != null);
checkState(!isCorrupted());
checkState(digest == null);

String execPath = relativePath.getPathString();
if (discoversInputs() && saveExecPath) {
Expand All @@ -117,8 +243,8 @@ public void addFile(PathFragment relativePath, FileArtifactValue md, boolean sav
mdMap.put(execPath, md);
}

public void addFile(PathFragment relativePath, FileArtifactValue md) {
addFile(relativePath, md, /* saveExecPath= */ true);
public void addInputFile(PathFragment relativePath, FileArtifactValue md) {
addInputFile(relativePath, md, /*saveExecPath=*/ true);
}

/**
Expand Down Expand Up @@ -197,6 +323,21 @@ public String toString() {
builder.append(" ").append(info).append("\n");
}
}

for (Map.Entry<String, RemoteFileArtifactValue> entry : outputFileMetadata.entrySet()) {
builder
.append(" ")
.append(entry.getKey())
.append(" = ")
.append(entry.getValue())
.append("\n");
}

for (Map.Entry<String, SerializableTreeArtifactValue> entry : outputTreeMetadata.entrySet()) {
SerializableTreeArtifactValue metadata = entry.getValue();
builder.append(" ").append(entry.getKey()).append(" = ").append(metadata).append("\n");
}

return builder.toString();
}
}
Expand Down
Loading

0 comments on commit 4e29042

Please sign in to comment.