Skip to content

Commit

Permalink
Only report repository return values if containing new information
Browse files Browse the repository at this point in the history
Repository rules have the option to return a new set of arguments that would
allow them to reproduce identically (whereas the arguments could request
following a branch). Currently, whenever a non-None value is returned, this
is reported to the user; change this to only report, if the return value
contains new information---and in this case, only return what has changed.

While there, also remove the double digesting of the generated directory
in case hashes are verified.

Change-Id: I8bcff45891c755adac72f24c6d07909ed7eb9f3a
PiperOrigin-RevId: 220252511
  • Loading branch information
aehlig authored and Copybara-Service committed Nov 6, 2018
1 parent c54ede7 commit 6a30ed6
Show file tree
Hide file tree
Showing 4 changed files with 200 additions and 26 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,12 @@
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.StructImpl;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.Printer;
import com.google.devtools.build.lib.syntax.Runtime;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.vfs.Path;
import java.io.IOException;
import java.util.List;
import java.util.Map;

/**
Expand All @@ -52,6 +55,9 @@ public class RepositoryResolvedEvent implements ProgressLike {
private final Object resolvedInformation;

private final String name;
private final boolean informationReturned;
private final String message;
private final String directoryDigest;

public RepositoryResolvedEvent(Rule rule, StructImpl attrs, Path outputDirectory, Object result) {
ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
Expand All @@ -61,15 +67,20 @@ public RepositoryResolvedEvent(Rule rule, StructImpl attrs, Path outputDirectory
builder.put(ORIGINAL_RULE_CLASS, originalClass);

ImmutableMap.Builder<String, Object> origAttrBuilder = ImmutableMap.builder();
ImmutableMap.Builder<String, Object> defaults = ImmutableMap.builder();
for (Attribute attr : rule.getAttributes()) {
if (rule.isAttributeValueExplicitlySpecified(attr)) {
String name = attr.getPublicName();
try {
Object value = attrs.getValue(name, Object.class);
origAttrBuilder.put(name, value);
} catch (EvalException e) {
// Do nothing, just ignore the value.
String name = attr.getPublicName();
try {
Object value = attrs.getValue(name, Object.class);
if (value != null) {
if (rule.isAttributeValueExplicitlySpecified(attr)) {
origAttrBuilder.put(name, value);
} else {
defaults.put(name, value);
}
}
} catch (EvalException e) {
// Do nothing, just ignore the value.
}
}
ImmutableMap<String, Object> origAttr = origAttrBuilder.build();
Expand All @@ -78,28 +89,64 @@ public RepositoryResolvedEvent(Rule rule, StructImpl attrs, Path outputDirectory
ImmutableMap.Builder<String, Object> repositoryBuilder =
ImmutableMap.<String, Object>builder().put(RULE_CLASS, originalClass);

String digest = "[unavailable]";
try {
repositoryBuilder.put(OUTPUT_TREE_HASH, outputDirectory.getDirectoryDigest());
digest = outputDirectory.getDirectoryDigest();
repositoryBuilder.put(OUTPUT_TREE_HASH, digest);
} catch (IOException e) {
// Digest not available, but we still have to report that a repository rule
// was invoked. So we can do nothing, but ignore the event.
}
this.directoryDigest = digest;

if (result == Runtime.NONE) {
// Rule claims to be already reproducible, so wants to be called as is.
builder.put(
REPOSITORIES,
ImmutableList.<Object>of(repositoryBuilder.put(ATTRIBUTES, origAttr).build()));
this.informationReturned = false;
this.message = "Repository rule '" + rule.getName() + "' finished.";
} else if (result instanceof Map) {
// Rule claims that the returned (probably changed) arguments are a reproducible
// version of itself.
builder.put(
REPOSITORIES,
ImmutableList.<Object>of(repositoryBuilder.put(ATTRIBUTES, result).build()));
Pair<Map<String, Object>, List<String>> diff =
compare(origAttr, defaults.build(), (Map<?, ?>) result);
if (diff.getFirst().isEmpty() && diff.getSecond().isEmpty()) {
this.informationReturned = false;
this.message = "Repository rule '" + rule.getName() + "' finished.";
} else {
this.informationReturned = true;
if (diff.getFirst().isEmpty()) {
this.message =
"Rule '"
+ rule.getName()
+ "' dropped arguments "
+ Printer.getPrinter().repr(diff.getSecond());
} else if (diff.getSecond().isEmpty()) {
this.message =
"Rule '"
+ rule.getName()
+ "' modified arguments "
+ Printer.getPrinter().repr(diff.getFirst());
} else {
this.message =
"Rule '"
+ rule.getName()
+ "' modified arguments "
+ Printer.getPrinter().repr(diff.getFirst())
+ " and dropped "
+ Printer.getPrinter().repr(diff.getSecond());
}
}
} else {
// TODO(aehlig): handle strings specially to allow encodings of the former
// values to be accepted as well.
builder.put(REPOSITORIES, result);
this.informationReturned = true;
this.message = "Repository rule '" + rule.getName() + "' returned: " + result;
}

this.resolvedInformation = builder.build();
Expand All @@ -115,4 +162,55 @@ public Object getResolvedInformation() {
public String getName() {
return name;
}

public String getDirectoryDigest() {
return directoryDigest;
}

/**
* True, if the return value of the repository rule contained new information with respect to the
* way it was called.
*/
public boolean isNewInformationReturned() {
return informationReturned;
}

/** Message describing the event */
public String getMessage() {
return message;
}

/**
* Compare two maps from Strings to objects, returning a pair of the map with all entries not in
* the original map or in the original map, but with a different value, and the keys dropped from
* the original map. However, ignore changes where a value is explicitly set to its default.
*/
static Pair<Map<String, Object>, List<String>> compare(
Map<String, Object> orig, Map<String, Object> defaults, Map<?, ?> modified) {
ImmutableMap.Builder<String, Object> valuesChanged = ImmutableMap.<String, Object>builder();
for (Map.Entry<?, ?> entry : modified.entrySet()) {
if (entry.getKey() instanceof String) {
Object value = entry.getValue();
String key = (String) entry.getKey();
Object old = orig.get(key);
if (old == null) {
Object defaultValue = defaults.get(key);
if (defaultValue == null || !defaultValue.equals(value)) {
valuesChanged.put(key, value);
}
} else {
if (!old.equals(entry.getValue())) {
valuesChanged.put(key, value);
}
}
}
}
ImmutableList.Builder<String> keysDropped = ImmutableList.<String>builder();
for (String key : orig.keySet()) {
if (!modified.containsKey(key)) {
keysDropped.add(key);
}
}
return Pair.of(valuesChanged.build(), keysDropped.build());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import com.google.devtools.build.lib.syntax.BaseFunction;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.Runtime;
import com.google.devtools.build.lib.syntax.SkylarkSemantics;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
Expand Down Expand Up @@ -128,35 +127,28 @@ public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory,
/*kwargs=*/ ImmutableMap.of(),
null,
buildEnv);
if (retValue != Runtime.NONE) {
env.getListener()
.handle(Event.info("Repository rule '" + rule.getName() + "' returned: " + retValue));
RepositoryResolvedEvent resolved =
new RepositoryResolvedEvent(
rule, skylarkRepositoryContext.getAttr(), outputDirectory, retValue);
if (resolved.isNewInformationReturned()) {
env.getListener().handle(Event.info(resolved.getMessage()));
}

String ruleClass =
rule.getRuleClassObject().getRuleDefinitionEnvironmentLabel() + "%" + rule.getRuleClass();
if (verificationRules.contains(ruleClass)) {
String expectedHash = resolvedHashes.get(rule.getName());
if (expectedHash != null) {
try {
String actualHash = outputDirectory.getDirectoryDigest();
if (!expectedHash.equals(actualHash)) {
throw new RepositoryFunctionException(
new IOException(
rule + " failed to create a directory with expected hash " + expectedHash),
Transience.PERSISTENT);
}
} catch (IOException e) {
String actualHash = resolved.getDirectoryDigest();
if (!expectedHash.equals(actualHash)) {
throw new RepositoryFunctionException(
new IOException("Rule failed to produce a directory with computable hash", e),
new IOException(
rule + " failed to create a directory with expected hash " + expectedHash),
Transience.PERSISTENT);
}
}
}
env.getListener()
.post(
new RepositoryResolvedEvent(
rule, skylarkRepositoryContext.getAttr(), outputDirectory, retValue));
env.getListener().post(resolved);
} catch (EvalException e) {
if (e.getCause() instanceof RepositoryMissingDependencyException) {
// A dependency is missing, cleanup and returns null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ java_test(
"//src/main/java/com/google/devtools/build/lib:os_util",
"//src/main/java/com/google/devtools/build/lib:packages-internal",
"//src/main/java/com/google/devtools/build/lib:syntax",
"//src/main/java/com/google/devtools/build/lib:util",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/collect",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.bazel.repository;

import static com.google.common.truth.Truth.assertThat;

import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.util.Pair;
import java.util.List;
import java.util.Map;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Test for {@link RepositoryOptions}. */
@RunWith(JUnit4.class)
public class RepositoryResolvedEventTest {

@Test
public void testCompareReplace() {
Pair<Map<String, Object>, List<String>> result =
RepositoryResolvedEvent.compare(
ImmutableMap.of("foo", "orig"),
ImmutableMap.<String, Object>of(),
ImmutableMap.of("foo", "changed"));
assertThat(result.getFirst()).containsExactly("foo", "changed");
assertThat(result.getSecond()).isEmpty();
}

@Test
public void testCompareDrop() {
Pair<Map<String, Object>, List<String>> result =
RepositoryResolvedEvent.compare(
ImmutableMap.of("foo", "orig"), ImmutableMap.<String, Object>of(), ImmutableMap.of());
assertThat(result.getFirst()).isEmpty();
assertThat(result.getSecond()).containsExactly("foo");
}

@Test
public void testCompareAdd() {
Pair<Map<String, Object>, List<String>> result =
RepositoryResolvedEvent.compare(
ImmutableMap.<String, Object>of(),
ImmutableMap.<String, Object>of(),
ImmutableMap.of("foo", "new"));
assertThat(result.getFirst()).containsExactly("foo", "new");
assertThat(result.getSecond()).isEmpty();
}

@Test
public void testCompareAddDefault() {
Pair<Map<String, Object>, List<String>> result =
RepositoryResolvedEvent.compare(
ImmutableMap.<String, Object>of(),
ImmutableMap.of("bar", "default", "unreleated", "xyz"),
ImmutableMap.of("foo", "new", "bar", "default"));
assertThat(result.getFirst()).containsExactly("foo", "new");
assertThat(result.getSecond()).isEmpty();
}

@Test
public void testCompareAddDifferentFromDefault() {
Pair<Map<String, Object>, List<String>> result =
RepositoryResolvedEvent.compare(
ImmutableMap.<String, Object>of(),
ImmutableMap.of("bar", "default", "unreleated", "xyz"),
ImmutableMap.of("foo", "new", "bar", "otherValue"));
assertThat(result.getFirst()).containsExactly("foo", "new", "bar", "otherValue");
assertThat(result.getSecond()).isEmpty();
}
}

0 comments on commit 6a30ed6

Please sign in to comment.