Skip to content

Commit

Permalink
Refresh registry content every hour in refresh mode
Browse files Browse the repository at this point in the history
  • Loading branch information
fmeum committed May 13, 2024
1 parent df3781a commit d9bca3c
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 4 deletions.
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
"//src/main/java/com/google/devtools/build/lib/bazel/rules/android",
"//src/main/java/com/google/devtools/build/lib/clock",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/pkgcache",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryRule;
import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryFunction;
import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryRule;
import com.google.devtools.build.lib.clock.Clock;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.pkgcache.PackageOptions;
Expand Down Expand Up @@ -121,6 +122,7 @@
import com.google.devtools.common.options.OptionsParsingResult;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -159,6 +161,8 @@ public class BazelRepositoryModule extends BlazeModule {
private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
private LockfileMode bazelLockfileMode = LockfileMode.UPDATE;
private Clock clock;
private Instant lastRegistryInvalidation = Instant.EPOCH;

private Optional<Path> vendorDirectory;
private List<String> allowedYankedVersions = ImmutableList.of();
Expand Down Expand Up @@ -526,6 +530,22 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
starlarkRepositoryFunction.setRepositoryRemoteExecutor(remoteExecutor);
singleExtensionEvalFunction.setRepositoryRemoteExecutor(remoteExecutor);
delegatingDownloader.setDelegate(env.getRuntime().getDownloaderSupplier().get());

clock = env.getClock();
try {
var lastYankedVersionsInvalidationValue =
(PrecomputedValue)
env.getSkyframeExecutor()
.getEvaluator()
.getExistingValue(RegistryFunction.LAST_INVALIDATION.getKey());
if (lastYankedVersionsInvalidationValue != null) {
lastRegistryInvalidation =
Instant.ofEpochMilli((long) lastYankedVersionsInvalidationValue.get());
}
} catch (InterruptedException e) {
// Not thrown in Bazel.
throw new IllegalStateException(e);
}
}
}

Expand Down Expand Up @@ -556,6 +576,10 @@ private String getAbsolutePath(String path, CommandEnvironment env) {

@Override
public ImmutableList<Injected> getPrecomputedValues() {
Instant now = clock.now();
if (now.isAfter(lastRegistryInvalidation.plus(RegistryFunction.INVALIDATION_INTERVAL))) {
lastRegistryInvalidation = now;
}
return ImmutableList.of(
PrecomputedValue.injected(RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, overrides),
PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, moduleOverrides),
Expand All @@ -582,7 +606,9 @@ public ImmutableList<Injected> getPrecomputedValues() {
PrecomputedValue.injected(
YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions),
PrecomputedValue.injected(
RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, disableNativeRepoRules));
RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, disableNativeRepoRules),
PrecomputedValue.injected(
RegistryFunction.LAST_INVALIDATION, lastRegistryInvalidation.toEpochMilli()));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,22 @@

import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.net.URISyntaxException;
import java.time.Duration;
import javax.annotation.Nullable;

/** A simple SkyFunction that creates a {@link Registry} with a given URL. */
public class RegistryFunction implements SkyFunction {
public static final Precomputed<Long> LAST_INVALIDATION =
new Precomputed<>("last_registry_invalidation");
public static final Duration INVALIDATION_INTERVAL = Duration.ofHours(1);

private final RegistryFactory registryFactory;
private final Path workspaceRoot;

Expand All @@ -41,6 +47,12 @@ public SkyValue compute(SkyKey skyKey, Environment env)
throws InterruptedException, RegistryException {
LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);

if (lockfileMode == LockfileMode.REFRESH) {
// Set to the current time in BazelRepositoryFunction after INVALIDATION_INTERVAL has passed.
// This is used to refresh the mutable registry contents cached in memory from time to time.
RegistryFunction.LAST_INVALIDATION.get(env);
}

BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY);
if (lockfile == null) {
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,9 +321,10 @@ public Converter() {
help =
"Specifies how and whether or not to use the lockfile. Valid values are `update` to"
+ " use the lockfile and update it if there are changes, `refresh` to additionally"
+ " download mutable information (yanked versions and previously missing modules)"
+ " from remote registries again, `error` to use the lockfile but throw an error"
+ " if it's not up-to-date, or `off` to neither read from or write to the lockfile.")
+ " refresh mutable information (yanked versions and previously missing modules)"
+ " from remote registries from time to time, `error` to use the lockfile but throw"
+ " an error if it's not up-to-date, or `off` to neither read from or write to the"
+ " lockfile.")
public LockfileMode lockfileMode;

@Option(
Expand Down

0 comments on commit d9bca3c

Please sign in to comment.