From d9bca3c809aa607b1765f63d02f2c9f94d6893ed Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Sun, 12 May 2024 21:01:11 +0200 Subject: [PATCH] Refresh registry content every hour in refresh mode --- .../com/google/devtools/build/lib/bazel/BUILD | 1 + .../lib/bazel/BazelRepositoryModule.java | 28 ++++++++++++++++++- .../lib/bazel/bzlmod/RegistryFunction.java | 12 ++++++++ .../bazel/repository/RepositoryOptions.java | 7 +++-- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 959505efd4d01e..3296f3201f3b7e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -41,6 +41,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/bazel/rules/android", + "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/pkgcache", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 58ce1c66eef484..a42c668e09e621 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -80,6 +80,7 @@ import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryRule; import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryFunction; import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryRule; +import com.google.devtools.build.lib.clock.Clock; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.pkgcache.PackageOptions; @@ -121,6 +122,7 @@ import com.google.devtools.common.options.OptionsParsingResult; import java.io.IOException; import java.lang.reflect.InvocationTargetException; +import java.time.Instant; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -159,6 +161,8 @@ public class BazelRepositoryModule extends BlazeModule { private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; private LockfileMode bazelLockfileMode = LockfileMode.UPDATE; + private Clock clock; + private Instant lastRegistryInvalidation = Instant.EPOCH; private Optional vendorDirectory; private List allowedYankedVersions = ImmutableList.of(); @@ -526,6 +530,22 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { starlarkRepositoryFunction.setRepositoryRemoteExecutor(remoteExecutor); singleExtensionEvalFunction.setRepositoryRemoteExecutor(remoteExecutor); delegatingDownloader.setDelegate(env.getRuntime().getDownloaderSupplier().get()); + + clock = env.getClock(); + try { + var lastYankedVersionsInvalidationValue = + (PrecomputedValue) + env.getSkyframeExecutor() + .getEvaluator() + .getExistingValue(RegistryFunction.LAST_INVALIDATION.getKey()); + if (lastYankedVersionsInvalidationValue != null) { + lastRegistryInvalidation = + Instant.ofEpochMilli((long) lastYankedVersionsInvalidationValue.get()); + } + } catch (InterruptedException e) { + // Not thrown in Bazel. + throw new IllegalStateException(e); + } } } @@ -556,6 +576,10 @@ private String getAbsolutePath(String path, CommandEnvironment env) { @Override public ImmutableList getPrecomputedValues() { + Instant now = clock.now(); + if (now.isAfter(lastRegistryInvalidation.plus(RegistryFunction.INVALIDATION_INTERVAL))) { + lastRegistryInvalidation = now; + } return ImmutableList.of( PrecomputedValue.injected(RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, overrides), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, moduleOverrides), @@ -582,7 +606,9 @@ public ImmutableList getPrecomputedValues() { PrecomputedValue.injected( YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions), PrecomputedValue.injected( - RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, disableNativeRepoRules)); + RepositoryDelegatorFunction.DISABLE_NATIVE_REPO_RULES, disableNativeRepoRules), + PrecomputedValue.injected( + RegistryFunction.LAST_INVALIDATION, lastRegistryInvalidation.toEpochMilli())); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index 121027c658e2aa..d12b555a6d7cba 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -17,16 +17,22 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import java.net.URISyntaxException; +import java.time.Duration; import javax.annotation.Nullable; /** A simple SkyFunction that creates a {@link Registry} with a given URL. */ public class RegistryFunction implements SkyFunction { + public static final Precomputed LAST_INVALIDATION = + new Precomputed<>("last_registry_invalidation"); + public static final Duration INVALIDATION_INTERVAL = Duration.ofHours(1); + private final RegistryFactory registryFactory; private final Path workspaceRoot; @@ -41,6 +47,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RegistryException { LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + if (lockfileMode == LockfileMode.REFRESH) { + // Set to the current time in BazelRepositoryFunction after INVALIDATION_INTERVAL has passed. + // This is used to refresh the mutable registry contents cached in memory from time to time. + RegistryFunction.LAST_INVALIDATION.get(env); + } + BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); if (lockfile == null) { return null; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index c20890b4596648..fdd08a97b192ca 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -321,9 +321,10 @@ public Converter() { help = "Specifies how and whether or not to use the lockfile. Valid values are `update` to" + " use the lockfile and update it if there are changes, `refresh` to additionally" - + " download mutable information (yanked versions and previously missing modules)" - + " from remote registries again, `error` to use the lockfile but throw an error" - + " if it's not up-to-date, or `off` to neither read from or write to the lockfile.") + + " refresh mutable information (yanked versions and previously missing modules)" + + " from remote registries from time to time, `error` to use the lockfile but throw" + + " an error if it's not up-to-date, or `off` to neither read from or write to the" + + " lockfile.") public LockfileMode lockfileMode; @Option(