From 7fa7d37faa13c000ada6deb8e61a7e66e77010a2 Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Mon, 13 May 2024 18:07:58 +0000
Subject: [PATCH 01/10] Add support for remote_files to bzlmod

This is a continuation of #22155 that adds the newly added
'remote_files' attribute for http_archive to the bzlmod functionality.

The end goal is to then update BCR to this new functionality to overlay
files rather than use patch files when providing MODULE/WORKSPACE/BUILD
files.

https://github.com/bazelbuild/bazel-central-registry/issues/1566 has a
good discussion of the rationale.
---
 .../bazel/bzlmod/ArchiveRepoSpecBuilder.java  | 18 +++++++++
 .../devtools/build/lib/bazel/bzlmod/BUILD     |  1 +
 .../build/lib/bazel/bzlmod/IndexRegistry.java |  7 ++++
 .../build/lib/bazel/bzlmod/RemoteFile.java    | 20 ++++++++++
 .../lib/bazel/bzlmod/IndexRegistryTest.java   | 38 +++++++++++++++++++
 5 files changed, 84 insertions(+)
 create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
index d6ffe2e46c577b..2dd1deccd332a0 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -19,6 +19,11 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
 import net.starlark.java.eval.StarlarkInt;
 
 /**
@@ -77,6 +82,19 @@ public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remo
     return this;
   }
 
+  @CanIgnoreReturnValue
+  public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overlay) {
+    final Map<String, List<String>> remoteFiles = overlay.entrySet().stream().collect(Collectors.toMap(
+        Entry::getKey,
+        e -> e.getValue().urls.stream().map(URL::toString).collect(Collectors.toList())
+    ));
+    final Map<String, String> remoteFilesIntegrity = overlay.entrySet().stream()
+        .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().integrity));
+    attrBuilder.put("remote_file_urls", remoteFiles);
+    attrBuilder.put("remote_file_integrity", remoteFilesIntegrity);
+    return this;
+  }
+
   @CanIgnoreReturnValue
   public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) {
     attrBuilder.put("remote_patch_strip", StarlarkInt.of(remotePatchStrip));
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
index cb5d26d6062348..e150dd9374009c 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
@@ -21,6 +21,7 @@ java_library(
         "GitRepoSpecBuilder.java",
         "ModuleFile.java",
         "ModuleKey.java",
+        "RemoteFile.java",
         "RepoSpec.java",
         "Version.java",
     ],
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index 9de56c594550a1..8b4ba67d5c303c 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -196,6 +196,7 @@ private static class ArchiveSourceJson {
     String integrity;
     String stripPrefix;
     Map<String, String> patches;
+    Map<String, RemoteFile> overlay;
     int patchStrip;
     String archiveType;
   }
@@ -386,11 +387,17 @@ private RepoSpec createArchiveRepoSpec(
       }
     }
 
+    ImmutableMap.Builder<String, RemoteFile> overlay = new ImmutableMap.Builder<>();
+    if (sourceJson .overlay != null) {
+      overlay.putAll(sourceJson.overlay);
+    }
+
     return new ArchiveRepoSpecBuilder()
         .setUrls(urls.build())
         .setIntegrity(sourceJson.integrity)
         .setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix))
         .setRemotePatches(remotePatches.buildOrThrow())
+        .setOverlay(overlay.buildOrThrow())
         .setRemotePatchStrip(sourceJson.patchStrip)
         .setArchiveType(sourceJson.archiveType)
         .build();
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
new file mode 100644
index 00000000000000..85786baab6d734
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
@@ -0,0 +1,20 @@
+package com.google.devtools.build.lib.bazel.bzlmod;
+import java.net.URL;
+import java.util.List;
+
+/**
+ * For use with IndexRegistry and associated files. A simple pojo to track remote files that are
+ * offered at multiple urls (mirrors) with a single integrity.
+ * We split up the file here to simplify the dependency.
+ */
+class RemoteFile {
+
+  RemoteFile(String integrity, List<URL> urls) {
+    this.integrity = integrity;
+    this.urls = urls;
+  }
+
+  List<URL> urls;
+  String integrity;
+}
+
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
index 806c8ebc58fac2..86f8827c9924e9 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -41,6 +41,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
+import java.net.URL;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
@@ -182,6 +183,18 @@ public void testGetArchiveRepoSpec() throws Exception {
         "  },",
         "  \"patch_strip\": 3",
         "}");
+    server.serve(
+        "/modules/baz/3.0/source.json",
+        "{",
+        "  \"url\": \"https://example.com/archive.jar?with=query\",",
+        "  \"integrity\": \"sha256-bleh\",",
+        "  \"overlay\": {",
+        "    \"BUILD.bazel\": {",
+        "       \"urls\": [\"http://mirror1\", \"http://mirror2\"],",
+        "       \"integrity\": \"sha256-bleh-overlay\"",
+        "   }",
+        "  }",
+        "}");
     server.start();
 
     Registry registry =
@@ -198,6 +211,7 @@ public void testGetArchiveRepoSpec() throws Exception {
                 .setIntegrity("sha256-blah")
                 .setStripPrefix("pref")
                 .setRemotePatches(ImmutableMap.of())
+                .setOverlay(ImmutableMap.of())
                 .setRemotePatchStrip(0)
                 .build());
     assertThat(registry.getRepoSpec(createModuleKey("bar", "2.0"), reporter))
@@ -216,6 +230,28 @@ public void testGetArchiveRepoSpec() throws Exception {
                         server.getUrl() + "/modules/bar/2.0/patches/2.fix-that.patch",
                             "sha256-kek"))
                 .setRemotePatchStrip(3)
+                .setOverlay(ImmutableMap.of())
+                .build());
+    assertThat(registry.getRepoSpec(createModuleKey("baz", "3.0"), reporter))
+        .isEqualTo(
+            new ArchiveRepoSpecBuilder()
+                .setUrls(
+                    ImmutableList.of(
+                        "https://mirror.bazel.build/example.com/archive.jar?with=query",
+                        "file:///home/bazel/mymirror/example.com/archive.jar?with=query",
+                        "https://example.com/archive.jar?with=query"))
+                .setIntegrity("sha256-bleh")
+                .setStripPrefix("")
+                .setOverlay(
+                    ImmutableMap.of(
+                        "BUILD.bazel", new RemoteFile(
+                            "sha256-bleh-overlay",
+                            List.of(new URL("http://mirror1"), new URL("http://mirror2"))
+                        )
+                    )
+                )
+                .setRemotePatches(ImmutableMap.of())
+                .setRemotePatchStrip(0)
                 .build());
   }
 
@@ -264,6 +300,7 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception {
                 .setIntegrity("sha256-blah")
                 .setStripPrefix("pref")
                 .setRemotePatches(ImmutableMap.of())
+                .setOverlay(ImmutableMap.of())
                 .setRemotePatchStrip(0)
                 .build());
   }
@@ -351,6 +388,7 @@ public void testArchiveWithExplicitType() throws Exception {
                 .setArchiveType("zip")
                 .setRemotePatches(ImmutableMap.of())
                 .setRemotePatchStrip(0)
+                .setOverlay(ImmutableMap.of())
                 .build());
   }
 

From 3288fc822f533692e9d4134b84b673def6a7ae40 Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Tue, 14 May 2024 00:20:22 +0000
Subject: [PATCH 02/10] Addressed feedback fmeum@

---
 .../bazel/bzlmod/ArchiveRepoSpecBuilder.java  |  4 +--
 .../build/lib/bazel/bzlmod/IndexRegistry.java | 27 +++++++++++++++----
 .../build/lib/bazel/bzlmod/RemoteFile.java    | 10 +------
 .../lib/bazel/bzlmod/IndexRegistryTest.java   | 25 ++++++++++-------
 4 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
index 2dd1deccd332a0..10f04deab609ca 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -86,10 +86,10 @@ public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remo
   public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overlay) {
     final Map<String, List<String>> remoteFiles = overlay.entrySet().stream().collect(Collectors.toMap(
         Entry::getKey,
-        e -> e.getValue().urls.stream().map(URL::toString).collect(Collectors.toList())
+        e -> e.getValue().urls().stream().map(URL::toString).collect(Collectors.toList())
     ));
     final Map<String, String> remoteFilesIntegrity = overlay.entrySet().stream()
-        .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().integrity));
+        .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().integrity()));
     attrBuilder.put("remote_file_urls", remoteFiles);
     attrBuilder.put("remote_file_integrity", remoteFilesIntegrity);
     return this;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index 8b4ba67d5c303c..9634fdee58fdba 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -21,6 +21,7 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.reflect.TypeToken;
 import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum.MissingChecksumException;
@@ -35,12 +36,18 @@
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.lang.reflect.Type;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
@@ -88,6 +95,18 @@ public IndexRegistry(
     this.gson =
         new GsonBuilder()
             .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
+            // record types in JDK15+ causes issues with GSON since fields are now final
+            // create custom deserializer
+            // https://github.com/google/gson/issues/1794#issuecomment-706532333
+            .registerTypeAdapter(RemoteFile.class, new JsonDeserializer<RemoteFile>() {
+              @Override
+              public RemoteFile deserialize(JsonElement json, Type type,
+                  JsonDeserializationContext ctxt) throws JsonParseException {
+                final JsonObject object = json.getAsJsonObject();
+                return new RemoteFile(object.get("integrity").getAsString(),
+                    ctxt.deserialize(object.get("urls"), new TypeToken<List<URL>>(){}.getType()));
+              }
+            })
             .create();
     this.knownFileHashes = knownFileHashes;
     this.knownFileHashesMode = knownFileHashesMode;
@@ -387,17 +406,15 @@ private RepoSpec createArchiveRepoSpec(
       }
     }
 
-    ImmutableMap.Builder<String, RemoteFile> overlay = new ImmutableMap.Builder<>();
-    if (sourceJson .overlay != null) {
-      overlay.putAll(sourceJson.overlay);
-    }
+    var overlay = sourceJson.overlay != null ?
+        ImmutableMap.copyOf(sourceJson.overlay) : ImmutableMap.<String, RemoteFile>of();
 
     return new ArchiveRepoSpecBuilder()
         .setUrls(urls.build())
         .setIntegrity(sourceJson.integrity)
         .setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix))
         .setRemotePatches(remotePatches.buildOrThrow())
-        .setOverlay(overlay.buildOrThrow())
+        .setOverlay(overlay)
         .setRemotePatchStrip(sourceJson.patchStrip)
         .setArchiveType(sourceJson.archiveType)
         .build();
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
index 85786baab6d734..3f62698b8a16d1 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
@@ -7,14 +7,6 @@
  * offered at multiple urls (mirrors) with a single integrity.
  * We split up the file here to simplify the dependency.
  */
-class RemoteFile {
-
-  RemoteFile(String integrity, List<URL> urls) {
-    this.integrity = integrity;
-    this.urls = urls;
-  }
-
-  List<URL> urls;
-  String integrity;
+record RemoteFile(String integrity, List<URL> urls) {
 }
 
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
index 86f8827c9924e9..2c10a3e2dcaf0a 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -185,16 +185,21 @@ public void testGetArchiveRepoSpec() throws Exception {
         "}");
     server.serve(
         "/modules/baz/3.0/source.json",
-        "{",
-        "  \"url\": \"https://example.com/archive.jar?with=query\",",
-        "  \"integrity\": \"sha256-bleh\",",
-        "  \"overlay\": {",
-        "    \"BUILD.bazel\": {",
-        "       \"urls\": [\"http://mirror1\", \"http://mirror2\"],",
-        "       \"integrity\": \"sha256-bleh-overlay\"",
-        "   }",
-        "  }",
-        "}");
+        """
+        {
+            "url": "https://example.com/archive.jar?with=query",
+            "integrity": "sha256-bleh",
+            "overlay": {
+                "BUILD.bazel": {
+                    "urls": [
+                        "http://mirror1",
+                        "http://mirror2"
+                    ],
+                    "integrity": "sha256-bleh-overlay"
+                }
+            }
+        }       
+        """);
     server.start();
 
     Registry registry =

From 858b9113640358a937f8dde5e6dc0ab566a8cd6c Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Tue, 14 May 2024 00:21:50 +0000
Subject: [PATCH 03/10] add static import for collectors

---
 .../build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
index 10f04deab609ca..f22ab1585fa094 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -15,6 +15,9 @@
 
 package com.google.devtools.build.lib.bazel.bzlmod;
 
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toList;
+
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -23,7 +26,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.stream.Collectors;
 import net.starlark.java.eval.StarlarkInt;
 
 /**
@@ -84,12 +86,12 @@ public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remo
 
   @CanIgnoreReturnValue
   public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overlay) {
-    final Map<String, List<String>> remoteFiles = overlay.entrySet().stream().collect(Collectors.toMap(
+    final Map<String, List<String>> remoteFiles = overlay.entrySet().stream().collect(toMap(
         Entry::getKey,
-        e -> e.getValue().urls().stream().map(URL::toString).collect(Collectors.toList())
+        e -> e.getValue().urls().stream().map(URL::toString).collect(toList())
     ));
     final Map<String, String> remoteFilesIntegrity = overlay.entrySet().stream()
-        .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().integrity()));
+        .collect(toMap(Entry::getKey, e -> e.getValue().integrity()));
     attrBuilder.put("remote_file_urls", remoteFiles);
     attrBuilder.put("remote_file_integrity", remoteFilesIntegrity);
     return this;

From ece6861a9aeef9b487300848cfd8f3de8491c235 Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Tue, 14 May 2024 03:26:36 +0000
Subject: [PATCH 04/10] Fix mod_command_test.py

---
 src/test/py/bazel/bzlmod/mod_command_test.py | 34 ++++++++++++--------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/test/py/bazel/bzlmod/mod_command_test.py b/src/test/py/bazel/bzlmod/mod_command_test.py
index cc4c844f5a1c62..eac9852c4a378d 100644
--- a/src/test/py/bazel/bzlmod/mod_command_test.py
+++ b/src/test/py/bazel/bzlmod/mod_command_test.py
@@ -397,13 +397,15 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
     )
     self.assertRegex(stdout.pop(4), r'^  urls = \[".*"\],$')
     self.assertRegex(stdout.pop(4), r'^  integrity = ".*",$')
-    stdout.pop(11)
-    self.assertRegex(stdout.pop(16), r'^  path = ".*",$')
-    stdout.pop(29)
-    stdout.pop(39)
-    self.assertRegex(stdout.pop(44), r'^  urls = \[".*"\],$')
-    self.assertRegex(stdout.pop(44), r'^  integrity = ".*",$')
-    stdout.pop(51)
+    self.assertRegex(stdout.pop(19), r'^  path = ".*",$')
+    # lines after 'Rule data_repo defined at (most recent call last):'
+    stdout.pop(32)
+    stdout.pop(42)
+    self.assertRegex(stdout.pop(47), r'^  urls = \[".*"\],$')
+    self.assertRegex(stdout.pop(47), r'^  integrity = ".*",$')
+    # lines after '# Rule http_archive defined at (most recent call last):'
+    stdout.pop(13)
+    stdout.pop(55)
     self.assertListEqual(
         stdout,
         [
@@ -414,19 +416,21 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
             # pop(4) -- urls=[...]
             # pop(4) -- integrity=...
             '  strip_prefix = "",',
+            '  remote_file_urls = {},',
+            '  remote_file_integrity = {},',
             '  remote_patches = {},',
             '  remote_patch_strip = 0,',
             ')',
             '# Rule bar~ instantiated at (most recent call last):',
             '#   <builtin> in <toplevel>',
             '# Rule http_archive defined at (most recent call last):',
-            # pop(11)
+            # pop(13)
             '',
             '## ext@1.0:',
             '# <builtin>',
             'local_repository(',
             '  name = "ext~",',
-            # pop(16) -- path=...
+            # pop(19) -- path=...
             ')',
             '# Rule ext~ instantiated at (most recent call last):',
             '#   <builtin> in <toplevel>',
@@ -440,7 +444,7 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
             '# Rule ext~~ext~repo3 instantiated at (most recent call last):',
             '#   <builtin> in <toplevel>',
             '# Rule data_repo defined at (most recent call last):',
-            # pop(29)
+            # pop(32)
             '',
             '## @my_repo4:',
             '# <builtin>',
@@ -451,22 +455,24 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
             '# Rule ext~~ext~repo4 instantiated at (most recent call last):',
             '#   <builtin> in <toplevel>',
             '# Rule data_repo defined at (most recent call last):',
-            # pop(39)
+            # pop(42)
             '',
             '## bar@2.0:',
             '# <builtin>',
             'http_archive(',
             '  name = "bar~",',
-            # pop(44) -- urls=[...]
-            # pop(44) -- integrity=...
+            # pop(47) -- urls=[...]
+            # pop(47) -- integrity=...
             '  strip_prefix = "",',
+            '  remote_file_urls = {},',
+            '  remote_file_integrity = {},',
             '  remote_patches = {},',
             '  remote_patch_strip = 0,',
             ')',
             '# Rule bar~ instantiated at (most recent call last):',
             '#   <builtin> in <toplevel>',
             '# Rule http_archive defined at (most recent call last):',
-            # pop(51)
+            # pop(55)
             '',
         ],
         'wrong output in the show query for module and extension-generated'

From 25a4869b985b09e718459dd1df7a36dcf6245f66 Mon Sep 17 00:00:00 2001
From: Fabian Meumertzheim <fabian@meumertzhe.im>
Date: Tue, 14 May 2024 09:48:44 +0200
Subject: [PATCH 05/10] Stylistic changes and GSON update

---
 MODULE.bazel                                  |  2 +-
 MODULE.bazel.lock                             | 21 ++++----------
 maven_install.json                            | 10 +++----
 .../bazel/bzlmod/ArchiveRepoSpecBuilder.java  | 22 +++++++++-----
 .../devtools/build/lib/bazel/bzlmod/BUILD     |  1 -
 .../build/lib/bazel/bzlmod/IndexRegistry.java | 29 ++++---------------
 .../build/lib/bazel/bzlmod/RemoteFile.java    | 12 --------
 .../lib/bazel/bzlmod/IndexRegistryTest.java   | 14 ++++-----
 8 files changed, 38 insertions(+), 73 deletions(-)
 delete mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java

diff --git a/MODULE.bazel b/MODULE.bazel
index 5963710b3da581..d1f06553d8e0fd 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -94,7 +94,7 @@ maven.install(
         "com.google.auto.value:auto-value:1.8.2",
         "com.google.auto.value:auto-value-annotations:1.9",
         "com.google.code.findbugs:jsr305:3.0.2",
-        "com.google.code.gson:gson:2.9.0",
+        "com.google.code.gson:gson:2.10.1",
         "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter:3.3.0",
         "com.google.errorprone:error_prone_annotation:2.23.0",
         "com.google.errorprone:error_prone_annotations:2.23.0",
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
index 1311d736d0a7e4..972aa71bd71061 100644
--- a/MODULE.bazel.lock
+++ b/MODULE.bazel.lock
@@ -1,6 +1,6 @@
 {
   "lockFileVersion": 6,
-  "moduleFileHash": "ec2861042ece461ae1d704f7f8643f5dbc353e5683f6138cfedf9b63b3d63343",
+  "moduleFileHash": "1c3fc0fb6e813794fe5fda4d81528691b500b7276ca8ef26b8057c388bb1a073",
   "flags": {
     "cmdRegistries": [
       "https://bcr.bazel.build/"
@@ -68,7 +68,7 @@
                   "com.google.auto.value:auto-value:1.8.2",
                   "com.google.auto.value:auto-value-annotations:1.9",
                   "com.google.code.findbugs:jsr305:3.0.2",
-                  "com.google.code.gson:gson:2.9.0",
+                  "com.google.code.gson:gson:2.10.1",
                   "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter:3.3.0",
                   "com.google.errorprone:error_prone_annotation:2.23.0",
                   "com.google.errorprone:error_prone_annotations:2.23.0",
@@ -5459,7 +5459,7 @@
         "recordedFileInputs": {
           "@@//src/tools/android/maven_android_install.json": "09bff3e33d291336046f7c9201630fb5e014f0e60b78b6f09b84e4f5f73ed04f",
           "@@rules_jvm_external~//rules_jvm_external_deps_install.json": "cafb5d2d8119391eb2b322ce3840d3352ea82d496bdb8cbd4b6779ec4d044dda",
-          "@@//maven_install.json": "36ad2194679bab5fd1cd8d09f256650cffeedda18e488ac9f3569aa5b5561277"
+          "@@//maven_install.json": "4e6024ecc17223430b4f2777d5f7d58b0ef4a337727ad8ecd6dd98c7fa241f37"
         },
         "recordedDirentsInputs": {},
         "envVariables": {},
@@ -6070,7 +6070,7 @@
                 "{ \"group\": \"com.google.auto.value\", \"artifact\": \"auto-value\", \"version\": \"1.8.2\" }",
                 "{ \"group\": \"com.google.auto.value\", \"artifact\": \"auto-value-annotations\", \"version\": \"1.9\" }",
                 "{ \"group\": \"com.google.code.findbugs\", \"artifact\": \"jsr305\", \"version\": \"3.0.2\" }",
-                "{ \"group\": \"com.google.code.gson\", \"artifact\": \"gson\", \"version\": \"2.9.0\" }",
+                "{ \"group\": \"com.google.code.gson\", \"artifact\": \"gson\", \"version\": \"2.10.1\" }",
                 "{ \"group\": \"com.google.code.java-allocation-instrumenter\", \"artifact\": \"java-allocation-instrumenter\", \"version\": \"3.3.0\" }",
                 "{ \"group\": \"com.google.errorprone\", \"artifact\": \"error_prone_annotation\", \"version\": \"2.23.0\" }",
                 "{ \"group\": \"com.google.errorprone\", \"artifact\": \"error_prone_annotations\", \"version\": \"2.23.0\" }",
@@ -8225,17 +8225,6 @@
               "downloaded_file_path": "v1/io/grpc/grpc-services/1.56.1/grpc-services-1.56.1.jar"
             }
           },
-          "com_google_code_gson_gson_2_9_0": {
-            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
-            "ruleClassName": "http_file",
-            "attributes": {
-              "sha256": "c96d60551331a196dac54b745aa642cd078ef89b6f267146b705f2c2cbef052d",
-              "urls": [
-                "https://repo1.maven.org/maven2/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar"
-              ],
-              "downloaded_file_path": "v1/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar"
-            }
-          },
           "software_amazon_awssdk_regions_2_20_128": {
             "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
             "ruleClassName": "http_file",
@@ -8708,7 +8697,7 @@
                 "{ \"group\": \"com.google.auto.value\", \"artifact\": \"auto-value\", \"version\": \"1.8.2\" }",
                 "{ \"group\": \"com.google.auto.value\", \"artifact\": \"auto-value-annotations\", \"version\": \"1.9\" }",
                 "{ \"group\": \"com.google.code.findbugs\", \"artifact\": \"jsr305\", \"version\": \"3.0.2\" }",
-                "{ \"group\": \"com.google.code.gson\", \"artifact\": \"gson\", \"version\": \"2.9.0\" }",
+                "{ \"group\": \"com.google.code.gson\", \"artifact\": \"gson\", \"version\": \"2.10.1\" }",
                 "{ \"group\": \"com.google.code.java-allocation-instrumenter\", \"artifact\": \"java-allocation-instrumenter\", \"version\": \"3.3.0\" }",
                 "{ \"group\": \"com.google.errorprone\", \"artifact\": \"error_prone_annotation\", \"version\": \"2.23.0\" }",
                 "{ \"group\": \"com.google.errorprone\", \"artifact\": \"error_prone_annotations\", \"version\": \"2.23.0\" }",
diff --git a/maven_install.json b/maven_install.json
index d32877bf11a99d..02a914a45d99f6 100644
--- a/maven_install.json
+++ b/maven_install.json
@@ -1,10 +1,10 @@
 {
   "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
-  "__INPUT_ARTIFACTS_HASH": -1536314734,
-  "__RESOLVED_ARTIFACTS_HASH": -1123103492,
+  "__INPUT_ARTIFACTS_HASH": 1275969355,
+  "__RESOLVED_ARTIFACTS_HASH": -209014774,
   "conflict_resolution": {
     "com.google.auto.value:auto-value-annotations:1.9": "com.google.auto.value:auto-value-annotations:1.10.4",
-    "com.google.code.gson:gson:2.8.9": "com.google.code.gson:gson:2.9.0",
+    "com.google.code.gson:gson:2.8.9": "com.google.code.gson:gson:2.10.1",
     "com.google.errorprone:error_prone_annotations:2.23.0": "com.google.errorprone:error_prone_annotations:2.24.1",
     "com.google.errorprone:error_prone_annotations:2.3.2": "com.google.errorprone:error_prone_annotations:2.24.1",
     "com.google.guava:failureaccess:1.0.1": "com.google.guava:failureaccess:1.0.2",
@@ -122,9 +122,9 @@
     },
     "com.google.code.gson:gson": {
       "shasums": {
-        "jar": "c96d60551331a196dac54b745aa642cd078ef89b6f267146b705f2c2cbef052d"
+        "jar": "4241c14a7727c34feea6507ec801318a3d4a90f070e4525681079fb94ee4c593"
       },
-      "version": "2.9.0"
+      "version": "2.10.1"
     },
     "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter": {
       "shasums": {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
index f22ab1585fa094..255b95a6c68100 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -15,8 +15,8 @@
 
 package com.google.devtools.build.lib.bazel.bzlmod;
 
-import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
@@ -86,12 +86,14 @@ public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remo
 
   @CanIgnoreReturnValue
   public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overlay) {
-    final Map<String, List<String>> remoteFiles = overlay.entrySet().stream().collect(toMap(
-        Entry::getKey,
-        e -> e.getValue().urls().stream().map(URL::toString).collect(toList())
-    ));
-    final Map<String, String> remoteFilesIntegrity = overlay.entrySet().stream()
-        .collect(toMap(Entry::getKey, e -> e.getValue().integrity()));
+    Map<String, List<String>> remoteFiles =
+        overlay.entrySet().stream()
+            .collect(
+                toMap(
+                    Entry::getKey,
+                    e -> e.getValue().urls().stream().map(URL::toString).collect(toList())));
+    Map<String, String> remoteFilesIntegrity =
+        overlay.entrySet().stream().collect(toMap(Entry::getKey, e -> e.getValue().integrity()));
     attrBuilder.put("remote_file_urls", remoteFiles);
     attrBuilder.put("remote_file_integrity", remoteFilesIntegrity);
     return this;
@@ -118,4 +120,10 @@ public RepoSpec build() {
         .setAttributes(AttributeValues.create(attrBuilder.buildOrThrow()))
         .build();
   }
+
+  /**
+   * A simple pojo to track remote files that are offered at multiple urls (mirrors) with a single
+   * integrity. We split up the file here to simplify the dependency.
+   */
+  public record RemoteFile(String integrity, List<URL> urls) {}
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
index e150dd9374009c..cb5d26d6062348 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
@@ -21,7 +21,6 @@ java_library(
         "GitRepoSpecBuilder.java",
         "ModuleFile.java",
         "ModuleKey.java",
-        "RemoteFile.java",
         "RepoSpec.java",
         "Version.java",
     ],
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index 9634fdee58fdba..a37ea8268e9f57 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -21,7 +21,6 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.reflect.TypeToken;
 import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum.MissingChecksumException;
@@ -36,18 +35,12 @@
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.lang.reflect.Type;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
@@ -95,18 +88,6 @@ public IndexRegistry(
     this.gson =
         new GsonBuilder()
             .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
-            // record types in JDK15+ causes issues with GSON since fields are now final
-            // create custom deserializer
-            // https://github.com/google/gson/issues/1794#issuecomment-706532333
-            .registerTypeAdapter(RemoteFile.class, new JsonDeserializer<RemoteFile>() {
-              @Override
-              public RemoteFile deserialize(JsonElement json, Type type,
-                  JsonDeserializationContext ctxt) throws JsonParseException {
-                final JsonObject object = json.getAsJsonObject();
-                return new RemoteFile(object.get("integrity").getAsString(),
-                    ctxt.deserialize(object.get("urls"), new TypeToken<List<URL>>(){}.getType()));
-              }
-            })
             .create();
     this.knownFileHashes = knownFileHashes;
     this.knownFileHashesMode = knownFileHashesMode;
@@ -215,7 +196,7 @@ private static class ArchiveSourceJson {
     String integrity;
     String stripPrefix;
     Map<String, String> patches;
-    Map<String, RemoteFile> overlay;
+    Map<String, ArchiveRepoSpecBuilder.RemoteFile> overlay;
     int patchStrip;
     String archiveType;
   }
@@ -406,15 +387,15 @@ private RepoSpec createArchiveRepoSpec(
       }
     }
 
-    var overlay = sourceJson.overlay != null ?
-        ImmutableMap.copyOf(sourceJson.overlay) : ImmutableMap.<String, RemoteFile>of();
-
     return new ArchiveRepoSpecBuilder()
         .setUrls(urls.build())
         .setIntegrity(sourceJson.integrity)
         .setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix))
         .setRemotePatches(remotePatches.buildOrThrow())
-        .setOverlay(overlay)
+        .setOverlay(
+            sourceJson.overlay != null
+                ? ImmutableMap.copyOf(sourceJson.overlay)
+                : ImmutableMap.of())
         .setRemotePatchStrip(sourceJson.patchStrip)
         .setArchiveType(sourceJson.archiveType)
         .build();
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
deleted file mode 100644
index 3f62698b8a16d1..00000000000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package com.google.devtools.build.lib.bazel.bzlmod;
-import java.net.URL;
-import java.util.List;
-
-/**
- * For use with IndexRegistry and associated files. A simple pojo to track remote files that are
- * offered at multiple urls (mirrors) with a single integrity.
- * We split up the file here to simplify the dependency.
- */
-record RemoteFile(String integrity, List<URL> urls) {
-}
-
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
index 2c10a3e2dcaf0a..3c7b6ebf2ebda8 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -41,7 +41,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
-import java.net.URL;
+import java.net.URI;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
@@ -198,7 +198,7 @@ public void testGetArchiveRepoSpec() throws Exception {
                     "integrity": "sha256-bleh-overlay"
                 }
             }
-        }       
+        }
         """);
     server.start();
 
@@ -249,12 +249,12 @@ public void testGetArchiveRepoSpec() throws Exception {
                 .setStripPrefix("")
                 .setOverlay(
                     ImmutableMap.of(
-                        "BUILD.bazel", new RemoteFile(
+                        "BUILD.bazel",
+                        new ArchiveRepoSpecBuilder.RemoteFile(
                             "sha256-bleh-overlay",
-                            List.of(new URL("http://mirror1"), new URL("http://mirror2"))
-                        )
-                    )
-                )
+                            ImmutableList.of(
+                                URI.create("http://mirror1").toURL(),
+                                URI.create("http://mirror2").toURL()))))
                 .setRemotePatches(ImmutableMap.of())
                 .setRemotePatchStrip(0)
                 .build());

From 2afc01abf78858101383cc88f34a7f3d58278445 Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Tue, 14 May 2024 16:28:46 +0000
Subject: [PATCH 06/10] Update lock file

---
 MODULE.bazel.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
index 4940743ba7258d..66b524bc980839 100644
--- a/MODULE.bazel.lock
+++ b/MODULE.bazel.lock
@@ -2927,7 +2927,7 @@
       "general": {
         "bzlTransitiveDigest": "VBORN+IBXeLBoYFlmaQRhi4EgJwXxJ9i+65BUjwwG18=",
         "recordedFileInputs": {
-          "@@//MODULE.bazel": "ec2861042ece461ae1d704f7f8643f5dbc353e5683f6138cfedf9b63b3d63343",
+          "@@//MODULE.bazel": "1c3fc0fb6e813794fe5fda4d81528691b500b7276ca8ef26b8057c388bb1a073",
           "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "c2da7815256d6bb92f953b05334419abb9fc698d13710d129a3ca9346eb04a4e"
         },
         "recordedDirentsInputs": {},

From e7f58e3058fa31196a3041c27d2be3973764cfb9 Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Wed, 15 May 2024 17:54:32 +0000
Subject: [PATCH 07/10] Changed overlay setup

* Used mirrors from bazel_registry.json to determine mirrors
* Changed overlay in source.json to be more simple
---
 MODULE.bazel                                  |  2 +-
 .../bazel/bzlmod/ArchiveRepoSpecBuilder.java  |  4 +-
 .../build/lib/bazel/bzlmod/IndexRegistry.java | 46 +++++++++++++++++--
 .../lib/bazel/bzlmod/IndexRegistryTest.java   | 15 +++---
 4 files changed, 50 insertions(+), 17 deletions(-)

diff --git a/MODULE.bazel b/MODULE.bazel
index 6055201bca7a57..cd2a634cb543e7 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -94,7 +94,7 @@ maven.install(
         "com.google.auto.value:auto-value:1.10.4",
         "com.google.auto.value:auto-value-annotations:1.10.4",
         "com.google.code.findbugs:jsr305:3.0.2",
-        "com.google.code.gson:gson:2.10.1",
+        "com.google.code.gson:gson:2.9.0",
         "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter:3.3.0",
         "com.google.errorprone:error_prone_annotation:2.23.0",
         "com.google.errorprone:error_prone_annotations:2.23.0",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
index 255b95a6c68100..d24849beb46452 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -91,7 +91,7 @@ public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overla
             .collect(
                 toMap(
                     Entry::getKey,
-                    e -> e.getValue().urls().stream().map(URL::toString).collect(toList())));
+                    e -> e.getValue().urls()));
     Map<String, String> remoteFilesIntegrity =
         overlay.entrySet().stream().collect(toMap(Entry::getKey, e -> e.getValue().integrity()));
     attrBuilder.put("remote_file_urls", remoteFiles);
@@ -125,5 +125,5 @@ public RepoSpec build() {
    * A simple pojo to track remote files that are offered at multiple urls (mirrors) with a single
    * integrity. We split up the file here to simplify the dependency.
    */
-  public record RemoteFile(String integrity, List<URL> urls) {}
+  public record RemoteFile(String integrity, List<String> urls) {}
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index 8732e93c045485..75ee60d287e005 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -17,6 +17,10 @@
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
+import static java.util.stream.Collectors.toList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+
+
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
@@ -44,6 +48,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 /**
  * Represents a Bazel module registry that serves a list of module metadata from a static HTTP
@@ -222,7 +227,7 @@ private static class ArchiveSourceJson {
     String integrity;
     String stripPrefix;
     Map<String, String> patches;
-    Map<String, ArchiveRepoSpecBuilder.RemoteFile> overlay;
+    Map<String, String> overlay;
     int patchStrip;
     String archiveType;
   }
@@ -413,15 +418,46 @@ private RepoSpec createArchiveRepoSpec(
       }
     }
 
+    //This should be the list of all base URLs: main URI + mirrors
+    ImmutableList.Builder<String> baseUrls = new ImmutableList.Builder<>();
+    baseUrls.add(this.getUrl());
+    if (bazelRegistryJson.isPresent() && bazelRegistryJson.get().mirrors != null) {
+      for (String mirror : bazelRegistryJson.get().mirrors) {
+        try {
+          var unused = new URL(mirror);
+        } catch (MalformedURLException e) {
+          throw new IOException("Malformed mirror URL", e);
+        }
+        baseUrls.add(mirror);
+      }
+    }
+
+    Map<String, String> sourceJsonOverlay = sourceJson.overlay != null
+        ? ImmutableMap.copyOf(sourceJson.overlay)
+        : ImmutableMap.of();
+    ImmutableMap<String, ArchiveRepoSpecBuilder.RemoteFile> overlay = sourceJsonOverlay
+        .entrySet()
+        .stream()
+        .collect(toImmutableMap(
+            entry -> entry.getKey(),
+            entry -> new ArchiveRepoSpecBuilder.RemoteFile(
+                entry.getValue(), // integrity
+                baseUrls.build().stream().map(url ->
+                    constructUrl(
+                        url,
+                        "modules",
+                        key.getName(),
+                        key.getVersion().toString(),
+                        entry.getKey())
+                ).collect(toList())
+            )));
+
     return new ArchiveRepoSpecBuilder()
         .setUrls(urls.build())
         .setIntegrity(sourceJson.integrity)
         .setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix))
         .setRemotePatches(remotePatches.buildOrThrow())
-        .setOverlay(
-            sourceJson.overlay != null
-                ? ImmutableMap.copyOf(sourceJson.overlay)
-                : ImmutableMap.of())
+        .setOverlay(overlay)
         .setRemotePatchStrip(sourceJson.patchStrip)
         .setArchiveType(sourceJson.archiveType)
         .build();
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
index 3c7b6ebf2ebda8..ced0c343fdbc43 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -190,13 +190,7 @@ public void testGetArchiveRepoSpec() throws Exception {
             "url": "https://example.com/archive.jar?with=query",
             "integrity": "sha256-bleh",
             "overlay": {
-                "BUILD.bazel": {
-                    "urls": [
-                        "http://mirror1",
-                        "http://mirror2"
-                    ],
-                    "integrity": "sha256-bleh-overlay"
-                }
+                "BUILD.bazel": "sha256-bleh-overlay"
             }
         }
         """);
@@ -253,8 +247,11 @@ public void testGetArchiveRepoSpec() throws Exception {
                         new ArchiveRepoSpecBuilder.RemoteFile(
                             "sha256-bleh-overlay",
                             ImmutableList.of(
-                                URI.create("http://mirror1").toURL(),
-                                URI.create("http://mirror2").toURL()))))
+                                server.getUrl() + "/modules/baz/3.0/BUILD.bazel",
+                                "https://mirror.bazel.build/modules/baz/3.0/BUILD.bazel",
+                                "file:///home/bazel/mymirror/modules/baz/3.0/BUILD.bazel"
+                            ))
+                    ))
                 .setRemotePatches(ImmutableMap.of())
                 .setRemotePatchStrip(0)
                 .build());

From a1e679140dadc2a9a0048801f5ad5423c9c5a496 Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Wed, 15 May 2024 18:15:20 +0000
Subject: [PATCH 08/10] Refactor creation of mirror urls into helper method

---
 .../bazel/bzlmod/ArchiveRepoSpecBuilder.java  |  1 -
 .../build/lib/bazel/bzlmod/IndexRegistry.java | 47 +++++++++----------
 2 files changed, 22 insertions(+), 26 deletions(-)

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
index d24849beb46452..14a5ae77307377 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -22,7 +22,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
-import java.net.URL;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index 75ee60d287e005..b9a3d4c189f4bf 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -42,9 +42,11 @@
 import com.google.gson.JsonParseException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
@@ -374,6 +376,21 @@ private RepoSpec createLocalPathRepoSpec(
         .build();
   }
 
+  private List<String> getMirrorUrls(Optional<BazelRegistryJson> registry) {
+    return registry.map(r -> r.mirrors)
+        .map(mirrors -> {
+          for (String mirror : mirrors) {
+            try {
+              var unused = new URL(mirror);
+            } catch (MalformedURLException e) {
+              throw new UncheckedIOException("Malformed mirror URL", e);
+            }
+          }
+          return List.of(mirrors);
+        })
+        .orElse(List.of());
+  }
+
   private RepoSpec createArchiveRepoSpec(
       ArchiveSourceJson sourceJson, Optional<BazelRegistryJson> bazelRegistryJson, ModuleKey key)
       throws IOException {
@@ -386,18 +403,8 @@ private RepoSpec createArchiveRepoSpec(
     }
 
     ImmutableList.Builder<String> urls = new ImmutableList.Builder<>();
-    // For each mirror specified in bazel_registry.json, add a URL that's essentially the mirror
-    // URL concatenated with the source URL.
-    if (bazelRegistryJson.isPresent() && bazelRegistryJson.get().mirrors != null) {
-      for (String mirror : bazelRegistryJson.get().mirrors) {
-        try {
-          var unused = new URL(mirror);
-        } catch (MalformedURLException e) {
-          throw new IOException("Malformed mirror URL", e);
-        }
-
-        urls.add(constructUrl(mirror, sourceUrl.getAuthority(), sourceUrl.getFile()));
-      }
+    for (String mirror : getMirrorUrls(bazelRegistryJson)) {
+      urls.add(constructUrl(mirror, sourceUrl.getAuthority(), sourceUrl.getFile()));
     }
     // Finally add the original source URL itself.
     urls.add(sourceUrl.toString());
@@ -419,18 +426,8 @@ private RepoSpec createArchiveRepoSpec(
     }
 
     //This should be the list of all base URLs: main URI + mirrors
-    ImmutableList.Builder<String> baseUrls = new ImmutableList.Builder<>();
-    baseUrls.add(this.getUrl());
-    if (bazelRegistryJson.isPresent() && bazelRegistryJson.get().mirrors != null) {
-      for (String mirror : bazelRegistryJson.get().mirrors) {
-        try {
-          var unused = new URL(mirror);
-        } catch (MalformedURLException e) {
-          throw new IOException("Malformed mirror URL", e);
-        }
-        baseUrls.add(mirror);
-      }
-    }
+    ImmutableList<String> baseUrls = new ImmutableList.Builder<String>()
+      .add(this.getUrl()).addAll(getMirrorUrls(bazelRegistryJson)).build();
 
     Map<String, String> sourceJsonOverlay = sourceJson.overlay != null
         ? ImmutableMap.copyOf(sourceJson.overlay)
@@ -442,7 +439,7 @@ private RepoSpec createArchiveRepoSpec(
             entry -> entry.getKey(),
             entry -> new ArchiveRepoSpecBuilder.RemoteFile(
                 entry.getValue(), // integrity
-                baseUrls.build().stream().map(url ->
+                baseUrls.stream().map(url ->
                     constructUrl(
                         url,
                         "modules",

From 1a2d3ca93b696af91c74b3dc4cd7f1755e0b5ecd Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Wed, 15 May 2024 18:35:30 +0000
Subject: [PATCH 09/10] Remove mirror from overlay

---
 .../build/lib/bazel/bzlmod/IndexRegistry.java | 23 ++++++++-----------
 .../lib/bazel/bzlmod/IndexRegistryTest.java   |  6 ++---
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index b9a3d4c189f4bf..a5ba787fb1ff63 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -425,10 +425,6 @@ private RepoSpec createArchiveRepoSpec(
       }
     }
 
-    //This should be the list of all base URLs: main URI + mirrors
-    ImmutableList<String> baseUrls = new ImmutableList.Builder<String>()
-      .add(this.getUrl()).addAll(getMirrorUrls(bazelRegistryJson)).build();
-
     Map<String, String> sourceJsonOverlay = sourceJson.overlay != null
         ? ImmutableMap.copyOf(sourceJson.overlay)
         : ImmutableMap.of();
@@ -439,15 +435,16 @@ private RepoSpec createArchiveRepoSpec(
             entry -> entry.getKey(),
             entry -> new ArchiveRepoSpecBuilder.RemoteFile(
                 entry.getValue(), // integrity
-                baseUrls.stream().map(url ->
-                    constructUrl(
-                        url,
-                        "modules",
-                        key.getName(),
-                        key.getVersion().toString(),
-                        entry.getKey())
-                ).collect(toList())
-            )));
+                // Registry itself is not mirror'd
+                // Confirmed with @Wyverland
+                List.of(constructUrl(
+                    getUrl(),
+                    "modules",
+                    key.getName(),
+                    key.getVersion().toString(),
+                    entry.getKey())
+                ))
+            ));
 
     return new ArchiveRepoSpecBuilder()
         .setUrls(urls.build())
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
index ced0c343fdbc43..1beee8de6a3d60 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -246,10 +246,10 @@ public void testGetArchiveRepoSpec() throws Exception {
                         "BUILD.bazel",
                         new ArchiveRepoSpecBuilder.RemoteFile(
                             "sha256-bleh-overlay",
+                            // Registry itself is not mirror'd
+                            // Confirmed with @Wyverland
                             ImmutableList.of(
-                                server.getUrl() + "/modules/baz/3.0/BUILD.bazel",
-                                "https://mirror.bazel.build/modules/baz/3.0/BUILD.bazel",
-                                "file:///home/bazel/mymirror/modules/baz/3.0/BUILD.bazel"
+                                server.getUrl() + "/modules/baz/3.0/BUILD.bazel"
                             ))
                     ))
                 .setRemotePatches(ImmutableMap.of())

From 3e27ccee48a6be687449d1ae276897d468c823e0 Mon Sep 17 00:00:00 2001
From: Farid Zakaria <fmzakari@google.com>
Date: Wed, 15 May 2024 21:13:55 +0000
Subject: [PATCH 10/10] Feedback from Wyverald addressed

---
 .../bazel/bzlmod/ArchiveRepoSpecBuilder.java  |  7 ++--
 .../build/lib/bazel/bzlmod/IndexRegistry.java | 35 ++++++++-----------
 .../lib/bazel/bzlmod/IndexRegistryTest.java   |  3 +-
 3 files changed, 18 insertions(+), 27 deletions(-)

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
index 14a5ae77307377..3cf7fd78cb9f79 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -15,8 +15,7 @@
 
 package com.google.devtools.build.lib.bazel.bzlmod;
 
-import static java.util.stream.Collectors.toList;
-import static java.util.stream.Collectors.toMap;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
@@ -88,11 +87,11 @@ public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overla
     Map<String, List<String>> remoteFiles =
         overlay.entrySet().stream()
             .collect(
-                toMap(
+                toImmutableMap(
                     Entry::getKey,
                     e -> e.getValue().urls()));
     Map<String, String> remoteFilesIntegrity =
-        overlay.entrySet().stream().collect(toMap(Entry::getKey, e -> e.getValue().integrity()));
+        overlay.entrySet().stream().collect(toImmutableMap(Entry::getKey, e -> e.getValue().integrity()));
     attrBuilder.put("remote_file_urls", remoteFiles);
     attrBuilder.put("remote_file_integrity", remoteFilesIntegrity);
     return this;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
index a5ba787fb1ff63..a184003e8ce588 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -17,7 +17,6 @@
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import static java.util.stream.Collectors.toList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 
 
@@ -376,21 +375,6 @@ private RepoSpec createLocalPathRepoSpec(
         .build();
   }
 
-  private List<String> getMirrorUrls(Optional<BazelRegistryJson> registry) {
-    return registry.map(r -> r.mirrors)
-        .map(mirrors -> {
-          for (String mirror : mirrors) {
-            try {
-              var unused = new URL(mirror);
-            } catch (MalformedURLException e) {
-              throw new UncheckedIOException("Malformed mirror URL", e);
-            }
-          }
-          return List.of(mirrors);
-        })
-        .orElse(List.of());
-  }
-
   private RepoSpec createArchiveRepoSpec(
       ArchiveSourceJson sourceJson, Optional<BazelRegistryJson> bazelRegistryJson, ModuleKey key)
       throws IOException {
@@ -403,8 +387,18 @@ private RepoSpec createArchiveRepoSpec(
     }
 
     ImmutableList.Builder<String> urls = new ImmutableList.Builder<>();
-    for (String mirror : getMirrorUrls(bazelRegistryJson)) {
-      urls.add(constructUrl(mirror, sourceUrl.getAuthority(), sourceUrl.getFile()));
+    // For each mirror specified in bazel_registry.json, add a URL that's essentially the mirror
+    // URL concatenated with the source URL.
+    if (bazelRegistryJson.isPresent() && bazelRegistryJson.get().mirrors != null) {
+      for (String mirror : bazelRegistryJson.get().mirrors) {
+        try {
+          var unused = new URL(mirror);
+        } catch (MalformedURLException e) {
+          throw new IOException("Malformed mirror URL", e);
+        }
+
+        urls.add(constructUrl(mirror, sourceUrl.getAuthority(), sourceUrl.getFile()));
+      }
     }
     // Finally add the original source URL itself.
     urls.add(sourceUrl.toString());
@@ -435,9 +429,8 @@ private RepoSpec createArchiveRepoSpec(
             entry -> entry.getKey(),
             entry -> new ArchiveRepoSpecBuilder.RemoteFile(
                 entry.getValue(), // integrity
-                // Registry itself is not mirror'd
-                // Confirmed with @Wyverland
-                List.of(constructUrl(
+                // URLs in the registry itself are not mirrored.
+                ImmutableList.of(constructUrl(
                     getUrl(),
                     "modules",
                     key.getName(),
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
index 1beee8de6a3d60..29cac3189eecd4 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -246,8 +246,7 @@ public void testGetArchiveRepoSpec() throws Exception {
                         "BUILD.bazel",
                         new ArchiveRepoSpecBuilder.RemoteFile(
                             "sha256-bleh-overlay",
-                            // Registry itself is not mirror'd
-                            // Confirmed with @Wyverland
+                            // URLs in the registry itself are not mirrored.
                             ImmutableList.of(
                                 server.getUrl() + "/modules/baz/3.0/BUILD.bazel"
                             ))