Skip to content

Commit

Permalink
Add support for overlays in source.json
Browse files Browse the repository at this point in the history 
This is a continuation of #22155 that adds the newly added 'remote_files' attribute for http_archive to the bzlmod functionality.

The end goal is to then update BCR to this new functionality to overlay files rather than use patch files when providing MODULE/WORKSPACE/BUILD files.

bazelbuild/bazel-central-registry#1566 has a good discussion of the rationale.

Co-authored-by: Fabian Meumertzheim <fabian@meumertzhe.im>

Closes #22349.

PiperOrigin-RevId: 636682112
Change-Id: Ief070985598a7c0f427a98cd3daeb69a0984f7be
  • Loading branch information
@fzakaria @copybara-github
fzakaria authored and copybara-github committed May 23, 2024
1 parent c902fbd commit c4167e3
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 14 deletions.
23 changes: 23 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@

package com.google.devtools.build.lib.bazel.bzlmod;

import static com.google.common.collect.ImmutableMap.toImmutableMap;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.util.List;
import java.util.Map.Entry;
import net.starlark.java.eval.StarlarkInt;

/**
Expand Down Expand Up @@ -78,6 +82,19 @@ public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remo
return this;
}

@CanIgnoreReturnValue
public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overlay) {
ImmutableMap<String, List<String>> remoteFiles =
overlay.entrySet().stream()
.collect(toImmutableMap(Entry::getKey, e -> e.getValue().urls()));
ImmutableMap<String, String> remoteFilesIntegrity =
overlay.entrySet().stream()
.collect(toImmutableMap(Entry::getKey, e -> e.getValue().integrity()));
attrBuilder.put("remote_file_urls", remoteFiles);
attrBuilder.put("remote_file_integrity", remoteFilesIntegrity);
return this;
}

@CanIgnoreReturnValue
public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) {
attrBuilder.put("remote_patch_strip", StarlarkInt.of(remotePatchStrip));
Expand All @@ -99,4 +116,10 @@ public RepoSpec build() {
.setAttributes(AttributeValues.create(attrBuilder.buildOrThrow()))
.build();
}

/**
* A simple pojo to track remote files that are offered at multiple urls (mirrors) with a single
* integrity. We split up the file here to simplify the dependency.
*/
public record RemoteFile(String integrity, List<String> urls) {}
}
22 changes: 22 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

package com.google.devtools.build.lib.bazel.bzlmod;

import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.base.Preconditions;
Expand Down Expand Up @@ -222,6 +223,7 @@ private static class ArchiveSourceJson {
String integrity;
String stripPrefix;
Map<String, String> patches;
Map<String, String> overlay;
int patchStrip;
String archiveType;
}
Expand Down Expand Up @@ -412,11 +414,31 @@ private RepoSpec createArchiveRepoSpec(
}
}

ImmutableMap<String, String> sourceJsonOverlay =
sourceJson.overlay != null ? ImmutableMap.copyOf(sourceJson.overlay) : ImmutableMap.of();
ImmutableMap<String, ArchiveRepoSpecBuilder.RemoteFile> overlay =
sourceJsonOverlay.entrySet().stream()
.collect(
toImmutableMap(
entry -> entry.getKey(),
entry ->
new ArchiveRepoSpecBuilder.RemoteFile(
entry.getValue(), // integrity
// URLs in the registry itself are not mirrored.
ImmutableList.of(
constructUrl(
getUrl(),
"modules",
key.getName(),
key.getVersion().toString(),
entry.getKey())))));

return new ArchiveRepoSpecBuilder()
.setUrls(urls.build())
.setIntegrity(sourceJson.integrity)
.setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix))
.setRemotePatches(remotePatches.buildOrThrow())
.setOverlay(overlay)
.setRemotePatchStrip(sourceJson.patchStrip)
.setArchiveType(sourceJson.archiveType)
.build();
Expand Down
35 changes: 35 additions & 0 deletions src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,17 @@ public void testGetArchiveRepoSpec() throws Exception {
" },",
" \"patch_strip\": 3",
"}");
server.serve(
"/modules/baz/3.0/source.json",
"""
{
"url": "https://example.com/archive.jar?with=query",
"integrity": "sha256-bleh",
"overlay": {
"BUILD.bazel": "sha256-bleh-overlay"
}
}
""");
server.start();

Registry registry =
Expand All @@ -198,6 +209,7 @@ public void testGetArchiveRepoSpec() throws Exception {
.setIntegrity("sha256-blah")
.setStripPrefix("pref")
.setRemotePatches(ImmutableMap.of())
.setOverlay(ImmutableMap.of())
.setRemotePatchStrip(0)
.build());
assertThat(registry.getRepoSpec(createModuleKey("bar", "2.0"), reporter))
Expand All @@ -216,6 +228,27 @@ public void testGetArchiveRepoSpec() throws Exception {
server.getUrl() + "/modules/bar/2.0/patches/2.fix-that.patch",
"sha256-kek"))
.setRemotePatchStrip(3)
.setOverlay(ImmutableMap.of())
.build());
assertThat(registry.getRepoSpec(createModuleKey("baz", "3.0"), reporter))
.isEqualTo(
new ArchiveRepoSpecBuilder()
.setUrls(
ImmutableList.of(
"https://mirror.bazel.build/example.com/archive.jar?with=query",
"file:///home/bazel/mymirror/example.com/archive.jar?with=query",
"https://example.com/archive.jar?with=query"))
.setIntegrity("sha256-bleh")
.setStripPrefix("")
.setOverlay(
ImmutableMap.of(
"BUILD.bazel",
new ArchiveRepoSpecBuilder.RemoteFile(
"sha256-bleh-overlay",
// URLs in the registry itself are not mirrored.
ImmutableList.of(server.getUrl() + "/modules/baz/3.0/BUILD.bazel"))))
.setRemotePatches(ImmutableMap.of())
.setRemotePatchStrip(0)
.build());
}

Expand Down Expand Up @@ -264,6 +297,7 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception {
.setIntegrity("sha256-blah")
.setStripPrefix("pref")
.setRemotePatches(ImmutableMap.of())
.setOverlay(ImmutableMap.of())
.setRemotePatchStrip(0)
.build());
}
Expand Down Expand Up @@ -351,6 +385,7 @@ public void testArchiveWithExplicitType() throws Exception {
.setArchiveType("zip")
.setRemotePatches(ImmutableMap.of())
.setRemotePatchStrip(0)
.setOverlay(ImmutableMap.of())
.build());
}

Expand Down
34 changes: 20 additions & 14 deletions src/test/py/bazel/bzlmod/mod_command_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,13 +397,15 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
)
self.assertRegex(stdout.pop(4), r'^ urls = \[".*"\],$')
self.assertRegex(stdout.pop(4), r'^ integrity = ".*",$')
stdout.pop(11)
self.assertRegex(stdout.pop(16), r'^ path = ".*",$')
stdout.pop(29)
stdout.pop(39)
self.assertRegex(stdout.pop(44), r'^ urls = \[".*"\],$')
self.assertRegex(stdout.pop(44), r'^ integrity = ".*",$')
stdout.pop(51)
self.assertRegex(stdout.pop(19), r'^ path = ".*",$')
# lines after 'Rule data_repo defined at (most recent call last):'
stdout.pop(32)
stdout.pop(42)
self.assertRegex(stdout.pop(47), r'^ urls = \[".*"\],$')
self.assertRegex(stdout.pop(47), r'^ integrity = ".*",$')
# lines after '# Rule http_archive defined at (most recent call last):'
stdout.pop(13)
stdout.pop(55)
self.assertListEqual(
stdout,
[
Expand All @@ -414,19 +416,21 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
# pop(4) -- urls=[...]
# pop(4) -- integrity=...
' strip_prefix = "",',
' remote_file_urls = {},',
' remote_file_integrity = {},',
' remote_patches = {},',
' remote_patch_strip = 0,',
')',
'# Rule bar~ instantiated at (most recent call last):',
'# <builtin> in <toplevel>',
'# Rule http_archive defined at (most recent call last):',
# pop(11)
# pop(13)
'',
'## ext@1.0:',
'# <builtin>',
'local_repository(',
' name = "ext~",',
# pop(16) -- path=...
# pop(19) -- path=...
')',
'# Rule ext~ instantiated at (most recent call last):',
'# <builtin> in <toplevel>',
Expand All @@ -440,7 +444,7 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
'# Rule ext~~ext~repo3 instantiated at (most recent call last):',
'# <builtin> in <toplevel>',
'# Rule data_repo defined at (most recent call last):',
# pop(29)
# pop(32)
'',
'## @my_repo4:',
'# <builtin>',
Expand All @@ -451,22 +455,24 @@ def testShowModuleAndExtensionReposFromBaseModule(self):
'# Rule ext~~ext~repo4 instantiated at (most recent call last):',
'# <builtin> in <toplevel>',
'# Rule data_repo defined at (most recent call last):',
# pop(39)
# pop(42)
'',
'## bar@2.0:',
'# <builtin>',
'http_archive(',
' name = "bar~",',
# pop(44) -- urls=[...]
# pop(44) -- integrity=...
# pop(47) -- urls=[...]
# pop(47) -- integrity=...
' strip_prefix = "",',
' remote_file_urls = {},',
' remote_file_integrity = {},',
' remote_patches = {},',
' remote_patch_strip = 0,',
')',
'# Rule bar~ instantiated at (most recent call last):',
'# <builtin> in <toplevel>',
'# Rule http_archive defined at (most recent call last):',
# pop(51)
# pop(55)
'',
],
'wrong output in the show query for module and extension-generated'
Expand Down

0 comments on commit c4167e3

Please sign in to comment.