Configurable override of url generation for External Tool versions (#…

…20074)

Adds support for an optional 5th argument in the `known_version` strings
to override the normally-generated url. This allows setting different
urls for architecture/version via user-defined options.

See #19940 for context.

src/python/pants/core/util_rules/external_tool.py

@@ -65,14 +65,20 @@ class ExternalToolVersion:
     platform: str
     sha256: str
     filesize: int
+    url_override: str | None = None
 
     def encode(self) -> str:
-        return "|".join([self.version, self.platform, self.sha256, str(self.filesize)])
+        parts = [self.version, self.platform, self.sha256, str(self.filesize)]
+        if self.url_override:
+            parts.append(self.url_override)
+        return "|".join(parts)
 
     @classmethod
     def decode(cls, version_str: str) -> ExternalToolVersion:
-        version, platform, sha256, filesize = [x.strip() for x in version_str.split("|")]
-        return cls(version, platform, sha256, int(filesize))
+        parts = [x.strip() for x in version_str.split("|")]
+        version, platform, sha256, filesize = parts[:4]
+        url_override = parts[4] if len(parts) > 4 else None
+        return cls(version, platform, sha256, int(filesize), url_override=url_override)
 
 
 class ExternalToolOptionsMixin:
@@ -112,16 +118,20 @@ def name(cls):
             f"""
         Known versions to verify downloads against.
 
-        Each element is a pipe-separated string of `version|platform|sha256|length`, where:
+        Each element is a pipe-separated string of `version|platform|sha256|length` or
+        `version|platform|sha256|length|url_override`, where:
 
           - `version` is the version string
           - `platform` is one of `[{','.join(Platform.__members__.keys())}]`
           - `sha256` is the 64-character hex representation of the expected sha256
             digest of the download file, as emitted by `shasum -a 256`
           - `length` is the expected length of the download file in bytes, as emitted by
             `wc -c`
+          - (Optional) `url_override` is a specific url to use instead of the normally
+            generated url for this version
 
         E.g., `3.1.2|macos_x86_64|6d0f18cd84b918c7b3edd0203e75569e0c7caecb1367bbbe409b44e28514f5be|42813`.
+        and `3.1.2|macos_arm64 |aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291|50926|https://example.mac.org/bin/v3.1.2/mac-aarch64-v3.1.2.tgz`.
 
         Values are space-stripped, so pipes can be indented for readability if necessary.
         """
@@ -204,7 +214,12 @@ def get_request(self, plat: Platform) -> ExternalToolRequest:
         for known_version in self.known_versions:
             version = self.decode_known_version(known_version)
             if plat.value == version.platform and version.version == self.version:
-                return self.get_request_for(version.platform, version.sha256, version.filesize)
+                return self.get_request_for(
+                    version.platform,
+                    version.sha256,
+                    version.filesize,
+                    url_override=version.url_override,
+                )
         raise UnknownVersion(
             softwrap(
                 f"""
@@ -228,12 +243,14 @@ def split_known_version_str(cls, known_version: str) -> tuple[str, str, str, int
         version = cls.decode_known_version(known_version)
         return version.version, version.platform, version.sha256, version.filesize
 
-    def get_request_for(self, plat_val: str, sha256: str, length: int) -> ExternalToolRequest:
+    def get_request_for(
+        self, plat_val: str, sha256: str, length: int, url_override: str | None = None
+    ) -> ExternalToolRequest:
         """Generate a request for this tool from the given info."""
         plat = Platform(plat_val)
         digest = FileDigest(fingerprint=sha256, serialized_bytes_length=length)
         try:
-            url = self.generate_url(plat)
+            url = url_override or self.generate_url(plat)
             exe = self.generate_exe(plat)
         except ExternalToolError as e:
             raise ExternalToolError(

src/python/pants/core/util_rules/external_tool_test.py

@@ -32,13 +32,16 @@ class FooBar(ExternalTool):
         "3.2.0|linux_x86_64|c0c667fb679a8221bed01bffeed1f80727c6c7827d0cbd8f162195efb12df9e4|121212",
         "3.4.7|macos_x86_64|9d0e18cd74b918c7b3edd0203e75569e0c8caecb1367b3be409b45e28514f5be|123321",
         "3.4.7|linux_x86_64|a019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5|134213",
+        "3.4.7|macos_arm64 |aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291|145678|https://macfoo.org/bin/v3.4.7/mac-m1-v3.4.7.tgz",
     ]
 
     def generate_url(self, plat: Platform) -> str:
         if plat == Platform.macos_x86_64:
             plat_str = "osx-x86_64"
         elif plat == Platform.linux_x86_64:
             plat_str = "linux-x86_64"
+        elif plat == Platform.macos_arm64:
+            plat_str = "osx-aarch64"
         else:
             raise ExternalToolError()
         return f"https://foobar.org/bin/v{self.version}/foobar-{self.version}-{plat_str}.tgz"
@@ -57,11 +60,12 @@ class TemplatedFooBar(TemplatedExternalTool):
         "3.2.0|linux_x86_64|c0c667fb679a8221bed01bffeed1f80727c6c7827d0cbd8f162195efb12df9e4|121212",
         "3.4.7|macos_x86_64|9d0e18cd74b918c7b3edd0203e75569e0c8caecb1367b3be409b45e28514f5be|123321",
         "3.4.7|linux_x86_64|a019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5|134213",
+        "3.4.7|macos_arm64 |aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291|145678|https://macfoo.org/bin/v3.4.7/mac-m1-v3.4.7.tgz",
     ]
     default_url_template = "https://foobar.org/bin/v{version}/foobar-{version}-{platform}.tgz"
     default_url_platform_mapping = {
         "macos_x86_64": "osx-x86_64",
-        "macos_arm64": "osx-x86_64",
+        "macos_arm64": "osx-aarch64",
         "linux_x86_64": "linux-x86_64",
     }
 
@@ -108,6 +112,13 @@ def do_test(
         Platform.linux_x86_64,
         "3.4.7",
     )
+    do_test(
+        "https://macfoo.org/bin/v3.4.7/mac-m1-v3.4.7.tgz",
+        145678,
+        "aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291",
+        Platform.macos_arm64,
+        "3.4.7",
+    )
 
     with pytest.raises(UnknownVersion):
         create_subsystem(