Merge branch 'master' into ENG-585

.gitignore

@@ -173,3 +173,4 @@ cython_debug/
 requirements.txt
 
 .coverage-*.xml
+

charts/platform-registry/templates/deployment.yml

@@ -64,6 +64,8 @@ spec:
           value: {{ .Values.upstreamRegistry.url | quote }}
         - name: NP_REGISTRY_UPSTREAM_PROJECT
           value: {{ .Values.upstreamRegistry.project | quote }}
+        - name: NP_REGISTRY_UPSTREAM_REPO
+          value: {{ .Values.upstreamRegistry.repo | quote }}
         - name: NP_REGISTRY_UPSTREAM_MAX_CATALOG_ENTRIES
           value: {{ .Values.upstreamRegistry.maxCatalogEntries | quote }}
         - name: NP_CLUSTER_NAME

platform_registry_api/api.py

@@ -134,13 +134,17 @@ def _parse(cls, url: URL) -> tuple[str, str, URL]:
             mounted_repo = path_suffix.query["from"]
         return match.group("repo"), mounted_repo, path_suffix
 
-    def with_project(self, project: str) -> "RepoURL":
+    def with_project(
+        self, project: str, upstream_repo: Optional[str] = None
+    ) -> "RepoURL":
         _, _, url_suffix = self._parse(self.url)
         new_mounted_repo = ""
         if self.mounted_repo:
             new_mounted_repo = f"{project}/{self.mounted_repo}"
             url_suffix = url_suffix.update_query([("from", new_mounted_repo)])
-        new_repo = f"{project}/{self.repo}"
+        new_repo = (
+            f"{project}/{upstream_repo + '/' if upstream_repo else ''}{self.repo}"
+        )
         rel_url = URL(f"/v2/{new_repo}/").join(url_suffix)
         url = self.url.join(rel_url)
         # TODO: dataclasses.replace turns out out be buggy :D
@@ -172,10 +176,12 @@ def __init__(
         registry_endpoint_url: URL,
         upstream_endpoint_url: URL,
         upstream_project: str,
+        upstream_repo: Optional[str] = None,  # for registries that have repo like GAR
     ) -> None:
         self._registry_endpoint_url = registry_endpoint_url
         self._upstream_endpoint_url = upstream_endpoint_url
         self._upstream_project = upstream_project
+        self._upstream_repo = upstream_repo
 
     @property
     def registry_host(self) -> str | None:
@@ -189,12 +195,17 @@ def upstream_host(self) -> str | None:
     def upstream_project(self) -> str:
         return self._upstream_project
 
+    @property
+    def upstream_repo(self) -> Optional[str]:
+        return self._upstream_repo
+
     @classmethod
     def from_config(cls, registry_endpoint_url: URL, config: Config) -> "URLFactory":
         return cls(
             registry_endpoint_url=registry_endpoint_url,
             upstream_endpoint_url=config.upstream_registry.endpoint_url,
             upstream_project=config.upstream_registry.project,
+            upstream_repo=config.upstream_registry.repo,
         )
 
     def create_registry_version_check_url(self) -> URL:
@@ -207,9 +218,9 @@ def create_registry_catalog_url(self, query: dict[str, str]) -> URL:
         return self._registry_endpoint_url.with_path("/v2/_catalog").with_query(query)
 
     def create_upstream_repo_url(self, registry_url: RepoURL) -> RepoURL:
-        return registry_url.with_project(self._upstream_project).with_origin(
-            self._upstream_endpoint_url
-        )
+        return registry_url.with_project(
+            self._upstream_project, self._upstream_repo
+        ).with_origin(self._upstream_endpoint_url)
 
     def create_registry_repo_url(self, upstream_url: RepoURL) -> RepoURL:
         upstream_repo = upstream_url.repo
@@ -304,9 +315,14 @@ def parse_catalog_repositories(cls, payload: dict[str, Any]) -> list[str]:
 
     @classmethod
     def filter_images_1_indexed(
-        cls, images_names: Iterable[str], tree: ClientSubTreeViewRoot, project_name: str
+        cls,
+        images_names: Iterable[str],
+        tree: ClientSubTreeViewRoot,
+        project_name: str,
+        upstream_repo: Optional[str] = None,
     ) -> Iterator[tuple[int, str]]:
-        project_prefix = project_name + "/"
+        upstream_repo_prefix = f"{upstream_repo}/" if upstream_repo else ""
+        project_prefix = f"{project_name}/{upstream_repo_prefix}"
         len_project_prefix = len(project_prefix)
         for index, image in enumerate(images_names, 1):
             if image.startswith(project_prefix):
@@ -364,7 +380,7 @@ async def handle_catalog(self, request: Request) -> Response:  # noqa: C901
             if not images_list:
                 break
             for index, image in self.filter_images_1_indexed(
-                images_list, tree, project_name
+                images_list, tree, project_name, url_factory.upstream_repo
             ):
                 filtered.append(image)
                 if len(filtered) == page.number:

platform_registry_api/config.py

@@ -31,6 +31,7 @@ class UpstreamType(str, Enum):
 class UpstreamRegistryConfig:
     endpoint_url: URL
     project: str
+    repo: str | None = None
 
     type: UpstreamType = UpstreamType.OAUTH
 
@@ -118,6 +119,8 @@ def create_upstream_registry(self) -> UpstreamRegistryConfig:
                     ],
                 }
             )
+            if "NP_REGISTRY_UPSTREAM_REPO" in self._environ:
+                upstream["repo"] = self._environ["NP_REGISTRY_UPSTREAM_REPO"]
             if "NP_REGISTRY_UPSTREAM_TOKEN_REGISTRY_SCOPE" in self._environ:
                 upstream["token_registry_catalog_scope"] = self._environ[
                     "NP_REGISTRY_UPSTREAM_TOKEN_REGISTRY_SCOPE"

tests/unit/test_api.py

@@ -74,6 +74,13 @@ def test_with_project(self) -> None:
             url=URL("https://example.com/v2/neuro/this/img/tags/list?what=ever"),
         )
 
+        # Test with project and upstream repo as it is in Google Artifact Registry
+        reg_url = RepoURL.from_url(url).with_project("neuro", "repo")
+        assert reg_url == RepoURL(
+            repo="neuro/repo/this/img",
+            url=URL("https://example.com/v2/neuro/repo/this/img/tags/list?what=ever"),
+        )
+
     def test_with_project_and_cross_repo_blob_mount(self) -> None:
         url = URL(
             "https://example.com/v2/this/img/blobs/uploads/?what=ever&from=another/img"