support projects

.gitignore

@@ -43,6 +43,7 @@ htmlcov/
 .nox/
 .coverage
 .coverage.*
+.coverage-*
 .cache
 nosetests.xml
 coverage.xml

platform_secrets/api.py

@@ -106,57 +106,65 @@ def _secret_cluster_uri(self) -> str:
     def _get_org_secrets_uri(self, org_name: str) -> str:
         return f"{self._secret_cluster_uri}/{org_name}"
 
-    def _get_user_secrets_uri(self, user: User, org_name: Optional[str]) -> str:
+    def _get_secrets_uri(self, project_name: str, org_name: Optional[str]) -> str:
         if org_name is None:
             base = self._secret_cluster_uri
         else:
             base = self._get_org_secrets_uri(org_name)
-        return f"{base}/{user.name}"
+        return f"{base}/{project_name}"
 
-    def _get_user_secrets_read_perm(
-        self, user: User, org_name: Optional[str]
-    ) -> Permission:
-        return Permission(self._get_user_secrets_uri(user, org_name), "read")
+    def _get_secret_uri(self, secret: Secret) -> str:
+        base = self._get_secrets_uri(secret.project_name, secret.org_name)
+        if secret.owner == secret.project_name:
+            return f"{base}/{secret.key}"
+        return base
+
+    def _get_secret_read_perm(self, secret: Secret) -> Permission:
+        return Permission(self._get_secret_uri(secret), "read")
 
-    def _get_user_secrets_write_perm(
-        self, user: User, org_name: Optional[str]
+    def _get_secrets_write_perm(
+        self, project_name: str, org_name: Optional[str]
     ) -> Permission:
-        return Permission(self._get_user_secrets_uri(user, org_name), "write")
+        return Permission(
+            self._get_secrets_uri(project_name, org_name),
+            "write",
+        )
 
     def _convert_secret_to_payload(self, secret: Secret) -> dict[str, Optional[str]]:
-        return {"key": secret.key, "owner": secret.owner, "org_name": secret.org_name}
+        return {
+            "key": secret.key,
+            "org_name": secret.org_name,
+            "project_name": secret.project_name,
+            # NOTE: We store all user/project keys in one k8s secret.
+            # Project k8s secret can contain keys from multiple users, so
+            # there is no single owner of k8s secret, we loose owner when we work
+            # with project secrets.
+            # For backward compatibility leave `owner` field. It will
+            # contain username for secrets without project.
+            "owner": secret.project_name,
+        }
 
     def _check_secret_read_perm(
         self, secret: Secret, tree: ClientSubTreeViewRoot
     ) -> bool:
-        node = tree.sub_tree
-        if node.can_read():
-            return True
-        parts = secret.owner.split("/") + [secret.key]
-        if secret.org_name:
-            parts = [secret.org_name] + parts
-        try:
-            for part in parts:
-                if node.can_read():
-                    return True
-                node = node.children[part]
-            return node.can_read()
-        except KeyError:
-            return False
+        return tree.allows(self._get_secret_read_perm(secret))
 
     async def handle_post(self, request: Request) -> Response:
         user = await self._get_untrusted_user(request)
         payload = await request.json()
         payload = secret_request_validator.check(payload)
         org_name = payload.get("org_name")
+        project_name = payload.get("project_name", user.name)
         await check_permissions(
-            request, [self._get_user_secrets_write_perm(user, org_name)]
+            request,
+            [self._get_secrets_write_perm(project_name, org_name)],
         )
         secret = Secret(
             key=payload["key"],
             value=payload["value"],
             owner=user.name,
             org_name=org_name,
+            project_name=project_name,
         )
         await self._service.add_secret(secret)
         resp_payload = self._convert_secret_to_payload(secret)
@@ -165,12 +173,16 @@ async def handle_post(self, request: Request) -> Response:
 
     async def handle_get_all(self, request: Request) -> Response:
         username = await check_authorized(request)
+        org_name = request.query.get("org_name")
+        project_name = request.query.get("project_name")
         tree = await self._auth_client.get_permissions_tree(
             username, self._secret_cluster_uri
         )
         secrets = [
             secret
-            for secret in await self._service.get_all_secrets()
+            for secret in await self._service.get_all_secrets(
+                org_name=org_name, project_name=project_name
+            )
             if self._check_secret_read_perm(secret, tree)
         ]
         resp_payload = [self._convert_secret_to_payload(secret) for secret in secrets]
@@ -180,12 +192,18 @@ async def handle_get_all(self, request: Request) -> Response:
     async def handle_delete(self, request: Request) -> Response:
         user = await self._get_untrusted_user(request)
         org_name = request.query.get("org_name")
+        project_name = request.query.get("project_name") or user.name
         await check_permissions(
-            request, [self._get_user_secrets_write_perm(user, org_name)]
+            request,
+            [self._get_secrets_write_perm(project_name, org_name)],
         )
         secret_key = request.match_info["key"]
         secret_key = secret_key_validator.check(secret_key)
-        secret = Secret(key=secret_key, owner=user.name, org_name=org_name)
+        secret = Secret(
+            key=secret_key,
+            org_name=org_name,
+            project_name=project_name,
+        )
         try:
             await self._service.remove_secret(secret)
         except SecretNotFound as exc:

platform_secrets/kube_client.py

@@ -7,6 +7,7 @@
 from urllib.parse import urlsplit
 
 import aiohttp
+from yarl import URL
 
 from .config import KubeClientAuthType
 
@@ -239,8 +240,12 @@ async def get_secret(
         self._cleanup_secret_payload(payload)
         return payload
 
-    async def list_secrets(self) -> list[dict[str, Any]]:
-        url = self._generate_all_secrets_url()
+    async def list_secrets(
+        self, label_selector: Optional[str] = None
+    ) -> list[dict[str, Any]]:
+        url = URL(self._generate_all_secrets_url())
+        if label_selector:
+            url = url.with_query(labelSelector=label_selector)
         payload = await self._request(method="GET", url=url)
         self._raise_for_status(payload)
         items = payload.get("items", [])

platform_secrets/service.py

@@ -12,6 +12,8 @@
 
 logger = logging.getLogger()
 
+USER_LABEL = "platform.neuromation.io/user"
+PROJECT_LABEL = "platform.neuromation.io/project"
 SECRET_API_ORG_LABEL = "platform.neuromation.io/secret-api-org-name"
 
 
@@ -24,7 +26,8 @@ def create(cls, secret_key: str) -> "SecretNotFound":
 @dataclass(frozen=True)
 class Secret:
     key: str
-    owner: str
+    project_name: str
+    owner: Optional[str] = None
     value: str = field(repr=False, default="")
     org_name: Optional[str] = None
 
@@ -33,17 +36,15 @@ class Service:
     def __init__(self, kube_client: KubeClient) -> None:
         self._kube_client = kube_client
 
-    def _get_kube_secret_name(self, owner: str, org_name: Optional[str]) -> str:
-        path = owner
+    def _get_kube_secret_name(self, project_name: str, org_name: Optional[str]) -> str:
+        path = project_name
         if org_name:
             path = f"{org_name}/{path}"
         return f"user--{path.replace('/', '--')}--secrets"
 
-    def _get_org_owner_from_secret_name(
-        self,
-        secret_name: str,
-        org_name: Optional[str],
-    ) -> tuple[Optional[str], Optional[str]]:
+    def _get_owner_from_secret_name(
+        self, secret_name: str, org_name: Optional[str]
+    ) -> Optional[str]:
         match = re.fullmatch(r"user--(?P<user_name>.*)--secrets", secret_name)
         if match:
             path: str = match.group("user_name").replace("--", "/")
@@ -52,11 +53,11 @@ def _get_org_owner_from_secret_name(
                 _, username = path.split("/", 1)
             else:
                 username = path
-            return username, org_name
-        return None, None
+            return username
+        return None
 
     async def add_secret(self, secret: Secret) -> None:
-        secret_name = self._get_kube_secret_name(secret.owner, secret.org_name)
+        secret_name = self._get_kube_secret_name(secret.project_name, secret.org_name)
         try:
             try:
                 await self._kube_client.add_secret_key(
@@ -66,6 +67,10 @@ async def add_secret(self, secret: Secret) -> None:
                 labels = {}
                 if secret.org_name:
                     labels[SECRET_API_ORG_LABEL] = secret.org_name
+                if secret.project_name == secret.owner:
+                    labels[USER_LABEL] = secret.owner.replace("/", "--")
+                else:
+                    labels[PROJECT_LABEL] = secret.project_name
                 await self._kube_client.create_secret(
                     secret_name, {secret.key: secret.value}, labels=labels
                 )
@@ -74,30 +79,57 @@ async def add_secret(self, secret: Secret) -> None:
             raise ValueError(f"Secret key {secret.key!r} or its value not valid")
 
     async def remove_secret(self, secret: Secret) -> None:
-        secret_name = self._get_kube_secret_name(secret.owner, secret.org_name)
+        secret_name = self._get_kube_secret_name(secret.project_name, secret.org_name)
         try:
             await self._kube_client.remove_secret_key(secret_name, secret.key)
         except (ResourceNotFound, ResourceInvalid):
             raise SecretNotFound.create(secret.key)
 
-    async def get_all_secrets(self, with_values: bool = False) -> list[Secret]:
-        payload = await self._kube_client.list_secrets()
+    async def get_all_secrets(
+        self,
+        with_values: bool = False,
+        org_name: Optional[str] = None,
+        project_name: Optional[str] = None,
+    ) -> list[Secret]:
+        label_selectors = []
+        if org_name:
+            label_selectors += [f"{SECRET_API_ORG_LABEL}={org_name}"]
+        if project_name:
+            label_selectors += [f"{PROJECT_LABEL}={project_name}"]
+        label_selector = ",".join(label_selectors) if label_selectors else None
+        payload = await self._kube_client.list_secrets(label_selector)
         result = []
         for item in payload:
-            owner, org_name = self._get_org_owner_from_secret_name(
-                item["metadata"]["name"],
-                org_name=item["metadata"].get("labels", {}).get(SECRET_API_ORG_LABEL),
-            )
-            if not owner:
+            labels = item["metadata"].get("labels", {})
+            owner = None
+            org_name = labels.get(SECRET_API_ORG_LABEL)
+            project_name = labels.get(PROJECT_LABEL)
+            if not project_name:
+                owner = self._get_owner_from_secret_name(
+                    item["metadata"]["name"], org_name
+                )
+                project_name = owner
+            if not project_name:
                 continue
             if with_values:
                 result += [
-                    Secret(key=key, value=value, owner=owner, org_name=org_name)
+                    Secret(
+                        key=key,
+                        value=value,
+                        owner=owner,
+                        project_name=project_name,
+                        org_name=org_name,
+                    )
                     for key, value in item.get("data", {}).items()
                 ]
             else:
                 result += [
-                    Secret(key=key, owner=owner, org_name=org_name)
+                    Secret(
+                        key=key,
+                        owner=owner,
+                        project_name=project_name,
+                        org_name=org_name,
+                    )
                     for key in item.get("data", {}).keys()
                 ]
         return result

platform_secrets/validators.py

@@ -31,9 +31,15 @@ def check_base64(value: str) -> str:
         "value": secret_value_validator,
         t.Key("org_name", optional=True): t.String(min_length=1, max_length=253)
         | t.Null(),
+        t.Key("project_name", optional=True): t.String(min_length=1, max_length=253),
     }
 )
 secret_response_validator = t.Dict(
-    {"key": t.String, "owner": t.String, "org_name": t.String() | t.Null()}
+    {
+        "key": t.String,
+        "owner": t.String,
+        "org_name": t.String() | t.Null(),
+        "project_name": t.String,
+    }
 )
 secret_list_response_validator = t.List(secret_response_validator)

setup.cfg

@@ -40,6 +40,10 @@ dev =
 max-line-length = 88
 ignore = N801,N802,N803,E252,W503,E133,E203,F541
 
+[isort]
+profile = black
+combine_as_imports = True
+
 [tool:pytest]
 testpaths = tests
 asyncio_mode = auto