[WIP] Add remaining endpoint APIs in SDK

cvat-sdk/cvat_sdk/core/client.py

@@ -19,6 +19,7 @@
 from cvat_sdk.api_client import ApiClient, Configuration, exceptions, models
 from cvat_sdk.core.exceptions import IncompatibleVersionException, InvalidHostException
 from cvat_sdk.core.helpers import expect_status
+from cvat_sdk.core.proxies.cloudstorages import CloudStoragesRepo
 from cvat_sdk.core.proxies.issues import CommentsRepo, IssuesRepo
 from cvat_sdk.core.proxies.jobs import JobsRepo
 from cvat_sdk.core.proxies.model_proxy import Repo
@@ -158,13 +159,15 @@ def wait_for_completion(
         post_params: Optional[Dict[str, Any]] = None,
         method: str = "POST",
         positive_statuses: Optional[Sequence[int]] = None,
+        max_checks: int = 0,
     ) -> urllib3.HTTPResponse:
         if status_check_period is None:
             status_check_period = self.config.status_check_period
 
         positive_statuses = set(positive_statuses) | {success_status}
 
-        while True:
+        attempt = 0
+        while not max_checks or attempt < max_checks:
             sleep(status_check_period)
 
             response = self.api_client.rest_client.request(
@@ -180,6 +183,8 @@ def wait_for_completion(
             if response.status == success_status:
                 break
 
+            attempt += 1
+
         return response
 
     def check_server_version(self, fail_if_unsupported: Optional[bool] = None) -> None:
@@ -228,6 +233,7 @@ def _get_repo(self, key: str) -> Repo:
             "users": UsersRepo,
             "issues": IssuesRepo,
             "comments": CommentsRepo,
+            "cloud_storages": CloudStoragesRepo,
         }
 
         repo = self._repos.get(key, None)
@@ -260,6 +266,10 @@ def issues(self) -> IssuesRepo:
     def comments(self) -> CommentsRepo:
         return self._get_repo("comments")
 
+    @property
+    def cloud_storages(self) -> CloudStoragesRepo:
+        return self._get_repo("cloud_storages")
+
 
 class CVAT_API_V2:
     """Build parameterized API URLs"""

cvat-sdk/cvat_sdk/core/proxies/cloudstorages.py

@@ -0,0 +1,57 @@
+# Copyright (C) 2022 CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import io
+from typing import List, Optional, Set
+
+from cvat_sdk.api_client import apis, models
+from cvat_sdk.core.proxies.model_proxy import (
+    ModelCreateMixin,
+    ModelDeleteMixin,
+    ModelListMixin,
+    ModelRetrieveMixin,
+    ModelUpdateMixin,
+    build_model_bases,
+)
+
+_CloudStorageEntityBase, _CloudStorageRepoBase = build_model_bases(
+    models.CloudStorageRead, apis.CloudstoragesApi, api_member_name="cloudstorages_api"
+)
+
+
+class CloudStorage(
+    _CloudStorageEntityBase,
+    models.ICloudStorageRead,
+    ModelUpdateMixin[models.IPatchedCloudStorageWriteRequest],
+    ModelDeleteMixin,
+):
+    _model_partial_update_arg = "patched_cloud_storage_write_request"
+    _put_annotations_data_param = "cloudstorage_annotations_update_request"
+
+    def get_actions(self) -> Set[str]:
+        return set(self.api.retrieve_actions(self.id)[0])
+
+    def get_preview(self) -> io.RawIOBase:
+        (_, response) = self.api.retrieve_preview(self.id, _parse_response=False)
+        return io.BytesIO(response.data)
+
+    def get_status(self) -> str:
+        return self.api.retrieve_status(self.id)[0]
+
+    def get_content(self, manifest_path: Optional[str] = None) -> List[str]:
+        kwargs = {}
+        if manifest_path:
+            kwargs["manifest_path"] = manifest_path
+        return self.api.retrieve_content(self.id, **kwargs)[0]
+
+
+class CloudStoragesRepo(
+    _CloudStorageRepoBase,
+    ModelCreateMixin[CloudStorage, models.ICloudStorageWriteRequest],
+    ModelRetrieveMixin[CloudStorage],
+    ModelListMixin[CloudStorage],
+):
+    _entity_type = CloudStorage

cvat-sdk/cvat_sdk/core/proxies/jobs.py

@@ -98,14 +98,14 @@ def get_frame(
         quality: Optional[str] = None,
     ) -> io.RawIOBase:
         (_, response) = self.api.retrieve_data(
-            self.id, number=frame_id, quality=quality, type="frame"
+            self.id, number=frame_id, quality=quality, type="frame", _parse_response=False
         )
         return io.BytesIO(response.data)
 
     def get_preview(
         self,
     ) -> io.RawIOBase:
-        (_, response) = self.api.retrieve_data(self.id, type="preview")
+        (_, response) = self.api.retrieve_data(self.id, type="preview", _parse_response=False)
         return io.BytesIO(response.data)
 
     def download_frames(

cvat-sdk/cvat_sdk/core/proxies/model_proxy.py

@@ -144,10 +144,10 @@ def list(self: Repo, *, return_json: Literal[False] = False) -> List[_EntityT]:
         ...
 
     @overload
-    def list(self: Repo, *, return_json: Literal[True] = False) -> List[Any]:
+    def list(self: Repo, *, return_json: Literal[True] = False) -> List[Dict[str, Any]]:
         ...
 
-    def list(self: Repo, *, return_json: bool = False) -> List[Union[_EntityT, Any]]:
+    def list(self: Repo, *, return_json: bool = False) -> List[Union[_EntityT, Dict[str, Any]]]:
         """
         Retrieves all objects from the server and returns them in basic or JSON format.
         """

cvat-sdk/cvat_sdk/core/proxies/projects.py

@@ -6,7 +6,7 @@
 
 import json
 import os.path as osp
-from typing import Optional
+from typing import List, Optional
 
 from cvat_sdk.api_client import apis, models
 from cvat_sdk.core.downloading import Downloader
@@ -19,6 +19,7 @@
     ModelUpdateMixin,
     build_model_bases,
 )
+from cvat_sdk.core.proxies.tasks import Task
 from cvat_sdk.core.uploading import DatasetUploader, Uploader
 
 _ProjectEntityBase, _ProjectRepoBase = build_model_bases(
@@ -27,7 +28,10 @@
 
 
 class Project(
-    _ProjectEntityBase, models.IProjectRead, ModelUpdateMixin[models.IPatchedProjectWriteRequest]
+    _ProjectEntityBase,
+    models.IProjectRead,
+    ModelUpdateMixin[models.IPatchedProjectWriteRequest],
+    ModelDeleteMixin,
 ):
     _model_partial_update_arg = "patched_project_write_request"
 
@@ -107,13 +111,15 @@ def get_annotations(self) -> models.ILabeledData:
         (annotations, _) = self.api.retrieve_annotations(self.id)
         return annotations
 
+    def get_tasks(self) -> List[Task]:
+        return [Task(self._client, m) for m in self.api.list_tasks(id=self.id)[0].results]
+
 
 class ProjectsRepo(
     _ProjectRepoBase,
     ModelCreateMixin[Project, models.IProjectWriteRequest],
     ModelListMixin[Project],
     ModelRetrieveMixin[Project],
-    ModelDeleteMixin,
 ):
     _entity_type = Project
 
@@ -157,12 +163,12 @@ def create_from_backup(
         Import a project from a backup file
         """
         if status_check_period is None:
-            status_check_period = self.config.status_check_period
+            status_check_period = self._client.config.status_check_period
 
         params = {"filename": osp.basename(filename)}
-        url = self.api_map.make_endpoint_url(self.api.create_backup_endpoint.path)
+        url = self._client.api_map.make_endpoint_url(self.api.create_backup_endpoint.path)
 
-        uploader = Uploader(self)
+        uploader = Uploader(self._client)
         response = uploader.upload_file(
             url,
             filename,

cvat-sdk/cvat_sdk/core/proxies/tasks.py

@@ -142,16 +142,18 @@ def get_frame(
         *,
         quality: Optional[str] = None,
     ) -> io.RawIOBase:
-        params = {}
+        params = {"number": frame_id}
         if quality:
             params["quality"] = quality
-        (_, response) = self.api.retrieve_data(self.id, number=frame_id, **params, type="frame")
+        (_, response) = self.api.retrieve_data(
+            self.id, type="frame", **params, _parse_response=False
+        )
         return io.BytesIO(response.data)
 
     def get_preview(
         self,
     ) -> io.RawIOBase:
-        (_, response) = self.api.retrieve_data(self.id, type="preview")
+        (_, response) = self.api.retrieve_data(self.id, type="preview", _parse_response=False)
         return io.BytesIO(response.data)
 
     def download_frames(
@@ -255,7 +257,6 @@ class TasksRepo(
     ModelCreateMixin[Task, models.ITaskWriteRequest],
     ModelRetrieveMixin[Task],
     ModelListMixin[Task],
-    ModelDeleteMixin,
 ):
     _entity_type = Task
 

cvat-sdk/cvat_sdk/core/uploading.py

@@ -7,11 +7,12 @@
 import os
 import os.path as osp
 from contextlib import ExitStack, closing
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 import requests
 import urllib3
 
+from cvat_sdk.api_client import exceptions
 from cvat_sdk.api_client.api_client import ApiClient, Endpoint
 from cvat_sdk.api_client.rest import RESTClientObject
 from cvat_sdk.core.helpers import StreamWithProgress, expect_status
@@ -80,21 +81,9 @@ def _wait_for_completion(
         url: str,
         *,
         success_status: int,
-        status_check_period: Optional[int] = None,
-        query_params: Optional[Dict[str, Any]] = None,
-        post_params: Optional[Dict[str, Any]] = None,
-        method: str = "POST",
-        positive_statuses: Optional[Sequence[int]] = None,
+        **kwargs,
     ) -> urllib3.HTTPResponse:
-        return self._client.wait_for_completion(
-            url,
-            success_status=success_status,
-            status_check_period=status_check_period,
-            query_params=query_params,
-            post_params=post_params,
-            method=method,
-            positive_statuses=positive_statuses,
-        )
+        return self._client.wait_for_completion(url, success_status=success_status, **kwargs)
 
     def _split_files_by_requests(
         self, filenames: List[str]
@@ -212,6 +201,7 @@ def get_offset(self):
                 http request to the tus server to retrieve the offset.
                 """
                 # FIXME: traefik changes HEAD to GET for some reason, and it breaks the protocol
+                # https://github.com/opencv/cvat/issues/4839
 
                 # Assume we are starting from scratch. This effectively disallows us to resume
                 # old file uploading
@@ -315,12 +305,38 @@ def upload_file_and_wait(
         url_params: Optional[Dict[str, Any]] = None,
         pbar: Optional[ProgressReporter] = None,
         status_check_period: Optional[int] = None,
+        restart: bool = False,
     ):
         url = self._client.api_map.make_endpoint_url(endpoint.path, kwsub=url_params)
         params = {"format": format_name, "filename": osp.basename(filename)}
-        self.upload_file(
-            url, filename, pbar=pbar, query_params=params, meta={"filename": params["filename"]}
-        )
+
+        try:
+            self.upload_file(
+                url, filename, pbar=pbar, query_params=params, meta={"filename": params["filename"]}
+            )
+        except exceptions.ApiException as ex:
+            if ex.status == 409 and restart:
+                self._client.logger.debug(f"Failed to upload dataset: {ex}. Retrying...")
+
+                # Remove the previous request
+                # TODO: Not sure if it works at all
+                self._wait_for_completion(
+                    url,
+                    success_status=201,
+                    positive_statuses=[202],
+                    status_check_period=status_check_period,
+                    query_params=params,
+                    method="GET",
+                    max_checks=1,
+                )
+
+                self.upload_file(
+                    url,
+                    filename,
+                    pbar=pbar,
+                    query_params=params,
+                    meta={"filename": params["filename"]},
+                )
 
         self._wait_for_completion(
             url,

cvat-sdk/gen/generate.sh

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: MIT
 
 set -e
+set -x
 
 GENERATOR_VERSION="v6.0.1"
 

cvat/apps/engine/cloud_provider.py

@@ -25,20 +25,15 @@
 
 from cvat.apps.engine.log import slogger
 from cvat.apps.engine.models import CredentialsTypeChoice, CloudProviderChoice
+from cvat.apps.engine.utils import DjangoEnum, StrEnum
 
-class Status(str, Enum):
+class Status(DjangoEnum, StrEnum):
     AVAILABLE = 'AVAILABLE'
     NOT_FOUND = 'NOT_FOUND'
     FORBIDDEN = 'FORBIDDEN'
 
-    @classmethod
-    def choices(cls):
-        return tuple((x.value, x.name) for x in cls)
-
-    def __str__(self):
-        return self.value
 
-class Permissions(str, Enum):
+class Permissions(DjangoEnum, StrEnum):
     READ = 'read'
     WRITE = 'write'