From bbaa8915aaaa4f726c5b75a1615029a12aa262f1 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 00:02:40 +0200 Subject: [PATCH 001/140] Add missing pagination in some endpoints --- cvat/apps/engine/serializers.py | 10 +-- cvat/apps/engine/view_utils.py | 47 ++++++++++++++ cvat/apps/engine/views.py | 107 +++++++++++++------------------ cvat/apps/organizations/views.py | 1 - 4 files changed, 98 insertions(+), 67 deletions(-) create mode 100644 cvat/apps/engine/view_utils.py diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index abb29bd36e3e..a042c17f2f4d 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1,5 +1,5 @@ # Copyright (C) 2019-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -184,14 +184,14 @@ class JobReadSerializer(serializers.ModelSerializer): project_id = serializers.ReadOnlyField(source="get_project_id", allow_null=True) start_frame = serializers.ReadOnlyField(source="segment.start_frame") stop_frame = serializers.ReadOnlyField(source="segment.stop_frame") - assignee = BasicUserSerializer(allow_null=True) - dimension = serializers.CharField(max_length=2, source='segment.task.dimension') - labels = LabelSerializer(many=True, source='get_labels') + assignee = BasicUserSerializer(allow_null=True, read_only=True) + dimension = serializers.CharField(max_length=2, source='segment.task.dimension', read_only=True) + labels = LabelSerializer(many=True, source='get_labels', read_only=True) data_chunk_size = serializers.ReadOnlyField(source='segment.task.data.chunk_size') data_compressed_chunk_type = serializers.ReadOnlyField(source='segment.task.data.compressed_chunk_type') mode = serializers.ReadOnlyField(source='segment.task.mode') bug_tracker = serializers.CharField(max_length=2000, source='get_bug_tracker', - allow_null=True) + allow_null=True, read_only=True) class Meta: model = models.Job diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py new file mode 100644 index 000000000000..b614885bf6e7 --- /dev/null +++ b/cvat/apps/engine/view_utils.py @@ -0,0 +1,47 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +# NOTE: importing in the header leads to circular importing +from typing import Optional, Type +from django.db.models.query import QuerySet +from django.http.request import HttpRequest +from django.http.response import HttpResponse +from rest_framework.serializers import Serializer +from rest_framework.viewsets import GenericViewSet + +def make_paginated_response( + queryset: QuerySet, + *, + viewset: GenericViewSet, + response_type: Optional[Type[HttpResponse]] = None, + serializer_type: Optional[Type[Serializer]] = None, + request: Optional[Type[HttpRequest]] = None, + **serializer_params +): + # Adapted from the mixins.ListModelMixin.list() + + serializer_params.setdefault('many', True) + + if response_type is None: + from rest_framework.response import Response + response_type = Response + + if request is None: + request = getattr(viewset, 'request', None) + + if request is not None: + context = serializer_params.setdefault('context', {}) + context.setdefault('request', request) + + if serializer_type is None: + serializer_type = viewset.get_serializer + + page = viewset.paginate_queryset(queryset) + if page is not None: + serializer = serializer_type(page, **serializer_params) + return viewset.get_paginated_response(serializer.data) + + serializer = serializer_type(queryset, **serializer_params) + + return response_type(serializer.data) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index aa3ecd1438b2..1246c8ed79cb 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1,5 +1,5 @@ # Copyright (C) 2018-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -47,7 +47,7 @@ from cvat.apps.engine.frame_provider import FrameProvider from cvat.apps.engine.media_extractors import get_mime from cvat.apps.engine.models import ( - Job, Task, Project, Issue, Data, + Job, JobCommit, Task, Project, Issue, Data, Comment, StorageMethodChoice, StorageChoice, Image, CloudProviderChoice, Location ) @@ -63,6 +63,7 @@ ProjectFileSerializer, TaskFileSerializer) from utils.dataset_manifest import ImageManifestManager +from cvat.apps.engine.view_utils import make_paginated_response from cvat.apps.engine.utils import ( av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message ) @@ -343,23 +344,17 @@ def perform_create(self, serializer, **kwargs): @extend_schema( summary='Method returns information of the tasks of the project with the selected id', - responses={ - '200': TaskReadSerializer(many=True), - }) - @action(detail=True, methods=['GET'], serializer_class=TaskReadSerializer) + responses=TaskReadSerializer(many=True)) # Duplicate to still get 'list' op. nam + @action(detail=True, methods=['GET'], serializer_class=TaskReadSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + filter_fields=None, search_fields=None, ordering_fields=None) def tasks(self, request, pk): self.get_object() # force to call check_object_permissions - queryset = Task.objects.filter(project_id=pk).order_by('-id') - - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True, - context={"request": request}) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True, - context={"request": request}) - return Response(serializer.data) + return make_paginated_response(Task.objects.filter(project_id=pk).order_by('-id'), + viewset=self, serializer_type=self.serializer_class) # from @action @extend_schema(methods=['GET'], summary='Export project as a dataset in a specific format', @@ -899,17 +894,16 @@ def perform_destroy(self, instance): @extend_schema(summary='Method returns a list of jobs for a specific task', responses=JobReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer(many=True), - # Remove regular list() parameters from swagger schema + @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + filter_fields=None, search_fields=None, ordering_fields=None) def jobs(self, request, pk): self.get_object() # force to call check_object_permissions - queryset = Job.objects.filter(segment__task_id=pk) - serializer = JobReadSerializer(queryset, many=True, - context={"request": request}) - - return Response(serializer.data) + return make_paginated_response(Job.objects.filter(segment__task_id=pk).order_by('id'), + viewset=self, serializer_type=self.serializer_class) # from @action # UploadMixin method def get_upload_dir(self): @@ -1652,18 +1646,16 @@ def dataset_export(self, request, pk): @extend_schema(summary='Method returns list of issues for the job', responses=IssueReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer(many=True), - # Remove regular list() parameters from swagger schema + @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + filter_fields=None, search_fields=None, ordering_fields=None) def issues(self, request, pk): - db_job = self.get_object() - queryset = db_job.issues - serializer = IssueReadSerializer(queryset, - context={'request': request}, many=True) - - return Response(serializer.data) - + self.get_object() # force to call check_object_permissions + return make_paginated_response(Issue.objects.filter(job_id=pk).order_by('id'), + viewset=self, serializer_type=self.serializer_class) # from @action @extend_schema(summary='Method returns data for a specific job', parameters=[ @@ -1705,7 +1697,7 @@ def data(self, request, pk): @action(detail=True, methods=['GET', 'PATCH'], serializer_class=DataMetaReadSerializer, url_path='data/meta') def metadata(self, request, pk): - self.get_object() #force to call check_object_permissions + self.get_object() # force to call check_object_permissions db_job = models.Job.objects.prefetch_related( 'segment', 'segment__task', @@ -1765,21 +1757,17 @@ def metadata(self, request, pk): return Response(serializer.data) @extend_schema(summary='The action returns the list of tracked changes for the job', - responses={ - '200': JobCommitSerializer(many=True), - }) - @action(detail=True, methods=['GET'], serializer_class=None) + responses=JobCommitSerializer(many=True)) # Duplicate to still get 'list' op. name + @action(detail=True, methods=['GET'], serializer_class=JobCommitSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + filter_fields=None, search_fields=None, ordering_fields=None) def commits(self, request, pk): - db_job = self.get_object() - queryset = db_job.commits.order_by('-id') - - page = self.paginate_queryset(queryset) - if page is not None: - serializer = JobCommitSerializer(page, context={'request': request}, many=True) - return self.get_paginated_response(serializer.data) - - serializer = JobCommitSerializer(queryset, context={'request': request}, many=True) - return Response(serializer.data) + self.get_object() # force to call check_object_permissions + return make_paginated_response(JobCommit.objects.filter(job_id=pk).order_by('-id'), + viewset=self, serializer_type=self.serializer_class) # from @action @extend_schema(summary='Method returns a preview image for the job', responses={ @@ -1865,19 +1853,16 @@ def perform_create(self, serializer, **kwargs): @extend_schema(summary='The action returns all comments of a specific issue', responses=CommentReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer(many=True), - # Remove regular list() parameters from swagger schema + @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + filter_fields=None, search_fields=None, ordering_fields=None) def comments(self, request, pk): - # TODO: remove this endpoint? It is totally covered by issue body. - - db_issue = self.get_object() - queryset = db_issue.comments - serializer = CommentReadSerializer(queryset, - context={'request': request}, many=True) - - return Response(serializer.data) + self.get_object() # force to call check_object_permissions + return make_paginated_response(Comment.objects.filter(issue_id=pk).order_by('-id'), + viewset=self, serializer_type=self.serializer_class) # from @action @extend_schema(tags=['comments']) @extend_schema_view( diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index 6aaa974cd401..40bebe53df62 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -61,7 +61,6 @@ class OrganizationViewSet(viewsets.GenericViewSet, ordering_fields = filter_fields ordering = '-id' http_method_names = ['get', 'post', 'patch', 'delete', 'head', 'options'] - pagination_class = None iam_organization_field = None def get_queryset(self): From 9542c8dccfc401d23947b5d09ad5ec9b0b581a42 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 00:02:57 +0200 Subject: [PATCH 002/140] Update SDK --- cvat-sdk/cvat_sdk/core/proxies/issues.py | 7 ++++++- cvat-sdk/cvat_sdk/core/proxies/jobs.py | 4 ++-- cvat-sdk/cvat_sdk/core/proxies/projects.py | 5 +++-- cvat-sdk/cvat_sdk/core/proxies/tasks.py | 6 ++++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/proxies/issues.py b/cvat-sdk/cvat_sdk/core/proxies/issues.py index 5583fd083c13..a27271109f32 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/issues.py +++ b/cvat-sdk/cvat_sdk/core/proxies/issues.py @@ -1,10 +1,12 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT from __future__ import annotations +from typing import List from cvat_sdk.api_client import apis, models +from cvat_sdk.core.helpers import get_paginated_collection from cvat_sdk.core.proxies.model_proxy import ( ModelCreateMixin, ModelDeleteMixin, @@ -50,6 +52,9 @@ class Issue( ): _model_partial_update_arg = "patched_issue_write_request" + def get_comments(self) -> List[Comment]: + return [Comment(self._client, m) for m in get_paginated_collection(self.api.list_comments_endpoint, id=self.id)] + class IssuesRepo( _IssueRepoBase, diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index 22235d07a975..48b914df55ed 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -161,7 +161,7 @@ def remove_frames_by_ids(self, ids: Sequence[int]) -> None: ) def get_issues(self) -> List[Issue]: - return [Issue(self._client, m) for m in self.api.list_issues(id=self.id)[0]] + return [Issue(self._client, m) for m in get_paginated_collection(self.api.list_issues_endpoint, id=self.id)] def get_commits(self) -> List[models.IJobCommit]: return get_paginated_collection(self.api.list_commits_endpoint, id=self.id) diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index 0053c15bc443..9c4f481a3e22 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -11,6 +11,7 @@ from cvat_sdk.api_client import apis, models from cvat_sdk.core.downloading import Downloader +from cvat_sdk.core.helpers import get_paginated_collection from cvat_sdk.core.progress import ProgressReporter from cvat_sdk.core.proxies.model_proxy import ( ModelCreateMixin, @@ -124,7 +125,7 @@ def get_annotations(self) -> models.ILabeledData: return annotations def get_tasks(self) -> List[Task]: - return [Task(self._client, m) for m in self.api.list_tasks(id=self.id)[0].results] + return [Task(self._client, m) for m in get_paginated_collection(self.api.list_tasks_endpoint, id=self.id)] def get_preview( self, diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index 97dcbdbcbdc4..12ee0f9163b8 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -18,6 +18,7 @@ from cvat_sdk.api_client import apis, exceptions, models from cvat_sdk.core import git from cvat_sdk.core.downloading import Downloader +from cvat_sdk.core.helpers import get_paginated_collection from cvat_sdk.core.progress import ProgressReporter from cvat_sdk.core.proxies.annotations import AnnotationCrudMixin from cvat_sdk.core.proxies.jobs import Job @@ -301,7 +302,8 @@ def download_backup( self._client.logger.info(f"Backup for task {self.id} has been downloaded to {filename}") def get_jobs(self) -> List[Job]: - return [Job(self._client, m) for m in self.api.list_jobs(id=self.id)[0]] + return [Job(self._client, model=m) + for m in get_paginated_collection(self.api.list_jobs_endpoint, id=self.id)] def get_meta(self) -> models.IDataMetaRead: (meta, _) = self.api.retrieve_data_meta(self.id) From c498fa5d619e8c50ab932468c51733b079824c58 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 00:03:08 +0200 Subject: [PATCH 003/140] Update tests --- .../tests/test_rest_api_formats.py | 7 ++++-- cvat/apps/dataset_repo/tests.py | 7 ++++-- cvat/apps/engine/tests/test_rest_api.py | 6 +++-- cvat/apps/engine/tests/test_rest_api_3D.py | 8 ++++-- cvat/apps/engine/tests/utils.py | 25 +++++++++++++++++++ cvat/apps/lambda_manager/tests/test_lambda.py | 13 +++++++--- 6 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 cvat/apps/engine/tests/utils.py diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index cbefa3e38e19..e6fbdaa30306 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -26,6 +26,7 @@ from cvat.apps.dataset_manager.bindings import CvatTaskOrJobDataExtractor, TaskData from cvat.apps.dataset_manager.task import TaskAnnotation from cvat.apps.engine.models import Task +from cvat.apps.engine.tests.utils import get_paginated_collection projects_path = osp.join(osp.dirname(__file__), 'assets', 'projects.json') with open(projects_path) as file: @@ -174,8 +175,10 @@ def _create_project(self, data): def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): - response = self.client.get("/api/tasks/{}/jobs".format(task_id)) - return response.data + values = get_paginated_collection(lambda page: + self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page)) + ) + return values def _get_request(self, path, user): with ForceLogin(user, self.client): diff --git a/cvat/apps/dataset_repo/tests.py b/cvat/apps/dataset_repo/tests.py index 71611f0f4f6c..f851a1c5ac85 100644 --- a/cvat/apps/dataset_repo/tests.py +++ b/cvat/apps/dataset_repo/tests.py @@ -19,6 +19,7 @@ from cvat.apps.engine.models import Task from cvat.apps.dataset_repo.dataset_repo import (Git, initial_create, push, get) from cvat.apps.dataset_repo.models import GitData, GitStatusChoice +from cvat.apps.engine.tests.utils import get_paginated_collection orig_execute = git.cmd.Git.execute GIT_URL = "https://1.2.3.4/repo/exist.git" @@ -198,8 +199,10 @@ def _run_api_v2_job_id_annotation(self, jid, data, user): def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): - response = self.client.get("/api/tasks/{}/jobs".format(task_id)) - return response.data + values = get_paginated_collection(lambda page: + self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page)) + ) + return values def _create_task(self, init_repos=False): data = { diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 103d2f210ddb..b612dd591dba 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -35,6 +35,7 @@ Project, Segment, StageChoice, StatusChoice, Task, Label, StorageMethodChoice, StorageChoice, DimensionType, SortingMethod) from cvat.apps.engine.media_extractors import ValidateDimension, sort +from cvat.apps.engine.tests.utils import get_paginated_collection from utils.dataset_manifest import ImageManifestManager, VideoManifestManager #supress av warnings @@ -4225,8 +4226,9 @@ def _create_task(self, owner, assignee, annotation_format=""): response = self.client.get("/api/tasks/{}".format(tid)) task = response.data - response = self.client.get("/api/tasks/{}/jobs".format(tid)) - jobs = response.data + jobs = get_paginated_collection(lambda page: + self.client.get("/api/tasks/{}/jobs?page={}".format(tid, page)) + ) return (task, jobs) diff --git a/cvat/apps/engine/tests/test_rest_api_3D.py b/cvat/apps/engine/tests/test_rest_api_3D.py index 92e158301a01..977cf000da5f 100644 --- a/cvat/apps/engine/tests/test_rest_api_3D.py +++ b/cvat/apps/engine/tests/test_rest_api_3D.py @@ -24,6 +24,8 @@ from cvat.apps.dataset_manager.task import TaskAnnotation from datumaro.util.test_utils import TestDir +from cvat.apps.engine.tests.utils import get_paginated_collection + CREATE_ACTION = "create" UPDATE_ACTION = "update" DELETE_ACTION = "delete" @@ -140,8 +142,10 @@ def _get_tmp_annotation(task, annotation): def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): - response = self.client.get("/api/tasks/{}/jobs".format(task_id)) - return response.data + values = get_paginated_collection(lambda page: + self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page)) + ) + return values def _get_request(self, path, user): with ForceLogin(user, self.client): diff --git a/cvat/apps/engine/tests/utils.py b/cvat/apps/engine/tests/utils.py new file mode 100644 index 000000000000..f4c3a2ebfcf9 --- /dev/null +++ b/cvat/apps/engine/tests/utils.py @@ -0,0 +1,25 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + + +import itertools +from typing import Callable, Iterator, TypeVar + +from django.http.response import HttpResponse + +T = TypeVar('T') + +def get_paginated_collection( + request_chunk_callback: Callable[[int], HttpResponse] +) -> Iterator[T]: + values = [] + + for page in itertools.count(start=1): + response = request_chunk_callback(page) + data = response.json() + values.extend(data["results"]) + if not data.get('next'): + break + + return values diff --git a/cvat/apps/lambda_manager/tests/test_lambda.py b/cvat/apps/lambda_manager/tests/test_lambda.py index b72f3f9f04c3..8239c8f4b8fe 100644 --- a/cvat/apps/lambda_manager/tests/test_lambda.py +++ b/cvat/apps/lambda_manager/tests/test_lambda.py @@ -17,6 +17,8 @@ from rest_framework import status from rest_framework.test import APIClient, APITestCase +from cvat.apps.engine.tests.utils import get_paginated_collection + LAMBDA_ROOT_PATH = '/api/lambda' LAMBDA_FUNCTIONS_PATH = f'{LAMBDA_ROOT_PATH}/functions' LAMBDA_REQUESTS_PATH = f'{LAMBDA_ROOT_PATH}/requests' @@ -1044,10 +1046,13 @@ def setUp(self): ) self.task = task - jobs = self._get_request(f"/api/tasks/{self.task['id']}/jobs", self.admin, - org_id=self.org['id']) - assert jobs.status_code == status.HTTP_200_OK - self.job = jobs.json()[1] + jobs = get_paginated_collection(lambda page: + self._get_request( + f"/api/tasks/{self.task['id']}/jobs?page={page}", + self.admin, org_id=self.org['id'] + ) + ) + self.job = jobs[1] self.common_data = { "task": self.task['id'], From fbcc3458a2a7c2270b062115acca4961fc55bd5d Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 00:03:26 +0200 Subject: [PATCH 004/140] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb76509d893b..07f1e7f1959d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - \[SDK\] The `resource_type` args now have the default value of `local` in task creation functions. The corresponding arguments are keyword-only now. () +- \[Server API\] Added missing pagination or pagination parameters in + `/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`, + `/jobs/{id}/commits`, `/issues/{id}/comments` + () ### Deprecated - TDB From 26473e8d240f7f88897a0fd2330eb6343f80b6a2 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 00:12:16 +0200 Subject: [PATCH 005/140] Fix formatting --- cvat-sdk/cvat_sdk/core/proxies/issues.py | 6 +++++- cvat-sdk/cvat_sdk/core/proxies/jobs.py | 5 ++++- cvat-sdk/cvat_sdk/core/proxies/projects.py | 5 ++++- cvat-sdk/cvat_sdk/core/proxies/tasks.py | 6 ++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/proxies/issues.py b/cvat-sdk/cvat_sdk/core/proxies/issues.py index a27271109f32..aa349741d701 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/issues.py +++ b/cvat-sdk/cvat_sdk/core/proxies/issues.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT from __future__ import annotations + from typing import List from cvat_sdk.api_client import apis, models @@ -53,7 +54,10 @@ class Issue( _model_partial_update_arg = "patched_issue_write_request" def get_comments(self) -> List[Comment]: - return [Comment(self._client, m) for m in get_paginated_collection(self.api.list_comments_endpoint, id=self.id)] + return [ + Comment(self._client, m) + for m in get_paginated_collection(self.api.list_comments_endpoint, id=self.id) + ] class IssuesRepo( diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index 48b914df55ed..11bfe2438c52 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -161,7 +161,10 @@ def remove_frames_by_ids(self, ids: Sequence[int]) -> None: ) def get_issues(self) -> List[Issue]: - return [Issue(self._client, m) for m in get_paginated_collection(self.api.list_issues_endpoint, id=self.id)] + return [ + Issue(self._client, m) + for m in get_paginated_collection(self.api.list_issues_endpoint, id=self.id) + ] def get_commits(self) -> List[models.IJobCommit]: return get_paginated_collection(self.api.list_commits_endpoint, id=self.id) diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index 9c4f481a3e22..b48cff74acd3 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -125,7 +125,10 @@ def get_annotations(self) -> models.ILabeledData: return annotations def get_tasks(self) -> List[Task]: - return [Task(self._client, m) for m in get_paginated_collection(self.api.list_tasks_endpoint, id=self.id)] + return [ + Task(self._client, m) + for m in get_paginated_collection(self.api.list_tasks_endpoint, id=self.id) + ] def get_preview( self, diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index 12ee0f9163b8..94c4b71f6d25 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -302,8 +302,10 @@ def download_backup( self._client.logger.info(f"Backup for task {self.id} has been downloaded to {filename}") def get_jobs(self) -> List[Job]: - return [Job(self._client, model=m) - for m in get_paginated_collection(self.api.list_jobs_endpoint, id=self.id)] + return [ + Job(self._client, model=m) + for m in get_paginated_collection(self.api.list_jobs_endpoint, id=self.id) + ] def get_meta(self) -> models.IDataMetaRead: (meta, _) = self.api.retrieve_data_meta(self.id) From c20991adfab2446b2a714ccb65c1ce58ce085aab Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 00:13:22 +0200 Subject: [PATCH 006/140] Fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f1e7f1959d..44b6b5a82dc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 () - \[Server API\] Added missing pagination or pagination parameters in `/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`, - `/jobs/{id}/commits`, `/issues/{id}/comments` + `/jobs/{id}/commits`, `/issues/{id}/comments`, `/organizations` () ### Deprecated From fb2516c8ed6cf3529b4606faf33c94ccdff8645c Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 16:53:36 +0200 Subject: [PATCH 007/140] t --- cvat/apps/engine/utils.py | 43 +++++++++++++++++++++++++ cvat/apps/engine/views.py | 68 +++++++++++++++++++-------------------- 2 files changed, 77 insertions(+), 34 deletions(-) diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index c2dd82ce7d2e..b9879b028226 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT import ast +# from typing import Optional, Type import cv2 as cv from collections import namedtuple import hashlib @@ -17,6 +18,11 @@ from PIL import Image from django.core.exceptions import ValidationError +from django.urls import reverse as _django_reverse +from django.utils.http import urlencode +# from django.db.models.query import QuerySet +# from rest_framework.response import Response +# from rest_framework.viewsets import GenericViewSet Import = namedtuple("Import", ["module", "name", "alias"]) @@ -146,3 +152,40 @@ def configure_dependent_job(queue, rq_id, rq_func, db_storage, filename, key): job_id=rq_job_id_download_file ) return rq_job_download_file + +# def make_paginated_response(queryset: QuerySet, *, +# viewset: GenericViewSet, +# response_type: Type[Response] = Response, +# request: Optional[Request] = None, +# **serializer_params +# ) -> Response: +# # Adapted from the mixins.ListModelMixin.list() + +# serializer_params.setdefault('many', True) + +# if request is not None: +# context = serializer_params.setdefault('context', {}) +# context.setdefault('request', request) + +# make_serializer = viewset.get_serializer + +# page = viewset.paginate_queryset(queryset) +# if page is not None: +# serializer = make_serializer(page, **serializer_params) +# return viewset.get_paginated_response(serializer.data) + +# serializer = make_serializer(queryset, **serializer_params) + +# return response_type(serializer.data) + +def reverse(viewname, *, args=None, kwargs=None, query_params=None) -> str: + """ + The same as reverse(), but adds query params support. + """ + + url = _django_reverse(viewname, args=args, kwargs=kwargs) + + if query_params: + return f'{url}?{urlencode(query_params)}' + + return url diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index aa3ecd1438b2..8a1c4946da40 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -64,7 +64,7 @@ from utils.dataset_manifest import ImageManifestManager from cvat.apps.engine.utils import ( - av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message + av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message, reverse ) from cvat.apps.engine import backup from cvat.apps.engine.mixins import PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin, DestroyModelMixin, CreateModelMixin @@ -898,15 +898,19 @@ def perform_destroy(self, instance): @extend_schema(summary='Method returns a list of jobs for a specific task', - responses=JobReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer(many=True), - # Remove regular list() parameters from swagger schema - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + responses=JobReadSerializer(many=True)) + @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer) def jobs(self, request, pk): self.get_object() # force to call check_object_permissions - queryset = Job.objects.filter(segment__task_id=pk) - serializer = JobReadSerializer(queryset, many=True, + queryset = Job.objects.filter(segment__task_id=pk).order_by('id') + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True, + context={"request": request}) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True, context={"request": request}) return Response(serializer.data) @@ -1650,20 +1654,14 @@ def dataset_export(self, request, pk): callback=dm.views.export_job_as_dataset ) - @extend_schema(summary='Method returns list of issues for the job', - responses=IssueReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer(many=True), - # Remove regular list() parameters from swagger schema - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + @extend_schema(summary='Moved to GET api/issues', + deprecated=True) # TODO: to be removed in v2.5 + @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer) def issues(self, request, pk): - db_job = self.get_object() - queryset = db_job.issues - serializer = IssueReadSerializer(queryset, - context={'request': request}, many=True) - - return Response(serializer.data) - + # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other + return Response(status=status.HTTP_303_SEE_OTHER, headers={ + 'Location': reverse('issue-list', query_params={'job_id': pk}) + }) @extend_schema(summary='Method returns data for a specific job', parameters=[ @@ -1705,7 +1703,7 @@ def data(self, request, pk): @action(detail=True, methods=['GET', 'PATCH'], serializer_class=DataMetaReadSerializer, url_path='data/meta') def metadata(self, request, pk): - self.get_object() #force to call check_object_permissions + self.get_object() # force to call check_object_permissions db_job = models.Job.objects.prefetch_related( 'segment', 'segment__task', @@ -1768,17 +1766,17 @@ def metadata(self, request, pk): responses={ '200': JobCommitSerializer(many=True), }) - @action(detail=True, methods=['GET'], serializer_class=None) + @action(detail=True, methods=['GET'], serializer_class=JobCommitSerializer) def commits(self, request, pk): db_job = self.get_object() queryset = db_job.commits.order_by('-id') page = self.paginate_queryset(queryset) if page is not None: - serializer = JobCommitSerializer(page, context={'request': request}, many=True) + serializer = self.get_serializer(page, context={'request': request}, many=True) return self.get_paginated_response(serializer.data) - serializer = JobCommitSerializer(queryset, context={'request': request}, many=True) + serializer = self.get_serializer(queryset, context={'request': request}, many=True) return Response(serializer.data) @extend_schema(summary='Method returns a preview image for the job', @@ -1864,17 +1862,19 @@ def perform_create(self, serializer, **kwargs): super().perform_create(serializer, owner=self.request.user) @extend_schema(summary='The action returns all comments of a specific issue', - responses=CommentReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer(many=True), - # Remove regular list() parameters from swagger schema - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + responses=CommentReadSerializer(many=True)) + @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer) def comments(self, request, pk): - # TODO: remove this endpoint? It is totally covered by issue body. - db_issue = self.get_object() - queryset = db_issue.comments - serializer = CommentReadSerializer(queryset, + queryset = db_issue.comments.order_by('-id') + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True, + context={"request": request}) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, context={'request': request}, many=True) return Response(serializer.data) From 3e507bfb0290f4744ab1d57d9499adc7228f8841 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 10 Jan 2023 23:59:56 -0800 Subject: [PATCH 008/140] Added fetchAll aggregator --- cvat-core/src/server-proxy.ts | 66 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index b84898f9783a..76b306d76bce 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -5,7 +5,7 @@ import FormData from 'form-data'; import store from 'store'; -import Axios from 'axios'; +import Axios, { AxiosResponse } from 'axios'; import * as tus from 'tus-js-client'; import { Storage } from './storage'; import { StorageLocation, WebhookSourceType } from './enums'; @@ -1236,19 +1236,69 @@ async function getJobs(filter = {}) { return response.data; } +function fetchAll(url): Promise { + const pageSize = 500; + let collection = []; + return new Promise((resolve, reject) => { + Axios.get(url, { + params: { + page_size: pageSize, + page: 1, + }, + proxy: config.proxy, + }).then((initialData) => { + const { count, results } = initialData.data; + collection = collection.concat(results); + if (count <= pageSize) { + resolve(collection); + return; + } + + const pages = Math.ceil(count / pageSize); + const promises = Array(pages).fill(0).map((_: number, i: number) => { + if (i) { + return Axios.get(url, { + params: { + page_size: pageSize, + page: i + 1, + }, + proxy: config.proxy, + }); + } + + return Promise.resolve(null); + }); + + Promise.all(promises).then((responses: AxiosResponse[]) => { + responses.forEach((resp) => { + if (resp) { + collection = collection.concat(resp.data.results); + } + }); + + // removing possible dublicates + const obj = collection.reduce((acc: Record, item: any) => { + acc[item.id] = item; + return acc; + }, {}); + + resolve(Object.values(obj)); + }).catch((error) => reject(error)); + }).catch((error) => reject(error)); + }); +} + async function getJobIssues(jobID) { const { backendAPI } = config; let response = null; try { - response = await Axios.get(`${backendAPI}/jobs/${jobID}/issues`, { - proxy: config.proxy, - }); + response = await fetchAll(`${backendAPI}/jobs/${jobID}/issues`); } catch (errorData) { throw generateError(errorData); } - return response.data; + return response; } async function createComment(data) { @@ -1951,14 +2001,12 @@ async function getOrganizations() { let response = null; try { - response = await Axios.get(`${backendAPI}/organizations`, { - proxy: config.proxy, - }); + response = await fetchAll(`${backendAPI}/organizations?page_size`); } catch (errorData) { throw generateError(errorData); } - return response.data; + return response; } async function createOrganization(data) { From c65ac3281809e50c72ff5e343486a22f41c037ea Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 10 Jan 2023 17:13:46 +0200 Subject: [PATCH 009/140] Add simple redirects --- cvat/apps/engine/utils.py | 42 ----------------------------- cvat/apps/engine/view_utils.py | 32 ++++++++++++++++++++-- cvat/apps/engine/views.py | 49 ++++++++++++++++++++-------------- 3 files changed, 59 insertions(+), 64 deletions(-) diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index b9879b028226..e668bd2ed6c5 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -18,11 +18,6 @@ from PIL import Image from django.core.exceptions import ValidationError -from django.urls import reverse as _django_reverse -from django.utils.http import urlencode -# from django.db.models.query import QuerySet -# from rest_framework.response import Response -# from rest_framework.viewsets import GenericViewSet Import = namedtuple("Import", ["module", "name", "alias"]) @@ -152,40 +147,3 @@ def configure_dependent_job(queue, rq_id, rq_func, db_storage, filename, key): job_id=rq_job_id_download_file ) return rq_job_download_file - -# def make_paginated_response(queryset: QuerySet, *, -# viewset: GenericViewSet, -# response_type: Type[Response] = Response, -# request: Optional[Request] = None, -# **serializer_params -# ) -> Response: -# # Adapted from the mixins.ListModelMixin.list() - -# serializer_params.setdefault('many', True) - -# if request is not None: -# context = serializer_params.setdefault('context', {}) -# context.setdefault('request', request) - -# make_serializer = viewset.get_serializer - -# page = viewset.paginate_queryset(queryset) -# if page is not None: -# serializer = make_serializer(page, **serializer_params) -# return viewset.get_paginated_response(serializer.data) - -# serializer = make_serializer(queryset, **serializer_params) - -# return response_type(serializer.data) - -def reverse(viewname, *, args=None, kwargs=None, query_params=None) -> str: - """ - The same as reverse(), but adds query params support. - """ - - url = _django_reverse(viewname, args=args, kwargs=kwargs) - - if query_params: - return f'{url}?{urlencode(query_params)}' - - return url diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index b614885bf6e7..c32753bf8391 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -2,14 +2,19 @@ # # SPDX-License-Identifier: MIT -# NOTE: importing in the header leads to circular importing -from typing import Optional, Type +# NOTE: importing in the utils.py header leads to circular importing + +from typing import Any, Dict, Optional, Type + from django.db.models.query import QuerySet from django.http.request import HttpRequest from django.http.response import HttpResponse +from django.urls import reverse as _django_reverse +from django.utils.http import urlencode from rest_framework.serializers import Serializer from rest_framework.viewsets import GenericViewSet + def make_paginated_response( queryset: QuerySet, *, @@ -45,3 +50,26 @@ def make_paginated_response( serializer = serializer_type(queryset, **serializer_params) return response_type(serializer.data) + +def reverse(viewname, *, args=None, kwargs=None, query_params=None) -> str: + """ + The same as Django reverse(), but adds query params support. + """ + + url = _django_reverse(viewname, args=args, kwargs=kwargs) + + if query_params: + return f'{url}?{urlencode(query_params)}' + + return url + +def build_field_search_params(field: str, value: Any) -> Dict[str, str]: + """ + Builds a collection filter query params for a single field and value. + """ + # Uses Reverse Polish Notation + return { + 'filter': '{"==":[{"var":"%(field)s"},%(value)s]}' % { + 'field': field, 'value':value + } + } diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 88b7923a535c..c02ce795d221 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -63,9 +63,9 @@ ProjectFileSerializer, TaskFileSerializer) from utils.dataset_manifest import ImageManifestManager -from cvat.apps.engine.view_utils import make_paginated_response +from cvat.apps.engine.view_utils import build_field_search_params, make_paginated_response, reverse from cvat.apps.engine.utils import ( - av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message, reverse + av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message ) from cvat.apps.engine import backup from cvat.apps.engine.mixins import PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin, DestroyModelMixin, CreateModelMixin @@ -343,8 +343,9 @@ def perform_create(self, serializer, **kwargs): ) @extend_schema( - summary='Method returns information of the tasks of the project with the selected id', - responses=TaskReadSerializer(many=True)) # Duplicate to still get 'list' op. nam + summary='Moved to /tasks', + responses=TaskReadSerializer(many=True), # Duplicate to still get 'list' op. name + deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=TaskReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, # Remove regular list() parameters from the swagger schema. @@ -352,10 +353,11 @@ def perform_create(self, serializer, **kwargs): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, search_fields=None, ordering_fields=None) def tasks(self, request, pk): - self.get_object() # force to call check_object_permissions - return make_paginated_response(Task.objects.filter(project_id=pk).order_by('-id'), - viewset=self, serializer_type=self.serializer_class) # from @action - + # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other + return Response(status=status.HTTP_303_SEE_OTHER, headers={ + 'Location': reverse('task-list', + query_params=build_field_search_params('project_id', pk)) + }) @extend_schema(methods=['GET'], summary='Export project as a dataset in a specific format', parameters=[ @@ -892,8 +894,9 @@ def perform_destroy(self, instance): db_project.save() - @extend_schema(summary='Method returns a list of jobs for a specific task', - responses=JobReadSerializer(many=True)) # Duplicate to still get 'list' op. name + @extend_schema(summary='Moved to /jobs', + responses=JobReadSerializer(many=True), # Duplicate to still get 'list' op. name + deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, # Remove regular list() parameters from the swagger schema. @@ -901,9 +904,11 @@ def perform_destroy(self, instance): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, search_fields=None, ordering_fields=None) def jobs(self, request, pk): - self.get_object() # force to call check_object_permissions - return make_paginated_response(Job.objects.filter(segment__task_id=pk).order_by('id'), - viewset=self, serializer_type=self.serializer_class) # from @action + # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other + return Response(status=status.HTTP_303_SEE_OTHER, headers={ + 'Location': reverse('job-list', + query_params=build_field_search_params('task_id', pk)) + }) # UploadMixin method def get_upload_dir(self): @@ -1646,7 +1651,7 @@ def dataset_export(self, request, pk): @extend_schema(summary='Moved to GET /issues', responses=IssueReadSerializer(many=True), # Duplicate to still get 'list' op. name - deprecated=True) # TODO: to be removed in v2.5 + deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, # Remove regular list() parameters from the swagger schema. @@ -1656,7 +1661,8 @@ def dataset_export(self, request, pk): def issues(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ - 'Location': reverse('issue-list', query_params={'job_id': pk}) + 'Location': reverse('issue-list', + query_params=build_field_search_params('job_id', pk)) }) @extend_schema(summary='Method returns data for a specific job', @@ -1853,8 +1859,9 @@ def get_serializer_class(self): def perform_create(self, serializer, **kwargs): super().perform_create(serializer, owner=self.request.user) - @extend_schema(summary='The action returns all comments of a specific issue', - responses=CommentReadSerializer(many=True)) # Duplicate to still get 'list' op. name + @extend_schema(summary='Moved to /comments', + responses=CommentReadSerializer(many=True), # Duplicate to still get 'list' op. name + deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, # Remove regular list() parameters from the swagger schema. @@ -1862,9 +1869,11 @@ def perform_create(self, serializer, **kwargs): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, search_fields=None, ordering_fields=None) def comments(self, request, pk): - self.get_object() # force to call check_object_permissions - return make_paginated_response(Comment.objects.filter(issue_id=pk).order_by('-id'), - viewset=self, serializer_type=self.serializer_class) # from @action + # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other + return Response(status=status.HTTP_303_SEE_OTHER, headers={ + 'Location': reverse('comment-list', + query_params=build_field_search_params('issue_id', pk)) + }) @extend_schema(tags=['comments']) @extend_schema_view( From 3ca2026e822a5b4766700243307cad0f59b1e1cf Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 11 Jan 2023 14:39:22 +0200 Subject: [PATCH 010/140] Update test assets --- tests/python/shared/assets/organizations.json | 77 ++++++++++--------- tests/python/shared/fixtures/data.py | 2 +- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/tests/python/shared/assets/organizations.json b/tests/python/shared/assets/organizations.json index 40b6d8758a42..ad26620a27e0 100644 --- a/tests/python/shared/assets/organizations.json +++ b/tests/python/shared/assets/organizations.json @@ -1,38 +1,43 @@ -[ - { - "contact": { - "email": "org2@cvat.org" - }, - "created_date": "2021-12-14T19:51:38.667000Z", - "description": "", - "id": 2, - "name": "Organization #2", - "owner": { - "first_name": "Business", - "id": 10, - "last_name": "First", - "url": "http://localhost:8080/api/users/10", - "username": "business1" - }, - "slug": "org2", - "updated_date": "2021-12-14T19:51:38.667000Z" - }, - { - "contact": { - "email": "org1@cvat.org" - }, - "created_date": "2021-12-14T18:45:40.172000Z", - "description": "", - "id": 1, - "name": "organization #1", - "owner": { - "first_name": "User", +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "contact": { + "email": "org2@cvat.org" + }, + "created_date": "2021-12-14T19:51:38.667000Z", + "description": "", "id": 2, - "last_name": "First", - "url": "http://localhost:8080/api/users/2", - "username": "user1" + "name": "Organization #2", + "owner": { + "first_name": "Business", + "id": 10, + "last_name": "First", + "url": "http://localhost:8080/api/users/10", + "username": "business1" + }, + "slug": "org2", + "updated_date": "2021-12-14T19:51:38.667000Z" }, - "slug": "org1", - "updated_date": "2021-12-14T18:45:40.172000Z" - } -] \ No newline at end of file + { + "contact": { + "email": "org1@cvat.org" + }, + "created_date": "2021-12-14T18:45:40.172000Z", + "description": "", + "id": 1, + "name": "organization #1", + "owner": { + "first_name": "User", + "id": 2, + "last_name": "First", + "url": "http://localhost:8080/api/users/2", + "username": "user1" + }, + "slug": "org1", + "updated_date": "2021-12-14T18:45:40.172000Z" + } + ] +} \ No newline at end of file diff --git a/tests/python/shared/fixtures/data.py b/tests/python/shared/fixtures/data.py index 47c6038277c8..cdccb67497af 100644 --- a/tests/python/shared/fixtures/data.py +++ b/tests/python/shared/fixtures/data.py @@ -43,7 +43,7 @@ def users(): @pytest.fixture(scope="session") def organizations(): with open(ASSETS_DIR / "organizations.json") as f: - return Container(json.load(f)) + return Container(json.load(f)["results"]) @pytest.fixture(scope="session") From ca7c732c842e1a1e373c0ec999a7893c98bba5dd Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 11 Jan 2023 16:53:24 +0200 Subject: [PATCH 011/140] Update UI tests --- .../integration/remove_users_tasks_projects_organizations.js | 2 +- tests/cypress/support/commands_organizations.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cypress/integration/remove_users_tasks_projects_organizations.js b/tests/cypress/integration/remove_users_tasks_projects_organizations.js index 407e8a42e38a..561878917a41 100644 --- a/tests/cypress/integration/remove_users_tasks_projects_organizations.js +++ b/tests/cypress/integration/remove_users_tasks_projects_organizations.js @@ -70,7 +70,7 @@ describe('Delete users, tasks, projects, organizations created during the tests Authorization: `Token ${authKey}`, }, }).then((response) => { - const responseResult = response.body; + const responseResult = response.body.results; for (const org of responseResult) { const { id } = org; cy.request({ diff --git a/tests/cypress/support/commands_organizations.js b/tests/cypress/support/commands_organizations.js index 86c919126990..0e6d63719e1e 100644 --- a/tests/cypress/support/commands_organizations.js +++ b/tests/cypress/support/commands_organizations.js @@ -36,7 +36,7 @@ Cypress.Commands.add('deleteOrganizations', (authResponse, otrganizationsToDelet Authorization: `Token ${authKey}`, }, }).then((_response) => { - const responceResult = _response.body; + const responceResult = _response.body.results; for (const organization of responceResult) { const { id, slug } = organization; for (const organizationToDelete of otrganizationsToDelete) { From 97bfb9fed474e10cf88cc1b9c66384e428b49ead Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 11 Jan 2023 17:24:05 +0200 Subject: [PATCH 012/140] t --- cvat/apps/engine/filters.py | 107 +++++++++++++++++++++++++++++++++--- cvat/apps/engine/views.py | 1 + cvat/settings/base.py | 5 +- 3 files changed, 105 insertions(+), 8 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 20e571ffd1b6..4c8877de184d 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -1,19 +1,33 @@ # Copyright (C) 2022 Intel Corporation +# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT -from rest_framework import filters +from contextlib import contextmanager +from typing import Any, Dict from functools import reduce +from unittest.mock import patch import operator import json + from django.db.models import Q -from rest_framework.compat import coreapi, coreschema from django.utils.translation import gettext_lazy as _ from django.utils.encoding import force_str +from rest_framework import filters +from rest_framework.compat import coreapi, coreschema from rest_framework.exceptions import ValidationError +# from rest_framework.viewsets import ViewSet +from django_filters.rest_framework import DjangoFilterBackend -class SearchFilter(filters.SearchFilter): +def get_lookup_fields(view) -> Dict[str, str]: + filter_fields = getattr(view, 'filter_fields', []) + lookup_fields = {field:field for field in filter_fields} + lookup_fields.update(getattr(view, 'lookup_fields', {})) + + return lookup_fields + +class SearchFilter(filters.SearchFilter): def get_search_fields(self, view, request): search_fields = getattr(view, 'search_fields') or [] lookup_fields = {field:field for field in search_fields} @@ -211,8 +225,87 @@ def get_schema_operation_parameters(self, view): ] def _get_lookup_fields(self, request, view): - filter_fields = getattr(view, 'filter_fields', []) - lookup_fields = {field:field for field in filter_fields} - lookup_fields.update(getattr(view, 'lookup_fields', {})) + return get_lookup_fields(view) - return lookup_fields + +class SimpleFilter(DjangoFilterBackend): + def get_filterset_class(self, view, queryset=None): + filterset_class = getattr(view, 'filterset_class', None) + filterset_fields = getattr(view, 'filterset_fields', None) + if not filterset_class and not filterset_fields: + return None + return super().get_filterset_class(view, queryset) + + +# class SimpleFilter(filters.BaseFilterBackend): +# """ +# This filter allows to do simple queries with no more +# than 1 field and the equality check for 1 value. + +# The main purpose is to provide user-friendly interface for simple cases. +# """ + +# filter_param = 'filter' +# filter_title = _('Filter') +# filter_description = _('A filter term.') + +# def filter_queryset(self, request, queryset, view): +# json_rules = request.query_params.get(self.filter_param) +# if json_rules: +# try: +# rules = json.loads(json_rules) +# if not len(rules): +# raise ValidationError(f"filter shouldn't be empty") +# except json.decoder.JSONDecodeError: +# raise ValidationError(f'filter: Json syntax should be used') +# lookup_fields = self._get_lookup_fields(request, view) +# try: +# q_object = self._build_Q(rules, lookup_fields) +# except KeyError as ex: +# raise ValidationError(f'filter: {str(ex)} term is not supported') +# return queryset.filter(q_object) + +# return queryset + +# def get_schema_fields(self, view): +# assert coreapi is not None, 'coreapi must be installed to use `get_schema_fields()`' +# assert coreschema is not None, 'coreschema must be installed to use `get_schema_fields()`' + +# filter_fields = getattr(view, 'filter_fields', []) +# full_description = self.filter_description + \ +# f' Avaliable filter_fields: {filter_fields}' + +# return [ +# coreapi.Field( +# name=self.filter_param, +# required=False, +# location='query', +# schema=coreschema.String( +# title=force_str(self.filter_title), +# description=force_str(full_description) +# ) +# ) +# ] + +# def get_schema_operation_parameters(self, view): +# filter_fields = getattr(view, 'filter_fields', []) +# full_description = self.filter_description + \ +# f' Avaliable filter_fields: {filter_fields}' +# return [ +# { +# 'name': self.filter_param, +# 'required': False, +# 'in': 'query', +# 'description': force_str(full_description), +# 'schema': { +# 'type': 'string', +# }, +# }, +# ] + +# def _get_lookup_fields(self, request, view): +# lookup_fields = get_lookup_fields(view) + +# return { +# k: +# } \ No newline at end of file diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index c02ce795d221..8e33d43c543c 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -317,6 +317,7 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, ordering_fields = filter_fields ordering = "-id" lookup_fields = {'owner': 'owner__username', 'assignee': 'assignee__username'} + filterset_fields = { k: [v] for k, v in lookup_fields.items()} iam_organization_field = 'organization' def get_serializer_class(self): diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 1128817860ea..08815f78b107 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -108,6 +108,7 @@ def add_ssh_keys(): 'django.contrib.messages', 'django.contrib.staticfiles', 'django_rq', + 'django_filters', 'compressor', 'django_sendfile', 'dj_pagination', @@ -173,10 +174,12 @@ def add_ssh_keys(): 'cvat.apps.engine.pagination.CustomPagination', 'PAGE_SIZE': 10, 'DEFAULT_FILTER_BACKENDS': ( + 'cvat.apps.engine.filters.SimpleFilter', 'cvat.apps.engine.filters.SearchFilter', 'cvat.apps.engine.filters.OrderingFilter', 'cvat.apps.engine.filters.JsonLogicFilter', - 'cvat.apps.iam.filters.OrganizationFilterBackend'), + 'cvat.apps.iam.filters.OrganizationFilterBackend', + ), 'SEARCH_PARAM': 'search', # Disable default handling of the 'format' query parameter by REST framework From af23b0649dc71f3a82b213b67ba63c370f16d945 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 12 Jan 2023 00:24:29 +0200 Subject: [PATCH 013/140] Implement simple filters support --- cvat/apps/engine/filters.py | 168 +++++++++++++++++++----------------- cvat/apps/engine/views.py | 2 +- 2 files changed, 92 insertions(+), 78 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 4c8877de184d..eff4e43dbb12 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -13,20 +13,29 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ from django.utils.encoding import force_str +from django_filters.filterset import BaseFilterSet from rest_framework import filters from rest_framework.compat import coreapi, coreschema from rest_framework.exceptions import ValidationError -# from rest_framework.viewsets import ViewSet +from django_filters import FilterSet from django_filters.rest_framework import DjangoFilterBackend def get_lookup_fields(view) -> Dict[str, str]: filter_fields = getattr(view, 'filter_fields', []) + if not filter_fields: + return {} + lookup_fields = {field:field for field in filter_fields} lookup_fields.update(getattr(view, 'lookup_fields', {})) return lookup_fields +@contextmanager +def _patched_attr(obj: Any, name: str, value: Any) -> None: + with patch.object(obj, attribute=name, new=value, create=True): + yield + class SearchFilter(filters.SearchFilter): def get_search_fields(self, view, request): search_fields = getattr(view, 'search_fields') or [] @@ -229,83 +238,88 @@ def _get_lookup_fields(self, request, view): class SimpleFilter(DjangoFilterBackend): + filter_desc = _('A simple equality filter for the {field_name} field') + reserved_names = ( + JsonLogicFilter.filter_param, + OrderingFilter.ordering_param, + SearchFilter.search_param, + ) + + class MappingFiltersetBase(BaseFilterSet): + @classmethod + def get_filter_name(cls, field_name, lookup_expr): + filter_names = getattr(cls, 'filter_names', {}) + + field_name = super().get_filter_name(field_name, lookup_expr) + + if filter_names: + # Map names after a lookup suffix is applied to allow + # mapping specific filters with lookups + field_name = filter_names.get(field_name, field_name) + + if field_name in SimpleFilter.reserved_names: + raise ValueError(f'Field name {field_name} is reserved') + + return field_name + + filterset_base = MappingFiltersetBase + + def get_filterset_class(self, view, queryset=None): filterset_class = getattr(view, 'filterset_class', None) filterset_fields = getattr(view, 'filterset_fields', None) - if not filterset_class and not filterset_fields: + + if filterset_class or filterset_fields: + return super().get_filterset_class(view, queryset) + + lookup_fields = self.get_lookup_fields(view) + if not lookup_fields: return None - return super().get_filterset_class(view, queryset) - - -# class SimpleFilter(filters.BaseFilterBackend): -# """ -# This filter allows to do simple queries with no more -# than 1 field and the equality check for 1 value. - -# The main purpose is to provide user-friendly interface for simple cases. -# """ - -# filter_param = 'filter' -# filter_title = _('Filter') -# filter_description = _('A filter term.') - -# def filter_queryset(self, request, queryset, view): -# json_rules = request.query_params.get(self.filter_param) -# if json_rules: -# try: -# rules = json.loads(json_rules) -# if not len(rules): -# raise ValidationError(f"filter shouldn't be empty") -# except json.decoder.JSONDecodeError: -# raise ValidationError(f'filter: Json syntax should be used') -# lookup_fields = self._get_lookup_fields(request, view) -# try: -# q_object = self._build_Q(rules, lookup_fields) -# except KeyError as ex: -# raise ValidationError(f'filter: {str(ex)} term is not supported') -# return queryset.filter(q_object) - -# return queryset - -# def get_schema_fields(self, view): -# assert coreapi is not None, 'coreapi must be installed to use `get_schema_fields()`' -# assert coreschema is not None, 'coreschema must be installed to use `get_schema_fields()`' - -# filter_fields = getattr(view, 'filter_fields', []) -# full_description = self.filter_description + \ -# f' Avaliable filter_fields: {filter_fields}' - -# return [ -# coreapi.Field( -# name=self.filter_param, -# required=False, -# location='query', -# schema=coreschema.String( -# title=force_str(self.filter_title), -# description=force_str(full_description) -# ) -# ) -# ] - -# def get_schema_operation_parameters(self, view): -# filter_fields = getattr(view, 'filter_fields', []) -# full_description = self.filter_description + \ -# f' Avaliable filter_fields: {filter_fields}' -# return [ -# { -# 'name': self.filter_param, -# 'required': False, -# 'in': 'query', -# 'description': force_str(full_description), -# 'schema': { -# 'type': 'string', -# }, -# }, -# ] - -# def _get_lookup_fields(self, request, view): -# lookup_fields = get_lookup_fields(view) - -# return { -# k: -# } \ No newline at end of file + + class MappingFilterset(self.MappingFiltersetBase, metaclass=FilterSet.__class__): + filter_names = { v: k for k, v in lookup_fields.items() } + + with _patched_attr(view, 'filterset_fields', list(lookup_fields.values())): + with _patched_attr(self, 'filterset_base', MappingFilterset): + filterset_class = super().get_filterset_class(view, queryset) + + return filterset_class + + def get_lookup_fields(self, view): + return get_lookup_fields(view) + + def get_filter_fields(self, view): + return list(self.get_lookup_fields(view)) + + def get_schema_fields(self, view): + assert coreapi is not None, 'coreapi must be installed to use `get_schema_fields()`' + assert coreschema is not None, 'coreschema must be installed to use `get_schema_fields()`' + + filter_fields = self.get_filter_fields(view) + + return [ + coreapi.Field( + name=field_name, + location='query', + required=False, + schema={ + 'type': 'string', + } + ) for field_name in filter_fields + ] + + def get_schema_operation_parameters(self, view): + filter_fields = self.get_filter_fields(view) + + parameters = [] + for field_name in filter_fields: + parameters.append({ + 'name': field_name, + 'required': False, + 'in': 'query', + 'description': self.filter_desc.format_map({'field_name': field_name}), + 'schema': { + 'type': 'string', + }, + }) + return parameters diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 20c39ed0b0a7..952116336d4f 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -268,6 +268,7 @@ def advanced_authentication(request): } return Response(response) +import django_filters @extend_schema(tags=['projects']) @extend_schema_view( list=extend_schema( @@ -317,7 +318,6 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, ordering_fields = filter_fields ordering = "-id" lookup_fields = {'owner': 'owner__username', 'assignee': 'assignee__username'} - filterset_fields = { k: [v] for k, v in lookup_fields.items()} iam_organization_field = 'organization' def get_serializer_class(self): From 6689ca2bf8959a6f6c64393dad777cc8b0749846 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 12 Jan 2023 00:55:51 +0200 Subject: [PATCH 014/140] Simplify implementation --- cvat/apps/engine/filters.py | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index eff4e43dbb12..f6871f2662af 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -21,8 +21,10 @@ from django_filters.rest_framework import DjangoFilterBackend -def get_lookup_fields(view) -> Dict[str, str]: - filter_fields = getattr(view, 'filter_fields', []) +def get_lookup_fields(view, filter_fields=None) -> Dict[str, str]: + if filter_fields is None: + filter_fields = getattr(view, 'filter_fields', []) + if not filter_fields: return {} @@ -238,6 +240,16 @@ def _get_lookup_fields(self, request, view): class SimpleFilter(DjangoFilterBackend): + """ + A simple filter, useful for small search queries and manually-edited + requests. + + The only available check is equality. + Multiple terms are joined with '&'. + Other operators are not supported (e.g. or, less, greater, not etc.). + Argument types are numbers and strings. + """ + filter_desc = _('A simple equality filter for the {field_name} field') reserved_names = ( JsonLogicFilter.filter_param, @@ -266,27 +278,24 @@ def get_filter_name(cls, field_name, lookup_expr): def get_filterset_class(self, view, queryset=None): - filterset_class = getattr(view, 'filterset_class', None) - filterset_fields = getattr(view, 'filterset_fields', None) - - if filterset_class or filterset_fields: - return super().get_filterset_class(view, queryset) - lookup_fields = self.get_lookup_fields(view) - if not lookup_fields: + if not lookup_fields or not queryset: return None - class MappingFilterset(self.MappingFiltersetBase, metaclass=FilterSet.__class__): + MetaBase = getattr(self.filterset_base, 'Meta', object) + + class AutoFilterSet(self.filterset_base, metaclass=FilterSet.__class__): filter_names = { v: k for k, v in lookup_fields.items() } - with _patched_attr(view, 'filterset_fields', list(lookup_fields.values())): - with _patched_attr(self, 'filterset_base', MappingFilterset): - filterset_class = super().get_filterset_class(view, queryset) + class Meta(MetaBase): + model = queryset.model + fields = list(lookup_fields.values()) - return filterset_class + return AutoFilterSet def get_lookup_fields(self, view): - return get_lookup_fields(view) + simple_filters = getattr(view, 'simple_filters', None) + return get_lookup_fields(view, filter_fields=simple_filters) def get_filter_fields(self, view): return list(self.get_lookup_fields(view)) @@ -317,7 +326,7 @@ def get_schema_operation_parameters(self, view): 'name': field_name, 'required': False, 'in': 'query', - 'description': self.filter_desc.format_map({'field_name': field_name}), + 'description': force_str(self.filter_desc.format_map({'field_name': field_name})), 'schema': { 'type': 'string', }, From 44944e7396f244be5d6ace210d790ac01c8a7eb3 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 12 Jan 2023 18:12:57 +0200 Subject: [PATCH 015/140] Update filtering fields in views --- cvat/apps/engine/views.py | 44 +++++++++++++------- cvat/apps/organizations/views.py | 13 +++--- cvat/apps/webhooks/views.py | 70 ++++++++++++++++++++------------ 3 files changed, 82 insertions(+), 45 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 952116336d4f..2e8fe7f10538 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -268,7 +268,6 @@ def advanced_authentication(request): } return Response(response) -import django_filters @extend_schema(tags=['projects']) @extend_schema_view( list=extend_schema( @@ -315,7 +314,8 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, # type fields on the model,such as CharField or TextField search_fields = ('name', 'owner', 'assignee', 'status') filter_fields = list(search_fields) + ['id', 'updated_date'] - ordering_fields = filter_fields + simple_filters = list(search_fields) + ordering_fields = list(filter_fields) ordering = "-id" lookup_fields = {'owner': 'owner__username', 'assignee': 'assignee__username'} iam_organization_field = 'organization' @@ -789,10 +789,19 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, 'project__label_set__attributespec_set', 'label_set__sublabels__attributespec_set', 'project__label_set__sublabels__attributespec_set') - lookup_fields = {'project_name': 'project__name', 'owner': 'owner__username', 'assignee': 'assignee__username'} - search_fields = ('project_name', 'name', 'owner', 'status', 'assignee', 'subset', 'mode', 'dimension') + lookup_fields = { + 'project_name': 'project__name', + 'owner': 'owner__username', + 'assignee': 'assignee__username', + 'tracker_link': 'bug_tracker', + } + search_fields = ( + 'project_name', 'name', 'owner', 'status', 'assignee', + 'subset', 'mode', 'dimension', 'tracker_link' + ) filter_fields = list(search_fields) + ['id', 'project_id', 'updated_date'] - ordering_fields = filter_fields + simple_filters = list(search_fields) + ['project_id'] + ordering_fields = list(filter_fields) ordering = "-id" iam_organization_field = 'organization' @@ -1410,7 +1419,8 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, iam_organization_field = 'segment__task__organization' search_fields = ('task_name', 'project_name', 'assignee', 'state', 'stage') filter_fields = list(search_fields) + ['id', 'task_id', 'project_id', 'updated_date'] - ordering_fields = filter_fields + simple_filters = list(search_fields) + ['task_id', 'project_id'] + ordering_fields = list(filter_fields) ordering = "-id" lookup_fields = { 'dimension': 'segment__task__dimension', @@ -1854,14 +1864,15 @@ class IssueViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, queryset = Issue.objects.all().order_by('-id') iam_organization_field = 'job__segment__task__organization' search_fields = ('owner', 'assignee') - filter_fields = list(search_fields) + ['id', 'job_id', 'task_id', 'resolved'] + filter_fields = list(search_fields) + ['id', 'job_id', 'task_id', 'resolved', 'frame_id'] + simple_filters = list(search_fields) + ['job_id', 'task_id', 'resolved', 'frame_id'] + ordering_fields = list(filter_fields) lookup_fields = { 'owner': 'owner__username', 'assignee': 'assignee__username', 'job_id': 'job__id', 'task_id': 'job__segment__task__id', } - ordering_fields = filter_fields ordering = '-id' def get_queryset(self): @@ -1935,7 +1946,8 @@ class CommentViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, iam_organization_field = 'issue__job__segment__task__organization' search_fields = ('owner',) filter_fields = list(search_fields) + ['id', 'issue_id'] - ordering_fields = filter_fields + simple_filters = list(search_fields) + ['issue_id'] + ordering_fields = list(filter_fields) ordering = '-id' lookup_fields = {'owner': 'owner__username', 'issue_id': 'issue__id'} @@ -1991,11 +2003,12 @@ def perform_create(self, serializer, **kwargs): class UserViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, PartialUpdateModelMixin, mixins.DestroyModelMixin): queryset = User.objects.prefetch_related('groups').all() - search_fields = ('username', 'first_name', 'last_name') iam_organization_field = 'memberships__organization' - filter_fields = ('id', 'is_active', 'username') - ordering_fields = filter_fields + search_fields = ('username', 'first_name', 'last_name') + filter_fields = list(search_fields) + ['id', 'is_active'] + simple_filters = list(search_fields) + ['is_active'] + ordering_fields = list(filter_fields) ordering = "-id" def get_queryset(self): @@ -2074,12 +2087,13 @@ class CloudStorageViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, ): queryset = CloudStorageModel.objects.all().prefetch_related('data') - search_fields = ('provider_type', 'display_name', 'resource', + search_fields = ('provider_type', 'name', 'resource', 'credentials_type', 'owner', 'description') filter_fields = list(search_fields) + ['id'] - ordering_fields = filter_fields + simple_filters = list(set(search_fields) - {'description'}) + ordering_fields = list(filter_fields) ordering = "-id" - lookup_fields = {'owner': 'owner__username'} + lookup_fields = {'owner': 'owner__username', 'name': 'display_name'} iam_organization_field = 'organization' def get_serializer_class(self): diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index 40bebe53df62..2542010ad092 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -57,8 +57,9 @@ class OrganizationViewSet(viewsets.GenericViewSet, queryset = Organization.objects.all() search_fields = ('name', 'owner') filter_fields = list(search_fields) + ['id', 'slug'] + simple_filters = list(search_fields) + ['slug'] lookup_fields = {'owner': 'owner__username'} - ordering_fields = filter_fields + ordering_fields = list(filter_fields) ordering = '-id' http_method_names = ['get', 'post', 'patch', 'delete', 'head', 'options'] iam_organization_field = None @@ -113,9 +114,10 @@ class MembershipViewSet(mixins.RetrieveModelMixin, DestroyModelMixin, ordering = '-id' http_method_names = ['get', 'patch', 'delete', 'head', 'options'] search_fields = ('user_name', 'role') - filter_fields = list(search_fields) + ['id', 'user'] - ordering_fields = filter_fields - lookup_fields = {'user': 'user__id', 'user_name': 'user__username'} + filter_fields = list(search_fields) + ['id', 'user_id'] + simple_filters = list(search_fields) + ['user_id'] + ordering_fields = list(filter_fields) + lookup_fields = {'user_id': 'user__id', 'user_name': 'user__username'} iam_organization_field = 'organization' def get_serializer_class(self): @@ -174,7 +176,8 @@ class InvitationViewSet(viewsets.GenericViewSet, iam_organization_field = 'membership__organization' search_fields = ('owner',) - filter_fields = search_fields + filter_fields = list(search_fields) + simple_filters = list(search_fields) ordering_fields = list(filter_fields) + ['created_date'] ordering = '-created_date' lookup_fields = {'owner': 'owner__username'} diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index 1e84d338bf19..deec3f3979c6 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -13,6 +13,7 @@ from rest_framework.decorators import action from rest_framework.permissions import SAFE_METHODS from rest_framework.response import Response +from cvat.apps.engine.view_utils import make_paginated_response from cvat.apps.iam.permissions import WebhookPermission @@ -40,17 +41,23 @@ update=extend_schema( summary="Method updates a webhook by id", request=WebhookWriteSerializer, - responses={"200": WebhookReadSerializer}, # check WebhookWriteSerializer.to_representation + responses={ + "200": WebhookReadSerializer + }, # check WebhookWriteSerializer.to_representation ), partial_update=extend_schema( summary="Methods does a partial update of chosen fields in a webhook", request=WebhookWriteSerializer, - responses={"200": WebhookReadSerializer}, # check WebhookWriteSerializer.to_representation + responses={ + "200": WebhookReadSerializer + }, # check WebhookWriteSerializer.to_representation ), create=extend_schema( request=WebhookWriteSerializer, summary="Method creates a webhook", - responses={"201": WebhookReadSerializer} # check WebhookWriteSerializer.to_representation + responses={ + "201": WebhookReadSerializer + }, # check WebhookWriteSerializer.to_representation ), destroy=extend_schema( summary="Method deletes a webhook", @@ -64,16 +71,19 @@ class WebhookViewSet(viewsets.ModelViewSet): search_fields = ("target_url", "owner", "type", "description") filter_fields = list(search_fields) + ["id", "project_id", "updated_date"] - ordering_fields = filter_fields + simple_filters = list(set(search_fields) - {"description"} | {"project_id"}) + ordering_fields = list(filter_fields) lookup_fields = {"owner": "owner__username"} iam_organization_field = "organization" def get_serializer_class(self): # Early exit for drf-spectacular compatibility - if getattr(self, 'swagger_fake_view', False): + if getattr(self, "swagger_fake_view", False): return WebhookReadSerializer - if self.request.path.endswith("redelivery") or self.request.path.endswith("ping"): + if self.request.path.endswith("redelivery") or self.request.path.endswith( + "ping" + ): return None else: if self.request.method in SAFE_METHODS: @@ -128,29 +138,38 @@ def events(self, request): @extend_schema( summary="Method return a list of deliveries for a specific webhook", - responses={"200": WebhookDeliveryReadSerializer(many=True)}, + responses=WebhookDeliveryReadSerializer( + many=True + ), # Duplicate to still get 'list' op. name ) @action( - detail=True, methods=["GET"], serializer_class=WebhookDeliveryReadSerializer + detail=True, + methods=["GET"], + serializer_class=WebhookDeliveryReadSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + search_fields=["event", "request", "response", "changed_fields"], + filter_fields=[ + "event", + "request", + "response", + "changed_fields", + "status_code", + "redelivery", + "changed_fields", + ], + ordering_fields=["event", "status_code", "redelivery"], + simple_filters=["event", "status_code", "redelivery", "changed_fields"], ) def deliveries(self, request, pk): self.get_object() queryset = WebhookDelivery.objects.filter(webhook_id=pk).order_by( "-updated_date" ) - - page = self.paginate_queryset(queryset) - if page is not None: - serializer = WebhookDeliveryReadSerializer( - page, many=True, context={"request": request} - ) - return self.get_paginated_response(serializer.data) - - serializer = WebhookDeliveryReadSerializer( - queryset, many=True, context={"request": request} - ) - - return Response(serializer.data) + return make_paginated_response( + queryset, viewset=self, serializer_type=self.serializer_class + ) # from @action @extend_schema( summary="Method return a specific delivery for a specific webhook", @@ -170,15 +189,16 @@ def retrieve_delivery(self, request, pk, delivery_id): ) return Response(serializer.data) - @extend_schema(summary="Method redeliver a specific webhook delivery", + @extend_schema( + summary="Method redeliver a specific webhook delivery", request=None, - responses={200: None} + responses={200: None}, ) @action( detail=True, methods=["POST"], url_path=r"deliveries/(?P\d+)/redelivery", - serializer_class=None + serializer_class=None, ) def redelivery(self, request, pk, delivery_id): delivery = WebhookDelivery.objects.get(webhook_id=pk, id=delivery_id) From 3f0f5d16de7206383649bc2171cb3f9e18b26355 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 12 Jan 2023 18:56:50 +0200 Subject: [PATCH 016/140] Remove extra fields from schema --- cvat/apps/engine/filters.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index f6871f2662af..0ef0f897a725 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -310,7 +310,6 @@ def get_schema_fields(self, view): coreapi.Field( name=field_name, location='query', - required=False, schema={ 'type': 'string', } @@ -324,7 +323,6 @@ def get_schema_operation_parameters(self, view): for field_name in filter_fields: parameters.append({ 'name': field_name, - 'required': False, 'in': 'query', 'description': force_str(self.filter_desc.format_map({'field_name': field_name})), 'schema': { From afba94c86f622be10ea71e65cf93b56311ad619c Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 13 Jan 2023 12:36:02 +0200 Subject: [PATCH 017/140] Fix prefetch use --- cvat/apps/engine/views.py | 50 ++++++++++++++++++++++++++----------- cvat/apps/webhooks/views.py | 2 +- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 2e8fe7f10538..be043e20d996 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -305,10 +305,12 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin ): - queryset = models.Project.objects.select_related('assignee', 'owner', - 'target_storage', 'source_storage').prefetch_related( + queryset = models.Project.objects.select_related( + 'assignee', 'owner', 'target_storage', 'source_storage' + ).prefetch_related( 'tasks', 'label_set__sublabels__attributespec_set', - 'label_set__attributespec_set') + 'label_set__attributespec_set' + ).all() # NOTE: The search_fields attribute should be a list of names of text # type fields on the model,such as CharField or TextField @@ -783,12 +785,16 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin ): - queryset = Task.objects.all().select_related('data', 'assignee', 'owner', - 'target_storage', 'source_storage').prefetch_related( + queryset = Task.objects.select_related( + 'data', 'assignee', 'owner', + 'target_storage', 'source_storage' + ).prefetch_related( 'segment_set__job_set__assignee', 'label_set__attributespec_set', 'project__label_set__attributespec_set', 'label_set__sublabels__attributespec_set', - 'project__label_set__sublabels__attributespec_set') + 'project__label_set__sublabels__attributespec_set' + ).all() + lookup_fields = { 'project_name': 'project__name', 'owner': 'owner__username', @@ -1410,12 +1416,14 @@ def preview(self, request, pk): class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin ): - queryset = Job.objects.all().select_related('segment__task__data').prefetch_related( + queryset = Job.objects.select_related('segment__task__data').prefetch_related( 'segment__task__label_set', 'segment__task__project__label_set', 'segment__task__label_set__sublabels__attributespec_set', 'segment__task__project__label_set__sublabels__attributespec_set', 'segment__task__label_set__attributespec_set', - 'segment__task__project__label_set__attributespec_set') + 'segment__task__project__label_set__attributespec_set' + ).all() + iam_organization_field = 'segment__task__organization' search_fields = ('task_name', 'project_name', 'assignee', 'state', 'stage') filter_fields = list(search_fields) + ['id', 'task_id', 'project_id', 'updated_date'] @@ -1861,7 +1869,10 @@ class IssueViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, PartialUpdateModelMixin ): - queryset = Issue.objects.all().order_by('-id') + queryset = Issue.objects.prefetch_related( + 'job__segment__task', 'owner', 'assignee', 'job' + ).all() + iam_organization_field = 'job__segment__task__organization' search_fields = ('owner', 'assignee') filter_fields = list(search_fields) + ['id', 'job_id', 'task_id', 'resolved', 'frame_id'] @@ -1870,8 +1881,9 @@ class IssueViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, lookup_fields = { 'owner': 'owner__username', 'assignee': 'assignee__username', - 'job_id': 'job__id', + 'job_id': 'job', 'task_id': 'job__segment__task__id', + 'frame_id': 'frame', } ordering = '-id' @@ -1942,14 +1954,22 @@ class CommentViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, CreateModelMixin, DestroyModelMixin, PartialUpdateModelMixin ): - queryset = Comment.objects.all().order_by('-id') + queryset = Comment.objects.prefetch_related( + 'issue', 'issue__job', 'owner' + ).all() + iam_organization_field = 'issue__job__segment__task__organization' search_fields = ('owner',) - filter_fields = list(search_fields) + ['id', 'issue_id'] - simple_filters = list(search_fields) + ['issue_id'] + filter_fields = list(search_fields) + ['id', 'issue_id', 'frame_id', 'job_id'] + simple_filters = list(search_fields) + ['issue_id', 'frame_id', 'job_id'] ordering_fields = list(filter_fields) ordering = '-id' - lookup_fields = {'owner': 'owner__username', 'issue_id': 'issue__id'} + lookup_fields = { + 'owner': 'owner__username', + 'issue_id': 'issue__id', + 'job_id': 'issue__job__id', + 'frame_id': 'issue__frame', + } def get_queryset(self): queryset = super().get_queryset() @@ -2085,7 +2105,7 @@ class CloudStorageViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin ): - queryset = CloudStorageModel.objects.all().prefetch_related('data') + queryset = CloudStorageModel.objects.prefetch_related('data').all() search_fields = ('provider_type', 'name', 'resource', 'credentials_type', 'owner', 'description') diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index deec3f3979c6..f8a4dca963d8 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -65,7 +65,7 @@ ), ) class WebhookViewSet(viewsets.ModelViewSet): - queryset = Webhook.objects.all() + queryset = Webhook.objects.prefetch_related('owner').all() ordering = "-id" http_method_names = ["get", "post", "delete", "patch", "put"] From ecbb364860bd86a580b68b1e7993191ddb36d3b0 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 13 Jan 2023 16:36:32 +0200 Subject: [PATCH 018/140] Change string filter to inclusion --- cvat/apps/engine/filters.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 0ef0f897a725..45305b337fb7 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -10,6 +10,7 @@ import operator import json +from django.db import models from django.db.models import Q from django.utils.translation import gettext_lazy as _ from django.utils.encoding import force_str @@ -244,10 +245,10 @@ class SimpleFilter(DjangoFilterBackend): A simple filter, useful for small search queries and manually-edited requests. - The only available check is equality. - Multiple terms are joined with '&'. - Other operators are not supported (e.g. or, less, greater, not etc.). - Argument types are numbers and strings. + Argument types are numbers and strings. The only available check is + equality for numbers and case-independent inclusion for strings. + Operators are not supported (e.g. or, less, greater, not etc.). + Multiple filters are joined with '&' as separate query params. """ filter_desc = _('A simple equality filter for the {field_name} field') @@ -258,6 +259,15 @@ class SimpleFilter(DjangoFilterBackend): ) class MappingFiltersetBase(BaseFilterSet): + class Meta: + filter_overrides = { + models.CharField: { + 'extra': lambda f: { + 'lookup_expr': 'icontains', + }, + }, + }, + @classmethod def get_filter_name(cls, field_name, lookup_expr): filter_names = getattr(cls, 'filter_names', {}) From 7ebab92ae92399e9de9ee29b57d50f752fe78db6 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 17 Jan 2023 18:34:21 +0200 Subject: [PATCH 019/140] Update search fields and add tests --- cvat-sdk/cvat_sdk/core/helpers.py | 7 +- cvat/apps/engine/filters.py | 9 --- cvat/apps/organizations/views.py | 8 +-- tests/python/rest_api/test_cloud_storages.py | 33 ++++++++++ tests/python/rest_api/test_invitations.py | 24 +++++++ tests/python/rest_api/test_issues.py | 66 +++++++++++++++++++ tests/python/rest_api/test_jobs.py | 32 ++++++++- tests/python/rest_api/test_memberships.py | 24 +++++++ tests/python/rest_api/test_organizations.py | 24 +++++++ tests/python/rest_api/test_projects.py | 29 +++++++- tests/python/rest_api/test_tasks.py | 36 +++++++++- tests/python/rest_api/test_users.py | 20 ++++++ tests/python/rest_api/utils.py | 69 +++++++++++++++++++- tests/python/shared/assets/cvat_db/data.json | 6 +- tests/python/shared/assets/issues.json | 10 ++- tests/python/shared/assets/jobs.json | 2 +- tests/python/shared/assets/tasks.json | 2 +- 17 files changed, 375 insertions(+), 26 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/helpers.py b/cvat-sdk/cvat_sdk/core/helpers.py index b548d85cdb15..bfab7146d246 100644 --- a/cvat-sdk/cvat_sdk/core/helpers.py +++ b/cvat-sdk/cvat_sdk/core/helpers.py @@ -34,7 +34,12 @@ def get_paginated_collection( else: results.extend(page_contents.results) - if not page_contents.next: + if ( + page_contents is not None + and not page_contents.next + or page_contents is None + and not json.loads(response.data).get("next") + ): break page += 1 diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 45305b337fb7..fd89a78b8f5f 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -259,15 +259,6 @@ class SimpleFilter(DjangoFilterBackend): ) class MappingFiltersetBase(BaseFilterSet): - class Meta: - filter_overrides = { - models.CharField: { - 'extra': lambda f: { - 'lookup_expr': 'icontains', - }, - }, - }, - @classmethod def get_filter_name(cls, field_name, lookup_expr): filter_names = getattr(cls, 'filter_names', {}) diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index 2542010ad092..50fb07104d46 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -113,11 +113,11 @@ class MembershipViewSet(mixins.RetrieveModelMixin, DestroyModelMixin, queryset = Membership.objects.all() ordering = '-id' http_method_names = ['get', 'patch', 'delete', 'head', 'options'] - search_fields = ('user_name', 'role') - filter_fields = list(search_fields) + ['id', 'user_id'] - simple_filters = list(search_fields) + ['user_id'] + search_fields = ('user', 'role') + filter_fields = list(search_fields) + ['id'] + simple_filters = list(search_fields) ordering_fields = list(filter_fields) - lookup_fields = {'user_id': 'user__id', 'user_name': 'user__username'} + lookup_fields = {'user': 'user__username'} iam_organization_field = 'organization' def get_serializer_class(self): diff --git a/tests/python/rest_api/test_cloud_storages.py b/tests/python/rest_api/test_cloud_storages.py index a392c2a58a0e..1a4dcbbef8c5 100644 --- a/tests/python/rest_api/test_cloud_storages.py +++ b/tests/python/rest_api/test_cloud_storages.py @@ -5,13 +5,17 @@ import io from http import HTTPStatus +from typing import List import pytest +from cvat_sdk.api_client import ApiClient, models from deepdiff import DeepDiff from PIL import Image from shared.utils.config import get_method, patch_method, post_method +from .utils import CollectionSimpleFilterTestBase + # https://docs.pytest.org/en/7.1.x/example/markers.html#marking-whole-classes-or-modules pytestmark = [pytest.mark.with_external_services] @@ -97,6 +101,35 @@ def test_org_user_get_coud_storage( self._test_cannot_see(username, storage_id, org_id=org_id) +class TestCloudStoragesListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + "name": ["display_name"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, cloud_storages): + self.user = admin_user + self.samples = cloud_storages + + def _get_endpoint(self, api_client: ApiClient): + return api_client.cloudstorages_api.list_endpoint + + def _retrieve_collection(self, **kwargs) -> List: + # TODO: fix invalid serializer schema for manifests + results = super()._retrieve_collection(_parse_response=False, return_json=True, **kwargs) + for r in results: + r["manifests"] = [{"filename": m} for m in r["manifests"]] + return [models.CloudStorageRead._from_openapi_data(**r) for r in results] + + @pytest.mark.parametrize( + "field", + ("provider_type", "name", "resource", "credentials_type", "owner"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + @pytest.mark.usefixtures("restore_db_per_function") class TestPostCloudStorage: _SPEC = { diff --git a/tests/python/rest_api/test_invitations.py b/tests/python/rest_api/test_invitations.py index 4e8babc8b7f8..c383dafd4090 100644 --- a/tests/python/rest_api/test_invitations.py +++ b/tests/python/rest_api/test_invitations.py @@ -6,9 +6,12 @@ from http import HTTPStatus import pytest +from cvat_sdk.api_client.api_client import ApiClient from shared.utils.config import post_method +from .utils import CollectionSimpleFilterTestBase + @pytest.mark.usefixtures("restore_db_per_function") class TestCreateInvitations: @@ -84,3 +87,24 @@ def test_create_invitation(self, organizations, memberships, users, org_id, org_ {"role": "owner", "email": non_member_users[4]["email"]}, org_id=org_id, ) + + +class TestInvitationsListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, invitations): + self.user = admin_user + self.samples = invitations + + def _get_endpoint(self, api_client: ApiClient): + return api_client.invitations_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ("owner",), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index 87fbfbf11ebb..f99128fc5fcd 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -6,14 +6,18 @@ import json from copy import deepcopy from http import HTTPStatus +from typing import Any, Dict, List, Tuple import pytest from cvat_sdk import models from cvat_sdk.api_client import exceptions +from cvat_sdk.api_client.api_client import ApiClient from deepdiff import DeepDiff from shared.utils.config import make_api_client +from .utils import CollectionSimpleFilterTestBase + @pytest.mark.usefixtures("restore_db_per_function") class TestPostIssues: @@ -326,3 +330,65 @@ def test_org_member_delete_issue( username, issue_id = find_issue_staff_user(issues, users, issue_staff, issue_admin) self._test_check_response(username, issue_id, expect_success, org_id=org) + + +class TestIssuesListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + "assignee": ["assignee", "username"], + "job_id": ["job"], + "frame_id": ["frame"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, issues): + self.user = admin_user + self.samples = issues + + def _get_endpoint(self, api_client: ApiClient): + return api_client.issues_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ("owner", "assignee", "job_id", "resolved", "frame_id"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + +class TestCommentsListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + "issue_id": ["issue"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, comments, issues): + self.user = admin_user + self.samples = comments + self.sample_issues = issues + + def _get_endpoint(self, api_client: ApiClient): + return api_client.comments_api.list_endpoint + + def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: + if field == "job_id": + issue_id, issue_comments = super()._get_field_samples("issue_id") + issue = next((s for s in self.sample_issues if s["id"] == issue_id)) + return issue["job"], issue_comments + elif field == "frame_id": + frame_id = self._find_valid_field_value(self.sample_issues, ["frame"]) + issues = [s["id"] for s in self.sample_issues if s["frame"] == frame_id] + comments = [ + s for s in self.samples if self._get_field(s, self._map_field("issue_id")) in issues + ] + return frame_id, comments + else: + return super()._get_field_samples(field) + + @pytest.mark.parametrize( + "field", + ("owner", "issue_id", "job_id", "frame_id"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) diff --git a/tests/python/rest_api/test_jobs.py b/tests/python/rest_api/test_jobs.py index 31ec61046b64..f97d5ad1eb9e 100644 --- a/tests/python/rest_api/test_jobs.py +++ b/tests/python/rest_api/test_jobs.py @@ -12,13 +12,14 @@ from typing import List import pytest +from cvat_sdk.api_client.api_client import ApiClient from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image from shared.utils.config import make_api_client -from .utils import export_dataset +from .utils import CollectionSimpleFilterTestBase, export_dataset def get_job_staff(job, tasks, projects): @@ -146,6 +147,35 @@ def test_non_admin_list_jobs( self._test_list_jobs_403(user["username"], **kwargs) +class TestJobsListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "assignee": ["assignee", "username"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, jobs, tasks, projects): + self.user = admin_user + self.samples = jobs + self.sample_tasks = tasks + self.sample_projects = projects + + def _get_endpoint(self, api_client: ApiClient): + return api_client.jobs_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ( + "assignee", + "state", + "stage", + "task_id", + "project_id", + ), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + @pytest.mark.usefixtures("restore_db_per_class") class TestGetAnnotations: def _test_get_job_annotations_200(self, user, jid, data, **kwargs): diff --git a/tests/python/rest_api/test_memberships.py b/tests/python/rest_api/test_memberships.py index 298392ede2dd..16e820513dfa 100644 --- a/tests/python/rest_api/test_memberships.py +++ b/tests/python/rest_api/test_memberships.py @@ -6,10 +6,13 @@ from http import HTTPStatus import pytest +from cvat_sdk.api_client.api_client import ApiClient from deepdiff import DeepDiff from shared.utils.config import get_method, patch_method +from .utils import CollectionSimpleFilterTestBase + @pytest.mark.usefixtures("restore_db_per_class") class TestGetMemberships: @@ -44,6 +47,27 @@ def test_non_members_cannot_see_members_membership(self): self._test_cannot_see_memberships(user, org_id=1) +class TestMembershipsListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "user": ["user", "username"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, memberships): + self.user = admin_user + self.samples = memberships + + def _get_endpoint(self, api_client: ApiClient): + return api_client.memberships_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ("role", "user"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + @pytest.mark.usefixtures("restore_db_per_function") class TestPatchMemberships: _ORG = 2 diff --git a/tests/python/rest_api/test_organizations.py b/tests/python/rest_api/test_organizations.py index 90b902161c62..b992c56d0e4b 100644 --- a/tests/python/rest_api/test_organizations.py +++ b/tests/python/rest_api/test_organizations.py @@ -7,10 +7,13 @@ from http import HTTPStatus import pytest +from cvat_sdk.api_client.api_client import ApiClient from deepdiff import DeepDiff from shared.utils.config import delete_method, get_method, options_method, patch_method +from .utils import CollectionSimpleFilterTestBase + class TestMetadataOrganizations: _ORG = 2 @@ -76,6 +79,27 @@ def test_can_see_specific_organization( assert response.status_code == HTTPStatus.NOT_FOUND +class TestOrganizationsListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, organizations): + self.user = admin_user + self.samples = organizations + + def _get_endpoint(self, api_client: ApiClient): + return api_client.organizations_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ("name", "owner", "slug"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + @pytest.mark.usefixtures("restore_db_per_function") class TestPatchOrganizations: _ORG = 2 diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index b21ff6c1ed2e..8258a5d0b4b7 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -21,7 +21,7 @@ from shared.utils.config import BASE_URL, USER_PASS, get_method, make_api_client, patch_method -from .utils import export_dataset +from .utils import CollectionSimpleFilterTestBase, export_dataset @pytest.mark.usefixtures("restore_db_per_class") @@ -132,6 +132,33 @@ def test_if_org_member_supervisor_or_worker_can_see_project( self._test_response_200(user["username"], pid, org_id=user["org"]) +class TestProjectsListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + "assignee": ["assignee", "username"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, projects): + self.user = admin_user + self.samples = projects + + def _get_endpoint(self, api_client: ApiClient): + return api_client.projects_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ( + "name", + "owner", + "assignee", + "status", + ), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + class TestGetProjectBackup: def _test_can_get_project_backup(self, username, pid, **kwargs): for _ in range(30): diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 7b5bdc09b09d..7c2b1c477d34 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -15,6 +15,7 @@ import pytest from cvat_sdk.api_client import apis, models +from cvat_sdk.api_client.api_client import ApiClient from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image @@ -24,7 +25,7 @@ from shared.utils.config import get_method, make_api_client, patch_method from shared.utils.helpers import generate_image_files -from .utils import export_dataset +from .utils import CollectionSimpleFilterTestBase, export_dataset def get_cloud_storage_content(username, cloud_storage_id, manifest): @@ -150,6 +151,39 @@ def test_org_task_assigneed_to_see_task( self._test_assigned_users_to_see_task_data(tasks, users, is_task_staff, org=org["slug"]) +class TestListTasksFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + "assignee": ["assignee", "username"], + "tracker_link": ["bug_tracker"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, tasks): + self.user = admin_user + self.samples = tasks + + def _get_endpoint(self, api_client: ApiClient): + return api_client.tasks_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ( + "name", + "owner", + "status", + "assignee", + "subset", + "mode", + "dimension", + "project_id", + "tracker_link", + ), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + @pytest.mark.usefixtures("restore_db_per_function") class TestPostTasks: def _test_create_task_201(self, user, spec, **kwargs): diff --git a/tests/python/rest_api/test_users.py b/tests/python/rest_api/test_users.py index d62ffc0671b5..863aacb50c69 100644 --- a/tests/python/rest_api/test_users.py +++ b/tests/python/rest_api/test_users.py @@ -8,11 +8,14 @@ from http import HTTPStatus import pytest +from cvat_sdk.api_client.api_client import ApiClient from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from shared.utils.config import make_api_client +from .utils import CollectionSimpleFilterTestBase + @pytest.mark.usefixtures("restore_db_per_class") class TestGetUsers: @@ -93,3 +96,20 @@ def test_all_members_can_see_list_of_members(self, find_users, users): for member in org_members: self._test_can_see(member, data, org="org1") + + +class TestUsersListFilters(CollectionSimpleFilterTestBase): + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, users): + self.user = admin_user + self.samples = users + + def _get_endpoint(self, api_client: ApiClient): + return api_client.users_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ("is_active", "username"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) diff --git a/tests/python/rest_api/utils.py b/tests/python/rest_api/utils.py index a5bcfe835a0b..bfd1a23d27c4 100644 --- a/tests/python/rest_api/utils.py +++ b/tests/python/rest_api/utils.py @@ -1,13 +1,18 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT +from abc import ABCMeta, abstractmethod from http import HTTPStatus from time import sleep +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union -from cvat_sdk.api_client.api_client import Endpoint +from cvat_sdk.api_client.api_client import ApiClient, Endpoint +from cvat_sdk.core.helpers import get_paginated_collection from urllib3 import HTTPResponse +from shared.utils.config import make_api_client + def export_dataset( endpoint: Endpoint, *, max_retries: int = 20, interval: float = 0.1, **kwargs @@ -24,3 +29,63 @@ def export_dataset( assert response.status == HTTPStatus.OK return response + + +FieldPath = Sequence[str] + + +class CollectionSimpleFilterTestBase(metaclass=ABCMeta): + # These fields need to be defined in the subclass + user: str + samples: List[Dict[str, Any]] + field_lookups: Dict[str, FieldPath] = None + + @abstractmethod + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: + ... + + def _retrieve_collection(self, **kwargs) -> List: + with make_api_client(self.user) as api_client: + return get_paginated_collection(self._get_endpoint(api_client), **kwargs) + + @classmethod + def _get_field(cls, d: Dict[str, Any], path: Union[str, FieldPath]) -> Optional[Any]: + assert path + for key in path: + if isinstance(d, dict): + d = d.get(key) + else: + d = None + + return d + + def _map_field(self, name: str) -> FieldPath: + return (self.field_lookups or {}).get(name, [name]) + + @classmethod + def _find_valid_field_value( + cls, samples: Iterator[Dict[str, Any]], field_path: FieldPath + ) -> Any: + value = None + for sample in samples: + value = cls._get_field(sample, field_path) + if value: + break + + assert value, f"Failed to find a sample for the '{'.'.join(field_path)}' field" + return value + + def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: + field_path = self._map_field(field) + field_value = self._find_valid_field_value(self.samples, field_path) + + gt_objects = filter(lambda p: field_value == self._get_field(p, field_path), self.samples) + + return field_value, gt_objects + + def test_can_use_simple_filter_for_object_list(self, field): + value, gt_objects = self._get_field_samples(field) + + received_items = self._retrieve_collection(**{field: str(value)}) + + assert set(p["id"] for p in gt_objects) == set(p.id for p in received_items) diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index d7462eb3e7b5..d08b273499b2 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -2618,7 +2618,7 @@ "admin1" ], "assignee": null, - "bug_tracker": "", + "bug_tracker": "https://bugtracker.localhost/task/12345", "created_date": "2022-09-22T14:22:25.820Z", "updated_date": "2022-09-23T11:57:02.300Z", "overlap": 0, @@ -5860,7 +5860,7 @@ "assignee": null, "created_date": "2022-03-16T11:04:39.444Z", "updated_date": null, - "resolved": false + "resolved": true } }, { @@ -5905,7 +5905,7 @@ "owner": [ "user1" ], - "assignee": null, + "assignee": 2, "created_date": "2022-03-16T12:40:00.764Z", "updated_date": null, "resolved": false diff --git a/tests/python/shared/assets/issues.json b/tests/python/shared/assets/issues.json index 689c95cb1ac7..94b9c907e198 100644 --- a/tests/python/shared/assets/issues.json +++ b/tests/python/shared/assets/issues.json @@ -42,7 +42,13 @@ "updated_date": null }, { - "assignee": null, + "assignee": { + "first_name": "User", + "id": 2, + "last_name": "First", + "url": "http://localhost:8080/api/users/2", + "username": "user1" + }, "comments": [ { "created_date": "2022-03-16T12:40:00.767000Z", @@ -214,7 +220,7 @@ 244.58581235698148, 319.63386727689067 ], - "resolved": false, + "resolved": true, "updated_date": null } ] diff --git a/tests/python/shared/assets/jobs.json b/tests/python/shared/assets/jobs.json index bac17fdc03e8..abf18f80c623 100644 --- a/tests/python/shared/assets/jobs.json +++ b/tests/python/shared/assets/jobs.json @@ -43,7 +43,7 @@ }, { "assignee": null, - "bug_tracker": "", + "bug_tracker": "https://bugtracker.localhost/task/12345", "data_chunk_size": 72, "data_compressed_chunk_type": "imageset", "dimension": "2d", diff --git a/tests/python/shared/assets/tasks.json b/tests/python/shared/assets/tasks.json index dbdc7e4ebfa3..f6f3953151fa 100644 --- a/tests/python/shared/assets/tasks.json +++ b/tests/python/shared/assets/tasks.json @@ -81,7 +81,7 @@ }, { "assignee": null, - "bug_tracker": "", + "bug_tracker": "https://bugtracker.localhost/task/12345", "created_date": "2022-09-22T14:22:25.820000Z", "data": 13, "data_chunk_size": 72, From c3221366b55c4f5d3e178be018446c8b9e80c13a Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 17 Jan 2023 23:16:00 +0200 Subject: [PATCH 020/140] Refactor some code --- cvat/apps/engine/filters.py | 64 +++++++++++++++---------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index fd89a78b8f5f..e25bb04f8334 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -3,14 +3,11 @@ # # SPDX-License-Identifier: MIT -from contextlib import contextmanager -from typing import Any, Dict +from typing import Dict, Iterator, Optional from functools import reduce -from unittest.mock import patch import operator import json -from django.db import models from django.db.models import Q from django.utils.translation import gettext_lazy as _ from django.utils.encoding import force_str @@ -21,34 +18,25 @@ from django_filters import FilterSet from django_filters.rest_framework import DjangoFilterBackend +DEFAULT_FILTER_FIELDS_ATTR = 'filter_fields' +DEFAULT_LOOKUP_MAP_ATTR = 'lookup_fields' -def get_lookup_fields(view, filter_fields=None) -> Dict[str, str]: - if filter_fields is None: - filter_fields = getattr(view, 'filter_fields', []) - - if not filter_fields: - return {} - - lookup_fields = {field:field for field in filter_fields} - lookup_fields.update(getattr(view, 'lookup_fields', {})) +def get_lookup_fields(view, fields: Optional[Iterator[str]] = None) -> Dict[str, str]: + if fields is None: + fields = getattr(view, DEFAULT_FILTER_FIELDS_ATTR, None) or [] + lookup_overrides = getattr(view, DEFAULT_LOOKUP_MAP_ATTR, None) or {} + lookup_fields = { + field: lookup_overrides.get(field, field) + for field in fields + } return lookup_fields -@contextmanager -def _patched_attr(obj: Any, name: str, value: Any) -> None: - with patch.object(obj, attribute=name, new=value, create=True): - yield class SearchFilter(filters.SearchFilter): def get_search_fields(self, view, request): search_fields = getattr(view, 'search_fields') or [] - lookup_fields = {field:field for field in search_fields} - view_lookup_fields = getattr(view, 'lookup_fields', {}) - keys_to_update = set(search_fields) & set(view_lookup_fields.keys()) - for key in keys_to_update: - lookup_fields[key] = view_lookup_fields[key] - - return lookup_fields.values() + return get_lookup_fields(view, search_fields).values() def get_schema_fields(self, view): assert coreapi is not None, 'coreapi must be installed to use `get_schema_fields()`' @@ -87,6 +75,7 @@ def get_schema_operation_parameters(self, view): class OrderingFilter(filters.OrderingFilter): ordering_param = 'sort' + def get_ordering(self, request, queryset, view): ordering = [] lookup_fields = self._get_lookup_fields(request, queryset, view) @@ -101,10 +90,8 @@ def get_ordering(self, request, queryset, view): def _get_lookup_fields(self, request, queryset, view): ordering_fields = self.get_valid_fields(queryset, view, {'request': request}) - lookup_fields = {field:field for field, _ in ordering_fields} - lookup_fields.update(getattr(view, 'lookup_fields', {})) - - return lookup_fields + ordering_fields = [v[0] for v in ordering_fields] + return get_lookup_fields(view, ordering_fields) def get_schema_fields(self, view): assert coreapi is not None, 'coreapi must be installed to use `get_schema_fields()`' @@ -258,10 +245,14 @@ class SimpleFilter(DjangoFilterBackend): SearchFilter.search_param, ) + filter_fields_attr = 'simple_filters' + class MappingFiltersetBase(BaseFilterSet): + _filter_name_map_attr = 'filter_names' + @classmethod def get_filter_name(cls, field_name, lookup_expr): - filter_names = getattr(cls, 'filter_names', {}) + filter_names = getattr(cls, cls._filter_name_map_attr, {}) field_name = super().get_filter_name(field_name, lookup_expr) @@ -295,17 +286,14 @@ class Meta(MetaBase): return AutoFilterSet def get_lookup_fields(self, view): - simple_filters = getattr(view, 'simple_filters', None) - return get_lookup_fields(view, filter_fields=simple_filters) - - def get_filter_fields(self, view): - return list(self.get_lookup_fields(view)) + simple_filters = getattr(view, self.filter_fields_attr, None) + return get_lookup_fields(view, fields=simple_filters) def get_schema_fields(self, view): assert coreapi is not None, 'coreapi must be installed to use `get_schema_fields()`' assert coreschema is not None, 'coreschema must be installed to use `get_schema_fields()`' - filter_fields = self.get_filter_fields(view) + lookup_fields = self.get_lookup_fields(view) return [ coreapi.Field( @@ -314,14 +302,14 @@ def get_schema_fields(self, view): schema={ 'type': 'string', } - ) for field_name in filter_fields + ) for field_name in lookup_fields ] def get_schema_operation_parameters(self, view): - filter_fields = self.get_filter_fields(view) + lookup_fields = self.get_lookup_fields(view) parameters = [] - for field_name in filter_fields: + for field_name in lookup_fields: parameters.append({ 'name': field_name, 'in': 'query', From 92e6a696244eab35dbd362cd169cddc7ae0f9bab Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 17 Jan 2023 23:18:06 +0200 Subject: [PATCH 021/140] Update webhook filters --- cvat/apps/webhooks/views.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index f8a4dca963d8..2993b3143f84 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -13,8 +13,8 @@ from rest_framework.decorators import action from rest_framework.permissions import SAFE_METHODS from rest_framework.response import Response -from cvat.apps.engine.view_utils import make_paginated_response +from cvat.apps.engine.view_utils import make_paginated_response from cvat.apps.iam.permissions import WebhookPermission from .event_type import AllEvents, OrganizationEvents, ProjectEvents @@ -65,7 +65,7 @@ ), ) class WebhookViewSet(viewsets.ModelViewSet): - queryset = Webhook.objects.prefetch_related('owner').all() + queryset = Webhook.objects.prefetch_related("owner").all() ordering = "-id" http_method_names = ["get", "post", "delete", "patch", "put"] @@ -147,20 +147,12 @@ def events(self, request): methods=["GET"], serializer_class=WebhookDeliveryReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, + # These non-root list endpoints do not suppose extra options, just the basic output # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - search_fields=["event", "request", "response", "changed_fields"], - filter_fields=[ - "event", - "request", - "response", - "changed_fields", - "status_code", - "redelivery", - "changed_fields", - ], - ordering_fields=["event", "status_code", "redelivery"], - simple_filters=["event", "status_code", "redelivery", "changed_fields"], + search_fields=None, + filter_fields=None, + ordering_fields=None, ) def deliveries(self, request, pk): self.get_object() From 0815eca0b3398373146641a8d6d64bb9784fc801 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 17 Jan 2023 23:22:09 +0200 Subject: [PATCH 022/140] Polish some code --- tests/python/rest_api/test_cloud_storages.py | 3 ++- tests/python/rest_api/test_invitations.py | 10 +++++--- tests/python/rest_api/test_issues.py | 6 ++--- tests/python/rest_api/test_jobs.py | 4 +-- tests/python/rest_api/test_memberships.py | 4 +-- tests/python/rest_api/test_organizations.py | 4 +-- tests/python/rest_api/test_projects.py | 3 ++- tests/python/rest_api/test_tasks.py | 4 +-- tests/python/rest_api/test_users.py | 4 +-- tests/python/rest_api/test_webhooks.py | 26 ++++++++++++++++++++ 10 files changed, 50 insertions(+), 18 deletions(-) diff --git a/tests/python/rest_api/test_cloud_storages.py b/tests/python/rest_api/test_cloud_storages.py index 1a4dcbbef8c5..c1acd01f5c4b 100644 --- a/tests/python/rest_api/test_cloud_storages.py +++ b/tests/python/rest_api/test_cloud_storages.py @@ -9,6 +9,7 @@ import pytest from cvat_sdk.api_client import ApiClient, models +from cvat_sdk.api_client.api_client import Endpoint from deepdiff import DeepDiff from PIL import Image @@ -112,7 +113,7 @@ def setup(self, restore_db_per_class, admin_user, cloud_storages): self.user = admin_user self.samples = cloud_storages - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.cloudstorages_api.list_endpoint def _retrieve_collection(self, **kwargs) -> List: diff --git a/tests/python/rest_api/test_invitations.py b/tests/python/rest_api/test_invitations.py index c383dafd4090..87ec27358910 100644 --- a/tests/python/rest_api/test_invitations.py +++ b/tests/python/rest_api/test_invitations.py @@ -6,7 +6,7 @@ from http import HTTPStatus import pytest -from cvat_sdk.api_client.api_client import ApiClient +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from shared.utils.config import post_method @@ -99,7 +99,7 @@ def setup(self, restore_db_per_class, admin_user, invitations): self.user = admin_user self.samples = invitations - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.invitations_api.list_endpoint @pytest.mark.parametrize( @@ -107,4 +107,8 @@ def _get_endpoint(self, api_client: ApiClient): ("owner",), ) def test_can_use_simple_filter_for_object_list(self, field): - return super().test_can_use_simple_filter_for_object_list(field) + value, gt_objects = self._get_field_samples(field) + + received_items = self._retrieve_collection(**{field: str(value)}) + + assert set(p["key"] for p in gt_objects) == set(p.key for p in received_items) diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index f99128fc5fcd..3e39c47fe47c 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -11,7 +11,7 @@ import pytest from cvat_sdk import models from cvat_sdk.api_client import exceptions -from cvat_sdk.api_client.api_client import ApiClient +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from deepdiff import DeepDiff from shared.utils.config import make_api_client @@ -345,7 +345,7 @@ def setup(self, restore_db_per_class, admin_user, issues): self.user = admin_user self.samples = issues - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.issues_api.list_endpoint @pytest.mark.parametrize( @@ -368,7 +368,7 @@ def setup(self, restore_db_per_class, admin_user, comments, issues): self.samples = comments self.sample_issues = issues - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.comments_api.list_endpoint def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: diff --git a/tests/python/rest_api/test_jobs.py b/tests/python/rest_api/test_jobs.py index f97d5ad1eb9e..73c3f285435a 100644 --- a/tests/python/rest_api/test_jobs.py +++ b/tests/python/rest_api/test_jobs.py @@ -12,7 +12,7 @@ from typing import List import pytest -from cvat_sdk.api_client.api_client import ApiClient +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image @@ -159,7 +159,7 @@ def setup(self, restore_db_per_class, admin_user, jobs, tasks, projects): self.sample_tasks = tasks self.sample_projects = projects - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.jobs_api.list_endpoint @pytest.mark.parametrize( diff --git a/tests/python/rest_api/test_memberships.py b/tests/python/rest_api/test_memberships.py index 16e820513dfa..259dc64a9a6a 100644 --- a/tests/python/rest_api/test_memberships.py +++ b/tests/python/rest_api/test_memberships.py @@ -6,7 +6,7 @@ from http import HTTPStatus import pytest -from cvat_sdk.api_client.api_client import ApiClient +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from deepdiff import DeepDiff from shared.utils.config import get_method, patch_method @@ -57,7 +57,7 @@ def setup(self, restore_db_per_class, admin_user, memberships): self.user = admin_user self.samples = memberships - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.memberships_api.list_endpoint @pytest.mark.parametrize( diff --git a/tests/python/rest_api/test_organizations.py b/tests/python/rest_api/test_organizations.py index b992c56d0e4b..8eda616d2c44 100644 --- a/tests/python/rest_api/test_organizations.py +++ b/tests/python/rest_api/test_organizations.py @@ -7,7 +7,7 @@ from http import HTTPStatus import pytest -from cvat_sdk.api_client.api_client import ApiClient +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from deepdiff import DeepDiff from shared.utils.config import delete_method, get_method, options_method, patch_method @@ -89,7 +89,7 @@ def setup(self, restore_db_per_class, admin_user, organizations): self.user = admin_user self.samples = organizations - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.organizations_api.list_endpoint @pytest.mark.parametrize( diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 8258a5d0b4b7..2aa1cb3a6625 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -16,6 +16,7 @@ import pytest from cvat_sdk.api_client import ApiClient, Configuration, models +from cvat_sdk.api_client.api_client import Endpoint from deepdiff import DeepDiff from PIL import Image @@ -143,7 +144,7 @@ def setup(self, restore_db_per_class, admin_user, projects): self.user = admin_user self.samples = projects - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.projects_api.list_endpoint @pytest.mark.parametrize( diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 7c2b1c477d34..1f8060307475 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -15,7 +15,7 @@ import pytest from cvat_sdk.api_client import apis, models -from cvat_sdk.api_client.api_client import ApiClient +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff from PIL import Image @@ -163,7 +163,7 @@ def setup(self, restore_db_per_class, admin_user, tasks): self.user = admin_user self.samples = tasks - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.tasks_api.list_endpoint @pytest.mark.parametrize( diff --git a/tests/python/rest_api/test_users.py b/tests/python/rest_api/test_users.py index 863aacb50c69..54f48a0f6a25 100644 --- a/tests/python/rest_api/test_users.py +++ b/tests/python/rest_api/test_users.py @@ -8,7 +8,7 @@ from http import HTTPStatus import pytest -from cvat_sdk.api_client.api_client import ApiClient +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from cvat_sdk.core.helpers import get_paginated_collection from deepdiff import DeepDiff @@ -104,7 +104,7 @@ def setup(self, restore_db_per_class, admin_user, users): self.user = admin_user self.samples = users - def _get_endpoint(self, api_client: ApiClient): + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.users_api.list_endpoint @pytest.mark.parametrize( diff --git a/tests/python/rest_api/test_webhooks.py b/tests/python/rest_api/test_webhooks.py index 8c931005df70..7d6edce467f0 100644 --- a/tests/python/rest_api/test_webhooks.py +++ b/tests/python/rest_api/test_webhooks.py @@ -5,12 +5,16 @@ from copy import deepcopy from http import HTTPStatus from itertools import product +from typing import Any, Dict, List import pytest +from cvat_sdk.api_client.api_client import ApiClient, Endpoint from deepdiff import DeepDiff from shared.utils.config import delete_method, get_method, patch_method, post_method +from .utils import CollectionSimpleFilterTestBase + @pytest.mark.usefixtures("restore_db_per_function") class TestPostWebhooks: @@ -528,6 +532,28 @@ def test_member_can_get_project_webhook_in_org(self, role, webhooks, find_users, assert DeepDiff(webhook, response.json(), ignore_order=True) == {} +class TestWebhooksListFilters(CollectionSimpleFilterTestBase): + field_lookups = { + "owner": ["owner", "username"], + "project_id": ["project"], + } + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, webhooks): + self.user = admin_user + self.samples = webhooks + + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: + return api_client.webhooks_api.list_endpoint + + @pytest.mark.parametrize( + "field", + ("target_url", "owner", "type", "project_id"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) + + @pytest.mark.usefixtures("restore_db_per_class") class TestGetListWebhooks: def test_can_get_webhooks_list(self, webhooks): From 23eabd3e4fd5819890b17ceb28b9d0e79e184413 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 17 Jan 2023 23:22:52 +0200 Subject: [PATCH 023/140] Fix schema errors --- cvat/apps/engine/serializers.py | 6 ---- cvat/apps/engine/views.py | 46 +++++++++++--------------- cvat/apps/organizations/serializers.py | 5 ++- cvat/apps/organizations/views.py | 28 ++++++++-------- cvat/apps/webhooks/serializers.py | 7 ++++ 5 files changed, 43 insertions(+), 49 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index a042c17f2f4d..68cc1e1918d7 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -704,12 +704,6 @@ def validate(self, attrs): return attrs -class ProjectSearchSerializer(serializers.ModelSerializer): - class Meta: - model = models.Project - fields = ('id', 'name') - read_only_fields = ('name',) - class ProjectReadSerializer(serializers.ModelSerializer): labels = LabelSerializer(many=True, source='label_set', partial=True, default=[], read_only=True) owner = BasicUserSerializer(required=False, read_only=True) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index be043e20d996..4b5347cad77d 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -56,7 +56,7 @@ AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaReadSerializer, DataMetaWriteSerializer, DataSerializer, ExceptionSerializer, FileInfoSerializer, JobReadSerializer, JobWriteSerializer, LabeledDataSerializer, - LogEventSerializer, ProjectReadSerializer, ProjectWriteSerializer, ProjectSearchSerializer, + LogEventSerializer, ProjectReadSerializer, ProjectWriteSerializer, RqStatusSerializer, TaskReadSerializer, TaskWriteSerializer, UserSerializer, PluginsSerializer, IssueReadSerializer, IssueWriteSerializer, CommentReadSerializer, CommentWriteSerializer, CloudStorageWriteSerializer, CloudStorageReadSerializer, DatasetFileSerializer, JobCommitSerializer, @@ -271,16 +271,13 @@ def advanced_authentication(request): @extend_schema(tags=['projects']) @extend_schema_view( list=extend_schema( - summary='Returns a paginated list of projects according to query parameters (12 projects per page)', + summary='Returns a paginated list of projects', responses={ - '200': PolymorphicProxySerializer(component_name='PolymorphicProject', - serializers=[ - ProjectReadSerializer, ProjectSearchSerializer, - ], resource_type_field_name=None, many=True), + '200': ProjectReadSerializer(many=True), }), create=extend_schema( summary='Method creates a new project', - # request=ProjectWriteSerializer, + request=ProjectWriteSerializer, responses={ '201': ProjectReadSerializer, # check ProjectWriteSerializer.to_representation }), @@ -296,7 +293,7 @@ def advanced_authentication(request): }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in a project', - # request=ProjectWriteSerializer, + request=ProjectWriteSerializer(partial=True), responses={ '200': ProjectReadSerializer, # check ProjectWriteSerializer.to_representation }) @@ -323,13 +320,10 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, iam_organization_field = 'organization' def get_serializer_class(self): - if self.request.path.endswith('tasks'): - return TaskReadSerializer + if self.request.method in SAFE_METHODS: + return ProjectReadSerializer else: - if self.request.method in SAFE_METHODS: - return ProjectReadSerializer - else: - return ProjectWriteSerializer + return ProjectWriteSerializer def get_queryset(self): queryset = super().get_queryset() @@ -754,7 +748,7 @@ def __call__(self, request, start, stop, db_data): @extend_schema(tags=['tasks']) @extend_schema_view( list=extend_schema( - summary='Returns a paginated list of tasks according to query parameters (10 tasks per page)', + summary='Returns a paginated list of tasks', responses={ '200': TaskReadSerializer(many=True), }), @@ -1401,13 +1395,13 @@ def preview(self, request, pk): '200': JobReadSerializer, }), list=extend_schema( - summary='Method returns a paginated list of jobs according to query parameters', + summary='Method returns a paginated list of jobs', responses={ '200': JobReadSerializer(many=True), }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in a job', - request=JobWriteSerializer, + request=JobWriteSerializer(partial=True), responses={ '200': JobReadSerializer, # check JobWriteSerializer.to_representation }) @@ -1843,13 +1837,13 @@ def preview(self, request, pk): '200': IssueReadSerializer, }), list=extend_schema( - summary='Method returns a paginated list of issues according to query parameters', + summary='Method returns a paginated list of issues', responses={ '200': IssueReadSerializer(many=True), }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in an issue', - request=IssueWriteSerializer, + request=IssueWriteSerializer(partial=True), responses={ '200': IssueReadSerializer, # check IssueWriteSerializer.to_representation }), @@ -1928,13 +1922,13 @@ def comments(self, request, pk): '200': CommentReadSerializer, }), list=extend_schema( - summary='Method returns a paginated list of comments according to query parameters', + summary='Method returns a paginated list of comments', responses={ - '200':CommentReadSerializer(many=True), + '200': CommentReadSerializer(many=True), }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in a comment', - request=CommentWriteSerializer, + request=CommentWriteSerializer(partial=True), responses={ '200': CommentReadSerializer, # check CommentWriteSerializer.to_representation }), @@ -1991,7 +1985,7 @@ def perform_create(self, serializer, **kwargs): @extend_schema(tags=['users']) @extend_schema_view( list=extend_schema( - summary='Method provides a paginated list of users registered on the server', + summary='Method returns a paginated list of users', responses={ '200': PolymorphicProxySerializer(component_name='MetaUser', serializers=[ @@ -2011,7 +2005,7 @@ def perform_create(self, serializer, **kwargs): responses={ '200': PolymorphicProxySerializer(component_name='MetaUser', serializers=[ - UserSerializer, BasicUserSerializer, + UserSerializer(partial=True), BasicUserSerializer(partial=True), ], resource_type_field_name=None), }), destroy=extend_schema( @@ -2079,7 +2073,7 @@ def self(self, request): '200': CloudStorageReadSerializer, }), list=extend_schema( - summary='Returns a paginated list of storages according to query parameters', + summary='Returns a paginated list of storages', responses={ '200': CloudStorageReadSerializer(many=True), }), @@ -2090,7 +2084,7 @@ def self(self, request): }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in a cloud storage instance', - request=CloudStorageWriteSerializer, + request=CloudStorageWriteSerializer(partial=True), responses={ '200': CloudStorageReadSerializer, # check CloudStorageWriteSerializer.to_representation }), diff --git a/cvat/apps/organizations/serializers.py b/cvat/apps/organizations/serializers.py index f3c406750aa5..5678cb5ad357 100644 --- a/cvat/apps/organizations/serializers.py +++ b/cvat/apps/organizations/serializers.py @@ -23,13 +23,12 @@ def to_representation(self, instance): class Meta: model = Organization - fields = ['id', 'slug', 'name', 'description', 'created_date', - 'updated_date', 'contact', 'owner'] + fields = ['slug', 'name', 'description', 'contact', 'owner'] # TODO: at the moment isn't possible to change the owner. It should # be a separate feature. Need to change it together with corresponding # Membership. Also such operation should be well protected. - read_only_fields = ['created_date', 'updated_date', 'owner'] + read_only_fields = ['owner'] def create(self, validated_data): organization = super().create(validated_data) diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index 50fb07104d46..a9e114953b1c 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -27,19 +27,21 @@ '200': OrganizationReadSerializer, }), list=extend_schema( - summary='Method returns a paginated list of organizations according to query parameters', + summary='Method returns a paginated list of organizations', responses={ '200': OrganizationReadSerializer(many=True), }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in an organization', + request=OrganizationWriteSerializer(partial=True), responses={ - '200': OrganizationWriteSerializer, + '200': OrganizationReadSerializer, # check OrganizationWriteSerializer.to_representation }), create=extend_schema( summary='Method creates an organization', + request=OrganizationWriteSerializer, responses={ - '201': OrganizationWriteSerializer, + '201': OrganizationReadSerializer, # check OrganizationWriteSerializer.to_representation }), destroy=extend_schema( summary='Method deletes an organization', @@ -93,14 +95,15 @@ class Meta: '200': MembershipReadSerializer, }), list=extend_schema( - summary='Method returns a paginated list of memberships according to query parameters', + summary='Method returns a paginated list of memberships', responses={ '200': MembershipReadSerializer(many=True), }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in a membership', + request=MembershipWriteSerializer(partial=True), responses={ - '200': MembershipWriteSerializer, + '200': MembershipReadSerializer, # check MembershipWriteSerializer.to_representation }), destroy=extend_schema( summary='Method deletes a membership', @@ -139,24 +142,21 @@ def get_queryset(self): '200': InvitationReadSerializer, }), list=extend_schema( - summary='Method returns a paginated list of invitations according to query parameters', + summary='Method returns a paginated list of invitations', responses={ '200': InvitationReadSerializer(many=True), }), - update=extend_schema( - summary='Method updates an invitation by id', - responses={ - '200': InvitationWriteSerializer, - }), partial_update=extend_schema( summary='Methods does a partial update of chosen fields in an invitation', + request=InvitationWriteSerializer(partial=True), responses={ - '200': InvitationWriteSerializer, + '200': InvitationReadSerializer, # check InvitationWriteSerializer.to_representation }), create=extend_schema( summary='Method creates an invitation', + request=InvitationWriteSerializer, responses={ - '201': InvitationWriteSerializer, + '201': InvitationReadSerializer, # check InvitationWriteSerializer.to_representation }), destroy=extend_schema( summary='Method deletes an invitation', @@ -167,7 +167,7 @@ def get_queryset(self): class InvitationViewSet(viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin, - mixins.UpdateModelMixin, + PartialUpdateModelMixin, CreateModelMixin, DestroyModelMixin, ): diff --git a/cvat/apps/webhooks/serializers.py b/cvat/apps/webhooks/serializers.py index d06a5edf7539..1ab080a799c7 100644 --- a/cvat/apps/webhooks/serializers.py +++ b/cvat/apps/webhooks/serializers.py @@ -92,6 +92,10 @@ class Meta: "last_delivery_date", ) read_only_fields = fields + extra_kwargs = { + 'project': { 'allow_null': True }, + 'organization': { 'allow_null': True }, + } class WebhookWriteSerializer(WriteOnceMixin, serializers.ModelSerializer): @@ -144,3 +148,6 @@ class Meta: "response", ) read_only_fields = fields + extra_kwargs = { + 'status_code': { 'allow_null': True }, + } From fbc52cc5895c166fbe549a037f8708fc1dd51c2e Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 17 Jan 2023 23:37:05 +0200 Subject: [PATCH 024/140] Remove extra parameters from endpoints --- cvat/apps/engine/views.py | 19 ++++++++++++------- cvat/apps/webhooks/serializers.py | 6 +++--- cvat/apps/webhooks/views.py | 5 ++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 4b5347cad77d..a0477b010df2 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -345,10 +345,11 @@ def perform_create(self, serializer, **kwargs): deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=TaskReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, + # These non-root list endpoints do not suppose extra options, just the basic output # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_backends=[]) def tasks(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ @@ -917,10 +918,11 @@ def perform_destroy(self, instance): deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, + # These non-root list endpoints do not suppose extra options, just the basic output # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_backends=[]) def jobs(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ @@ -1420,8 +1422,8 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, iam_organization_field = 'segment__task__organization' search_fields = ('task_name', 'project_name', 'assignee', 'state', 'stage') - filter_fields = list(search_fields) + ['id', 'task_id', 'project_id', 'updated_date'] - simple_filters = list(search_fields) + ['task_id', 'project_id'] + filter_fields = list(search_fields) + ['id', 'task_id', 'project_id', 'updated_date', 'dimension'] + simple_filters = list(set(filter_fields) - {'id', 'updated_date'}) ordering_fields = list(filter_fields) ordering = "-id" lookup_fields = { @@ -1688,10 +1690,11 @@ def dataset_export(self, request, pk): deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, + # These non-root list endpoints do not suppose extra options, just the basic output # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_backends=[]) def issues(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ @@ -1802,10 +1805,11 @@ def metadata(self, request, pk): responses=JobCommitSerializer(many=True)) # Duplicate to still get 'list' op. name @action(detail=True, methods=['GET'], serializer_class=JobCommitSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, + # These non-root list endpoints do not suppose extra options, just the basic output # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_backends=[]) def commits(self, request, pk): self.get_object() # force to call check_object_permissions return make_paginated_response(JobCommit.objects.filter(job_id=pk).order_by('-id'), @@ -1903,10 +1907,11 @@ def perform_create(self, serializer, **kwargs): deprecated=True) # TODO: remove in v2.5 @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, + # These non-root list endpoints do not suppose extra options, just the basic output # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_backends=[]) def comments(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ diff --git a/cvat/apps/webhooks/serializers.py b/cvat/apps/webhooks/serializers.py index 1ab080a799c7..e44612b34486 100644 --- a/cvat/apps/webhooks/serializers.py +++ b/cvat/apps/webhooks/serializers.py @@ -93,8 +93,8 @@ class Meta: ) read_only_fields = fields extra_kwargs = { - 'project': { 'allow_null': True }, - 'organization': { 'allow_null': True }, + "project": {"allow_null": True}, + "organization": {"allow_null": True}, } @@ -149,5 +149,5 @@ class Meta: ) read_only_fields = fields extra_kwargs = { - 'status_code': { 'allow_null': True }, + "status_code": {"allow_null": True}, } diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index 2993b3143f84..d1854a64a5a8 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -148,11 +148,10 @@ def events(self, request): serializer_class=WebhookDeliveryReadSerializer, pagination_class=viewsets.GenericViewSet.pagination_class, # These non-root list endpoints do not suppose extra options, just the basic output + # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - search_fields=None, - filter_fields=None, - ordering_fields=None, + filter_backends=[], ) def deliveries(self, request, pk): self.get_object() From 0e235b42b4e208681d8356fba1465e4a51adea28 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 18 Jan 2023 13:43:28 +0200 Subject: [PATCH 025/140] Fix redirects and restore extra parameters --- cvat/apps/engine/view_utils.py | 14 ++++++++++++-- cvat/apps/engine/views.py | 20 ++++++++++---------- cvat/apps/webhooks/views.py | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index c32753bf8391..77e4f3ebaff9 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -51,19 +51,29 @@ def make_paginated_response( return response_type(serializer.data) -def reverse(viewname, *, args=None, kwargs=None, query_params=None) -> str: +def reverse(viewname, *, args=None, kwargs=None, + query_params: Optional[Dict[str, str]] = None, + request: Optional[HttpRequest] = None, +) -> str: """ The same as Django reverse(), but adds query params support. + The original request can be passed in the 'request' kwarg parameter to + forward parameters. """ url = _django_reverse(viewname, args=args, kwargs=kwargs) + if request: + new_query_params = query_params or {} + query_params = request.GET.dict() + query_params.update(new_query_params) + if query_params: return f'{url}?{urlencode(query_params)}' return url -def build_field_search_params(field: str, value: Any) -> Dict[str, str]: +def build_field_filter_params(field: str, value: Any) -> Dict[str, str]: """ Builds a collection filter query params for a single field and value. """ diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index a0477b010df2..a73e4ddd2e80 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -63,7 +63,7 @@ ProjectFileSerializer, TaskFileSerializer) from utils.dataset_manifest import ImageManifestManager -from cvat.apps.engine.view_utils import build_field_search_params, make_paginated_response, reverse +from cvat.apps.engine.view_utils import build_field_filter_params, make_paginated_response, reverse from cvat.apps.engine.utils import ( av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message ) @@ -349,12 +349,12 @@ def perform_create(self, serializer, **kwargs): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_backends=[]) + filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def tasks(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('task-list', - query_params=build_field_search_params('project_id', pk)) + query_params=build_field_filter_params('project_id', pk), request=request) }) @extend_schema(methods=['GET'], summary='Export project as a dataset in a specific format', @@ -922,12 +922,12 @@ def perform_destroy(self, instance): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_backends=[]) + filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def jobs(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('job-list', - query_params=build_field_search_params('task_id', pk)) + query_params=build_field_filter_params('task_id', pk), request=request) }) # UploadMixin method @@ -1694,12 +1694,12 @@ def dataset_export(self, request, pk): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_backends=[]) + filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def issues(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('issue-list', - query_params=build_field_search_params('job_id', pk)) + query_params=build_field_filter_params('job_id', pk), request=request) }) @extend_schema(summary='Method returns data for a specific job', @@ -1809,7 +1809,7 @@ def metadata(self, request, pk): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_backends=[]) + filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def commits(self, request, pk): self.get_object() # force to call check_object_permissions return make_paginated_response(JobCommit.objects.filter(job_id=pk).order_by('-id'), @@ -1911,12 +1911,12 @@ def perform_create(self, serializer, **kwargs): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_backends=[]) + filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def comments(self, request, pk): # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('comment-list', - query_params=build_field_search_params('issue_id', pk)) + query_params=build_field_filter_params('issue_id', pk), request=request) }) @extend_schema(tags=['comments']) diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index d1854a64a5a8..07586a201ee5 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -151,7 +151,7 @@ def events(self, request): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_backends=[], + filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None, ) def deliveries(self, request, pk): self.get_object() From 49e4b615678e59c8f85daa71fff95834376a5b68 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 18 Jan 2023 13:56:05 +0200 Subject: [PATCH 026/140] Restore permission check behavior --- cvat/apps/engine/views.py | 22 +++++++++++++--------- cvat/apps/webhooks/views.py | 6 +++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index a73e4ddd2e80..d3ef07e3954b 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -351,6 +351,7 @@ def perform_create(self, serializer, **kwargs): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def tasks(self, request, pk): + self.get_object() # force call of check_object_permissions() # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('task-list', @@ -409,7 +410,7 @@ def tasks(self, request, pk): @action(detail=True, methods=['GET', 'POST', 'OPTIONS'], serializer_class=None, url_path=r'dataset/?$') def dataset(self, request, pk): - self._object = self.get_object() # force to call check_object_permissions + self._object = self.get_object() # force call of check_object_permissions() rq_id = f"import:dataset-for-porject.id{pk}-by-{request.user}" if request.method in {'POST', 'OPTIONS'}: @@ -549,7 +550,7 @@ def upload_finished(self, request): @action(detail=True, methods=['GET'], serializer_class=LabeledDataSerializer) def annotations(self, request, pk): - self._object = self.get_object() # force to call check_object_permissions + self._object = self.get_object() # force call of check_object_permissions() return self.export_annotations( request=request, pk=pk, @@ -924,6 +925,7 @@ def perform_destroy(self, instance): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def jobs(self, request, pk): + self.get_object() # force call of check_object_permissions() # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('job-list', @@ -1164,7 +1166,7 @@ def append_data_chunk(self, request, pk, file_id): @action(detail=True, methods=['GET', 'DELETE', 'PUT', 'PATCH', 'POST', 'OPTIONS'], url_path=r'annotations/?$', serializer_class=None) def annotations(self, request, pk): - self._object = self.get_object() # force to call check_object_permissions + self._object = self.get_object() # force call of check_object_permissions() if request.method == 'GET': if self._object.data: return self.export_annotations( @@ -1250,7 +1252,7 @@ def append_annotations_chunk(self, request, pk, file_id): }) @action(detail=True, methods=['GET'], serializer_class=RqStatusSerializer) def status(self, request, pk): - self.get_object() # force to call check_object_permissions + self.get_object() # force call of check_object_permissions() response = self._get_rq_response( queue=settings.CVAT_QUEUES.IMPORT_DATA.value, job_id=f"create:task.id{pk}-by-{request.user}" @@ -1354,7 +1356,7 @@ def metadata(self, request, pk): @action(detail=True, methods=['GET'], serializer_class=None, url_path='dataset') def dataset_export(self, request, pk): - self._object = self.get_object() # force to call check_object_permissions + self._object = self.get_object() # force call of check_object_permissions() if self._object.data: return self.export_annotations( @@ -1564,7 +1566,7 @@ def upload_finished(self, request): @action(detail=True, methods=['GET', 'DELETE', 'PUT', 'PATCH', 'POST', 'OPTIONS'], url_path=r'annotations/?$', serializer_class=LabeledDataSerializer) def annotations(self, request, pk): - self._object = self.get_object() # force to call check_object_permissions + self._object = self.get_object() # force call of check_object_permissions() if request.method == 'GET': return self.export_annotations( request=request, @@ -1675,7 +1677,7 @@ def append_annotations_chunk(self, request, pk, file_id): @action(detail=True, methods=['GET'], serializer_class=None, url_path='dataset') def dataset_export(self, request, pk): - self._object = self.get_object() # force to call check_object_permissions + self._object = self.get_object() # force call of check_object_permissions() return self.export_annotations( request=request, @@ -1696,6 +1698,7 @@ def dataset_export(self, request, pk): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def issues(self, request, pk): + self.get_object() # force call of check_object_permissions() # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('issue-list', @@ -1742,7 +1745,7 @@ def data(self, request, pk): @action(detail=True, methods=['GET', 'PATCH'], serializer_class=DataMetaReadSerializer, url_path='data/meta') def metadata(self, request, pk): - self.get_object() # force to call check_object_permissions + self.get_object() # force call of check_object_permissions() db_job = models.Job.objects.prefetch_related( 'segment', 'segment__task', @@ -1811,7 +1814,7 @@ def metadata(self, request, pk): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def commits(self, request, pk): - self.get_object() # force to call check_object_permissions + self.get_object() # force call of check_object_permissions() return make_paginated_response(JobCommit.objects.filter(job_id=pk).order_by('-id'), viewset=self, serializer_type=self.serializer_class) # from @action @@ -1913,6 +1916,7 @@ def perform_create(self, serializer, **kwargs): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def comments(self, request, pk): + self.get_object() # force call of check_object_permissions() # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other return Response(status=status.HTTP_303_SEE_OTHER, headers={ 'Location': reverse('comment-list', diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index 07586a201ee5..ad82494bc219 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -154,7 +154,7 @@ def events(self, request): filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None, ) def deliveries(self, request, pk): - self.get_object() + self.get_object() # force call of check_object_permissions() queryset = WebhookDelivery.objects.filter(webhook_id=pk).order_by( "-updated_date" ) @@ -173,7 +173,7 @@ def deliveries(self, request, pk): serializer_class=WebhookDeliveryReadSerializer, ) def retrieve_delivery(self, request, pk, delivery_id): - self.get_object() + self.get_object() # force call of check_object_permissions() queryset = WebhookDelivery.objects.get(webhook_id=pk, id=delivery_id) serializer = WebhookDeliveryReadSerializer( queryset, context={"request": request} @@ -207,7 +207,7 @@ def redelivery(self, request, pk, delivery_id): detail=True, methods=["POST"], serializer_class=WebhookDeliveryReadSerializer ) def ping(self, request, pk): - instance = self.get_object() + instance = self.get_object() # force call of check_object_permissions() serializer = WebhookReadSerializer(instance, context={"request": request}) delivery = signal_ping.send(sender=self, serializer=serializer)[0][1] From f645f407999ef9ae6adfc8c987dbd0bbcc6f7c97 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 18 Jan 2023 16:09:51 +0200 Subject: [PATCH 027/140] Update issue tests --- tests/python/rest_api/test_issues.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index 3e39c47fe47c..8450c5e269bc 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -158,6 +158,8 @@ def get_data(issue_id): data.pop("updated_date") data.pop("id") data.pop("owner") + if assignee := data.get('assignee', None): + data['assignee'] = assignee['id'] return data return get_data From b58f121d7ab79a72f4a7cad5157735bd525030f1 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 18 Jan 2023 16:36:28 +0200 Subject: [PATCH 028/140] Fix linter problems --- tests/python/rest_api/test_issues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index 8450c5e269bc..ec27e6122f3c 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -158,8 +158,8 @@ def get_data(issue_id): data.pop("updated_date") data.pop("id") data.pop("owner") - if assignee := data.get('assignee', None): - data['assignee'] = assignee['id'] + if assignee := data.get("assignee", None): + data["assignee"] = assignee["id"] return data return get_data From 0bec88469b3be2a05542c230ad62512785dc901b Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 18 Jan 2023 16:46:19 +0200 Subject: [PATCH 029/140] Update license headers --- cvat-sdk/cvat_sdk/core/helpers.py | 2 +- cvat/apps/dataset_manager/tests/test_rest_api_formats.py | 2 +- cvat/apps/dataset_repo/tests.py | 1 + cvat/apps/engine/utils.py | 1 - cvat/apps/organizations/models.py | 2 +- cvat/apps/organizations/serializers.py | 1 + cvat/apps/organizations/views.py | 2 +- tests/python/rest_api/test_cloud_storages.py | 2 +- tests/python/rest_api/test_invitations.py | 2 +- tests/python/rest_api/test_issues.py | 2 +- tests/python/rest_api/test_jobs.py | 2 +- tests/python/rest_api/test_memberships.py | 2 +- tests/python/rest_api/test_organizations.py | 2 +- tests/python/rest_api/test_projects.py | 2 +- tests/python/rest_api/test_tasks.py | 2 +- tests/python/rest_api/test_users.py | 2 +- tests/python/rest_api/test_webhooks.py | 3 +-- 17 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/helpers.py b/cvat-sdk/cvat_sdk/core/helpers.py index bfab7146d246..36b739bebab5 100644 --- a/cvat-sdk/cvat_sdk/core/helpers.py +++ b/cvat-sdk/cvat_sdk/core/helpers.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index e6fbdaa30306..77757aabf178 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/dataset_repo/tests.py b/cvat/apps/dataset_repo/tests.py index f851a1c5ac85..8f36c62b2763 100644 --- a/cvat/apps/dataset_repo/tests.py +++ b/cvat/apps/dataset_repo/tests.py @@ -1,4 +1,5 @@ # Copyright (C) 2018-2022 Intel Corporation +# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index e668bd2ed6c5..c2dd82ce7d2e 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: MIT import ast -# from typing import Optional, Type import cv2 as cv from collections import namedtuple import hashlib diff --git a/cvat/apps/organizations/models.py b/cvat/apps/organizations/models.py index 131be94d49e8..45bd35634b34 100644 --- a/cvat/apps/organizations/models.py +++ b/cvat/apps/organizations/models.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/organizations/serializers.py b/cvat/apps/organizations/serializers.py index 5678cb5ad357..add215ca31c0 100644 --- a/cvat/apps/organizations/serializers.py +++ b/cvat/apps/organizations/serializers.py @@ -1,4 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index a9e114953b1c..4fe1985a2c7a 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_cloud_storages.py b/tests/python/rest_api/test_cloud_storages.py index c1acd01f5c4b..354db7c57062 100644 --- a/tests/python/rest_api/test_cloud_storages.py +++ b/tests/python/rest_api/test_cloud_storages.py @@ -1,5 +1,5 @@ # Copyright (C) 2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_invitations.py b/tests/python/rest_api/test_invitations.py index 87ec27358910..719af24b6d71 100644 --- a/tests/python/rest_api/test_invitations.py +++ b/tests/python/rest_api/test_invitations.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index ec27e6122f3c..7d1660e94308 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -1,5 +1,5 @@ # Copyright (C) 2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_jobs.py b/tests/python/rest_api/test_jobs.py index 73c3f285435a..9ef7adf09f32 100644 --- a/tests/python/rest_api/test_jobs.py +++ b/tests/python/rest_api/test_jobs.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_memberships.py b/tests/python/rest_api/test_memberships.py index 259dc64a9a6a..31f4b3447722 100644 --- a/tests/python/rest_api/test_memberships.py +++ b/tests/python/rest_api/test_memberships.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_organizations.py b/tests/python/rest_api/test_organizations.py index 8eda616d2c44..e988ada04ecd 100644 --- a/tests/python/rest_api/test_organizations.py +++ b/tests/python/rest_api/test_organizations.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 2aa1cb3a6625..dfbd8d140454 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -1,5 +1,5 @@ # Copyright (C) 2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 52a22acf5ab5..55362df63d3d 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -1,5 +1,5 @@ # Copyright (C) 2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_users.py b/tests/python/rest_api/test_users.py index 54f48a0f6a25..bec563e2e79b 100644 --- a/tests/python/rest_api/test_users.py +++ b/tests/python/rest_api/test_users.py @@ -1,5 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_webhooks.py b/tests/python/rest_api/test_webhooks.py index 7d6edce467f0..769372a0e2f2 100644 --- a/tests/python/rest_api/test_webhooks.py +++ b/tests/python/rest_api/test_webhooks.py @@ -1,11 +1,10 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT from copy import deepcopy from http import HTTPStatus from itertools import product -from typing import Any, Dict, List import pytest from cvat_sdk.api_client.api_client import ApiClient, Endpoint From ecf8dcb7e63c5adb41a9192dde28c69f74cefa8d Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 18 Jan 2023 19:58:49 +0200 Subject: [PATCH 030/140] Fix linter warning --- cvat/apps/engine/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index e25bb04f8334..8d6709907411 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -279,7 +279,7 @@ def get_filterset_class(self, view, queryset=None): class AutoFilterSet(self.filterset_base, metaclass=FilterSet.__class__): filter_names = { v: k for k, v in lookup_fields.items() } - class Meta(MetaBase): + class Meta(MetaBase): # pylint: disable=useless-object-inheritance model = queryset.model fields = list(lookup_fields.values()) From ee31c85addd0c1a9bdfa6983e57fdb0470b6d27f Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 18 Jan 2023 20:03:50 +0200 Subject: [PATCH 031/140] Update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36b9c70015de..8184fdab9a8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - YOLO v7 serverless feature added using ONNX backend () - Cypress test for social account authentication () - Dummy github and google authentication servers () +- \[Server API\] Simple filters for object collection endpoints + () ### Changed - The Docker Compose files now use the Compose Specification version @@ -38,7 +40,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The contour detection function for semantic segmentation () ### Deprecated -- TDB +- \[Server API\] Endpoints with collections are deprecated in favor of their full variants + `/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`, `/issues/{id}/comments` + () ### Removed - TDB @@ -48,6 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed HRNet serverless function runtime error on images with alpha channel () - Preview & chunk cache settings are ignored () - Export annotations to Azure container () +- \[Server API\] Various errors in schema () ### Security - Fixed vulnerability with social authentication () From f72024d0551ee7fe0354920c26f33ba936f45fa2 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 19 Jan 2023 19:29:30 +0200 Subject: [PATCH 032/140] Fix issue tests --- tests/python/rest_api/test_issues.py | 46 ++++++++++++++++++---------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index 7d1660e94308..357113547af8 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -127,10 +127,11 @@ def test_member_create_issue( @pytest.mark.usefixtures("restore_db_per_function") class TestPatchIssues: def _test_check_response(self, user, issue_id, data, is_allow, **kwargs): + request_data, expected_response_data = data with make_api_client(user) as client: (_, response) = client.issues_api.partial_update( issue_id, - patched_issue_write_request=models.PatchedIssueWriteRequest(**data), + patched_issue_write_request=models.PatchedIssueWriteRequest(**request_data), **kwargs, _parse_response=False, _check_status=False, @@ -140,7 +141,7 @@ def _test_check_response(self, user, issue_id, data, is_allow, **kwargs): assert response.status == HTTPStatus.OK assert ( DeepDiff( - data, + expected_response_data, json.loads(response.data), exclude_regex_paths=r"root\['created_date|updated_date|comments|id|owner'\]", ) @@ -150,17 +151,28 @@ def _test_check_response(self, user, issue_id, data, is_allow, **kwargs): assert response.status == HTTPStatus.FORBIDDEN @pytest.fixture(scope="class") - def request_data(self, issues): - def get_data(issue_id): - data = deepcopy(issues[issue_id]) - data["resolved"] = not data["resolved"] - data.pop("comments") - data.pop("updated_date") - data.pop("id") - data.pop("owner") - if assignee := data.get("assignee", None): - data["assignee"] = assignee["id"] - return data + def request_and_response_data(self, issues, users): + def get_data(issue_id, *, username: str = None): + request_data = deepcopy(issues[issue_id]) + request_data["resolved"] = not request_data["resolved"] + + response_data = deepcopy(request_data) + + request_data.pop("comments") + request_data.pop("updated_date") + request_data.pop("id") + request_data.pop("owner") + + if username: + assignee = next(u for u in users if u["username"] == username) + request_data["assignee"] = assignee["id"] + response_data["assignee"] = { + k: assignee[k] for k in ["id", "username", "url", "first_name", "last_name"] + } + else: + request_data["assignee"] = None + + return request_data, response_data return get_data @@ -189,13 +201,13 @@ def test_user_update_issue( find_issue_staff_user, find_users, issues_by_org, - request_data, + request_and_response_data, ): users = find_users(privilege=privilege) issues = issues_by_org[org] username, issue_id = find_issue_staff_user(issues, users, issue_staff, issue_admin) - data = request_data(issue_id) + data = request_and_response_data(issue_id, username=username) self._test_check_response(username, issue_id, data, is_allow) @pytest.mark.parametrize("org", [2]) @@ -223,13 +235,13 @@ def test_member_update_issue( find_issue_staff_user, find_users, issues_by_org, - request_data, + request_and_response_data, ): users = find_users(role=role, org=org) issues = issues_by_org[org] username, issue_id = find_issue_staff_user(issues, users, issue_staff, issue_admin) - data = request_data(issue_id) + data = request_and_response_data(issue_id, username=username) self._test_check_response(username, issue_id, data, is_allow, org_id=org) @pytest.mark.xfail( From 073037dfba277ea4a434b93eb2fb15afd6142e35 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 19 Jan 2023 23:18:39 +0200 Subject: [PATCH 033/140] Refactor some code --- cvat/apps/engine/view_utils.py | 24 +++++++++++++++++------ cvat/apps/engine/views.py | 36 ++++++++++++++-------------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index 77e4f3ebaff9..e500dd465ca1 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -11,6 +11,8 @@ from django.http.response import HttpResponse from django.urls import reverse as _django_reverse from django.utils.http import urlencode +from rest_framework import status +from rest_framework.response import Response from rest_framework.serializers import Serializer from rest_framework.viewsets import GenericViewSet @@ -77,9 +79,19 @@ def build_field_filter_params(field: str, value: Any) -> Dict[str, str]: """ Builds a collection filter query params for a single field and value. """ - # Uses Reverse Polish Notation - return { - 'filter': '{"==":[{"var":"%(field)s"},%(value)s]}' % { - 'field': field, 'value':value - } - } + return { field: value } + +def redirect_to_full_collection_endpoint(location: str, *, request: HttpRequest, + filter_field: str, filter_key: str +) -> HttpResponse: + """ + Builds a redirection response for a collection endpoint. + """ + + # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other + return Response(status=status.HTTP_303_SEE_OTHER, headers={ + 'Location': reverse(location, + query_params=build_field_filter_params(filter_field, filter_key), + request=request + ) + }) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 610d80b5ae5a..19b049cebff4 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -62,7 +62,9 @@ ProjectFileSerializer, TaskFileSerializer) from utils.dataset_manifest import ImageManifestManager -from cvat.apps.engine.view_utils import build_field_filter_params, make_paginated_response, reverse +from cvat.apps.engine.view_utils import (make_paginated_response, + redirect_to_full_collection_endpoint +) from cvat.apps.engine.utils import ( av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message ) @@ -318,11 +320,9 @@ def perform_create(self, serializer, **kwargs): filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def tasks(self, request, pk): self.get_object() # force call of check_object_permissions() - # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other - return Response(status=status.HTTP_303_SEE_OTHER, headers={ - 'Location': reverse('task-list', - query_params=build_field_filter_params('project_id', pk), request=request) - }) + return redirect_to_full_collection_endpoint('task-list', + filter_field='project_id', filter_key=pk, request=request + ) @extend_schema(methods=['GET'], summary='Export project as a dataset in a specific format', parameters=[ @@ -884,11 +884,9 @@ def perform_destroy(self, instance): filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def jobs(self, request, pk): self.get_object() # force call of check_object_permissions() - # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other - return Response(status=status.HTTP_303_SEE_OTHER, headers={ - 'Location': reverse('job-list', - query_params=build_field_filter_params('task_id', pk), request=request) - }) + return redirect_to_full_collection_endpoint('job-list', + filter_field='task_id', filter_key=pk, request=request + ) # UploadMixin method def get_upload_dir(self): @@ -1660,11 +1658,9 @@ def dataset_export(self, request, pk): filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def issues(self, request, pk): self.get_object() # force call of check_object_permissions() - # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other - return Response(status=status.HTTP_303_SEE_OTHER, headers={ - 'Location': reverse('issue-list', - query_params=build_field_filter_params('job_id', pk), request=request) - }) + return redirect_to_full_collection_endpoint('issue-list', + filter_field='job_id', filter_key=pk, request=request + ) @extend_schema(summary='Method returns data for a specific job', parameters=[ @@ -1878,11 +1874,9 @@ def perform_create(self, serializer, **kwargs): filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) def comments(self, request, pk): self.get_object() # force call of check_object_permissions() - # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other - return Response(status=status.HTTP_303_SEE_OTHER, headers={ - 'Location': reverse('comment-list', - query_params=build_field_filter_params('issue_id', pk), request=request) - }) + return redirect_to_full_collection_endpoint('comment-list', + filter_field='issue_id', filter_key=pk, request=request + ) @extend_schema(tags=['comments']) @extend_schema_view( From 7fd4e56fc9118bfde3901de0e7b1e737cb7b9779 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 19 Jan 2023 23:39:43 +0200 Subject: [PATCH 034/140] Fix merge --- cvat/apps/engine/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 19b049cebff4..862bd3e73719 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -377,7 +377,7 @@ def tasks(self, request, pk): url_path=r'dataset/?$') def dataset(self, request, pk): self._object = self.get_object() # force call of check_object_permissions() - rq_id = f"import:dataset-for-porject.id{pk}-by-{request.user}" + rq_id = f"import:dataset-for-project.id{pk}-by-{request.user}" if request.method in {'POST', 'OPTIONS'}: return self.import_annotations( From f12884ecd625b2690d2154aa8781ab079d777b74 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 20 Jan 2023 00:09:44 +0200 Subject: [PATCH 035/140] Fix linter --- cvat/apps/engine/view_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index e500dd465ca1..e86c0f9e84d3 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -31,7 +31,6 @@ def make_paginated_response( serializer_params.setdefault('many', True) if response_type is None: - from rest_framework.response import Response response_type = Response if request is None: From 056e8cc3bad07eddb7f6c7734a3312054c114c08 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 20 Jan 2023 14:10:57 +0200 Subject: [PATCH 036/140] Implement filters using JsonFilter --- cvat/apps/engine/filters.py | 97 ++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 8d6709907411..30c997bec9d0 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -3,20 +3,18 @@ # # SPDX-License-Identifier: MIT -from typing import Dict, Iterator, Optional +from typing import Any, Dict, Iterator, Optional from functools import reduce import operator import json from django.db.models import Q +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ from django.utils.encoding import force_str -from django_filters.filterset import BaseFilterSet from rest_framework import filters from rest_framework.compat import coreapi, coreschema from rest_framework.exceptions import ValidationError -from django_filters import FilterSet -from django_filters.rest_framework import DjangoFilterBackend DEFAULT_FILTER_FIELDS_ATTR = 'filter_fields' DEFAULT_LOOKUP_MAP_ATTR = 'lookup_fields' @@ -129,6 +127,7 @@ def get_schema_operation_parameters(self, view): }] class JsonLogicFilter(filters.BaseFilterBackend): + Rules = Dict[str, Any] filter_param = 'filter' filter_title = _('Filter') filter_description = _('A filter term.') @@ -169,21 +168,32 @@ def _build_Q(self, rules, lookup_fields): else: raise ValidationError(f'filter: {op} operation with {args} arguments is not implemented') + def _parse_query(self, json_rules: str) -> Rules: + try: + rules = json.loads(json_rules) + if not len(rules): + raise ValidationError(f"filter shouldn't be empty") + except json.decoder.JSONDecodeError: + raise ValidationError(f'filter: Json syntax should be used') + + return rules + + def apply_filter(self, + queryset: QuerySet, parsed_rules: Rules, *, lookup_fields: Dict[str, Any] + ) -> QuerySet: + try: + q_object = self._build_Q(parsed_rules, lookup_fields) + except KeyError as ex: + raise ValidationError(f'filter: {str(ex)} term is not supported') + + return queryset.filter(q_object) + def filter_queryset(self, request, queryset, view): json_rules = request.query_params.get(self.filter_param) if json_rules: - try: - rules = json.loads(json_rules) - if not len(rules): - raise ValidationError(f"filter shouldn't be empty") - except json.decoder.JSONDecodeError: - raise ValidationError(f'filter: Json syntax should be used') - lookup_fields = self._get_lookup_fields(request, view) - try: - q_object = self._build_Q(rules, lookup_fields) - except KeyError as ex: - raise ValidationError(f'filter: {str(ex)} term is not supported') - return queryset.filter(q_object) + parsed_rules = self._parse_query(json_rules) + lookup_fields = self._get_lookup_fields(view) + queryset = self.apply_filter(queryset, parsed_rules, lookup_fields=lookup_fields) return queryset @@ -223,11 +233,11 @@ def get_schema_operation_parameters(self, view): }, ] - def _get_lookup_fields(self, request, view): + def _get_lookup_fields(self, view): return get_lookup_fields(view) -class SimpleFilter(DjangoFilterBackend): +class SimpleFilter(filters.BaseFilterBackend): """ A simple filter, useful for small search queries and manually-edited requests. @@ -247,46 +257,31 @@ class SimpleFilter(DjangoFilterBackend): filter_fields_attr = 'simple_filters' - class MappingFiltersetBase(BaseFilterSet): - _filter_name_map_attr = 'filter_names' - - @classmethod - def get_filter_name(cls, field_name, lookup_expr): - filter_names = getattr(cls, cls._filter_name_map_attr, {}) - - field_name = super().get_filter_name(field_name, lookup_expr) - - if filter_names: - # Map names after a lookup suffix is applied to allow - # mapping specific filters with lookups - field_name = filter_names.get(field_name, field_name) - - if field_name in SimpleFilter.reserved_names: - raise ValueError(f'Field name {field_name} is reserved') + def _build_rules(self, query_params: Dict[str, Any]) -> JsonLogicFilter.Rules: + field_rules = [] + rules = {"and": field_rules} + for field, value in query_params.items(): + field_rules.append({"==": [{"var": field}, value]}) + return rules - return field_name - - filterset_base = MappingFiltersetBase - - - def get_filterset_class(self, view, queryset=None): + def filter_queryset(self, request, queryset, view): lookup_fields = self.get_lookup_fields(view) - if not lookup_fields or not queryset: - return None + query_params = { + k: request.query_params[k] for k in lookup_fields if k in request.query_params + } - MetaBase = getattr(self.filterset_base, 'Meta', object) + if query_params: + json_filter = JsonLogicFilter() + rules = self._build_rules(query_params) + queryset = json_filter.apply_filter(queryset, rules, lookup_fields=lookup_fields) - class AutoFilterSet(self.filterset_base, metaclass=FilterSet.__class__): - filter_names = { v: k for k, v in lookup_fields.items() } - - class Meta(MetaBase): # pylint: disable=useless-object-inheritance - model = queryset.model - fields = list(lookup_fields.values()) - - return AutoFilterSet + return queryset def get_lookup_fields(self, view): simple_filters = getattr(view, self.filter_fields_attr, None) + for k in self.reserved_names: + assert k not in simple_filters, f"Field '{k}' is reserved" + return get_lookup_fields(view, fields=simple_filters) def get_schema_fields(self, view): From 403b547fa37115a43280346380e8ed3f8c986915 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 20 Jan 2023 14:12:39 +0200 Subject: [PATCH 037/140] Remove extra import --- cvat/settings/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 32360a2524ff..ffb1bfbb7216 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -109,7 +109,6 @@ def add_ssh_keys(): 'django.contrib.messages', 'django.contrib.staticfiles', 'django_rq', - 'django_filters', 'compressor', 'django_sendfile', "dj_rest_auth", From a2792f2603bc390119780dcee97a1c92681d0415 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 20 Jan 2023 14:22:28 +0200 Subject: [PATCH 038/140] Fix check --- cvat/apps/engine/filters.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 30c997bec9d0..65cb1b0bda3b 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -279,8 +279,10 @@ def filter_queryset(self, request, queryset, view): def get_lookup_fields(self, view): simple_filters = getattr(view, self.filter_fields_attr, None) - for k in self.reserved_names: - assert k not in simple_filters, f"Field '{k}' is reserved" + if simple_filters: + for k in self.reserved_names: + assert k not in simple_filters, \ + f"Query parameter '{k}' is reserved, try to change the filter name." return get_lookup_fields(view, fields=simple_filters) From 90091794af92f5f367f0ed155ccb2155e8a641f4 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 20 Jan 2023 14:23:23 +0200 Subject: [PATCH 039/140] Fix type description --- cvat/apps/engine/filters.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 65cb1b0bda3b..725ae379e23d 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -242,8 +242,7 @@ class SimpleFilter(filters.BaseFilterBackend): A simple filter, useful for small search queries and manually-edited requests. - Argument types are numbers and strings. The only available check is - equality for numbers and case-independent inclusion for strings. + Argument types are numbers and strings. The only available check is equality. Operators are not supported (e.g. or, less, greater, not etc.). Multiple filters are joined with '&' as separate query params. """ From e3aa275976b5ce1d669c6d917bcad860b3267bf2 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 23 Jan 2023 15:24:44 +0200 Subject: [PATCH 040/140] Fix api tests --- cvat/apps/dataset_manager/tests/test_rest_api_formats.py | 1 - cvat/apps/dataset_repo/tests.py | 1 - tests/python/rest_api/test_webhooks.py | 1 - 3 files changed, 3 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 77757aabf178..13f76753fa48 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -1,5 +1,4 @@ # Copyright (C) 2021-2022 Intel Corporation -# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/dataset_repo/tests.py b/cvat/apps/dataset_repo/tests.py index 8f36c62b2763..f851a1c5ac85 100644 --- a/cvat/apps/dataset_repo/tests.py +++ b/cvat/apps/dataset_repo/tests.py @@ -1,5 +1,4 @@ # Copyright (C) 2018-2022 Intel Corporation -# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/tests/python/rest_api/test_webhooks.py b/tests/python/rest_api/test_webhooks.py index 3cb9d6ce2e8b..7ec30ae70bb4 100644 --- a/tests/python/rest_api/test_webhooks.py +++ b/tests/python/rest_api/test_webhooks.py @@ -534,7 +534,6 @@ def test_member_can_get_project_webhook_in_org(self, role, webhooks, find_users, class TestWebhooksListFilters(CollectionSimpleFilterTestBase): field_lookups = { "owner": ["owner", "username"], - "project_id": ["project"], } @pytest.fixture(autouse=True) From 67a0a5459b8925d47a7de8f67b48024369401b67 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 23 Jan 2023 17:27:39 +0200 Subject: [PATCH 041/140] Add project.labels endpoint --- cvat/apps/engine/views.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 54f0dec779db..a4ddb03784cc 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -54,7 +54,7 @@ from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaReadSerializer, DataMetaWriteSerializer, DataSerializer, ExceptionSerializer, - FileInfoSerializer, JobReadSerializer, JobWriteSerializer, LabeledDataSerializer, + FileInfoSerializer, JobReadSerializer, JobWriteSerializer, LabelSerializer, LabeledDataSerializer, LogEventSerializer, ProjectReadSerializer, ProjectWriteSerializer, ProjectSearchSerializer, RqStatusSerializer, TaskReadSerializer, TaskWriteSerializer, UserSerializer, PluginsSerializer, IssueReadSerializer, IssueWriteSerializer, CommentReadSerializer, CommentWriteSerializer, CloudStorageWriteSerializer, @@ -630,6 +630,20 @@ def _get_rq_response(queue, job_id): return response + @extend_schema(description="Return a paginated list of labels", + responses=LabelSerializer(many=True)) # Duplicate to still get 'list' op. name + @action(detail=True, methods=['GET'], serializer_class=LabelSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + filter_fields=None, search_fields=None, ordering_fields=None) + def labels(self, pk): + queryset = self.get_object().labels + return make_paginated_response(queryset, + viewset=self, serializer_type=self.serializer_class) # from @action + + class DataChunkGetter: def __init__(self, data_type, data_num, data_quality, task_dim): possible_data_type_values = ('chunk', 'frame', 'preview', 'context_image') From b59e6eadf28830f7fc69fe83cce1751f557b035d Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 23 Jan 2023 20:34:56 +0200 Subject: [PATCH 042/140] Revert switching to json filter --- cvat/apps/engine/filters.py | 53 ++++++++++++++++++++++++++----------- cvat/settings/base.py | 1 + 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 725ae379e23d..dc3fcc4b1019 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -8,6 +8,9 @@ import operator import json +from django_filters import FilterSet +from django_filters.filterset import BaseFilterSet +from django_filters.rest_framework import DjangoFilterBackend from django.db.models import Q from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ @@ -237,7 +240,7 @@ def _get_lookup_fields(self, view): return get_lookup_fields(view) -class SimpleFilter(filters.BaseFilterBackend): +class SimpleFilter(DjangoFilterBackend): """ A simple filter, useful for small search queries and manually-edited requests. @@ -256,25 +259,43 @@ class SimpleFilter(filters.BaseFilterBackend): filter_fields_attr = 'simple_filters' - def _build_rules(self, query_params: Dict[str, Any]) -> JsonLogicFilter.Rules: - field_rules = [] - rules = {"and": field_rules} - for field, value in query_params.items(): - field_rules.append({"==": [{"var": field}, value]}) - return rules + class MappingFiltersetBase(BaseFilterSet): + _filter_name_map_attr = 'filter_names' - def filter_queryset(self, request, queryset, view): + @classmethod + def get_filter_name(cls, field_name, lookup_expr): + filter_names = getattr(cls, cls._filter_name_map_attr, {}) + + field_name = super().get_filter_name(field_name, lookup_expr) + + if filter_names: + # Map names after a lookup suffix is applied to allow + # mapping specific filters with lookups + field_name = filter_names.get(field_name, field_name) + + if field_name in SimpleFilter.reserved_names: + raise ValueError(f'Field name {field_name} is reserved') + + return field_name + + filterset_base = MappingFiltersetBase + + + def get_filterset_class(self, view, queryset=None): lookup_fields = self.get_lookup_fields(view) - query_params = { - k: request.query_params[k] for k in lookup_fields if k in request.query_params - } + if not lookup_fields or not queryset: + return None - if query_params: - json_filter = JsonLogicFilter() - rules = self._build_rules(query_params) - queryset = json_filter.apply_filter(queryset, rules, lookup_fields=lookup_fields) + MetaBase = getattr(self.filterset_base, 'Meta', object) - return queryset + class AutoFilterSet(self.filterset_base, metaclass=FilterSet.__class__): + filter_names = { v: k for k, v in lookup_fields.items() } + + class Meta(MetaBase): # pylint: disable=useless-object-inheritance + model = queryset.model + fields = list(lookup_fields.values()) + + return AutoFilterSet def get_lookup_fields(self, view): simple_filters = getattr(view, self.filter_fields_attr, None) diff --git a/cvat/settings/base.py b/cvat/settings/base.py index ffb1bfbb7216..e0161a8ee9bb 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -114,6 +114,7 @@ def add_ssh_keys(): "dj_rest_auth", 'dj_rest_auth.registration', 'dj_pagination', + 'django_filters', 'rest_framework', 'rest_framework.authtoken', 'drf_spectacular', From 03325cb81fc73e88f90609bbbb73bd720d14a58b Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 23 Jan 2023 22:02:06 +0200 Subject: [PATCH 043/140] Fix tests --- cvat/apps/dataset_manager/tests/test_rest_api_formats.py | 2 +- cvat/apps/dataset_repo/tests.py | 2 +- cvat/apps/engine/tests/test_rest_api.py | 7 ++++--- cvat/apps/engine/tests/test_rest_api_3D.py | 2 +- cvat/apps/lambda_manager/tests/test_lambda.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 13f76753fa48..193a93787248 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -175,7 +175,7 @@ def _create_project(self, data): def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): values = get_paginated_collection(lambda page: - self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page)) + self.client.get("/api/jobs?task_id={}&page={}".format(task_id, page)) ) return values diff --git a/cvat/apps/dataset_repo/tests.py b/cvat/apps/dataset_repo/tests.py index f851a1c5ac85..92ea968c8dc9 100644 --- a/cvat/apps/dataset_repo/tests.py +++ b/cvat/apps/dataset_repo/tests.py @@ -200,7 +200,7 @@ def _run_api_v2_job_id_annotation(self, jid, data, user): def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): values = get_paginated_collection(lambda page: - self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page)) + self.client.get("/api/jobs?task_id={}&page={}".format(task_id, page)) ) return values diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index b612dd591dba..1c03e39e0a07 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -1224,7 +1224,7 @@ def setUpTestData(cls): def _run_api_v2_projects_id_tasks(self, user, pid): with ForceLogin(user, self.client): - response = self.client.get('/api/projects/{}/tasks'.format(pid)) + response = self.client.get('/api/tasks?project_id={}'.format(pid)) return response @@ -1247,7 +1247,8 @@ def test_api_v2_projects_id_tasks_user(self): def test_api_v2_projects_id_tasks_somebody(self): project = self.projects[1] response = self._run_api_v2_projects_id_tasks(self.somebody, project.id) - self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual([], response.data['results']) def test_api_v2_projects_id_tasks_no_auth(self): project = self.projects[1] @@ -4227,7 +4228,7 @@ def _create_task(self, owner, assignee, annotation_format=""): task = response.data jobs = get_paginated_collection(lambda page: - self.client.get("/api/tasks/{}/jobs?page={}".format(tid, page)) + self.client.get("/api/jobs?task_id={}&page={}".format(tid, page)) ) return (task, jobs) diff --git a/cvat/apps/engine/tests/test_rest_api_3D.py b/cvat/apps/engine/tests/test_rest_api_3D.py index 977cf000da5f..5b42c24c940c 100644 --- a/cvat/apps/engine/tests/test_rest_api_3D.py +++ b/cvat/apps/engine/tests/test_rest_api_3D.py @@ -143,7 +143,7 @@ def _get_tmp_annotation(task, annotation): def _get_jobs(self, task_id): with ForceLogin(self.admin, self.client): values = get_paginated_collection(lambda page: - self.client.get("/api/tasks/{}/jobs?page={}".format(task_id, page)) + self.client.get("/api/jobs?task_id={}&page={}".format(task_id, page)) ) return values diff --git a/cvat/apps/lambda_manager/tests/test_lambda.py b/cvat/apps/lambda_manager/tests/test_lambda.py index 8239c8f4b8fe..ef072e8b85a3 100644 --- a/cvat/apps/lambda_manager/tests/test_lambda.py +++ b/cvat/apps/lambda_manager/tests/test_lambda.py @@ -1048,7 +1048,7 @@ def setUp(self): jobs = get_paginated_collection(lambda page: self._get_request( - f"/api/tasks/{self.task['id']}/jobs?page={page}", + f"/api/jobs?task_id={self.task['id']}&page={page}", self.admin, org_id=self.org['id'] ) ) From fa790bba01b75765ad397cb4ed8b184e1c4447e8 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 24 Jan 2023 17:49:08 +0200 Subject: [PATCH 044/140] t --- cvat/apps/engine/serializers.py | 17 ++------ cvat/apps/engine/views.py | 73 ++++++++++++++++++++++++++++----- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 4256bfa9e28c..ec6554d178e1 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -187,7 +187,6 @@ class JobReadSerializer(serializers.ModelSerializer): stop_frame = serializers.ReadOnlyField(source="segment.stop_frame") assignee = BasicUserSerializer(allow_null=True, read_only=True) dimension = serializers.CharField(max_length=2, source='segment.task.dimension', read_only=True) - labels = LabelSerializer(many=True, source='get_labels', read_only=True) data_chunk_size = serializers.ReadOnlyField(source='segment.task.data.chunk_size') data_compressed_chunk_type = serializers.ReadOnlyField(source='segment.task.data.compressed_chunk_type') mode = serializers.ReadOnlyField(source='segment.task.mode') @@ -197,7 +196,7 @@ class JobReadSerializer(serializers.ModelSerializer): class Meta: model = models.Job fields = ('url', 'id', 'task_id', 'project_id', 'assignee', - 'dimension', 'labels', 'bug_tracker', 'status', 'stage', 'state', 'mode', + 'dimension', 'bug_tracker', 'status', 'stage', 'state', 'mode', 'start_frame', 'stop_frame', 'data_chunk_size', 'data_compressed_chunk_type', 'updated_date',) read_only_fields = fields @@ -512,8 +511,6 @@ class Meta: fields = ('id', 'location', 'cloud_storage_id') class TaskReadSerializer(serializers.ModelSerializer): - labels = LabelSerializer(many=True, source='get_labels') - segments = SegmentSerializer(many=True, source='segment_set', read_only=True) data_chunk_size = serializers.ReadOnlyField(source='data.chunk_size', required=False) data_compressed_chunk_type = serializers.ReadOnlyField(source='data.compressed_chunk_type', required=False) data_original_chunk_type = serializers.ReadOnlyField(source='data.original_chunk_type', required=False) @@ -531,7 +528,7 @@ class Meta: model = models.Task fields = ('url', 'id', 'name', 'project_id', 'mode', 'owner', 'assignee', 'bug_tracker', 'created_date', 'updated_date', 'overlap', 'segment_size', - 'status', 'labels', 'segments', 'data_chunk_size', 'data_compressed_chunk_type', + 'status', 'data_chunk_size', 'data_compressed_chunk_type', 'data_original_chunk_type', 'size', 'image_quality', 'data', 'dimension', 'subset', 'organization', 'target_storage', 'source_storage', ) @@ -767,7 +764,6 @@ class Meta: read_only_fields = ('name',) class ProjectReadSerializer(serializers.ModelSerializer): - labels = LabelSerializer(many=True, source='label_set', partial=True, default=[], read_only=True) owner = BasicUserSerializer(required=False, read_only=True) assignee = BasicUserSerializer(allow_null=True, required=False, read_only=True) task_subsets = serializers.ListField(child=serializers.CharField(), required=False, read_only=True) @@ -777,14 +773,11 @@ class ProjectReadSerializer(serializers.ModelSerializer): class Meta: model = models.Project - fields = ('url', 'id', 'name', 'labels', 'tasks', 'owner', 'assignee', + fields = ('url', 'id', 'name', 'owner', 'assignee', 'bug_tracker', 'task_subsets', 'created_date', 'updated_date', 'status', 'dimension', 'organization', 'target_storage', 'source_storage', ) - read_only_fields = ('created_date', 'updated_date', 'status', 'owner', - 'assignee', 'task_subsets', 'dimension', 'organization', 'tasks', - 'target_storage', 'source_storage', - ) + read_only_fields = fields extra_kwargs = { 'organization': { 'allow_null': True } } def to_representation(self, instance): @@ -933,7 +926,6 @@ class PluginsSerializer(serializers.Serializer): PREDICT = serializers.BooleanField() class DataMetaReadSerializer(serializers.ModelSerializer): - frames = FrameMetaSerializer(many=True, allow_null=True) image_quality = serializers.IntegerField(min_value=0, max_value=100) deleted_frames = serializers.ListField(child=serializers.IntegerField(min_value=0)) @@ -946,7 +938,6 @@ class Meta: 'start_frame', 'stop_frame', 'frame_filter', - 'frames', 'deleted_frames', ) read_only_fields = fields diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index a4ddb03784cc..eed5e3372945 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -54,9 +54,11 @@ from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaReadSerializer, DataMetaWriteSerializer, DataSerializer, ExceptionSerializer, - FileInfoSerializer, JobReadSerializer, JobWriteSerializer, LabelSerializer, LabeledDataSerializer, + FileInfoSerializer, FrameMetaSerializer, JobReadSerializer, JobWriteSerializer, + LabelSerializer, LabeledDataSerializer, LogEventSerializer, ProjectReadSerializer, ProjectWriteSerializer, ProjectSearchSerializer, - RqStatusSerializer, TaskReadSerializer, TaskWriteSerializer, UserSerializer, PluginsSerializer, IssueReadSerializer, + RqStatusSerializer, SegmentSerializer, TaskReadSerializer, TaskWriteSerializer, + UserSerializer, PluginsSerializer, IssueReadSerializer, IssueWriteSerializer, CommentReadSerializer, CommentWriteSerializer, CloudStorageWriteSerializer, CloudStorageReadSerializer, DatasetFileSerializer, JobCommitSerializer, ProjectFileSerializer, TaskFileSerializer) @@ -639,7 +641,7 @@ def _get_rq_response(queue, job_id): # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, search_fields=None, ordering_fields=None) def labels(self, pk): - queryset = self.get_object().labels + queryset = self.get_object().label_set return make_paginated_response(queryset, viewset=self, serializer_type=self.serializer_class) # from @action @@ -1256,9 +1258,7 @@ def _get_rq_response(queue, job_id): def metadata(self, request, pk): self.get_object() #force to call check_object_permissions db_task = models.Task.objects.prefetch_related( - Prefetch('data', queryset=models.Data.objects.select_related('video').prefetch_related( - Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) - )) + Prefetch('data', queryset=models.Data.objects.select_related('video')) ).get(pk=pk) if request.method == 'PATCH': @@ -1266,6 +1266,21 @@ def metadata(self, request, pk): if serializer.is_valid(raise_exception=True): db_task.data = serializer.save() + serializer = DataMetaReadSerializer(db_task.data) + return Response(serializer.data) + + @extend_schema(summary='Returns a paginated list of frame metadata', + responses=FrameMetaSerializer(many=True)) + @action(detail=True, methods=['GET'], serializer_class=DataMetaReadSerializer, + url_path='data/meta/frames') + def metadata_frames(self, request, pk): + self.get_object() #force to call check_object_permissions + db_task = models.Task.objects.prefetch_related( + Prefetch('data', queryset=models.Data.objects.select_related('video').prefetch_related( + Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) + )) + ).get(pk=pk) + if hasattr(db_task.data, 'video'): media = [db_task.data.video] else: @@ -1278,10 +1293,7 @@ def metadata(self, request, pk): 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0 } for item in media] - db_data = db_task.data - db_data.frames = frame_meta - - serializer = DataMetaReadSerializer(db_data) + serializer = FrameMetaSerializer(frame_meta, many=True) return Response(serializer.data) @extend_schema(summary='Export task as a dataset in a specific format', @@ -1348,6 +1360,33 @@ def preview(self, request, pk): return data_getter(request, self._object.data.start_frame, self._object.data.stop_frame, self._object.data) + @extend_schema(description="Return a paginated list of labels", + responses=LabelSerializer(many=True)) # Duplicate to still get 'list' op. name + @action(detail=True, methods=['GET'], serializer_class=LabelSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + filter_fields=None, search_fields=None, ordering_fields=None) + def labels(self, pk): + queryset = self.get_object().get_labels() + return make_paginated_response(queryset, + viewset=self, serializer_type=self.serializer_class) # from @action + + @extend_schema(description="Return a paginated list of segments", + responses=SegmentSerializer(many=True)) # Duplicate to still get 'list' op. name + @action(detail=True, methods=['GET'], serializer_class=SegmentSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + filter_fields=None, search_fields=None, ordering_fields=None) + def segments(self, pk): + queryset = self.get_object().segment_set + return make_paginated_response(queryset, + viewset=self, serializer_type=self.serializer_class) # from @action + + @extend_schema(tags=['jobs']) @extend_schema_view( retrieve=extend_schema( @@ -1784,6 +1823,20 @@ def preview(self, request, pk): return data_getter(request, self._object.segment.start_frame, self._object.segment.stop_frame, self._object.segment.task.data) + @extend_schema(description="Return a paginated list of labels", + responses=LabelSerializer(many=True)) # Duplicate to still get 'list' op. name + @action(detail=True, methods=['GET'], serializer_class=LabelSerializer, + pagination_class=viewsets.GenericViewSet.pagination_class, + # Remove regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + filter_fields=None, search_fields=None, ordering_fields=None) + def labels(self, pk): + queryset = self.get_object().get_labels() + return make_paginated_response(queryset, + viewset=self, serializer_type=self.serializer_class) # from @action + + @extend_schema(tags=['issues']) @extend_schema_view( retrieve=extend_schema( From 1474de4418e897c39eb225bf1df86268bf01ae20 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 26 Jan 2023 11:30:21 +0200 Subject: [PATCH 045/140] t --- cvat-sdk/cvat_sdk/core/proxies/jobs.py | 5 +- cvat-sdk/cvat_sdk/core/proxies/projects.py | 3 + cvat-sdk/cvat_sdk/core/proxies/tasks.py | 5 +- cvat/apps/engine/backup.py | 14 ++-- cvat/apps/engine/models.py | 9 +- cvat/apps/engine/serializers.py | 3 +- cvat/apps/engine/views.py | 96 ++++++++++++---------- tests/python/sdk/test_issues_comments.py | 4 +- tests/python/sdk/test_jobs.py | 43 +++++++--- tests/python/sdk/test_projects.py | 19 +++-- tests/python/sdk/test_tasks.py | 42 +++++++--- 11 files changed, 154 insertions(+), 89 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index 11bfe2438c52..770b238339b4 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -151,8 +151,11 @@ def get_meta(self) -> models.IDataMetaRead: (meta, _) = self.api.retrieve_data_meta(self.id) return meta + def get_labels(self) -> List[models.ILabel]: + return get_paginated_collection(self.api.list_labels_endpoint, id=self.id) + def get_frames_info(self) -> List[models.IFrameMeta]: - return self.get_meta().frames + return get_paginated_collection(self.api.list_data_meta_frames_endpoint, id=self.id) def remove_frames_by_ids(self, ids: Sequence[int]) -> None: self._client.api_client.tasks_api.jobs_partial_update_data_meta( diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index b48cff74acd3..72abaada6653 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -130,6 +130,9 @@ def get_tasks(self) -> List[Task]: for m in get_paginated_collection(self.api.list_tasks_endpoint, id=self.id) ] + def get_labels(self) -> List[models.ILabel]: + return get_paginated_collection(self.api.list_labels_endpoint, id=self.id) + def get_preview( self, ) -> io.RawIOBase: diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index 953fcd46693a..685a098ee024 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -312,8 +312,11 @@ def get_meta(self) -> models.IDataMetaRead: (meta, _) = self.api.retrieve_data_meta(self.id) return meta + def get_labels(self) -> List[models.ILabel]: + return get_paginated_collection(self.api.list_labels_endpoint, id=self.id) + def get_frames_info(self) -> List[models.IFrameMeta]: - return self.get_meta().frames + return get_paginated_collection(self.api.list_data_meta_frames_endpoint, id=self.id) def remove_frames_by_ids(self, ids: Sequence[int]) -> None: self.api.partial_update_data_meta( diff --git a/cvat/apps/engine/backup.py b/cvat/apps/engine/backup.py index 65c75da6b5f0..e1bee670e7cd 100644 --- a/cvat/apps/engine/backup.py +++ b/cvat/apps/engine/backup.py @@ -31,7 +31,7 @@ import cvat.apps.dataset_manager as dm from cvat.apps.engine import models from cvat.apps.engine.log import slogger -from cvat.apps.engine.serializers import (AttributeSerializer, DataSerializer, +from cvat.apps.engine.serializers import (AttributeSerializer, DataSerializer, LabelSerializer, LabeledDataSerializer, SegmentSerializer, SimpleJobSerializer, TaskReadSerializer, ProjectReadSerializer, ProjectFileSerializer, TaskFileSerializer) from cvat.apps.engine.utils import av_scan_paths, process_failed_job, configure_dependent_job @@ -312,11 +312,13 @@ def _write_task(self, zip_object, target_dir=None): def _write_manifest(self, zip_object, target_dir=None): def serialize_task(): task_serializer = TaskReadSerializer(self._db_task) - for field in ('url', 'owner', 'assignee', 'segments'): + for field in ('url', 'owner', 'assignee'): task_serializer.fields.pop(field) + task_labels = LabelSerializer(self._db_task.get_labels(), many=True) + task = self._prepare_task_meta(task_serializer.data) - task['labels'] = [self._prepare_label_meta(l) for l in task['labels'] if not l['has_parent']] + task['labels'] = [self._prepare_label_meta(l) for l in task_labels.data if not l['has_parent']] for label in task['labels']: label['attributes'] = [self._prepare_attribute_meta(a) for a in label['attributes']] @@ -662,11 +664,13 @@ def _write_tasks(self, zip_object): def _write_manifest(self, zip_object): def serialize_project(): project_serializer = ProjectReadSerializer(self._db_project) - for field in ('assignee', 'owner', 'tasks', 'url'): + for field in ('assignee', 'owner', 'url'): project_serializer.fields.pop(field) + project_labels = LabelSerializer(self._db_project.get_labels(), many=True).data + project = self._prepare_project_meta(project_serializer.data) - project['labels'] = [self._prepare_label_meta(l) for l in project['labels'] if not l['has_parent']] + project['labels'] = [self._prepare_label_meta(l) for l in project_labels if not l['has_parent']] for label in project['labels']: label['attributes'] = [self._prepare_attribute_meta(a) for a in label['attributes']] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index cd47e516c84e..4732929ba2a4 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -1,5 +1,5 @@ # Copyright (C) 2018-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -307,6 +307,9 @@ class Project(models.Model): target_storage = models.ForeignKey('Storage', null=True, default=None, blank=True, on_delete=models.SET_NULL, related_name='+') + def get_labels(self): + return self.label_set + def get_dirname(self): return os.path.join(settings.PROJECTS_ROOT, str(self.id)) @@ -364,7 +367,7 @@ class Meta: def get_labels(self): project = self.project - return project.label_set if project else self.label_set + return project.get_labels() if project else self.label_set def get_dirname(self): return os.path.join(settings.TASKS_ROOT, str(self.id)) @@ -489,7 +492,7 @@ def get_bug_tracker(self): def get_labels(self): task = self.segment.task project = task.project - return project.label_set if project else task.label_set + return project.get_labels() if project else task.get_labels() def save(self, *args, **kwargs): super().save(*args, **kwargs) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index ec6554d178e1..99175874ab57 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1093,12 +1093,11 @@ class IssueReadSerializer(serializers.ModelSerializer): position = serializers.ListField( child=serializers.FloatField(), allow_empty=False ) - comments = CommentReadSerializer(many=True) class Meta: model = models.Issue fields = ('id', 'frame', 'position', 'job', 'owner', 'assignee', - 'created_date', 'updated_date', 'comments', 'resolved') + 'created_date', 'updated_date', 'resolved') read_only_fields = fields extra_kwargs = { 'created_date': { 'allow_null': True }, diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index eed5e3372945..8c6373f6c5db 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -57,7 +57,7 @@ FileInfoSerializer, FrameMetaSerializer, JobReadSerializer, JobWriteSerializer, LabelSerializer, LabeledDataSerializer, LogEventSerializer, ProjectReadSerializer, ProjectWriteSerializer, ProjectSearchSerializer, - RqStatusSerializer, SegmentSerializer, TaskReadSerializer, TaskWriteSerializer, + RqStatusSerializer, TaskReadSerializer, TaskWriteSerializer, UserSerializer, PluginsSerializer, IssueReadSerializer, IssueWriteSerializer, CommentReadSerializer, CommentWriteSerializer, CloudStorageWriteSerializer, CloudStorageReadSerializer, DatasetFileSerializer, JobCommitSerializer, @@ -322,7 +322,7 @@ def perform_create(self, serializer, **kwargs): def tasks(self, request, pk): self.get_object() # force to call check_object_permissions return make_paginated_response(Task.objects.filter(project_id=pk).order_by('-id'), - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action @extend_schema(methods=['GET'], summary='Export project as a dataset in a specific format', @@ -640,10 +640,10 @@ def _get_rq_response(queue, job_id): # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, search_fields=None, ordering_fields=None) - def labels(self, pk): - queryset = self.get_object().label_set + def labels(self, request, pk): + queryset = self.get_object().get_labels().order_by('id') return make_paginated_response(queryset, - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action class DataChunkGetter: @@ -757,7 +757,7 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, ): queryset = Task.objects.all().select_related('data', 'assignee', 'owner', 'target_storage', 'source_storage').prefetch_related( - 'segment_set__job_set__assignee', 'label_set__attributespec_set', + 'label_set__attributespec_set', 'project__label_set__attributespec_set', 'label_set__sublabels__attributespec_set', 'project__label_set__sublabels__attributespec_set') @@ -886,7 +886,7 @@ def perform_destroy(self, instance): def jobs(self, request, pk): self.get_object() # force to call check_object_permissions return make_paginated_response(Job.objects.filter(segment__task_id=pk).order_by('id'), - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action # UploadMixin method def get_upload_dir(self): @@ -1271,7 +1271,7 @@ def metadata(self, request, pk): @extend_schema(summary='Returns a paginated list of frame metadata', responses=FrameMetaSerializer(many=True)) - @action(detail=True, methods=['GET'], serializer_class=DataMetaReadSerializer, + @action(detail=True, methods=['GET'], serializer_class=FrameMetaSerializer, url_path='data/meta/frames') def metadata_frames(self, request, pk): self.get_object() #force to call check_object_permissions @@ -1368,23 +1368,10 @@ def preview(self, request, pk): # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, search_fields=None, ordering_fields=None) - def labels(self, pk): - queryset = self.get_object().get_labels() + def labels(self, request, pk): + queryset = self.get_object().get_labels().order_by('id') return make_paginated_response(queryset, - viewset=self, serializer_type=self.serializer_class) # from @action - - @extend_schema(description="Return a paginated list of segments", - responses=SegmentSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=SegmentSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) - def segments(self, pk): - queryset = self.get_object().segment_set - return make_paginated_response(queryset, - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action @extend_schema(tags=['jobs']) @@ -1691,7 +1678,7 @@ def dataset_export(self, request, pk): def issues(self, request, pk): self.get_object() # force to call check_object_permissions return make_paginated_response(Issue.objects.filter(job_id=pk).order_by('id'), - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action @extend_schema(summary='Method returns data for a specific job', parameters=[ @@ -1737,9 +1724,7 @@ def metadata(self, request, pk): db_job = models.Job.objects.prefetch_related( 'segment', 'segment__task', - Prefetch('segment__task__data', queryset=models.Data.objects.select_related('video').prefetch_related( - Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) - )) + Prefetch('segment__task__data', queryset=models.Data.objects.select_related('video')) ).get(pk=pk) db_data = db_job.segment.task.data @@ -1763,14 +1748,6 @@ def metadata(self, request, pk): if db_job.segment.task.project: db_job.segment.task.project.save() - if hasattr(db_data, 'video'): - media = [db_data.video] - else: - media = list(db_data.images.filter( - frame__gte=data_start_frame, - frame__lte=data_stop_frame, - ).all()) - # Filter data with segment size # Should data.size also be cropped by segment size? db_data.deleted_frames = filter( @@ -1780,16 +1757,45 @@ def metadata(self, request, pk): db_data.start_frame = data_start_frame db_data.stop_frame = data_stop_frame + serializer = DataMetaReadSerializer(db_data) + return Response(serializer.data) + + @extend_schema(summary='Returns a paginated list of frame metadata', + responses=FrameMetaSerializer(many=True)) + @action(detail=True, methods=['GET'], serializer_class=FrameMetaSerializer, + url_path='data/meta/frames') + def metadata_frames(self, request, pk): + self.get_object() #force to call check_object_permissions + db_job = models.Job.objects.prefetch_related( + 'segment', + 'segment__task', + Prefetch('segment__task__data', queryset=models.Data.objects.select_related('video').prefetch_related( + Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) + )) + ).get(pk=pk) + + db_data = db_job.segment.task.data + start_frame = db_job.segment.start_frame + stop_frame = db_job.segment.stop_frame + data_start_frame = db_data.start_frame + start_frame * db_data.get_frame_step() + data_stop_frame = db_data.start_frame + stop_frame * db_data.get_frame_step() + + if hasattr(db_data, 'video'): + media = [db_data.video] + else: + media = list(db_data.images.filter( + frame__gte=data_start_frame, + frame__lte=data_stop_frame, + ).all()) + frame_meta = [{ 'width': item.width, 'height': item.height, 'name': item.path, - 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0 + 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0, } for item in media] - db_data.frames = frame_meta - - serializer = DataMetaReadSerializer(db_data) + serializer = FrameMetaSerializer(frame_meta, many=True) return Response(serializer.data) @extend_schema(summary='The action returns the list of tracked changes for the job', @@ -1803,7 +1809,7 @@ def metadata(self, request, pk): def commits(self, request, pk): self.get_object() # force to call check_object_permissions return make_paginated_response(JobCommit.objects.filter(job_id=pk).order_by('-id'), - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action @extend_schema(summary='Method returns a preview image for the job', responses={ @@ -1831,10 +1837,10 @@ def preview(self, request, pk): # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want filter_fields=None, search_fields=None, ordering_fields=None) - def labels(self, pk): - queryset = self.get_object().get_labels() + def labels(self, request, pk): + queryset = self.get_object().get_labels().order_by('id') return make_paginated_response(queryset, - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action @extend_schema(tags=['issues']) @@ -1912,7 +1918,7 @@ def perform_create(self, serializer, **kwargs): def comments(self, request, pk): self.get_object() # force to call check_object_permissions return make_paginated_response(Comment.objects.filter(issue_id=pk).order_by('-id'), - viewset=self, serializer_type=self.serializer_class) # from @action + viewset=self, serializer_type=self.serializer_class, request=request) # from @action @extend_schema(tags=['comments']) @extend_schema_view( diff --git a/tests/python/sdk/test_issues_comments.py b/tests/python/sdk/test_issues_comments.py index 4f3b4eecfb6e..ae5379d6b959 100644 --- a/tests/python/sdk/test_issues_comments.py +++ b/tests/python/sdk/test_issues_comments.py @@ -91,7 +91,7 @@ def test_can_list_comments(self, fxt_new_task: Task): comment = self.client.comments.create(models.CommentWriteRequest(issue.id, message="hi!")) issue.fetch() - comment_ids = {c.id for c in issue.comments} + comment_ids = {c.id for c in issue.get_comments()} assert len(comment_ids) == 2 assert comment.id in comment_ids @@ -129,7 +129,7 @@ def test_can_remove_issue(self, fxt_new_task: Task): with pytest.raises(exceptions.NotFoundException): issue.fetch() with pytest.raises(exceptions.NotFoundException): - self.client.comments.retrieve(issue.comments[0].id) + self.client.comments.retrieve(issue.get_comments()[0].id) assert self.stdout.getvalue() == "" diff --git a/tests/python/sdk/test_jobs.py b/tests/python/sdk/test_jobs.py index 6be70fa4b057..d090d5304026 100644 --- a/tests/python/sdk/test_jobs.py +++ b/tests/python/sdk/test_jobs.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -53,12 +53,13 @@ def fxt_new_task(self, fxt_image_file: Path): @pytest.fixture def fxt_task_with_shapes(self, fxt_new_task: Task): + labels = fxt_new_task.get_labels() fxt_new_task.set_annotations( models.LabeledDataRequest( shapes=[ models.LabeledShapeRequest( frame=0, - label_id=fxt_new_task.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[1, 1, 2, 2], ), @@ -96,6 +97,14 @@ def test_can_update_job_field_directly(self, fxt_new_task: Task): assert updated_job.assignee.id == new_assignee.id assert self.stdout.getvalue() == "" + def test_can_get_labels(self, fxt_new_task: Task): + expected_labels = {'car', 'person'} + + received_labels = fxt_new_task.get_jobs()[0].get_labels() + + assert {obj.name for obj in received_labels} == expected_labels + assert self.stdout.getvalue() == "" + @pytest.mark.parametrize("include_images", (True, False)) def test_can_download_dataset(self, fxt_new_task: Task, include_images: bool): pbar_out = io.StringIO() @@ -160,13 +169,20 @@ def test_can_get_meta(self, fxt_new_task: Task): assert meta.image_quality == 80 assert meta.size == 1 - assert len(meta.frames) == meta.size - assert meta.frames[0].name == "img.png" - assert meta.frames[0].width == 5 - assert meta.frames[0].height == 10 assert not meta.deleted_frames assert self.stdout.getvalue() == "" + def test_can_get_frame_info(self, fxt_new_task: Task): + job = meta = fxt_new_task.get_jobs()[0] + meta = job.get_meta() + frames = job.get_frames_info() + + assert len(frames) == meta.size + assert frames[0].name == "img.png" + assert frames[0].width == 5 + assert frames[0].height == 10 + assert self.stdout.getvalue() == "" + def test_can_remove_frames(self, fxt_new_task: Task): fxt_new_task.get_jobs()[0].remove_frames_by_ids([0]) @@ -197,9 +213,10 @@ def test_can_get_annotations(self, fxt_task_with_shapes: Task): assert self.stdout.getvalue() == "" def test_can_set_annotations(self, fxt_new_task: Task): + labels = fxt_new_task.get_labels() fxt_new_task.get_jobs()[0].set_annotations( models.LabeledDataRequest( - tags=[models.LabeledImageRequest(frame=0, label_id=fxt_new_task.labels[0].id)], + tags=[models.LabeledImageRequest(frame=0, label_id=labels[0].id)], ) ) @@ -218,18 +235,19 @@ def test_can_clear_annotations(self, fxt_task_with_shapes: Task): assert self.stdout.getvalue() == "" def test_can_remove_annotations(self, fxt_new_task: Task): + labels = fxt_new_task.get_labels() fxt_new_task.get_jobs()[0].set_annotations( models.LabeledDataRequest( shapes=[ models.LabeledShapeRequest( frame=0, - label_id=fxt_new_task.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[1, 1, 2, 2], ), models.LabeledShapeRequest( frame=0, - label_id=fxt_new_task.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[2, 2, 3, 3], ), @@ -247,12 +265,13 @@ def test_can_remove_annotations(self, fxt_new_task: Task): assert self.stdout.getvalue() == "" def test_can_update_annotations(self, fxt_task_with_shapes: Task): + labels = fxt_task_with_shapes.get_labels() fxt_task_with_shapes.get_jobs()[0].update_annotations( models.PatchedLabeledDataRequest( shapes=[ models.LabeledShapeRequest( frame=0, - label_id=fxt_task_with_shapes.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[0, 1, 2, 3], ), @@ -260,7 +279,7 @@ def test_can_update_annotations(self, fxt_task_with_shapes: Task): tracks=[ models.LabeledTrackRequest( frame=0, - label_id=fxt_task_with_shapes.labels[0].id, + label_id=labels[0].id, shapes=[ models.TrackedShapeRequest( frame=0, type="polygon", points=[3, 2, 2, 3, 3, 4] @@ -269,7 +288,7 @@ def test_can_update_annotations(self, fxt_task_with_shapes: Task): ) ], tags=[ - models.LabeledImageRequest(frame=0, label_id=fxt_task_with_shapes.labels[0].id) + models.LabeledImageRequest(frame=0, label_id=labels[0].id) ], ) ) diff --git a/tests/python/sdk/test_projects.py b/tests/python/sdk/test_projects.py index 14de38b444d2..429f3f14d886 100644 --- a/tests/python/sdk/test_projects.py +++ b/tests/python/sdk/test_projects.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -53,12 +53,13 @@ def fxt_new_task(self, fxt_image_file: Path): @pytest.fixture def fxt_task_with_shapes(self, fxt_new_task: Task): + labels = fxt_new_task.get_labels() fxt_new_task.set_annotations( models.LabeledDataRequest( shapes=[ models.LabeledShapeRequest( frame=0, - label_id=fxt_new_task.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[1, 1, 2, 2], ), @@ -92,7 +93,7 @@ def fxt_project_with_shapes(self, fxt_task_with_shapes: Task): models.PatchedLabelRequest( **filter_dict(label.to_dict(), drop=["id", "has_parent"]) ) - for label in fxt_task_with_shapes.labels + for label in fxt_task_with_shapes.get_labels() ], ) ) @@ -161,12 +162,18 @@ def test_can_delete_project(self, fxt_new_project: Project): assert self.stdout.getvalue() == "" def test_can_get_tasks(self, fxt_project_with_shapes: Project): - task_ids = set(fxt_project_with_shapes.tasks) - tasks = fxt_project_with_shapes.get_tasks() assert len(tasks) == 1 - assert {tasks[0].id} == task_ids + assert tasks[0].project_id == fxt_project_with_shapes.id + + def test_can_get_labels(self, fxt_project_with_shapes: Project): + expected_labels = {'car', 'person'} + + received_labels = fxt_project_with_shapes.get_labels() + + assert {obj.name for obj in received_labels} == expected_labels + assert self.stdout.getvalue() == "" def test_can_download_backup(self, fxt_project_with_shapes: Project): pbar_out = io.StringIO() diff --git a/tests/python/sdk/test_tasks.py b/tests/python/sdk/test_tasks.py index 24c377831e71..978a60c4ae88 100644 --- a/tests/python/sdk/test_tasks.py +++ b/tests/python/sdk/test_tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -66,12 +66,13 @@ def fxt_new_task(self, fxt_image_file: Path): @pytest.fixture def fxt_task_with_shapes(self, fxt_new_task: Task): + labels = fxt_new_task.get_labels() fxt_new_task.set_annotations( models.LabeledDataRequest( shapes=[ models.LabeledShapeRequest( frame=0, - label_id=fxt_new_task.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[1, 1, 2, 2], ), @@ -393,6 +394,14 @@ def counting_do_request(uploader): # make sure the upload was actually chunked assert num_requests > 1 + def test_can_get_labels(self, fxt_new_task: Task): + expected_labels = {'car', 'person'} + + received_labels = fxt_new_task.get_labels() + + assert {obj.name for obj in received_labels} == expected_labels + assert self.stdout.getvalue() == "" + def test_can_get_jobs(self, fxt_new_task: Task): jobs = fxt_new_task.get_jobs() @@ -404,13 +413,19 @@ def test_can_get_meta(self, fxt_new_task: Task): assert meta.image_quality == 80 assert meta.size == 1 - assert len(meta.frames) == meta.size - assert meta.frames[0].name == "img.png" - assert meta.frames[0].width == 5 - assert meta.frames[0].height == 10 assert not meta.deleted_frames assert self.stdout.getvalue() == "" + def test_can_get_frame_info(self, fxt_new_task: Task): + meta = fxt_new_task.get_meta() + frames = fxt_new_task.get_frames_info() + + assert len(frames) == meta.size + assert frames[0].name == "img.png" + assert frames[0].width == 5 + assert frames[0].height == 10 + assert self.stdout.getvalue() == "" + def test_can_remove_frames(self, fxt_new_task: Task): fxt_new_task.remove_frames_by_ids([0]) @@ -426,9 +441,10 @@ def test_can_get_annotations(self, fxt_task_with_shapes: Task): assert self.stdout.getvalue() == "" def test_can_set_annotations(self, fxt_new_task: Task): + labels = fxt_new_task.get_labels() fxt_new_task.set_annotations( models.LabeledDataRequest( - tags=[models.LabeledImageRequest(frame=0, label_id=fxt_new_task.labels[0].id)], + tags=[models.LabeledImageRequest(frame=0, label_id=labels[0].id)], ) ) @@ -447,18 +463,19 @@ def test_can_clear_annotations(self, fxt_task_with_shapes: Task): assert self.stdout.getvalue() == "" def test_can_remove_annotations(self, fxt_new_task: Task): + labels = fxt_new_task.get_labels() fxt_new_task.set_annotations( models.LabeledDataRequest( shapes=[ models.LabeledShapeRequest( frame=0, - label_id=fxt_new_task.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[1, 1, 2, 2], ), models.LabeledShapeRequest( frame=0, - label_id=fxt_new_task.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[2, 2, 3, 3], ), @@ -476,12 +493,13 @@ def test_can_remove_annotations(self, fxt_new_task: Task): assert self.stdout.getvalue() == "" def test_can_update_annotations(self, fxt_task_with_shapes: Task): + labels = fxt_task_with_shapes.get_labels() fxt_task_with_shapes.update_annotations( models.PatchedLabeledDataRequest( shapes=[ models.LabeledShapeRequest( frame=0, - label_id=fxt_task_with_shapes.labels[0].id, + label_id=labels[0].id, type="rectangle", points=[0, 1, 2, 3], ), @@ -489,7 +507,7 @@ def test_can_update_annotations(self, fxt_task_with_shapes: Task): tracks=[ models.LabeledTrackRequest( frame=0, - label_id=fxt_task_with_shapes.labels[0].id, + label_id=labels[0].id, shapes=[ models.TrackedShapeRequest( frame=0, type="polygon", points=[3, 2, 2, 3, 3, 4] @@ -498,7 +516,7 @@ def test_can_update_annotations(self, fxt_task_with_shapes: Task): ) ], tags=[ - models.LabeledImageRequest(frame=0, label_id=fxt_task_with_shapes.labels[0].id) + models.LabeledImageRequest(frame=0, label_id=labels[0].id) ], ) ) From 43795a0a2d71b2314ae2d02de55de0d4f4c95ef6 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 26 Jan 2023 11:45:44 +0200 Subject: [PATCH 046/140] Fix license headers --- cvat/apps/dataset_manager/tests/test_rest_api_formats.py | 1 + cvat/apps/dataset_repo/tests.py | 1 + cvat/apps/engine/tests/test_rest_api.py | 1 + cvat/apps/engine/tests/test_rest_api_3D.py | 1 + cvat/apps/lambda_manager/tests/test_lambda.py | 2 +- cvat/apps/webhooks/serializers.py | 6 +----- cvat/settings/base.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index 193a93787248..b24ccb43c212 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -1,4 +1,5 @@ # Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/dataset_repo/tests.py b/cvat/apps/dataset_repo/tests.py index 92ea968c8dc9..eb12923e7488 100644 --- a/cvat/apps/dataset_repo/tests.py +++ b/cvat/apps/dataset_repo/tests.py @@ -1,4 +1,5 @@ # Copyright (C) 2018-2022 Intel Corporation +# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 1c03e39e0a07..dd34ae6661d6 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -1,4 +1,5 @@ # Copyright (C) 2020-2022 Intel Corporation +# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/engine/tests/test_rest_api_3D.py b/cvat/apps/engine/tests/test_rest_api_3D.py index 5b42c24c940c..b6650ee22265 100644 --- a/cvat/apps/engine/tests/test_rest_api_3D.py +++ b/cvat/apps/engine/tests/test_rest_api_3D.py @@ -1,4 +1,5 @@ # Copyright (C) 2020-2022 Intel Corporation +# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/lambda_manager/tests/test_lambda.py b/cvat/apps/lambda_manager/tests/test_lambda.py index ef072e8b85a3..9409f2f9c400 100644 --- a/cvat/apps/lambda_manager/tests/test_lambda.py +++ b/cvat/apps/lambda_manager/tests/test_lambda.py @@ -1,5 +1,5 @@ - # Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/webhooks/serializers.py b/cvat/apps/webhooks/serializers.py index 740e4bcf4cd8..a0badd4467f2 100644 --- a/cvat/apps/webhooks/serializers.py +++ b/cvat/apps/webhooks/serializers.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -94,7 +94,6 @@ class Meta: ) read_only_fields = fields extra_kwargs = { - "project": {"allow_null": True}, "organization": {"allow_null": True}, } @@ -149,6 +148,3 @@ class Meta: "response", ) read_only_fields = fields - extra_kwargs = { - "status_code": {"allow_null": True}, - } diff --git a/cvat/settings/base.py b/cvat/settings/base.py index e0161a8ee9bb..989b00b5150c 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -1,5 +1,5 @@ # Copyright (C) 2018-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT From 101a8de02fc0c67280eee6cd6a9e0cdd3dc56ce8 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 27 Jan 2023 00:17:15 +0200 Subject: [PATCH 047/140] Implement extra fields in serializers, update test data --- cvat-sdk/cvat_sdk/core/proxies/issues.py | 4 +- cvat-sdk/cvat_sdk/core/proxies/jobs.py | 4 +- cvat-sdk/cvat_sdk/core/proxies/projects.py | 4 +- cvat-sdk/cvat_sdk/core/proxies/tasks.py | 4 +- cvat/apps/engine/serializers.py | 47 ++++++++-- cvat/apps/engine/view_utils.py | 37 +++----- cvat/apps/engine/views.py | 69 +-------------- tests/python/shared/assets/issues.json | 99 ++-------------------- tests/python/shared/assets/jobs.json | 13 +++ tests/python/shared/assets/projects.json | 26 ++---- tests/python/shared/assets/tasks.json | 11 +++ 11 files changed, 103 insertions(+), 215 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/proxies/issues.py b/cvat-sdk/cvat_sdk/core/proxies/issues.py index aa349741d701..5df1069c1178 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/issues.py +++ b/cvat-sdk/cvat_sdk/core/proxies/issues.py @@ -56,7 +56,9 @@ class Issue( def get_comments(self) -> List[Comment]: return [ Comment(self._client, m) - for m in get_paginated_collection(self.api.list_comments_endpoint, id=self.id) + for m in get_paginated_collection( + self._client.api_client.comments_api.list_endpoint, issue_id=self.id + ) ] diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index 11bfe2438c52..19897f443166 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -163,7 +163,9 @@ def remove_frames_by_ids(self, ids: Sequence[int]) -> None: def get_issues(self) -> List[Issue]: return [ Issue(self._client, m) - for m in get_paginated_collection(self.api.list_issues_endpoint, id=self.id) + for m in get_paginated_collection( + self._client.api_client.issues_api.list_endpoint, job_id=self.id + ) ] def get_commits(self) -> List[models.IJobCommit]: diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index b48cff74acd3..b1bed2e7d870 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -127,7 +127,9 @@ def get_annotations(self) -> models.ILabeledData: def get_tasks(self) -> List[Task]: return [ Task(self._client, m) - for m in get_paginated_collection(self.api.list_tasks_endpoint, id=self.id) + for m in get_paginated_collection( + self._client.api_client.tasks_api.list_endpoint, project_id=self.id + ) ] def get_preview( diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index 6be725c56b83..0174134f5abd 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -305,7 +305,9 @@ def download_backup( def get_jobs(self) -> List[Job]: return [ Job(self._client, model=m) - for m in get_paginated_collection(self.api.list_jobs_endpoint, id=self.id) + for m in get_paginated_collection( + self._client.api_client.jobs_api.list_endpoint, task_id=self.id + ) ] def get_meta(self) -> models.IDataMetaRead: diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index b3558733a0ef..4146bab01e89 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -22,6 +22,37 @@ from drf_spectacular.utils import OpenApiExample, extend_schema_field, extend_schema_serializer +from cvat.apps.engine.view_utils import build_field_filter_params, get_list_view_name, reverse + +class HyperlinkedModelViewSerializer(serializers.Serializer): + key_field = 'pk' + + def __init__(self, view_name=None, *, filter_key=None, **kwargs): + if issubclass(view_name, models.models.Model): + view_name = get_list_view_name(view_name) + else: + assert isinstance(view_name, str) + + kwargs['read_only'] = True + super().__init__(**kwargs) + + self.view_name = view_name + self.filter_key = filter_key + + def get_attribute(self, instance): + return instance + + def to_representation(self, instance): + request = self.context['request'] + return serializers.Hyperlink( + reverse(self.view_name, request=request, + query_params=build_field_filter_params( + self.filter_key, getattr(instance, self.key_field) + )), + instance + ) + + class BasicUserSerializer(serializers.ModelSerializer): def validate(self, attrs): if hasattr(self, 'initial_data'): @@ -193,13 +224,14 @@ class JobReadSerializer(serializers.ModelSerializer): mode = serializers.ReadOnlyField(source='segment.task.mode') bug_tracker = serializers.CharField(max_length=2000, source='get_bug_tracker', allow_null=True, read_only=True) + issues = HyperlinkedModelViewSerializer(models.Issue, filter_key='job_id') class Meta: model = models.Job fields = ('url', 'id', 'task_id', 'project_id', 'assignee', 'dimension', 'labels', 'bug_tracker', 'status', 'stage', 'state', 'mode', 'start_frame', 'stop_frame', 'data_chunk_size', 'data_compressed_chunk_type', - 'updated_date',) + 'updated_date', 'issues') read_only_fields = fields class JobWriteSerializer(serializers.ModelSerializer): @@ -526,6 +558,7 @@ class TaskReadSerializer(serializers.ModelSerializer): dimension = serializers.CharField(allow_blank=True, required=False) target_storage = StorageSerializer(required=False, allow_null=True) source_storage = StorageSerializer(required=False, allow_null=True) + jobs = HyperlinkedModelViewSerializer(models.Job, filter_key='task_id') class Meta: model = models.Task @@ -533,7 +566,7 @@ class Meta: 'bug_tracker', 'created_date', 'updated_date', 'overlap', 'segment_size', 'status', 'labels', 'segments', 'data_chunk_size', 'data_compressed_chunk_type', 'data_original_chunk_type', 'size', 'image_quality', 'data', 'dimension', - 'subset', 'organization', 'target_storage', 'source_storage', + 'subset', 'organization', 'target_storage', 'source_storage', 'jobs', ) read_only_fields = fields extra_kwargs = { @@ -768,17 +801,15 @@ class ProjectReadSerializer(serializers.ModelSerializer): dimension = serializers.CharField(max_length=16, required=False, read_only=True, allow_null=True) target_storage = StorageSerializer(required=False, allow_null=True, read_only=True) source_storage = StorageSerializer(required=False, allow_null=True, read_only=True) + tasks = HyperlinkedModelViewSerializer(models.Task, filter_key='project_id') class Meta: model = models.Project - fields = ('url', 'id', 'name', 'labels', 'tasks', 'owner', 'assignee', + fields = ('url', 'id', 'name', 'labels', 'tasks', 'owner', 'assignee', 'tasks', 'bug_tracker', 'task_subsets', 'created_date', 'updated_date', 'status', 'dimension', 'organization', 'target_storage', 'source_storage', ) - read_only_fields = ('created_date', 'updated_date', 'status', 'owner', - 'assignee', 'task_subsets', 'dimension', 'organization', 'tasks', - 'target_storage', 'source_storage', - ) + read_only_fields = fields extra_kwargs = { 'organization': { 'allow_null': True } } def to_representation(self, instance): @@ -1103,7 +1134,7 @@ class IssueReadSerializer(serializers.ModelSerializer): position = serializers.ListField( child=serializers.FloatField(), allow_empty=False ) - comments = CommentReadSerializer(many=True) + comments = HyperlinkedModelViewSerializer(models.Comment, filter_key='issue_id') class Meta: model = models.Issue diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index e86c0f9e84d3..ed0652ac3525 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -9,10 +9,10 @@ from django.db.models.query import QuerySet from django.http.request import HttpRequest from django.http.response import HttpResponse -from django.urls import reverse as _django_reverse from django.utils.http import urlencode -from rest_framework import status from rest_framework.response import Response +from rest_framework.reverse import reverse as _reverse +from rest_framework.utils.urls import remove_query_param from rest_framework.serializers import Serializer from rest_framework.viewsets import GenericViewSet @@ -57,17 +57,12 @@ def reverse(viewname, *, args=None, kwargs=None, request: Optional[HttpRequest] = None, ) -> str: """ - The same as Django reverse(), but adds query params support. - The original request can be passed in the 'request' kwarg parameter to - forward parameters. + The same as rest_framework's reverse(), but adds custom query params support. + The original request can be passed in the 'request' parameter to + return absolute URLs. """ - url = _django_reverse(viewname, args=args, kwargs=kwargs) - - if request: - new_query_params = query_params or {} - query_params = request.GET.dict() - query_params.update(new_query_params) + url = _reverse(viewname, args, kwargs, request) if query_params: return f'{url}?{urlencode(query_params)}' @@ -80,17 +75,13 @@ def build_field_filter_params(field: str, value: Any) -> Dict[str, str]: """ return { field: value } -def redirect_to_full_collection_endpoint(location: str, *, request: HttpRequest, - filter_field: str, filter_key: str -) -> HttpResponse: +def get_list_view_name(model): + # Implemented after + # rest_framework/utils/field_mapping.py.get_detail_view_name() """ - Builds a redirection response for a collection endpoint. + Given a model class, return the view name to use for URL relationships + that refer to instances of the model. """ - - # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other - return Response(status=status.HTTP_303_SEE_OTHER, headers={ - 'Location': reverse(location, - query_params=build_field_filter_params(filter_field, filter_key), - request=request - ) - }) + return '%(model_name)s-list' % { + 'model_name': model._meta.object_name.lower() + } diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 862bd3e73719..5de60905d81b 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -62,9 +62,7 @@ ProjectFileSerializer, TaskFileSerializer) from utils.dataset_manifest import ImageManifestManager -from cvat.apps.engine.view_utils import (make_paginated_response, - redirect_to_full_collection_endpoint -) +from cvat.apps.engine.view_utils import make_paginated_response from cvat.apps.engine.utils import ( av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message ) @@ -307,23 +305,6 @@ def perform_create(self, serializer, **kwargs): organization=self.request.iam_context['organization'] ) - @extend_schema( - summary='Moved to /tasks', - responses=TaskReadSerializer(many=True), # Duplicate to still get 'list' op. name - deprecated=True) # TODO: remove in v2.5 - @action(detail=True, methods=['GET'], serializer_class=TaskReadSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # These non-root list endpoints do not suppose extra options, just the basic output - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) - def tasks(self, request, pk): - self.get_object() # force call of check_object_permissions() - return redirect_to_full_collection_endpoint('task-list', - filter_field='project_id', filter_key=pk, request=request - ) - @extend_schema(methods=['GET'], summary='Export project as a dataset in a specific format', parameters=[ OpenApiParameter('format', description='Desired output format name\n' @@ -872,22 +853,6 @@ def perform_destroy(self, instance): db_project = instance.project db_project.save() - @extend_schema(summary='Moved to /jobs', - responses=JobReadSerializer(many=True), # Duplicate to still get 'list' op. name - deprecated=True) # TODO: remove in v2.5 - @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # These non-root list endpoints do not suppose extra options, just the basic output - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) - def jobs(self, request, pk): - self.get_object() # force call of check_object_permissions() - return redirect_to_full_collection_endpoint('job-list', - filter_field='task_id', filter_key=pk, request=request - ) - # UploadMixin method def get_upload_dir(self): if 'annotations' in self.action: @@ -1646,22 +1611,6 @@ def dataset_export(self, request, pk): callback=dm.views.export_job_as_dataset ) - @extend_schema(summary='Moved to GET /issues', - responses=IssueReadSerializer(many=True), # Duplicate to still get 'list' op. name - deprecated=True) # TODO: remove in v2.5 - @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # These non-root list endpoints do not suppose extra options, just the basic output - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) - def issues(self, request, pk): - self.get_object() # force call of check_object_permissions() - return redirect_to_full_collection_endpoint('issue-list', - filter_field='job_id', filter_key=pk, request=request - ) - @extend_schema(summary='Method returns data for a specific job', parameters=[ OpenApiParameter('type', description='Specifies the type of the requested data', @@ -1862,22 +1811,6 @@ def get_serializer_class(self): def perform_create(self, serializer, **kwargs): super().perform_create(serializer, owner=self.request.user) - @extend_schema(summary='Moved to /comments', - responses=CommentReadSerializer(many=True), # Duplicate to still get 'list' op. name - deprecated=True) # TODO: remove in v2.5 - @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # These non-root list endpoints do not suppose extra options, just the basic output - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) - def comments(self, request, pk): - self.get_object() # force call of check_object_permissions() - return redirect_to_full_collection_endpoint('comment-list', - filter_field='issue_id', filter_key=pk, request=request - ) - @extend_schema(tags=['comments']) @extend_schema_view( retrieve=extend_schema( diff --git a/tests/python/shared/assets/issues.json b/tests/python/shared/assets/issues.json index 94b9c907e198..3c9585cad3c1 100644 --- a/tests/python/shared/assets/issues.json +++ b/tests/python/shared/assets/issues.json @@ -5,22 +5,7 @@ "results": [ { "assignee": null, - "comments": [ - { - "created_date": "2022-03-16T12:49:29.372000Z", - "id": 6, - "issue": 5, - "message": "Wrong position", - "owner": { - "first_name": "User", - "id": 20, - "last_name": "Sixth", - "url": "http://localhost:8080/api/users/20", - "username": "user6" - }, - "updated_date": "2022-03-16T12:49:29.372000Z" - } - ], + "comments": "http://localhost:8080/api/comments?issue_id=5", "created_date": "2022-03-16T12:49:29.369000Z", "frame": 0, "id": 5, @@ -49,22 +34,7 @@ "url": "http://localhost:8080/api/users/2", "username": "user1" }, - "comments": [ - { - "created_date": "2022-03-16T12:40:00.767000Z", - "id": 5, - "issue": 4, - "message": "Issue with empty frame", - "owner": { - "first_name": "User", - "id": 2, - "last_name": "First", - "url": "http://localhost:8080/api/users/2", - "username": "user1" - }, - "updated_date": "2022-03-16T12:40:00.767000Z" - } - ], + "comments": "http://localhost:8080/api/comments?issue_id=4", "created_date": "2022-03-16T12:40:00.764000Z", "frame": 5, "id": 4, @@ -87,22 +57,7 @@ }, { "assignee": null, - "comments": [ - { - "created_date": "2022-03-16T11:08:18.370000Z", - "id": 4, - "issue": 3, - "message": "Another one issue", - "owner": { - "first_name": "Business", - "id": 11, - "last_name": "Second", - "url": "http://localhost:8080/api/users/11", - "username": "business2" - }, - "updated_date": "2022-03-16T11:08:18.370000Z" - } - ], + "comments": "http://localhost:8080/api/comments?issue_id=3", "created_date": "2022-03-16T11:08:18.367000Z", "frame": 5, "id": 3, @@ -125,22 +80,7 @@ }, { "assignee": null, - "comments": [ - { - "created_date": "2022-03-16T11:07:22.173000Z", - "id": 3, - "issue": 2, - "message": "Something should be here", - "owner": { - "first_name": "Business", - "id": 11, - "last_name": "Second", - "url": "http://localhost:8080/api/users/11", - "username": "business2" - }, - "updated_date": "2022-03-16T11:07:22.173000Z" - } - ], + "comments": "http://localhost:8080/api/comments?issue_id=2", "created_date": "2022-03-16T11:07:22.170000Z", "frame": 0, "id": 2, @@ -163,36 +103,7 @@ }, { "assignee": null, - "comments": [ - { - "created_date": "2022-03-16T11:04:39.447000Z", - "id": 1, - "issue": 1, - "message": "Why are we still here?", - "owner": { - "first_name": "User", - "id": 2, - "last_name": "First", - "url": "http://localhost:8080/api/users/2", - "username": "user1" - }, - "updated_date": "2022-03-16T11:04:39.447000Z" - }, - { - "created_date": "2022-03-16T11:04:49.821000Z", - "id": 2, - "issue": 1, - "message": "Just to suffer?", - "owner": { - "first_name": "User", - "id": 2, - "last_name": "First", - "url": "http://localhost:8080/api/users/2", - "username": "user1" - }, - "updated_date": "2022-03-16T11:04:49.821000Z" - } - ], + "comments": "http://localhost:8080/api/comments?issue_id=1", "created_date": "2022-03-16T11:04:39.444000Z", "frame": 0, "id": 1, diff --git a/tests/python/shared/assets/jobs.json b/tests/python/shared/assets/jobs.json index abf18f80c623..432c1d702087 100644 --- a/tests/python/shared/assets/jobs.json +++ b/tests/python/shared/assets/jobs.json @@ -10,6 +10,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 19, + "issues": "http://localhost:8080/api/issues?job_id=19", "labels": [ { "attributes": [], @@ -48,6 +49,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 18, + "issues": "http://localhost:8080/api/issues?job_id=18", "labels": [ { "attributes": [], @@ -245,6 +247,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 17, + "issues": "http://localhost:8080/api/issues?job_id=17", "labels": [ { "attributes": [], @@ -289,6 +292,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 16, + "issues": "http://localhost:8080/api/issues?job_id=16", "labels": [ { "attributes": [], @@ -327,6 +331,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 14, + "issues": "http://localhost:8080/api/issues?job_id=14", "labels": [ { "attributes": [ @@ -378,6 +383,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 13, + "issues": "http://localhost:8080/api/issues?job_id=13", "labels": [ { "attributes": [ @@ -429,6 +435,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 12, + "issues": "http://localhost:8080/api/issues?job_id=12", "labels": [ { "attributes": [ @@ -486,6 +493,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 11, + "issues": "http://localhost:8080/api/issues?job_id=11", "labels": [ { "attributes": [ @@ -543,6 +551,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 10, + "issues": "http://localhost:8080/api/issues?job_id=10", "labels": [ { "attributes": [], @@ -581,6 +590,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 9, + "issues": "http://localhost:8080/api/issues?job_id=9", "labels": [ { "attributes": [], @@ -619,6 +629,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "3d", "id": 8, + "issues": "http://localhost:8080/api/issues?job_id=8", "labels": [ { "attributes": [], @@ -654,6 +665,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 7, + "issues": "http://localhost:8080/api/issues?job_id=7", "labels": [ { "attributes": [], @@ -689,6 +701,7 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 2, + "issues": "http://localhost:8080/api/issues?job_id=2", "labels": [ { "attributes": [], diff --git a/tests/python/shared/assets/projects.json b/tests/python/shared/assets/projects.json index 2b40cfdd7a6e..bc0c302b8abb 100644 --- a/tests/python/shared/assets/projects.json +++ b/tests/python/shared/assets/projects.json @@ -50,9 +50,7 @@ "location": "local" }, "task_subsets": [], - "tasks": [ - 15 - ], + "tasks": "http://localhost:8080/api/tasks?project_id=8", "updated_date": "2022-12-01T12:53:34.917000Z", "url": "http://localhost:8080/api/projects/8" }, @@ -100,7 +98,7 @@ "location": "local" }, "task_subsets": [], - "tasks": [], + "tasks": "http://localhost:8080/api/tasks?project_id=7", "updated_date": "2022-09-28T12:26:29.285000Z", "url": "http://localhost:8080/api/projects/7" }, @@ -148,7 +146,7 @@ "location": "local" }, "task_subsets": [], - "tasks": [], + "tasks": "http://localhost:8080/api/tasks?project_id=6", "updated_date": "2022-09-28T12:25:54.563000Z", "url": "http://localhost:8080/api/projects/6" }, @@ -358,9 +356,7 @@ "location": "local" }, "task_subsets": [], - "tasks": [ - 14 - ], + "tasks": "http://localhost:8080/api/tasks?project_id=5", "updated_date": "2022-09-28T12:26:49.493000Z", "url": "http://localhost:8080/api/projects/5" }, @@ -403,9 +399,7 @@ "status": "annotation", "target_storage": null, "task_subsets": [], - "tasks": [ - 13 - ], + "tasks": "http://localhost:8080/api/tasks?project_id=4", "updated_date": "2022-12-05T07:47:01.518000Z", "url": "http://localhost:8080/api/projects/4" }, @@ -435,7 +429,7 @@ "status": "annotation", "target_storage": null, "task_subsets": [], - "tasks": [], + "tasks": "http://localhost:8080/api/tasks?project_id=3", "updated_date": "2022-03-28T13:06:09.283000Z", "url": "http://localhost:8080/api/projects/3" }, @@ -494,9 +488,7 @@ "task_subsets": [ "Train" ], - "tasks": [ - 11 - ], + "tasks": "http://localhost:8080/api/tasks?project_id=2", "updated_date": "2022-06-30T08:56:45.601000Z", "url": "http://localhost:8080/api/projects/2" }, @@ -558,9 +550,7 @@ "status": "annotation", "target_storage": null, "task_subsets": [], - "tasks": [ - 9 - ], + "tasks": "http://localhost:8080/api/tasks?project_id=1", "updated_date": "2022-11-03T13:57:25.895000Z", "url": "http://localhost:8080/api/projects/1" } diff --git a/tests/python/shared/assets/tasks.json b/tests/python/shared/assets/tasks.json index f6f3953151fa..14386442d534 100644 --- a/tests/python/shared/assets/tasks.json +++ b/tests/python/shared/assets/tasks.json @@ -14,6 +14,7 @@ "dimension": "2d", "id": 15, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=15", "labels": [ { "attributes": [], @@ -90,6 +91,7 @@ "dimension": "2d", "id": 14, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=14", "labels": [ { "attributes": [], @@ -325,6 +327,7 @@ "dimension": "2d", "id": 13, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=13", "labels": [ { "attributes": [], @@ -388,6 +391,7 @@ "created_date": "2022-03-14T13:24:05.852000Z", "dimension": "2d", "id": 12, + "jobs": "http://localhost:8080/api/jobs?task_id=12", "labels": [ { "attributes": [], @@ -437,6 +441,7 @@ "dimension": "2d", "id": 11, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=11", "labels": [ { "attributes": [], @@ -525,6 +530,7 @@ "dimension": "2d", "id": 9, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=9", "labels": [ { "attributes": [ @@ -660,6 +666,7 @@ "dimension": "2d", "id": 8, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=8", "labels": [ { "attributes": [], @@ -740,6 +747,7 @@ "dimension": "2d", "id": 7, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=7", "labels": [ { "attributes": [], @@ -808,6 +816,7 @@ "dimension": "3d", "id": 6, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=6", "labels": [ { "attributes": [], @@ -873,6 +882,7 @@ "dimension": "2d", "id": 5, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=5", "labels": [ { "attributes": [], @@ -944,6 +954,7 @@ "dimension": "2d", "id": 2, "image_quality": 70, + "jobs": "http://localhost:8080/api/jobs?task_id=2", "labels": [ { "attributes": [], From 2484bf9f9e6e385d3d175c1b5e395d16546cc064 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 02:53:06 -0800 Subject: [PATCH 048/140] Refactored previews & projects --- cvat-core/src/api-implementation.ts | 6 +-- cvat-core/src/cloud-storage.ts | 16 ++---- cvat-core/src/frames.ts | 30 +++++------- cvat-core/src/project-implementation.ts | 15 ++---- cvat-core/src/project.ts | 26 ++-------- cvat-core/src/server-proxy.ts | 59 ++++++++--------------- cvat-core/src/session-implementation.ts | 12 +++-- cvat-ui/src/components/common/preview.tsx | 2 +- 8 files changed, 52 insertions(+), 114 deletions(-) diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 5a018f14d518..e0981a3c28de 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -262,11 +262,7 @@ export default function implementAPI(cvat) { } const projectsData = await serverProxy.projects.get(searchParams); - const projects = projectsData.map((project) => { - project.task_ids = project.tasks; - return project; - }).map((project) => new Project(project)); - + const projects = projectsData.map((project) => new Project(project)); projects.count = projectsData.count; return projects; diff --git a/cvat-core/src/cloud-storage.ts b/cvat-core/src/cloud-storage.ts index 1aab6ef40598..0b83dd46f504 100644 --- a/cvat-core/src/cloud-storage.ts +++ b/cvat-core/src/cloud-storage.ts @@ -1,14 +1,14 @@ // Copyright (C) 2021-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT -import { isBrowser, isNode } from 'browser-or-node'; import PluginRegistry from './plugins'; import serverProxy from './server-proxy'; import { ArgumentError } from './exceptions'; import { CloudStorageCredentialsType, CloudStorageProviderType, CloudStorageStatus } from './enums'; import User from './user'; +import { decodePreview } from './frames'; function validateNotEmptyString(value: string): void { if (typeof value !== 'string') { @@ -362,17 +362,7 @@ Object.defineProperties(CloudStorage.prototype.getPreview, { return new Promise((resolve, reject) => { serverProxy.cloudStorages .getPreview(this.id) - .then((result) => { - if (isNode) { - resolve(global.Buffer.from(result, 'binary').toString('base64')); - } else if (isBrowser) { - const reader = new FileReader(); - reader.onload = () => { - resolve(reader.result); - }; - reader.readAsDataURL(result); - } - }) + .then((result) => decodePreview(result)) .catch((error) => { reject(error); }); diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index 1dc3ef24d35a..cd67e55ac3c0 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -618,26 +618,20 @@ export async function getContextImage(jobID, frame) { return frameDataCache[jobID].frameBuffer.getContextImage(frame); } -export async function getPreview(taskID = null, jobID = null) { +export function decodePreview(preview: Blob): Promise { return new Promise((resolve, reject) => { - // Just go to server and get preview (no any cache) - serverProxy.frames - .getPreview(taskID, jobID) - .then((result) => { - if (isNode) { - // eslint-disable-next-line no-undef - resolve(global.Buffer.from(result, 'binary').toString('base64')); - } else if (isBrowser) { - const reader = new FileReader(); - reader.onload = () => { - resolve(reader.result); - }; - reader.readAsDataURL(result); - } - }) - .catch((error) => { + if (isNode) { + resolve(global.Buffer.from(preview, 'binary').toString('base64')); + } else if (isBrowser) { + const reader = new FileReader(); + reader.onload = () => { + resolve(reader.result as string); + }; + reader.onerror = (error) => { reject(error); - }); + }; + reader.readAsDataURL(preview); + } }); } diff --git a/cvat-core/src/project-implementation.ts b/cvat-core/src/project-implementation.ts index adcefe8f35e9..cac2ce9232b9 100644 --- a/cvat-core/src/project-implementation.ts +++ b/cvat-core/src/project-implementation.ts @@ -6,7 +6,7 @@ import { Storage } from './storage'; import serverProxy from './server-proxy'; -import { getPreview } from './frames'; +import { decodePreview } from './frames'; import Project from './project'; import { exportDataset, importDataset } from './annotations'; @@ -16,7 +16,6 @@ export default function implementProject(projectClass) { if (typeof this.id !== 'undefined') { const projectData = this._updateTrigger.getUpdated(this, { bugTracker: 'bug_tracker', - trainingProject: 'training_project', assignee: 'assignee_id', }); if (projectData.assignee_id) { @@ -41,10 +40,6 @@ export default function implementProject(projectClass) { projectSpec.bug_tracker = this.bugTracker; } - if (this.trainingProject) { - projectSpec.training_project = this.trainingProject; - } - if (this.targetStorage) { projectSpec.target_storage = this.targetStorage.toJSON(); } @@ -63,11 +58,9 @@ export default function implementProject(projectClass) { }; projectClass.prototype.preview.implementation = async function () { - if (!this._internalData.task_ids.length) { - return ''; - } - const frameData = await getPreview(this._internalData.task_ids[0]); - return frameData; + const preview = await serverProxy.projects.getPreview(this.id); + const decoded = await decodePreview(preview); + return decoded; }; projectClass.prototype.annotations.exportDataset.implementation = async function ( diff --git a/cvat-core/src/project.ts b/cvat-core/src/project.ts index 636701de402d..6e7252ece29b 100644 --- a/cvat-core/src/project.ts +++ b/cvat-core/src/project.ts @@ -19,13 +19,12 @@ export default class Project { name: undefined, status: undefined, assignee: undefined, + organization: undefined, owner: undefined, bug_tracker: undefined, created_date: undefined, updated_date: undefined, task_subsets: undefined, - training_project: undefined, - task_ids: undefined, dimension: undefined, source_storage: undefined, target_storage: undefined, @@ -47,10 +46,6 @@ export default class Project { .map((labelData) => new Label(labelData)).filter((label) => !label.hasParent); } - if (typeof initialData.training_project === 'object') { - data.training_project = { ...initialData.training_project }; - } - Object.defineProperties( this, Object.freeze({ @@ -83,6 +78,9 @@ export default class Project { owner: { get: () => data.owner, }, + organization: { + get: () => data.organization, + }, bugTracker: { get: () => data.bug_tracker, set: (tracker) => { @@ -125,22 +123,6 @@ export default class Project { subsets: { get: () => [...data.task_subsets], }, - trainingProject: { - get: () => { - if (typeof data.training_project === 'object') { - return { ...data.training_project }; - } - return data.training_project; - }, - set: (updatedProject) => { - if (typeof training === 'object') { - data.training_project = { ...updatedProject }; - } else { - data.training_project = updatedProject; - } - updateTrigger.update('trainingProject'); - }, - }, sourceStorage: { get: () => ( new Storage({ diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 725e87fe960a..ed706709d176 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1423,22 +1423,24 @@ async function getUsers(filter = { page_size: 'all' }) { return response.data.results; } -async function getPreview(tid, jid) { - const { backendAPI } = config; +function getPreview(instance) { + return async function (id: number) { + const { backendAPI } = config; - let response = null; - try { - const url = `${backendAPI}/${jid !== null ? 'jobs' : 'tasks'}/${jid || tid}/preview`; - response = await Axios.get(url, { - proxy: config.proxy, - responseType: 'blob', - }); - } catch (errorData) { - const code = errorData.response ? errorData.response.status : errorData.code; - throw new ServerError(`Could not get preview frame for the task ${tid} from the server`, code); - } + let response = null; + try { + const url = `${backendAPI}/${instance}/${id}/preview`; + response = await Axios.get(url, { + proxy: config.proxy, + responseType: 'blob', + }); + } catch (errorData) { + const code = errorData.response ? errorData.response.status : errorData.code; + throw new ServerError(`Could not get preview for "${instance}/${id}"`, code); + } - return response.data; + return response.data; + }; } async function getImageContext(jid, frame) { @@ -1982,30 +1984,6 @@ async function getCloudStorageContent(id, manifestPath) { return response.data; } -async function getCloudStoragePreview(id) { - const { backendAPI } = config; - - let response = null; - try { - const url = `${backendAPI}/cloudstorages/${id}/preview`; - response = await workerAxios.get(url, { - params: enableOrganization(), - proxy: config.proxy, - responseType: 'arraybuffer', - }); - } catch (errorData) { - throw generateError({ - message: '', - response: { - ...errorData.response, - data: String.fromCharCode.apply(null, new Uint8Array(errorData.response.data)), - }, - }); - } - - return new Blob([new Uint8Array(response)]); -} - async function getCloudStorageStatus(id) { const { backendAPI } = config; @@ -2373,6 +2351,7 @@ export default Object.freeze({ create: createProject, delete: deleteProject, exportDataset: exportDataset('projects'), + getPreview: getPreview('projects'), backup: backupProject, restore: restoreProject, importDataset, @@ -2384,12 +2363,14 @@ export default Object.freeze({ create: createTask, delete: deleteTask, exportDataset: exportDataset('tasks'), + getPreview: getPreview('tasks'), backup: backupTask, restore: restoreTask, }), jobs: Object.freeze({ get: getJobs, + getPreview: getPreview('jobs'), save: saveJob, exportDataset: exportDataset('jobs'), }), @@ -2446,7 +2427,7 @@ export default Object.freeze({ cloudStorages: Object.freeze({ get: getCloudStorages, getContent: getCloudStorageContent, - getPreview: getCloudStoragePreview, + getPreview: getPreview('cloudstorages'), getStatus: getCloudStorageStatus, create: createCloudStorage, delete: deleteCloudStorage, diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index 578f7cf7da2d..b582116ac81e 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -7,12 +7,12 @@ import { deleteFrame, restoreFrame, getRanges, - getPreview, clear as clearFrames, findNotDeletedFrame, getContextImage, patchMeta, getDeletedFrames, + decodePreview, } from './frames'; import Issue from './issue'; import { checkObjectType } from './common'; @@ -146,8 +146,9 @@ export function implementJob(Job) { return ''; } - const frameData = await getPreview(this.taskId, this.id); - return frameData; + const preview = await serverProxy.jobs.getPreview(this.id); + const decoded = await decodePreview(preview); + return decoded; }; Job.prototype.frames.contextImage.implementation = async function (frameId) { @@ -547,8 +548,9 @@ export function implementTask(Task) { return ''; } - const frameData = await getPreview(this.id); - return frameData; + const preview = await serverProxy.tasks.getPreview(this.id); + const decoded = await decodePreview(preview); + return decoded; }; Task.prototype.frames.delete.implementation = async function (frame) { diff --git a/cvat-ui/src/components/common/preview.tsx b/cvat-ui/src/components/common/preview.tsx index 0faf0a6300ff..481ae10e1000 100644 --- a/cvat-ui/src/components/common/preview.tsx +++ b/cvat-ui/src/components/common/preview.tsx @@ -79,7 +79,7 @@ export default function Preview(props: Props): JSX.Element { if (preview.initialized && !preview.preview) { return ( -
+
); From 31686a749a3a4ac95ca6cdb77073999dff64e260 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 04:45:27 -0800 Subject: [PATCH 049/140] Fetching jobs workarounded --- cvat-core/src/api-implementation.ts | 43 +++++++------- cvat-core/src/server-proxy.ts | 91 +++++++++++++++++------------ cvat-core/src/session.ts | 51 ++++++++-------- 3 files changed, 97 insertions(+), 88 deletions(-) diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index e0981a3c28de..50b2d8f74973 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -20,7 +20,6 @@ import { import User from './user'; import { AnnotationFormats } from './annotation-formats'; -import { ArgumentError } from './exceptions'; import { Task, Job } from './session'; import Project from './project'; import CloudStorage from './cloud-storage'; @@ -166,8 +165,8 @@ export default function implementAPI(cvat) { return users; }; - cvat.jobs.get.implementation = async (filter) => { - checkFilter(filter, { + cvat.jobs.get.implementation = async (query) => { + checkFilter(query, { page: isInteger, filter: isString, sort: isString, @@ -176,30 +175,24 @@ export default function implementAPI(cvat) { jobID: isInteger, }); - if ('taskID' in filter && 'jobID' in filter) { - throw new ArgumentError('Filter fields "taskID" and "jobID" are not permitted to be used at the same time'); - } - - if ('taskID' in filter) { - const [task] = await serverProxy.tasks.get({ id: filter.taskID }); - if (task) { - return new Task(task).jobs; + checkExclusiveFields(query, ['jobID', 'taskID', 'filter', 'search'], ['page', 'sort']); + if ('jobID' in query) { + const job = await serverProxy.jobs.get({ id: query.jobID }); + if (job) { + return [new Job(job)]; } return []; } - if ('jobID' in filter) { - const job = await serverProxy.jobs.get({ id: filter.jobID }); - if (job) { - return [new Job(job)]; - } + if ('taskID' in query) { + query.filter = JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, query.taskID] }] }); } const searchParams = {}; - for (const key of Object.keys(filter)) { + for (const key of Object.keys(query)) { if (['page', 'sort', 'search', 'filter'].includes(key)) { - searchParams[key] = filter[key]; + searchParams[key] = query[key]; } } @@ -228,8 +221,7 @@ export default function implementAPI(cvat) { } } - let tasksData = null; - if (filter.projectId) { + if ('projectId' in filter) { if (searchParams.filter) { const parsed = JSON.parse(searchParams.filter); searchParams.filter = JSON.stringify({ and: [parsed, { '==': [{ var: 'project_id' }, filter.projectId] }] }); @@ -238,8 +230,15 @@ export default function implementAPI(cvat) { } } - tasksData = await serverProxy.tasks.get(searchParams); - const tasks = tasksData.map((task) => new Task(task)); + const tasksData = await serverProxy.tasks.get(searchParams); + const tasks = await Promise.all(tasksData.map(async (taskItem) => { + // Temporary workaround for UI + const jobs = await serverProxy.jobs.get({ + filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, taskItem.id] }] }), + }, true); + return new Task({ ...taskItem, jobs: jobs.results }); + })); + tasks.count = tasksData.count; return tasks; }; diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index ed706709d176..871dead20ac6 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1232,47 +1232,25 @@ async function createTask(taskSpec, taskDataSpec, onUpdate) { return createdTask[0]; } -async function getJobs(filter = {}) { - const { backendAPI } = config; - const id = filter.id || null; - - let response = null; - try { - if (id !== null) { - response = await Axios.get(`${backendAPI}/jobs/${id}`, { - proxy: config.proxy, - }); - } else { - response = await Axios.get(`${backendAPI}/jobs`, { - proxy: config.proxy, - params: { - ...filter, - page_size: 12, - }, - }); - } - } catch (errorData) { - throw generateError(errorData); - } - - return response.data; -} - -function fetchAll(url): Promise { - const pageSize = 500; - let collection = []; +function fetchAll(url, filter = {}): Promise { + const pageSize = 2; + const result = { + count: 0, + results: [], + }; return new Promise((resolve, reject) => { Axios.get(url, { params: { + ...filter, page_size: pageSize, page: 1, }, proxy: config.proxy, }).then((initialData) => { const { count, results } = initialData.data; - collection = collection.concat(results); + result.results = result.results.concat(results); if (count <= pageSize) { - resolve(collection); + resolve(result); return; } @@ -1281,6 +1259,7 @@ function fetchAll(url): Promise { if (i) { return Axios.get(url, { params: { + ...filter, page_size: pageSize, page: i + 1, }, @@ -1294,33 +1273,69 @@ function fetchAll(url): Promise { Promise.all(promises).then((responses: AxiosResponse[]) => { responses.forEach((resp) => { if (resp) { - collection = collection.concat(resp.data.results); + result.results = result.results.concat(resp.data.results); } }); // removing possible dublicates - const obj = collection.reduce((acc: Record, item: any) => { + const obj = result.results.reduce((acc: Record, item: any) => { acc[item.id] = item; return acc; }, {}); - resolve(Object.values(obj)); + result.results = Object.values(obj); + result.count = result.results.length; + + resolve(result); }).catch((error) => reject(error)); }).catch((error) => reject(error)); }); } +async function getJobs(filter = {}, aggregate = false) { + const { backendAPI } = config; + const id = filter.id || null; + + let response = null; + try { + if (id !== null) { + response = await Axios.get(`${backendAPI}/jobs/${id}`, { + proxy: config.proxy, + }); + } else { + if (aggregate) { + return await fetchAll(`${backendAPI}/jobs`, { + ...filter, + ...enableOrganization(), + }); + } + + response = await Axios.get(`${backendAPI}/jobs`, { + proxy: config.proxy, + params: { + ...filter, + page_size: 12, + }, + }); + } + } catch (errorData) { + throw generateError(errorData); + } + + return response.data; +} + async function getJobIssues(jobID) { const { backendAPI } = config; let response = null; try { - response = await fetchAll(`${backendAPI}/jobs/${jobID}/issues`); + response = await fetchAll(`${backendAPI}/jobs/${jobID}/issues`, { ...enableOrganization() }); } catch (errorData) { throw generateError(errorData); } - return response; + return response.results; } async function createComment(data) { @@ -2017,12 +2032,12 @@ async function getOrganizations() { let response = null; try { - response = await fetchAll(`${backendAPI}/organizations?page_size`); + response = await fetchAll(`${backendAPI}/organizations`); } catch (errorData) { throw generateError(errorData); } - return response; + return response.results; } async function createOrganization(data) { diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 7b86f4b29508..1a4494ab32ec 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -564,7 +564,6 @@ export class Task extends Session { data_chunk_size: undefined, data_compressed_chunk_type: undefined, data_original_chunk_type: undefined, - deleted_frames: undefined, use_zip_chunks: undefined, use_cache: undefined, copy_data: undefined, @@ -599,33 +598,29 @@ export class Task extends Session { .map((labelData) => new Label(labelData)).filter((label) => !label.hasParent); } - if (Array.isArray(initialData.segments)) { - for (const segment of initialData.segments) { - if (Array.isArray(segment.jobs)) { - for (const job of segment.jobs) { - const jobInstance = new Job({ - url: job.url, - id: job.id, - assignee: job.assignee, - state: job.state, - stage: job.stage, - start_frame: segment.start_frame, - stop_frame: segment.stop_frame, - // following fields also returned when doing API request /jobs/ - // here we know them from task and append to constructor - task_id: data.id, - project_id: data.project_id, - labels: data.labels, - bug_tracker: data.bug_tracker, - mode: data.mode, - dimension: data.dimension, - data_compressed_chunk_type: data.data_compressed_chunk_type, - data_chunk_size: data.data_chunk_size, - }); - - data.jobs.push(jobInstance); - } - } + if (Array.isArray(initialData.jobs)) { + for (const job of initialData.jobs) { + const jobInstance = new Job({ + url: job.url, + id: job.id, + assignee: job.assignee, + state: job.state, + stage: job.stage, + start_frame: job.start_frame, + stop_frame: job.stop_frame, + // following fields also returned when doing API request /jobs/ + // here we know them from task and append to constructor + task_id: data.id, + project_id: data.project_id, + labels: data.labels, + bug_tracker: data.bug_tracker, + mode: data.mode, + dimension: data.dimension, + data_compressed_chunk_type: data.data_compressed_chunk_type, + data_chunk_size: data.data_chunk_size, + }); + + data.jobs.push(jobInstance); } } From bf7e42019f72f4c1140269b8e0cbca6fca3c0ac5 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 04:51:56 -0800 Subject: [PATCH 050/140] Fixed code --- cvat-core/src/server-proxy.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 871dead20ac6..9cd1919b5fb6 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1330,7 +1330,12 @@ async function getJobIssues(jobID) { let response = null; try { - response = await fetchAll(`${backendAPI}/jobs/${jobID}/issues`, { ...enableOrganization() }); + response = await fetchAll(`${backendAPI}/issues`, { + params: { + job_id: jobID, + ...enableOrganization(), + }, + }); } catch (errorData) { throw generateError(errorData); } From db797c48afbad7e8b28f8c9c6aed5f6e0e7ee817 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 10:57:09 -0800 Subject: [PATCH 051/140] Fixed issue comments --- cvat-core/src/issue.ts | 22 ++++++++++------ cvat-core/src/server-proxy.ts | 17 ++++++++++++ .../review/hidden-issue-label.tsx | 26 ++++++++++++++----- .../annotation-page/review/issue-dialog.tsx | 24 ++++++++++++----- .../review/issues-aggregator.tsx | 8 +++--- cvat-ui/src/cvat-core-wrapper.ts | 4 +++ 6 files changed, 75 insertions(+), 26 deletions(-) diff --git a/cvat-core/src/issue.ts b/cvat-core/src/issue.ts index 82e6969f74da..aafdca8ba973 100644 --- a/cvat-core/src/issue.ts +++ b/cvat-core/src/issue.ts @@ -16,7 +16,6 @@ interface RawIssueData { id?: number; job?: any; position?: number[]; - comments?: any; frame?: number; owner?: any; resolved?: boolean; @@ -26,19 +25,19 @@ interface RawIssueData { export default class Issue { public readonly id: number; public readonly job: Job; - public readonly comments: Comment[]; public readonly frame: number; public readonly owner: User; + public readonly comments: Promise; public readonly resolved: boolean; public readonly createdDate: string; public position: number[]; constructor(initialData: RawIssueData) { + let comments: Comment[]; const data: RawIssueData = { id: undefined, job: undefined, position: undefined, - comments: [], frame: undefined, created_date: undefined, owner: undefined, @@ -53,10 +52,6 @@ export default class Issue { if (data.owner && !(data.owner instanceof User)) data.owner = new User(data.owner); - if (data.comments) { - data.comments = data.comments.map((comment) => new Comment(comment)); - } - if (typeof data.created_date === 'undefined') { data.created_date = new Date().toISOString(); } @@ -80,7 +75,18 @@ export default class Issue { get: () => data.job, }, comments: { - get: () => [...data.comments], + get: () => new Promise((resolve, reject) => { + if (comments) { + resolve(comments); + } + + serverProxy.comments.get(this.id).then((_comments: RawCommentData[]) => { + comments = _comments.map((comment: RawCommentData): Comment => new Comment(comment)); + resolve(comments); + }).catch((error) => { + reject(error); + }); + }), }, frame: { get: () => data.frame, diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 9cd1919b5fb6..2ec3820e8250 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1343,6 +1343,22 @@ async function getJobIssues(jobID) { return response.results; } +async function getComments(issueID: number) { + const { backendAPI } = config; + + let response = null; + try { + response = await fetchAll(`${backendAPI}/comments`, { + issue_id: issueID, + ...enableOrganization(), + }); + } catch (errorData) { + throw generateError(errorData); + } + + return response.results; +} + async function createComment(data) { const { backendAPI } = config; @@ -2436,6 +2452,7 @@ export default Object.freeze({ }), comments: Object.freeze({ + get: getComments, create: createComment, }), diff --git a/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx b/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx index c69896506957..b4892ab9e1bc 100644 --- a/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx +++ b/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx @@ -1,17 +1,20 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT -import React, { ReactPortal, useEffect, useRef } from 'react'; +import React, { + ReactPortal, useEffect, useRef, useState, +} from 'react'; import ReactDOM from 'react-dom'; import Tag from 'antd/lib/tag'; -import { CheckCircleOutlined, CloseCircleOutlined } from '@ant-design/icons'; +import { CheckCircleOutlined, CloseCircleOutlined, LoadingOutlined } from '@ant-design/icons'; +import { Issue, Comment } from 'cvat-core-wrapper'; import CVATTooltip from 'components/common/cvat-tooltip'; interface Props { - id: number; - message: string; + issue: Issue; top: number; left: number; angle: number; @@ -24,10 +27,13 @@ interface Props { export default function HiddenIssueLabel(props: Props): ReactPortal { const { - id, message, top, left, angle, scale, resolved, onClick, highlight, blur, + issue, top, left, angle, scale, resolved, onClick, highlight, blur, } = props; + const { id } = issue; + const ref = useRef(null); + const [firstComment, setFirstComments] = useState(null); useEffect(() => { if (!resolved) { @@ -37,9 +43,15 @@ export default function HiddenIssueLabel(props: Props): ReactPortal { } }, [resolved]); + useEffect(() => { + issue.comments.then((_comments: Comment[]) => { + setFirstComments(_comments[0]); + }); + }, []); + const elementID = `cvat-hidden-issue-label-${id}`; return ReactDOM.createPortal( - + )} - {message} + {firstComment ? firstComment.message : } , window.document.getElementById('cvat_canvas_attachment_board') as HTMLElement, diff --git a/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx b/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx index e5d72c1bf8ac..eed714495c2d 100644 --- a/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx +++ b/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -17,14 +18,15 @@ import Comment from 'antd/lib/comment'; import Text from 'antd/lib/typography/Text'; import Title from 'antd/lib/typography/Title'; import Button from 'antd/lib/button'; +import Spin from 'antd/lib/spin'; import Input from 'antd/lib/input'; import moment from 'moment'; import CVATTooltip from 'components/common/cvat-tooltip'; +import { Issue, Comment as CommentModel } from 'cvat-core-wrapper'; import { deleteIssueAsync } from 'actions/review-actions'; interface Props { - id: number; - comments: any[]; + issue: Issue; left: number; top: number; resolved: boolean; @@ -42,10 +44,10 @@ interface Props { export default function IssueDialog(props: Props): JSX.Element { const ref = useRef(null); const [currentText, setCurrentText] = useState(''); + const [comments, setComments] = useState([]); const dispatch = useDispatch(); const { - comments, - id, + issue, left, top, scale, @@ -60,6 +62,8 @@ export default function IssueDialog(props: Props): JSX.Element { blur, } = props; + const { id } = issue; + useEffect(() => { if (!resolved) { setTimeout(highlight); @@ -68,6 +72,12 @@ export default function IssueDialog(props: Props): JSX.Element { } }, [resolved]); + useEffect(() => { + issue.comments.then((_comments: CommentModel[]) => { + setComments(_comments); + }); + }, []); + const onDeleteIssue = useCallback((): void => { Modal.confirm({ title: `The issue${id >= 0 ? ` #${id}` : ''} will be deleted.`, @@ -85,7 +95,7 @@ export default function IssueDialog(props: Props): JSX.Element { }, []); const lines = comments.map( - (_comment: any): JSX.Element => { + (_comment: CommentModel): JSX.Element => { const created = _comment.createdDate ? moment(_comment.createdDate) : moment(moment.now()); const diff = created.fromNow(); @@ -128,7 +138,9 @@ export default function IssueDialog(props: Props): JSX.Element { - {lines} + { + lines.length > 0 ? {lines} : + } diff --git a/cvat-ui/src/components/annotation-page/review/issues-aggregator.tsx b/cvat-ui/src/components/annotation-page/review/issues-aggregator.tsx index 55782936a633..860ae9e21b44 100644 --- a/cvat-ui/src/components/annotation-page/review/issues-aggregator.tsx +++ b/cvat-ui/src/components/annotation-page/review/issues-aggregator.tsx @@ -118,13 +118,12 @@ export default function IssueAggregatorComponent(): JSX.Element | null { issueDialogs.push( , ); - } else if (issue.comments.length) { + } else { issueLabels.push( { diff --git a/cvat-ui/src/cvat-core-wrapper.ts b/cvat-ui/src/cvat-core-wrapper.ts index a04e5c6ecc6c..ce49bf66e133 100644 --- a/cvat-ui/src/cvat-core-wrapper.ts +++ b/cvat-ui/src/cvat-core-wrapper.ts @@ -10,6 +10,8 @@ import { } from 'cvat-core/src/labels'; import { ShapeType, LabelType } from 'cvat-core/src/enums'; import { Storage, StorageData } from 'cvat-core/src/storage'; +import Issue from 'cvat-core/src/issue'; +import Comment from 'cvat-core/src/comment'; import { SocialAuthMethods, SocialAuthMethod } from 'cvat-core/src/auth-methods'; const cvat: any = _cvat; @@ -33,6 +35,8 @@ export { Storage, Webhook, SocialAuthMethod, + Issue, + Comment, }; export type { From 1b806e047429525228658deeffa8017828f9f723 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 27 Jan 2023 21:02:22 +0200 Subject: [PATCH 052/140] update some tests --- cvat-sdk/cvat_sdk/core/proxies/issues.py | 2 +- cvat-sdk/cvat_sdk/core/proxies/jobs.py | 2 +- cvat-sdk/cvat_sdk/core/proxies/projects.py | 2 +- cvat-sdk/cvat_sdk/core/proxies/tasks.py | 2 +- tests/python/rest_api/test_issues.py | 6 ++++- tests/python/rest_api/test_jobs.py | 4 +-- tests/python/rest_api/test_projects.py | 11 ++++---- tests/python/rest_api/test_tasks.py | 8 +++--- tests/python/sdk/test_issues_comments.py | 5 ++-- tests/python/sdk/test_projects.py | 2 +- tests/python/shared/assets/cvat_db/data.json | 28 +++++++++++++++++++- 11 files changed, 51 insertions(+), 21 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/proxies/issues.py b/cvat-sdk/cvat_sdk/core/proxies/issues.py index 5df1069c1178..cc3be5a6bd64 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/issues.py +++ b/cvat-sdk/cvat_sdk/core/proxies/issues.py @@ -57,7 +57,7 @@ def get_comments(self) -> List[Comment]: return [ Comment(self._client, m) for m in get_paginated_collection( - self._client.api_client.comments_api.list_endpoint, issue_id=self.id + self._client.api_client.comments_api.list_endpoint, issue_id=str(self.id) ) ] diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index 19897f443166..4c6047edf883 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -164,7 +164,7 @@ def get_issues(self) -> List[Issue]: return [ Issue(self._client, m) for m in get_paginated_collection( - self._client.api_client.issues_api.list_endpoint, job_id=self.id + self._client.api_client.issues_api.list_endpoint, job_id=str(self.id) ) ] diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index b1bed2e7d870..b8a3764b9945 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -128,7 +128,7 @@ def get_tasks(self) -> List[Task]: return [ Task(self._client, m) for m in get_paginated_collection( - self._client.api_client.tasks_api.list_endpoint, project_id=self.id + self._client.api_client.tasks_api.list_endpoint, project_id=str(self.id) ) ] diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index 0174134f5abd..46c04612ce72 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -306,7 +306,7 @@ def get_jobs(self) -> List[Job]: return [ Job(self._client, model=m) for m in get_paginated_collection( - self._client.api_client.jobs_api.list_endpoint, task_id=self.id + self._client.api_client.jobs_api.list_endpoint, task_id=str(self.id) ) ] diff --git a/tests/python/rest_api/test_issues.py b/tests/python/rest_api/test_issues.py index 357113547af8..2a9644eb7754 100644 --- a/tests/python/rest_api/test_issues.py +++ b/tests/python/rest_api/test_issues.py @@ -34,7 +34,11 @@ def _test_check_response(self, user, data, is_allow, **kwargs): assert response.status == HTTPStatus.CREATED response_json = json.loads(response.data) assert user == response_json["owner"]["username"] - assert data["message"] == response_json["comments"][0]["message"] + + with make_api_client(user) as client: + (comments, _) = client.comments_api.list(issue_id=str(response_json["id"])) + assert data["message"] == comments.results[0].message + assert ( DeepDiff( data, diff --git a/tests/python/rest_api/test_jobs.py b/tests/python/rest_api/test_jobs.py index 9ef7adf09f32..000e9e7157e6 100644 --- a/tests/python/rest_api/test_jobs.py +++ b/tests/python/rest_api/test_jobs.py @@ -153,11 +153,9 @@ class TestJobsListFilters(CollectionSimpleFilterTestBase): } @pytest.fixture(autouse=True) - def setup(self, restore_db_per_class, admin_user, jobs, tasks, projects): + def setup(self, restore_db_per_class, admin_user, jobs): self.user = admin_user self.samples = jobs - self.sample_tasks = tasks - self.sample_projects = projects def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.jobs_api.list_endpoint diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index dfbd8d140454..3008000b7867 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -575,12 +575,13 @@ def test_exported_project_dataset_structure( "name": project["name"], "tasks": [ { - "id": tid, - "name": (task := tasks[tid])["name"], + "id": task["id"], + "name": task["name"], "size": str(task["size"]), "mode": task["mode"], } - for tid in project["tasks"] + for task in tasks + if task["project_id"] == project["id"] ], } @@ -823,9 +824,9 @@ def test_project_preview_owner_accessibility(self, projects): project_with_assignee["assignee"]["username"], project_with_assignee["id"] ) - def test_project_preview_not_found(self, projects): + def test_project_preview_not_found(self, projects, tasks): for p in projects: - if p["tasks"]: + if any(t["project_id"] == p["id"] for t in tasks): continue if p["owner"] is not None: project_with_owner = p diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 55362df63d3d..6d01bc634010 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -45,17 +45,17 @@ class TestGetTasks: def _test_task_list_200(self, user, project_id, data, exclude_paths="", **kwargs): with make_api_client(user) as api_client: results = get_paginated_collection( - api_client.projects_api.list_tasks_endpoint, + api_client.tasks_api.list_endpoint, return_json=True, - id=project_id, + project_id=str(project_id), **kwargs, ) assert DeepDiff(data, results, ignore_order=True, exclude_paths=exclude_paths) == {} def _test_task_list_403(self, user, project_id, **kwargs): with make_api_client(user) as api_client: - (_, response) = api_client.projects_api.list_tasks( - project_id, **kwargs, _parse_response=False, _check_status=False + (_, response) = api_client.tasks_api.list( + project_id=str(project_id), **kwargs, _parse_response=False, _check_status=False ) assert response.status == HTTPStatus.FORBIDDEN diff --git a/tests/python/sdk/test_issues_comments.py b/tests/python/sdk/test_issues_comments.py index 4f3b4eecfb6e..12047c75a1f0 100644 --- a/tests/python/sdk/test_issues_comments.py +++ b/tests/python/sdk/test_issues_comments.py @@ -91,7 +91,7 @@ def test_can_list_comments(self, fxt_new_task: Task): comment = self.client.comments.create(models.CommentWriteRequest(issue.id, message="hi!")) issue.fetch() - comment_ids = {c.id for c in issue.comments} + comment_ids = {c.id for c in issue.get_comments()} assert len(comment_ids) == 2 assert comment.id in comment_ids @@ -123,13 +123,14 @@ def test_can_remove_issue(self, fxt_new_task: Task): message="hello", ) ) + comments = issue.get_comments() issue.remove() with pytest.raises(exceptions.NotFoundException): issue.fetch() with pytest.raises(exceptions.NotFoundException): - self.client.comments.retrieve(issue.comments[0].id) + self.client.comments.retrieve(comments[0].id) assert self.stdout.getvalue() == "" diff --git a/tests/python/sdk/test_projects.py b/tests/python/sdk/test_projects.py index 14de38b444d2..446c79c8de95 100644 --- a/tests/python/sdk/test_projects.py +++ b/tests/python/sdk/test_projects.py @@ -161,7 +161,7 @@ def test_can_delete_project(self, fxt_new_project: Project): assert self.stdout.getvalue() == "" def test_can_get_tasks(self, fxt_project_with_shapes: Project): - task_ids = set(fxt_project_with_shapes.tasks) + task_ids = set(t.id for t in fxt_project_with_shapes.get_tasks()) tasks = fxt_project_with_shapes.get_tasks() diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index d08b273499b2..6e1ca2035322 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -659,6 +659,30 @@ "name": "example.com" } }, +{ + "model": "account.emailaddress", + "pk": 1, + "fields": { + "user": [ + "admin1" + ], + "email": "admin1@cvat.org", + "verified": true, + "primary": true + } +}, +{ + "model": "account.emailaddress", + "pk": 2, + "fields": { + "user": [ + "admin2" + ], + "email": "admin2@cvat.org", + "verified": true, + "primary": true + } +}, { "model": "organizations.organization", "pk": 1, @@ -5905,7 +5929,9 @@ "owner": [ "user1" ], - "assignee": 2, + "assignee": [ + "user1" + ], "created_date": "2022-03-16T12:40:00.764Z", "updated_date": null, "resolved": false From 94516c3ce706b9b74fe537d062c9235e898db290 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 11:48:17 -0800 Subject: [PATCH 053/140] Fixed comments --- cvat-core/src/issue.ts | 82 +++++++++++-------- .../review/hidden-issue-label.tsx | 4 +- .../annotation-page/review/issue-dialog.tsx | 4 +- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/cvat-core/src/issue.ts b/cvat-core/src/issue.ts index aafdca8ba973..9e47ed542f36 100644 --- a/cvat-core/src/issue.ts +++ b/cvat-core/src/issue.ts @@ -1,11 +1,10 @@ // Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT import quickhull from 'quickhull'; -import { Job } from 'session'; import PluginRegistry from './plugins'; import Comment, { RawCommentData } from './comment'; import User from './user'; @@ -13,28 +12,28 @@ import { ArgumentError } from './exceptions'; import serverProxy from './server-proxy'; interface RawIssueData { + job: number; + position: number[]; + frame: number; id?: number; - job?: any; - position?: number[]; - frame?: number; owner?: any; resolved?: boolean; created_date?: string; } export default class Issue { - public readonly id: number; - public readonly job: Job; + public readonly id?: number; + public readonly job: number; public readonly frame: number; - public readonly owner: User; - public readonly comments: Promise; - public readonly resolved: boolean; - public readonly createdDate: string; - public position: number[]; + public readonly owner?: User; + public readonly comments?: Comment[]; + public readonly resolved?: boolean; + public readonly createdDate?: string; + public position?: number[]; + private readonly __internal: RawIssueData & { comments: Comment[] }; constructor(initialData: RawIssueData) { - let comments: Comment[]; - const data: RawIssueData = { + const data: RawIssueData & { comments: Comment[] } = { id: undefined, job: undefined, position: undefined, @@ -42,6 +41,7 @@ export default class Issue { created_date: undefined, owner: undefined, resolved: undefined, + comments: undefined, }; for (const property in data) { @@ -75,18 +75,7 @@ export default class Issue { get: () => data.job, }, comments: { - get: () => new Promise((resolve, reject) => { - if (comments) { - resolve(comments); - } - - serverProxy.comments.get(this.id).then((_comments: RawCommentData[]) => { - comments = _comments.map((comment: RawCommentData): Comment => new Comment(comment)); - resolve(comments); - }).catch((error) => { - reject(error); - }); - }), + get: () => data.comments, }, frame: { get: () => data.frame, @@ -122,6 +111,12 @@ export default class Issue { return coordinates; } + // Method fetches comments list from the server + public async initComments(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, Issue.prototype.initComments); + return result; + } + // Method appends a comment to the issue // For a new issue it saves comment locally, for a saved issue it saves comment on the server public async comment(data: RawCommentData): Promise { @@ -150,19 +145,15 @@ export default class Issue { } public serialize(): RawIssueData { - const { comments } = this; const data: RawIssueData = { + job: this.job, position: this.position, frame: this.frame, - comments: comments.map((comment) => comment.serialize()), }; if (typeof this.id === 'number') { data.id = this.id; } - if (typeof this.job === 'number') { - data.job = this.job; - } if (typeof this.createdDate === 'string') { data.created_date = this.createdDate; } @@ -177,11 +168,29 @@ export default class Issue { } } +Object.defineProperties(Issue.prototype.initComments, { + implementation: { + writable: false, + enumerable: false, + value: async function implementation(this: Issue) { + const internalData = Object.getOwnPropertyDescriptor(this, '__internal').get(); + if (this.id) { + const rawComments = await serverProxy.comments.get(this.id); + internalData.comments = rawComments.map((comment: RawCommentData): Comment => new Comment(comment)); + return [...internalData.comments]; + } + + internalData.comments = []; + return [...internalData.comments]; + }, + }, +}); + Object.defineProperties(Issue.prototype.comment, { implementation: { writable: false, enumerable: false, - value: async function implementation(data: RawCommentData) { + value: async function implementation(this: Issue, data: RawCommentData) { if (typeof data !== 'object' || data === null) { throw new ArgumentError(`The argument "data" must be an object. Got "${data}"`); } @@ -189,15 +198,20 @@ Object.defineProperties(Issue.prototype.comment, { throw new ArgumentError(`Comment message must be a not empty string. Got "${data.message}"`); } + const internalData = Object.getOwnPropertyDescriptor(this, '__internal').get(); + if (!internalData.comments) { + await Object.getOwnPropertyDescriptor(this.initComments, 'implementation').value(); + } + const comment = new Comment(data); if (typeof this.id === 'number') { const serialized = comment.serialize(); serialized.issue = this.id; const response = await serverProxy.comments.create(serialized); const savedComment = new Comment(response); - this.__internal.comments.push(savedComment); + internalData.comments.push(savedComment); } else { - this.__internal.comments.push(comment); + internalData.comments.push(comment); } }, }, diff --git a/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx b/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx index b4892ab9e1bc..ea5b2ee46670 100644 --- a/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx +++ b/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx @@ -44,8 +44,8 @@ export default function HiddenIssueLabel(props: Props): ReactPortal { }, [resolved]); useEffect(() => { - issue.comments.then((_comments: Comment[]) => { - setFirstComments(_comments[0]); + issue.initComments().then(() => { + setFirstComments((issue.comments as Comment[])[0]); }); }, []); diff --git a/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx b/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx index eed714495c2d..d3121a1ff2cb 100644 --- a/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx +++ b/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx @@ -73,8 +73,8 @@ export default function IssueDialog(props: Props): JSX.Element { }, [resolved]); useEffect(() => { - issue.comments.then((_comments: CommentModel[]) => { - setComments(_comments); + issue.initComments().then(() => { + setComments(issue.comments as CommentModel[]); }); }, []); From 9eb136f309936d4f8f17144dd583d714eb0ab903 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 11:50:55 -0800 Subject: [PATCH 054/140] Fixed page size --- cvat-core/src/server-proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 2ec3820e8250..8bea694fc623 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1233,7 +1233,7 @@ async function createTask(taskSpec, taskDataSpec, onUpdate) { } function fetchAll(url, filter = {}): Promise { - const pageSize = 2; + const pageSize = 500; const result = { count: 0, results: [], From 81508b0fee02a47e226ddc74c46ab4d91bc6ac88 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 12:36:34 -0800 Subject: [PATCH 055/140] Reworked fetching issue comments --- cvat-core/src/issue.ts | 37 +++----------- cvat-core/src/server-proxy.ts | 50 ++++++++++++------- .../review/hidden-issue-label.tsx | 21 +++----- .../annotation-page/review/issue-dialog.tsx | 14 +----- 4 files changed, 47 insertions(+), 75 deletions(-) diff --git a/cvat-core/src/issue.ts b/cvat-core/src/issue.ts index 9e47ed542f36..f1c09cf29ba7 100644 --- a/cvat-core/src/issue.ts +++ b/cvat-core/src/issue.ts @@ -16,6 +16,7 @@ interface RawIssueData { position: number[]; frame: number; id?: number; + comments?: RawCommentData[]; owner?: any; resolved?: boolean; created_date?: string; @@ -26,7 +27,7 @@ export default class Issue { public readonly job: number; public readonly frame: number; public readonly owner?: User; - public readonly comments?: Comment[]; + public readonly comments: Comment[]; public readonly resolved?: boolean; public readonly createdDate?: string; public position?: number[]; @@ -56,6 +57,12 @@ export default class Issue { data.created_date = new Date().toISOString(); } + if (Array.isArray(initialData.comments)) { + data.comments = initialData.comments.map((comment: RawCommentData): Comment => new Comment(comment)); + } else { + data.comments = []; + } + Object.defineProperties( this, Object.freeze({ @@ -111,12 +118,6 @@ export default class Issue { return coordinates; } - // Method fetches comments list from the server - public async initComments(): Promise { - const result = await PluginRegistry.apiWrapper.call(this, Issue.prototype.initComments); - return result; - } - // Method appends a comment to the issue // For a new issue it saves comment locally, for a saved issue it saves comment on the server public async comment(data: RawCommentData): Promise { @@ -168,24 +169,6 @@ export default class Issue { } } -Object.defineProperties(Issue.prototype.initComments, { - implementation: { - writable: false, - enumerable: false, - value: async function implementation(this: Issue) { - const internalData = Object.getOwnPropertyDescriptor(this, '__internal').get(); - if (this.id) { - const rawComments = await serverProxy.comments.get(this.id); - internalData.comments = rawComments.map((comment: RawCommentData): Comment => new Comment(comment)); - return [...internalData.comments]; - } - - internalData.comments = []; - return [...internalData.comments]; - }, - }, -}); - Object.defineProperties(Issue.prototype.comment, { implementation: { writable: false, @@ -199,10 +182,6 @@ Object.defineProperties(Issue.prototype.comment, { } const internalData = Object.getOwnPropertyDescriptor(this, '__internal').get(); - if (!internalData.comments) { - await Object.getOwnPropertyDescriptor(this.initComments, 'implementation').value(); - } - const comment = new Comment(data); if (typeof this.id === 'number') { const serialized = comment.serialize(); diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 8bea694fc623..df78accfdddd 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1325,33 +1325,37 @@ async function getJobs(filter = {}, aggregate = false) { return response.data; } -async function getJobIssues(jobID) { +async function getJobIssues(jobID: number) { const { backendAPI } = config; let response = null; try { + const organization = enableOrganization(); response = await fetchAll(`${backendAPI}/issues`, { - params: { - job_id: jobID, - ...enableOrganization(), - }, + job_id: jobID, + ...organization, }); - } catch (errorData) { - throw generateError(errorData); - } - return response.results; -} + const commentsResponse = await fetchAll(`${backendAPI}/comments`, { + job_id: jobID, + ...organization, + }); -async function getComments(issueID: number) { - const { backendAPI } = config; + const issuesById = response.results.reduce((acc, val: { id: number }) => { + acc[val.id] = val; + return acc; + }, {}); - let response = null; - try { - response = await fetchAll(`${backendAPI}/comments`, { - issue_id: issueID, - ...enableOrganization(), - }); + const commentsByIssue = commentsResponse.results.reduce((acc, val) => { + acc[val.issue] = acc[val.issue] || []; + acc[val.issue].push(val); + return acc; + }, {}); + + for (const issue of Object.keys(commentsByIssue)) { + commentsByIssue[issue] = [...commentsResponse.results].sort((a, b) => a.id - b.id); + issuesById[issue].comments = commentsByIssue[issue]; + } } catch (errorData) { throw generateError(errorData); } @@ -1382,12 +1386,21 @@ async function createIssue(data) { let response = null; try { + const organization = enableOrganization(); response = await Axios.post(`${backendAPI}/issues`, JSON.stringify(data), { proxy: config.proxy, + params: { ...organization }, headers: { 'Content-Type': 'application/json', }, }); + + const commentsResponse = await fetchAll(`${backendAPI}/comments`, { + issue_id: response.data.id, + ...organization, + }); + + response.data.comments = commentsResponse.results; } catch (errorData) { throw generateError(errorData); } @@ -2452,7 +2465,6 @@ export default Object.freeze({ }), comments: Object.freeze({ - get: getComments, create: createComment, }), diff --git a/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx b/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx index ea5b2ee46670..076c531b75af 100644 --- a/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx +++ b/cvat-ui/src/components/annotation-page/review/hidden-issue-label.tsx @@ -4,13 +4,13 @@ // SPDX-License-Identifier: MIT import React, { - ReactPortal, useEffect, useRef, useState, + ReactPortal, useEffect, useRef, } from 'react'; import ReactDOM from 'react-dom'; import Tag from 'antd/lib/tag'; -import { CheckCircleOutlined, CloseCircleOutlined, LoadingOutlined } from '@ant-design/icons'; +import { CheckCircleOutlined, CloseCircleOutlined, WarningOutlined } from '@ant-design/icons'; -import { Issue, Comment } from 'cvat-core-wrapper'; +import { Issue } from 'cvat-core-wrapper'; import CVATTooltip from 'components/common/cvat-tooltip'; interface Props { @@ -30,11 +30,8 @@ export default function HiddenIssueLabel(props: Props): ReactPortal { issue, top, left, angle, scale, resolved, onClick, highlight, blur, } = props; - const { id } = issue; - + const { id, comments } = issue; const ref = useRef(null); - const [firstComment, setFirstComments] = useState(null); - useEffect(() => { if (!resolved) { setTimeout(highlight); @@ -43,15 +40,9 @@ export default function HiddenIssueLabel(props: Props): ReactPortal { } }, [resolved]); - useEffect(() => { - issue.initComments().then(() => { - setFirstComments((issue.comments as Comment[])[0]); - }); - }, []); - const elementID = `cvat-hidden-issue-label-${id}`; return ReactDOM.createPortal( - + )} - {firstComment ? firstComment.message : } + {comments[0]?.message || } , window.document.getElementById('cvat_canvas_attachment_board') as HTMLElement, diff --git a/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx b/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx index d3121a1ff2cb..de0153f06f0a 100644 --- a/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx +++ b/cvat-ui/src/components/annotation-page/review/issue-dialog.tsx @@ -4,10 +4,7 @@ // SPDX-License-Identifier: MIT import React, { - useState, - useEffect, - useRef, - useCallback, + useState, useEffect, useRef, useCallback, } from 'react'; import ReactDOM from 'react-dom'; import { useDispatch } from 'react-redux'; @@ -44,7 +41,6 @@ interface Props { export default function IssueDialog(props: Props): JSX.Element { const ref = useRef(null); const [currentText, setCurrentText] = useState(''); - const [comments, setComments] = useState([]); const dispatch = useDispatch(); const { issue, @@ -62,7 +58,7 @@ export default function IssueDialog(props: Props): JSX.Element { blur, } = props; - const { id } = issue; + const { id, comments } = issue; useEffect(() => { if (!resolved) { @@ -72,12 +68,6 @@ export default function IssueDialog(props: Props): JSX.Element { } }, [resolved]); - useEffect(() => { - issue.initComments().then(() => { - setComments(issue.comments as CommentModel[]); - }); - }, []); - const onDeleteIssue = useCallback((): void => { Modal.confirm({ title: `The issue${id >= 0 ? ` #${id}` : ''} will be deleted.`, From f69d708687fb0e7e9a1cd1ed97818181ce1ffc2a Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 27 Jan 2023 12:54:59 -0800 Subject: [PATCH 056/140] Simplified sorting --- cvat-core/src/server-proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index df78accfdddd..7c70e596792c 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1353,7 +1353,7 @@ async function getJobIssues(jobID: number) { }, {}); for (const issue of Object.keys(commentsByIssue)) { - commentsByIssue[issue] = [...commentsResponse.results].sort((a, b) => a.id - b.id); + commentsByIssue[issue].sort((a, b) => a.id - b.id); issuesById[issue].comments = commentsByIssue[issue]; } } catch (errorData) { From afdf454df944ffc357d856763a091d003c556d34 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 30 Jan 2023 15:11:03 +0200 Subject: [PATCH 057/140] Fix tests --- cvat/apps/engine/serializers.py | 5 ++++- tests/python/rest_api/test_projects.py | 2 +- tests/python/rest_api/test_tasks.py | 17 ++++++----------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 4146bab01e89..7098c99a081a 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -43,7 +43,10 @@ def get_attribute(self, instance): return instance def to_representation(self, instance): - request = self.context['request'] + request = self.context.get('request') + if not request: + return None + return serializers.Hyperlink( reverse(self.view_name, request=request, query_params=build_field_filter_params( diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 3008000b7867..924852a04e5a 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -607,7 +607,7 @@ def test_can_import_export_annotations_with_rotation(self): self._test_import_project(username, project_id, "CVAT 1.1", import_data) - response = get_method(username, f"/projects/{project_id}/tasks") + response = get_method(username, f"/tasks?project_id={project_id}") assert response.status_code == HTTPStatus.OK tasks = response.json()["results"] diff --git a/tests/python/rest_api/test_tasks.py b/tests/python/rest_api/test_tasks.py index 6d01bc634010..a815b0c0e8a8 100644 --- a/tests/python/rest_api/test_tasks.py +++ b/tests/python/rest_api/test_tasks.py @@ -52,13 +52,6 @@ def _test_task_list_200(self, user, project_id, data, exclude_paths="", **kwargs ) assert DeepDiff(data, results, ignore_order=True, exclude_paths=exclude_paths) == {} - def _test_task_list_403(self, user, project_id, **kwargs): - with make_api_client(user) as api_client: - (_, response) = api_client.tasks_api.list( - project_id=str(project_id), **kwargs, _parse_response=False, _check_status=False - ) - assert response.status == HTTPStatus.FORBIDDEN - def _test_users_to_see_task_list( self, project_id, tasks, users, is_staff, is_allow, is_project_staff, **kwargs ): @@ -69,10 +62,12 @@ def _test_users_to_see_task_list( assert len(users) for user in users: - if is_allow: - self._test_task_list_200(user["username"], project_id, tasks, **kwargs) - else: - self._test_task_list_403(user["username"], project_id, **kwargs) + if not is_allow: + # Users outside project or org should not know if one exists. + # Thus, no error should be produced on a list request. + tasks = [] + + self._test_task_list_200(user["username"], project_id, tasks, **kwargs) def _test_assigned_users_to_see_task_data(self, tasks, users, is_task_staff, **kwargs): for task in tasks: From cfcfec55892bf8ae161b4af06d58ac92a6acc197 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 30 Jan 2023 15:59:03 +0200 Subject: [PATCH 058/140] Fix merge --- cvat/apps/engine/views.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 3d4ed3b37d90..2df30b6357aa 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -841,17 +841,6 @@ def perform_create(self, serializer, **kwargs): db_project.save() assert serializer.instance.organization == db_project.organization - def perform_destroy(self, instance): - task_dirname = instance.get_dirname() - super().perform_destroy(instance) - shutil.rmtree(task_dirname, ignore_errors=True) - if instance.data and not instance.data.tasks.all(): - shutil.rmtree(instance.data.get_data_dirname(), ignore_errors=True) - instance.data.delete() - if instance.project: - db_project = instance.project - db_project.save() - # UploadMixin method def get_upload_dir(self): if 'annotations' in self.action: From b489edcaea0cbb128f7c3d1037acf73f0b113c07 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 30 Jan 2023 16:01:46 +0200 Subject: [PATCH 059/140] Fix linter issues --- cvat/apps/engine/view_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index ed0652ac3525..a54b146844f4 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -12,7 +12,6 @@ from django.utils.http import urlencode from rest_framework.response import Response from rest_framework.reverse import reverse as _reverse -from rest_framework.utils.urls import remove_query_param from rest_framework.serializers import Serializer from rest_framework.viewsets import GenericViewSet From 2539dc436696091deaf3e4741662a0ad455e8915 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 30 Jan 2023 16:10:33 +0200 Subject: [PATCH 060/140] Fix test --- cvat/apps/engine/tests/test_rest_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index dd34ae6661d6..f509300b516d 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -2976,6 +2976,7 @@ def _run_api_v2_tasks_id_export_import(self, user): "data", "source_storage", "target_storage", + "jobs", ), ) From f7c1cdcea612be4102c9fec8acaade544e2ef954 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 30 Jan 2023 22:05:55 +0200 Subject: [PATCH 061/140] Fix pytorch adapter --- cvat-sdk/cvat_sdk/pytorch/caching.py | 90 ++++++++++++++++++-- cvat-sdk/cvat_sdk/pytorch/project_dataset.py | 2 +- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/cvat-sdk/cvat_sdk/pytorch/caching.py b/cvat-sdk/cvat_sdk/pytorch/caching.py index 47f46d759e93..aba0e4b9a6e5 100644 --- a/cvat-sdk/cvat_sdk/pytorch/caching.py +++ b/cvat-sdk/cvat_sdk/pytorch/caching.py @@ -8,7 +8,9 @@ from abc import ABCMeta, abstractmethod from enum import Enum, auto from pathlib import Path -from typing import Callable, Mapping, Type, TypeVar +from typing import Any, Callable, Dict, List, Mapping, Type, TypeVar, Union, cast + +from attrs import define import cvat_sdk.models as models from cvat_sdk.api_client.model_utils import OpenApiModel, to_json @@ -37,7 +39,21 @@ class UpdatePolicy(Enum): """ -_ModelType = TypeVar("_ModelType", bound=OpenApiModel) +_CacheObject = Dict[str, Any] + + +class _CacheObjectModel(metaclass=ABCMeta): + @abstractmethod + def dump(self) -> _CacheObject: + ... + + @classmethod + @abstractmethod + def load(cls, obj: _CacheObject): + ... + + +_ModelType = TypeVar("_ModelType", bound=Union[OpenApiModel, _CacheObjectModel]) class CacheManager(metaclass=ABCMeta): @@ -67,15 +83,37 @@ def project_dir(self, project_id: int) -> Path: def project_json_path(self, project_id: int) -> Path: return self.project_dir(project_id) / "project.json" - def load_model(self, path: Path, model_type: Type[_ModelType]) -> _ModelType: + def _load_object(self, path: Path) -> _CacheObject: with open(path, "rb") as f: - return model_type._new_from_openapi_data(**json.load(f)) + return json.load(f) - def save_model(self, path: Path, model: OpenApiModel) -> None: + def _save_object(self, path: Path, obj: _CacheObject) -> None: with atomic_writer(path, "w", encoding="UTF-8") as f: - json.dump(to_json(model), f, indent=4) + json.dump(obj, f, indent=4) print(file=f) # add final newline + def _deserialize_model(self, obj: _CacheObject, model_type: _ModelType) -> _ModelType: + if issubclass(model_type, OpenApiModel): + return cast(OpenApiModel, model_type)._new_from_openapi_data(**obj) + elif issubclass(model_type, _CacheObjectModel): + return cast(_CacheObjectModel, model_type).load(obj) + else: + raise NotImplementedError("Unexpected model type") + + def _serialize_model(self, model: _ModelType) -> _CacheObject: + if isinstance(model, OpenApiModel): + return to_json(model) + elif isinstance(model, _CacheObjectModel): + return model.dump() + else: + raise NotImplementedError("Unexpected model type") + + def load_model(self, path: Path, model_type: Type[_ModelType]) -> _ModelType: + return self._deserialize_model(self._load_object(path), model_type) + + def save_model(self, path: Path, model: _ModelType) -> None: + return self._save_object(path, self._serialize_model(model)) + @abstractmethod def retrieve_task(self, task_id: int) -> Task: ... @@ -178,7 +216,7 @@ def retrieve_project(self, project_id: int) -> Project: # There are currently no files cached alongside project.json, # so we don't need to check if we need to purge them. - self.save_model(project_json_path, project._model) + self.save_model(project_json_path, _OfflineProjectModel.from_entity(project)) return project @@ -207,10 +245,44 @@ def ensure_chunk(self, task: Task, chunk_index: int) -> None: def retrieve_project(self, project_id: int) -> Project: self._logger.info(f"Retrieving project {project_id} from cache...") - return Project( - self._client, self.load_model(self.project_json_path(project_id), models.ProjectRead) + cached_model = self.load_model(self.project_json_path(project_id), _OfflineProjectModel) + return _OfflineProjectProxy(self._client, cached_model, cache_manager=self) + + +@define +class _OfflineProjectModel(_CacheObjectModel): + api_model: models.IProjectRead + task_ids: List[int] + + def dump(self) -> _CacheObject: + return { + "model": to_json(self.api_model), + "tasks": self.task_ids, + } + + @classmethod + def load(cls, obj: _CacheObject): + return cls( + api_model=obj["model"], + task_ids=obj["tasks"], ) + @classmethod + def from_entity(cls, entity: Project): + return cls(api_model=entity._model, task_ids=[t.id for t in entity.get_tasks()]) + + +class _OfflineProjectProxy(Project): + def __init__( + self, client: Client, cached_model: _OfflineProjectModel, *, cache_manager: CacheManager + ) -> None: + super().__init__(client, cached_model.api_model) + self._offline_model = cached_model + self._cache_manager = cache_manager + + def get_tasks(self) -> List[Task]: + return [self._cache_manager.retrieve_task(t) for t in self._offline_model.task_ids] + _CACHE_MANAGER_CLASSES: Mapping[UpdatePolicy, Type[CacheManager]] = { UpdatePolicy.IF_MISSING_OR_STALE: _CacheManagerOnline, diff --git a/cvat-sdk/cvat_sdk/pytorch/project_dataset.py b/cvat-sdk/cvat_sdk/pytorch/project_dataset.py index 421a17ff6421..be834b1cedd9 100644 --- a/cvat-sdk/cvat_sdk/pytorch/project_dataset.py +++ b/cvat-sdk/cvat_sdk/pytorch/project_dataset.py @@ -79,7 +79,7 @@ def __init__( ) self._logger.info("Fetching project tasks...") - tasks = [cache_manager.retrieve_task(task_id) for task_id in project.tasks] + tasks = project.get_tasks() if task_filter is not None: tasks = list(filter(task_filter, tasks)) From c7c6cf7926ac83fa453ca8dc927183ede55b019c Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 30 Jan 2023 22:06:06 +0200 Subject: [PATCH 062/140] Update changelog --- CHANGELOG.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 734f90d9fc3c..32c4ebc23a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,20 +33,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 The corresponding arguments are keyword-only now. () - \[Server API\] Added missing pagination or pagination parameters in - `/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`, - `/jobs/{id}/commits`, `/issues/{id}/comments`, `/organizations` + `/jobs/{id}/commits`, `/organizations` () - Windows Installation Instructions adjusted to work around - The contour detection function for semantic segmentation () - Delete newline character when generating a webhook signature () ### Deprecated -- \[Server API\] Endpoints with collections are deprecated in favor of their full variants - `/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`, `/issues/{id}/comments` - () +- TBD ### Removed -- TDB +- \[Server API\] Endpoints with collections are removed in favor of their full variants + `/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`, `/issues/{id}/comments`. + Corresponding fields are added or changed to provide a link to the child collection + in `/projects/{id}`, `/tasks/{id}`, `/jobs/{id}`, `/issues/{id}` + () ### Fixed - Helm: Empty password for Redis () From e49b578d269a402e02036c1c38b2f97e08c26e1e Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 31 Jan 2023 11:34:21 +0200 Subject: [PATCH 063/140] Fix test --- tests/python/rest_api/test_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index 924852a04e5a..fcab5ff45650 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -607,7 +607,7 @@ def test_can_import_export_annotations_with_rotation(self): self._test_import_project(username, project_id, "CVAT 1.1", import_data) - response = get_method(username, f"/tasks?project_id={project_id}") + response = get_method(username, f"/tasks", project_id=project_id) assert response.status_code == HTTPStatus.OK tasks = response.json()["results"] From 327fc9fc896d603a26ec45f3348e8638ac8fb852 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 31 Jan 2023 13:01:53 +0200 Subject: [PATCH 064/140] Fix ui problem with jobs --- cvat-core/src/session.ts | 51 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 1a4494ab32ec..7b86f4b29508 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -564,6 +564,7 @@ export class Task extends Session { data_chunk_size: undefined, data_compressed_chunk_type: undefined, data_original_chunk_type: undefined, + deleted_frames: undefined, use_zip_chunks: undefined, use_cache: undefined, copy_data: undefined, @@ -598,29 +599,33 @@ export class Task extends Session { .map((labelData) => new Label(labelData)).filter((label) => !label.hasParent); } - if (Array.isArray(initialData.jobs)) { - for (const job of initialData.jobs) { - const jobInstance = new Job({ - url: job.url, - id: job.id, - assignee: job.assignee, - state: job.state, - stage: job.stage, - start_frame: job.start_frame, - stop_frame: job.stop_frame, - // following fields also returned when doing API request /jobs/ - // here we know them from task and append to constructor - task_id: data.id, - project_id: data.project_id, - labels: data.labels, - bug_tracker: data.bug_tracker, - mode: data.mode, - dimension: data.dimension, - data_compressed_chunk_type: data.data_compressed_chunk_type, - data_chunk_size: data.data_chunk_size, - }); - - data.jobs.push(jobInstance); + if (Array.isArray(initialData.segments)) { + for (const segment of initialData.segments) { + if (Array.isArray(segment.jobs)) { + for (const job of segment.jobs) { + const jobInstance = new Job({ + url: job.url, + id: job.id, + assignee: job.assignee, + state: job.state, + stage: job.stage, + start_frame: segment.start_frame, + stop_frame: segment.stop_frame, + // following fields also returned when doing API request /jobs/ + // here we know them from task and append to constructor + task_id: data.id, + project_id: data.project_id, + labels: data.labels, + bug_tracker: data.bug_tracker, + mode: data.mode, + dimension: data.dimension, + data_compressed_chunk_type: data.data_compressed_chunk_type, + data_chunk_size: data.data_chunk_size, + }); + + data.jobs.push(jobInstance); + } + } } } From a379b2b6d25139aabf5687341a8de52fec8a0f2b Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 1 Feb 2023 10:13:41 +0200 Subject: [PATCH 065/140] Fix some tests --- cvat-core/src/common.ts | 8 +++--- cvat-core/src/server-proxy.ts | 2 +- cvat-core/tests/api/jobs.js | 2 +- cvat-core/tests/api/projects.js | 2 -- cvat-core/tests/mocks/server-proxy.mock.js | 32 ++++++++++++++++++++-- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/cvat-core/src/common.ts b/cvat-core/src/common.ts index 10eed7cf0002..ab07c5b96ca5 100644 --- a/cvat-core/src/common.ts +++ b/cvat-core/src/common.ts @@ -51,10 +51,10 @@ export function checkExclusiveFields(obj, exclusive, ignore): void { exclusive: [], other: [], }; - for (const field in Object.keys(obj)) { - if (!(field in ignore)) { - if (field in exclusive) { - if (fields.other.length) { + for (const field in obj) { + if (!(ignore.includes(field))) { + if (exclusive.includes(field)) { + if (fields.other.length || fields.exclusive.length) { throw new ArgumentError(`Do not use the filter field "${field}" with others`); } fields.exclusive.push(field); diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 601356475c89..83f0948bc921 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1322,7 +1322,7 @@ async function getJobs(filter = {}, aggregate = false) { throw generateError(errorData); } - return response.data; + return response.results; } async function getJobIssues(jobID: number) { diff --git a/cvat-core/tests/api/jobs.js b/cvat-core/tests/api/jobs.js index 7d457f172828..b8a9197f55b6 100644 --- a/cvat-core/tests/api/jobs.js +++ b/cvat-core/tests/api/jobs.js @@ -51,7 +51,7 @@ describe('Feature: get a list of jobs', () => { test('get jobs by an unknown job id', async () => { const result = await window.cvat.jobs.get({ - taskID: 50, + jobID: 50, }); expect(Array.isArray(result)).toBeTruthy(); expect(result).toHaveLength(0); diff --git a/cvat-core/tests/api/projects.js b/cvat-core/tests/api/projects.js index da5d61d1204e..95fa5692baaf 100644 --- a/cvat-core/tests/api/projects.js +++ b/cvat-core/tests/api/projects.js @@ -35,8 +35,6 @@ describe('Feature: get projects', () => { expect(result).toHaveLength(1); expect(result[0]).toBeInstanceOf(Project); expect(result[0].id).toBe(2); - // eslint-disable-next-line no-underscore-dangle - expect(result[0]._internalData.task_ids).toHaveLength(1); }); test('get a project by an unknown id', async () => { diff --git a/cvat-core/tests/mocks/server-proxy.mock.js b/cvat-core/tests/mocks/server-proxy.mock.js index 9491f9c0bcf6..1f9fb8ca90cb 100644 --- a/cvat-core/tests/mocks/server-proxy.mock.js +++ b/cvat-core/tests/mocks/server-proxy.mock.js @@ -215,6 +215,24 @@ class ServerProxy { } async function getJobs(filter = {}) { + function make_json_filter(json_expr) { + if (!json_expr) { + return (job) => true; + } + + // This function only covers test cases. Extend it if needed. + function escapeRegExp(string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + let pattern = JSON.stringify({ + and: [{ '==': [{ var: 'task_id' }, ''] }] + }); + pattern = escapeRegExp(pattern).replace('""', '(\\d+)'); + const matches = json_expr.match(pattern); + const task_id = Number.parseInt(matches[1]); + return (job) => job.task_id === task_id; + }; + const id = filter.id || null; const jobs = tasksDummyData.results .reduce((acc, task) => { @@ -237,10 +255,18 @@ class ServerProxy { return acc; }, []) - .filter((job) => job.id === id); + .filter(make_json_filter(filter.filter || null)); + + if (id !== null) { + // A specific object is requested + return jobs.filter((job) => job.id === id)[0] || null; + } return ( - jobs[0] || { + jobs ? { + results: jobs, + count: jobs.length, + } : { detail: 'Not found.', } ); @@ -510,6 +536,7 @@ class ServerProxy { save: saveTask, create: createTask, delete: deleteTask, + getPreview: getPreview, }), writable: false, }, @@ -518,6 +545,7 @@ class ServerProxy { value: Object.freeze({ get: getJobs, save: saveJob, + getPreview: getPreview, }), writable: false, }, From dfea00dbc6f90986460b7a7f622ee04e15eb0db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cklakhov=E2=80=9D?= Date: Wed, 1 Feb 2023 12:11:58 +0300 Subject: [PATCH 066/140] Fixed jest tests --- cvat-core/src/session-implementation.ts | 6 +- cvat-core/src/session.ts | 50 +- cvat-core/tests/mocks/dummy-data.mock.js | 604 +++++++++------------ cvat-core/tests/mocks/server-proxy.mock.js | 52 +- 4 files changed, 315 insertions(+), 397 deletions(-) diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index b582116ac81e..ef4b301207fa 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -419,8 +419,12 @@ export function implementTask(Task) { } const data = await serverProxy.tasks.save(this.id, taskData); + // Temporary workaround for UI + const jobs = await serverProxy.jobs.get({ + filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, data.id] }] }), + }, true); this._updateTrigger.reset(); - return new Task(data); + return new Task({ ...data, jobs: jobs.results }); } const taskSpec: any = { diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 7b86f4b29508..b13451f7c369 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -599,33 +599,29 @@ export class Task extends Session { .map((labelData) => new Label(labelData)).filter((label) => !label.hasParent); } - if (Array.isArray(initialData.segments)) { - for (const segment of initialData.segments) { - if (Array.isArray(segment.jobs)) { - for (const job of segment.jobs) { - const jobInstance = new Job({ - url: job.url, - id: job.id, - assignee: job.assignee, - state: job.state, - stage: job.stage, - start_frame: segment.start_frame, - stop_frame: segment.stop_frame, - // following fields also returned when doing API request /jobs/ - // here we know them from task and append to constructor - task_id: data.id, - project_id: data.project_id, - labels: data.labels, - bug_tracker: data.bug_tracker, - mode: data.mode, - dimension: data.dimension, - data_compressed_chunk_type: data.data_compressed_chunk_type, - data_chunk_size: data.data_chunk_size, - }); - - data.jobs.push(jobInstance); - } - } + if (Array.isArray(initialData.jobs)) { + for (const job of initialData.jobs) { + const jobInstance = new Job({ + url: job.url, + id: job.id, + assignee: job.assignee, + state: job.state, + stage: job.stage, + start_frame: job.start_frame, + stop_frame: job.stop_frame, + // following fields also returned when doing API request /jobs/ + // here we know them from task and append to constructor + task_id: data.id, + project_id: data.project_id, + labels: data.labels, + bug_tracker: data.bug_tracker, + mode: data.mode, + dimension: data.dimension, + data_compressed_chunk_type: data.data_compressed_chunk_type, + data_chunk_size: data.data_chunk_size, + }); + + data.jobs.push(jobInstance); } } diff --git a/cvat-core/tests/mocks/dummy-data.mock.js b/cvat-core/tests/mocks/dummy-data.mock.js index 0bc7642d9533..7198da19b5fd 100644 --- a/cvat-core/tests/mocks/dummy-data.mock.js +++ b/cvat-core/tests/mocks/dummy-data.mock.js @@ -233,78 +233,7 @@ const projectsDummyData = { ], }, ], - segments: [ - { - start_frame: 0, - stop_frame: 99, - jobs: [ - { - url: 'http://192.168.0.139:7000/api/jobs/1', - id: 1, - assignee: null, - status: 'completed', - stage: 'acceptance', - state: 'completed', - }, - ], - }, - { - start_frame: 95, - stop_frame: 194, - jobs: [ - { - url: 'http://192.168.0.139:7000/api/jobs/2', - id: 2, - assignee: null, - status: 'completed', - stage: 'acceptance', - state: 'completed', - }, - ], - }, - { - start_frame: 190, - stop_frame: 289, - jobs: [ - { - url: 'http://192.168.0.139:7000/api/jobs/3', - id: 3, - assignee: null, - status: 'completed', - stage: 'acceptance', - state: 'completed', - }, - ], - }, - { - start_frame: 285, - stop_frame: 384, - jobs: [ - { - url: 'http://192.168.0.139:7000/api/jobs/4', - id: 4, - assignee: null, - status: 'completed', - stage: 'acceptance', - state: 'completed', - }, - ], - }, - { - start_frame: 380, - stop_frame: 431, - jobs: [ - { - url: 'http://192.168.0.139:7000/api/jobs/5', - id: 5, - assignee: null, - status: 'completed', - stage: 'acceptance', - state: 'completed', - }, - ], - }, - ], + jobs: "http://localhost:7000/api/jobs?task_id=2", data_chunk_size: 36, data_compressed_chunk_type: 'imageset', data_original_chunk_type: 'video', @@ -360,22 +289,7 @@ const tasksDummyData = { attributes: [], }, ], - segments: [ - { - start_frame: 0, - stop_frame: 0, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/112', - id: 112, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - ], + jobs: "http://localhost:7000/api/jobs?task_id=102", image_quality: 50, start_frame: 0, stop_frame: 0, @@ -414,22 +328,7 @@ const tasksDummyData = { attributes: [], }, ], - segments: [ - { - start_frame: 0, - stop_frame: 8, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/100', - id: 100, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - ], + jobs: "http://localhost:7000/api/jobs?task_id=100", image_quality: 50, start_frame: 0, stop_frame: 0, @@ -622,162 +521,7 @@ const tasksDummyData = { attributes: [], }, ], - segments: [ - { - start_frame: 0, - stop_frame: 499, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/10', - id: 101, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 495, - stop_frame: 994, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/11', - id: 102, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 990, - stop_frame: 1489, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/12', - id: 103, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 1485, - stop_frame: 1984, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/13', - id: 104, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 1980, - stop_frame: 2479, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/14', - id: 105, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 2475, - stop_frame: 2974, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/15', - id: 106, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 2970, - stop_frame: 3469, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/16', - id: 107, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 3465, - stop_frame: 3964, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/17', - id: 108, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 3960, - stop_frame: 4459, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/18', - id: 109, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 4455, - stop_frame: 4954, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/19', - id: 110, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 4950, - stop_frame: 5001, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/20', - id: 111, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - ], + jobs: "http://localhost:7000/api/jobs?task_id=101", image_quality: 50, start_frame: 0, stop_frame: 5001, @@ -858,18 +602,7 @@ const tasksDummyData = { ` }], - segments: [{ - start_frame: 0, - stop_frame: 3, - jobs: [{ - url: 'http://localhost:7000/api/jobs/40', - id: 40, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }] - }], + jobs: "http://localhost:7000/api/jobs?task_id=40", data_chunk_size: 17, data_compressed_chunk_type: 'imageset', data_original_chunk_type: 'imageset', @@ -1069,36 +802,7 @@ const tasksDummyData = { attributes: [], }, ], - segments: [ - { - start_frame: 0, - stop_frame: 4999, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/3', - id: 3, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - { - start_frame: 4995, - stop_frame: 5001, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/4', - id: 4, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - ], + jobs: "http://localhost:7000/api/jobs?task_id=3", image_quality: 50, }, { @@ -1289,22 +993,7 @@ const tasksDummyData = { attributes: [], }, ], - segments: [ - { - start_frame: 0, - stop_frame: 74, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/2', - id: 2, - assignee: null, - status: 'annotation', - stage: 'annotation', - state: 'new', - }, - ], - }, - ], + jobs: "http://localhost:7000/api/jobs?task_id=2", image_quality: 50, }, { @@ -1494,27 +1183,273 @@ const tasksDummyData = { attributes: [], }, ], - segments: [ - { - start_frame: 0, - stop_frame: 8, - jobs: [ - { - url: 'http://localhost:7000/api/jobs/1', - id: 1, - assignee: null, - status: 'annotation', - stage: "annotation", - state: "new", - }, - ], - }, - ], + jobs: "http://localhost:7000/api/jobs?task_id=1", image_quality: 95, }, ], }; +const jobsDummyData = { + count: 2, + next: null, + previous: null, + results: [ + { + url: 'http://localhost:7000/api/jobs/112', + id: 112, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 0, + stop_frame: 0, + task_id: 102, + }, + { + url: 'http://localhost:7000/api/jobs/100', + id: 100, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 0, + stop_frame: 8, + task_id: 100, + }, + { + url: 'http://localhost:7000/api/jobs/40', + id: 40, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 0, + stop_frame: 3, + task_id: 40, + }, + { + url: 'http://localhost:7000/api/jobs/20', + id: 111, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 4950, + stop_frame: 5001, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/19', + id: 110, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 4455, + stop_frame: 4954, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/18', + id: 109, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 3960, + stop_frame: 4459, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/17', + id: 108, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 3465, + stop_frame: 3964, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/16', + id: 107, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 2970, + stop_frame: 3469, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/15', + id: 106, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 2475, + stop_frame: 2974, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/14', + id: 105, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 1980, + stop_frame: 2479, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/13', + id: 104, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 1485, + stop_frame: 1984, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/12', + id: 103, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 990, + stop_frame: 1489, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/11', + id: 102, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 495, + stop_frame: 994, + task_id: 101, + }, + { + url: 'http://localhost:7000/api/jobs/10', + id: 101, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 0, + stop_frame: 499, + task_id: 101, + }, + { + url: 'http://192.168.0.139:7000/api/jobs/9', + id: 9, + assignee: null, + status: 'completed', + stage: 'acceptance', + state: 'completed', + start_frame: 0, + stop_frame: 99, + task_id: 2, + }, + { + url: 'http://192.168.0.139:7000/api/jobs/8', + id: 8, + assignee: null, + status: 'completed', + stage: 'acceptance', + state: 'completed', + start_frame: 95, + stop_frame: 194, + task_id: 2, + }, + { + url: 'http://192.168.0.139:7000/api/jobs/7', + id: 7, + assignee: null, + status: 'completed', + stage: 'acceptance', + state: 'completed', + start_frame: 190, + stop_frame: 289, + task_id: 2, + }, + { + url: 'http://192.168.0.139:7000/api/jobs/6', + id: 6, + assignee: null, + status: 'completed', + stage: 'acceptance', + state: 'completed', + start_frame: 285, + stop_frame: 384, + task_id: 2, + }, + { + url: 'http://192.168.0.139:7000/api/jobs/5', + id: 5, + assignee: null, + status: 'completed', + stage: 'acceptance', + state: 'completed', + start_frame: 380, + stop_frame: 431, + task_id: 2, + }, + { + url: 'http://localhost:7000/api/jobs/4', + id: 4, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 4995, + stop_frame: 5001, + task_id: 3, + }, + { + url: 'http://localhost:7000/api/jobs/3', + id: 3, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 0, + stop_frame: 4999, + task_id: 3, + }, + { + url: 'http://localhost:7000/api/jobs/2', + id: 2, + assignee: null, + status: 'annotation', + stage: 'annotation', + state: 'new', + start_frame: 0, + stop_frame: 74, + task_id: 2, + }, + { + url: 'http://localhost:7000/api/jobs/1', + id: 1, + assignee: null, + status: 'annotation', + stage: "annotation", + state: "new", + start_frame: 0, + stop_frame: 8, + task_id: 1, + }, + ] +} + const taskAnnotationsDummyData = { 112: { version: 21, @@ -3454,4 +3389,5 @@ module.exports = { cloudStoragesDummyData, webhooksDummyData, webhooksEventsDummyData, + jobsDummyData, }; diff --git a/cvat-core/tests/mocks/server-proxy.mock.js b/cvat-core/tests/mocks/server-proxy.mock.js index 1f9fb8ca90cb..3cc25825075f 100644 --- a/cvat-core/tests/mocks/server-proxy.mock.js +++ b/cvat-core/tests/mocks/server-proxy.mock.js @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -15,6 +16,7 @@ const { cloudStoragesDummyData, webhooksDummyData, webhooksEventsDummyData, + jobsDummyData, } = require('./dummy-data.mock'); function QueryStringToJSON(query, ignoreList = []) { @@ -215,8 +217,8 @@ class ServerProxy { } async function getJobs(filter = {}) { - function make_json_filter(json_expr) { - if (!json_expr) { + function makeJsonFilter(jsonExpr) { + if (!jsonExpr) { return (job) => true; } @@ -228,34 +230,23 @@ class ServerProxy { and: [{ '==': [{ var: 'task_id' }, ''] }] }); pattern = escapeRegExp(pattern).replace('""', '(\\d+)'); - const matches = json_expr.match(pattern); + const matches = jsonExpr.match(pattern); const task_id = Number.parseInt(matches[1]); return (job) => job.task_id === task_id; }; const id = filter.id || null; - const jobs = tasksDummyData.results - .reduce((acc, task) => { - for (const segment of task.segments) { - for (const job of segment.jobs) { - const copy = JSON.parse(JSON.stringify(job)); - copy.start_frame = segment.start_frame; - copy.stop_frame = segment.stop_frame; - copy.task_id = task.id; - copy.dimension = task.dimension; - copy.data_compressed_chunk_type = task.data_compressed_chunk_type; - copy.data_chunk_size = task.data_chunk_size; - copy.bug_tracker = task.bug_tracker; - copy.mode = task.mode; - copy.labels = task.labels; - - acc.push(copy); - } - } - - return acc; - }, []) - .filter(make_json_filter(filter.filter || null)); + const jobs = jobsDummyData.results.filter(makeJsonFilter(filter.filter || null)); + + for (const job of jobs) { + const task = tasksDummyData.results.find((task) => task.id === job.task_id); + job.dimension = task.dimension; + job.data_compressed_chunk_type = task.data_compressed_chunk_type; + job.data_chunk_size = task.data_chunk_size; + job.bug_tracker = task.bug_tracker; + job.mode = task.mode; + job.labels = task.labels; + } if (id !== null) { // A specific object is requested @@ -273,16 +264,7 @@ class ServerProxy { } async function saveJob(id, jobData) { - const object = tasksDummyData.results - .reduce((acc, task) => { - for (const segment of task.segments) { - for (const job of segment.jobs) { - acc.push(job); - } - } - - return acc; - }, []) + const object = jobsDummyData.results .filter((job) => job.id === id)[0]; for (const prop in jobData) { From 8448c9859405dc3f31a088dc4aee6920167a402a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cklakhov=E2=80=9D?= Date: Wed, 1 Feb 2023 13:32:10 +0300 Subject: [PATCH 067/140] fixed getJobs method --- cvat-core/src/server-proxy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 83f0948bc921..601356475c89 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1322,7 +1322,7 @@ async function getJobs(filter = {}, aggregate = false) { throw generateError(errorData); } - return response.results; + return response.data; } async function getJobIssues(jobID: number) { From cfae321c35344d8596e4879ee65c2364d5c536b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cklakhov=E2=80=9D?= Date: Wed, 1 Feb 2023 16:01:26 +0300 Subject: [PATCH 068/140] Fixed remaining tests --- cvat-core/src/session-implementation.ts | 6 +++++- .../case_117_paste_labels_from_another_task.js | 2 ++ tests/cypress/support/commands_review_pipeline.js | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index ef4b301207fa..e05b90067d56 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -486,7 +486,11 @@ export function implementTask(Task) { } const task = await serverProxy.tasks.create(taskSpec, taskDataSpec, onUpdate); - return new Task(task); + // Temporary workaround for UI + const jobs = await serverProxy.jobs.get({ + filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, task.id] }] }), + }, true); + return new Task({ ...task, jobs: jobs.results }); }; Task.prototype.delete.implementation = async function () { diff --git a/tests/cypress/integration/actions_tasks/case_117_paste_labels_from_another_task.js b/tests/cypress/integration/actions_tasks/case_117_paste_labels_from_another_task.js index 1d07f8c2ca4b..b17d06f076b7 100644 --- a/tests/cypress/integration/actions_tasks/case_117_paste_labels_from_another_task.js +++ b/tests/cypress/integration/actions_tasks/case_117_paste_labels_from_another_task.js @@ -1,4 +1,5 @@ // Copyright (C) 2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -81,6 +82,7 @@ context('Paste labels from one task to another.', { browser: '!firefox' }, () => }); cy.wait('@patchTaskLabels').its('response.statusCode').should('equal', 200); cy.get('.cvat-modal-confirm-remove-existing-labels').should('not.exist'); + cy.get('.cvat-spinner').should('not.exist'); cy.get('.cvat-raw-labels-viewer').then((raw) => { expect(raw.text()).contain('"id":'); }); diff --git a/tests/cypress/support/commands_review_pipeline.js b/tests/cypress/support/commands_review_pipeline.js index aea1708df484..888d37b3ce05 100644 --- a/tests/cypress/support/commands_review_pipeline.js +++ b/tests/cypress/support/commands_review_pipeline.js @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -152,6 +153,7 @@ Cypress.Commands.add('createIssueFromControlButton', (createIssueParams) => { cy.get('[type="submit"]').click(); }); cy.wait('@issues').its('response.statusCode').should('equal', 201); + cy.get('.cvat-create-issue-dialog').should('not.exist'); cy.checkIssueRegion(); }); From 27c5e16ba6a085d4319b038b785dd38fc23776ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cklakhov=E2=80=9D?= Date: Wed, 1 Feb 2023 17:39:24 +0300 Subject: [PATCH 069/140] progress count by segments, removed jobs fetching --- cvat-core/src/api-implementation.ts | 10 +++++++--- cvat-core/src/session.ts | 19 +++++++++++++++++++ .../src/components/task-page/task-page.tsx | 4 ++-- .../src/components/tasks-page/task-item.tsx | 4 ++-- .../src/containers/tasks-page/task-item.tsx | 2 +- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index efc8d5f869e8..7a77b3f6247c 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -234,9 +234,13 @@ export default function implementAPI(cvat) { const tasksData = await serverProxy.tasks.get(searchParams); const tasks = await Promise.all(tasksData.map(async (taskItem) => { // Temporary workaround for UI - const jobs = await serverProxy.jobs.get({ - filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, taskItem.id] }] }), - }, true); + // Fixme: too much requests on tasks page + let jobs = { results: [] }; + if ('id' in filter) { + jobs = await serverProxy.jobs.get({ + filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, taskItem.id] }] }), + }, true); + } return new Task({ ...taskItem, jobs: jobs.results }); })); diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index b13451f7c369..e776c9c2a207 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -573,6 +573,7 @@ export class Task extends Session { sorting_method: undefined, source_storage: undefined, target_storage: undefined, + progress: undefined, }; const updateTrigger = new FieldUpdateTrigger(); @@ -588,6 +589,21 @@ export class Task extends Session { data.labels = []; data.jobs = []; + + // FIX ME: progress shoud come from server, not from segments + const progress = { + completedJobs: 0, + totalJobs: 0, + }; + if (Array.isArray(initialData.segments)) { + for (const segment of initialData.segments) { + for (const job of segment.jobs) { + progress.totalJobs += 1; + if (job.stage === 'acceptance') progress.completedJobs += 1; + } + } + } + data.progress = progress; data.files = Object.freeze({ server_files: [], client_files: [], @@ -918,6 +934,9 @@ export class Task extends Session { }) ), }, + progress: { + get: () => data.progress, + }, _internalData: { get: () => data, }, diff --git a/cvat-ui/src/components/task-page/task-page.tsx b/cvat-ui/src/components/task-page/task-page.tsx index 92e6640d0d16..b98b4cb1f510 100644 --- a/cvat-ui/src/components/task-page/task-page.tsx +++ b/cvat-ui/src/components/task-page/task-page.tsx @@ -33,9 +33,9 @@ type Props = TaskPageComponentProps & RouteComponentProps<{ id: string }>; class TaskPageComponent extends React.PureComponent { public componentDidMount(): void { - const { task, fetching, getTask } = this.props; + const { fetching, getTask } = this.props; - if (task === null && !fetching) { + if (!fetching) { getTask(); } } diff --git a/cvat-ui/src/components/tasks-page/task-item.tsx b/cvat-ui/src/components/tasks-page/task-item.tsx index 24b6e61714bb..aa5fe17a52a4 100644 --- a/cvat-ui/src/components/tasks-page/task-item.tsx +++ b/cvat-ui/src/components/tasks-page/task-item.tsx @@ -75,8 +75,8 @@ class TaskItemComponent extends React.PureComponent job.stage === 'acceptance').length; + const numOfJobs = taskInstance.progress.totalJobs; + const numOfCompleted = taskInstance.progress.completedJobs; // Progress appearance depends on number of jobs let progressColor = null; diff --git a/cvat-ui/src/containers/tasks-page/task-item.tsx b/cvat-ui/src/containers/tasks-page/task-item.tsx index 220cf0444731..992c5e761458 100644 --- a/cvat-ui/src/containers/tasks-page/task-item.tsx +++ b/cvat-ui/src/containers/tasks-page/task-item.tsx @@ -35,7 +35,7 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { const id = own.taskID; return { - hidden: state.tasks.hideEmpty && task.jobs.length === 0, + hidden: state.tasks.hideEmpty && task.progress.totalJobs === 0, deleted: id in deletes ? deletes[id] === true : false, taskInstance: task, activeInference: state.models.inferences[id] || null, From 7920f29a25a409d66a789b9db40b8982ba7ad318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cklakhov=E2=80=9D?= Date: Wed, 1 Feb 2023 18:32:17 +0300 Subject: [PATCH 070/140] added waiting for task load --- tests/cypress/support/commands.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index 47b8f2c5cd99..d95cf6e70bf2 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -322,6 +322,7 @@ Cypress.Commands.add('pressSplitControl', () => { Cypress.Commands.add('openTaskJob', (taskName, jobID = 0, removeAnnotations = true, expectedFail = false) => { cy.openTask(taskName); + cy.get('.cvat-spinner').should('not.exist'); cy.openJob(jobID, removeAnnotations, expectedFail); }); From c5d00322cbd6d781339e8c4bb87595cfea4ccc7e Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 1 Feb 2023 17:47:20 +0200 Subject: [PATCH 071/140] Add new fields to the server schema --- cvat/apps/engine/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 7098c99a081a..96755cfe54c8 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -24,6 +24,7 @@ from cvat.apps.engine.view_utils import build_field_filter_params, get_list_view_name, reverse +@extend_schema_field(serializers.URLField) class HyperlinkedModelViewSerializer(serializers.Serializer): key_field = 'pk' From 2f80481e11b9a106a4c54ed848e44f19a3da906f Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 2 Feb 2023 12:14:52 +0200 Subject: [PATCH 072/140] Add label endpoints and fields --- cvat/apps/engine/serializers.py | 18 +++++++++++------- cvat/apps/engine/views.py | 6 +++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 05f027cb6400..059dba554f83 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -25,7 +25,7 @@ from cvat.apps.engine.view_utils import build_field_filter_params, get_list_view_name, reverse @extend_schema_field(serializers.URLField) -class HyperlinkedModelViewSerializer(serializers.Serializer): +class HyperlinkedEndpointSerializer(serializers.Serializer): key_field = 'pk' def __init__(self, view_name=None, *, filter_key=None, **kwargs): @@ -227,14 +227,15 @@ class JobReadSerializer(serializers.ModelSerializer): mode = serializers.ReadOnlyField(source='segment.task.mode') bug_tracker = serializers.CharField(max_length=2000, source='get_bug_tracker', allow_null=True, read_only=True) - issues = HyperlinkedModelViewSerializer(models.Issue, filter_key='job_id') + labels = HyperlinkedEndpointSerializer('job-labels') + issues = HyperlinkedEndpointSerializer(models.Issue, filter_key='job_id') class Meta: model = models.Job fields = ('url', 'id', 'task_id', 'project_id', 'assignee', 'dimension', 'bug_tracker', 'status', 'stage', 'state', 'mode', 'start_frame', 'stop_frame', 'data_chunk_size', 'data_compressed_chunk_type', - 'updated_date', 'issues') + 'updated_date', 'issues', 'labels') read_only_fields = fields class JobWriteSerializer(serializers.ModelSerializer): @@ -559,7 +560,8 @@ class TaskReadSerializer(serializers.ModelSerializer): dimension = serializers.CharField(allow_blank=True, required=False) target_storage = StorageSerializer(required=False, allow_null=True) source_storage = StorageSerializer(required=False, allow_null=True) - jobs = HyperlinkedModelViewSerializer(models.Job, filter_key='task_id') + labels = HyperlinkedEndpointSerializer('task-labels') + jobs = HyperlinkedEndpointSerializer(models.Job, filter_key='task_id') class Meta: model = models.Task @@ -567,7 +569,7 @@ class Meta: 'bug_tracker', 'created_date', 'updated_date', 'overlap', 'segment_size', 'status', 'data_chunk_size', 'data_compressed_chunk_type', 'data_original_chunk_type', 'size', 'image_quality', 'data', 'dimension', - 'subset', 'organization', 'target_storage', 'source_storage', 'jobs', + 'subset', 'organization', 'target_storage', 'source_storage', 'jobs', 'labels', ) read_only_fields = fields extra_kwargs = { @@ -801,13 +803,15 @@ class ProjectReadSerializer(serializers.ModelSerializer): dimension = serializers.CharField(max_length=16, required=False, read_only=True, allow_null=True) target_storage = StorageSerializer(required=False, allow_null=True, read_only=True) source_storage = StorageSerializer(required=False, allow_null=True, read_only=True) - tasks = HyperlinkedModelViewSerializer(models.Task, filter_key='project_id') + labels = HyperlinkedEndpointSerializer('project-labels') + tasks = HyperlinkedEndpointSerializer(models.Task, filter_key='project_id') class Meta: model = models.Project fields = ('url', 'id', 'name', 'owner', 'assignee', 'bug_tracker', 'task_subsets', 'created_date', 'updated_date', 'status', 'dimension', 'organization', 'target_storage', 'source_storage', + 'tasks', 'labels', ) read_only_fields = fields extra_kwargs = { 'organization': { 'allow_null': True } } @@ -1132,7 +1136,7 @@ class IssueReadSerializer(serializers.ModelSerializer): position = serializers.ListField( child=serializers.FloatField(), allow_empty=False ) - comments = HyperlinkedModelViewSerializer(models.Comment, filter_key='issue_id') + comments = HyperlinkedEndpointSerializer(models.Comment, filter_key='issue_id') class Meta: model = models.Issue diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index d555ef97cada..0614f45072ee 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -620,7 +620,7 @@ def _get_rq_response(queue, job_id): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_fields=None, search_fields=None, ordering_fields=None, simple_filters=None) def labels(self, request, pk): queryset = self.get_object().get_labels().order_by('id') return make_paginated_response(queryset, @@ -1336,7 +1336,7 @@ def preview(self, request, pk): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_fields=None, search_fields=None, ordering_fields=None, simple_filters=None) def labels(self, request, pk): queryset = self.get_object().get_labels().order_by('id') return make_paginated_response(queryset, @@ -1796,7 +1796,7 @@ def preview(self, request, pk): # Remove regular list() parameters from the swagger schema. # Unset, they would be taken from the enclosing class, which is wrong. # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None) + filter_fields=None, search_fields=None, ordering_fields=None, simple_filters=None) def labels(self, request, pk): queryset = self.get_object().get_labels().order_by('id') return make_paginated_response(queryset, From 6a55cd96ee48d18dfeb251c0ac69ddd33bdc8d69 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 2 Feb 2023 21:12:23 +0200 Subject: [PATCH 073/140] Add label viewset, update collection fields --- cvat/apps/engine/serializers.py | 59 +++++++++++--- cvat/apps/engine/urls.py | 1 + cvat/apps/engine/view_utils.py | 19 +++++ cvat/apps/engine/views.py | 136 ++++++++++++++++++-------------- cvat/apps/webhooks/views.py | 14 +--- 5 files changed, 145 insertions(+), 84 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 059dba554f83..b12c7aaaf790 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: MIT +from inspect import isclass import os import re import shutil @@ -29,10 +30,10 @@ class HyperlinkedEndpointSerializer(serializers.Serializer): key_field = 'pk' def __init__(self, view_name=None, *, filter_key=None, **kwargs): - if issubclass(view_name, models.models.Model): + if isclass(view_name) and issubclass(view_name, models.models.Model): view_name = get_list_view_name(view_name) - else: - assert isinstance(view_name, str) + elif not isinstance(view_name, str): + raise TypeError(view_name) kwargs['read_only'] = True super().__init__(**kwargs) @@ -57,6 +58,42 @@ def to_representation(self, instance): ) +class CollectionSummarySerializer(serializers.Serializer): + count = serializers.ReadOnlyField() + + def __init__(self, model, *, url_filter_key, **kwargs): + super().__init__(**kwargs) + self._collection_key = self.source + self._model = model + self._url_filter_key = url_filter_key + + def bind(self, field_name, parent): + super().bind(field_name, parent) + self._collection_key = self._collection_key or self.source + self._model = self._model or type(self.parent) + + def get_fields(self): + fields = super().get_fields() + fields['url'] = HyperlinkedEndpointSerializer(self._model, filter_key=self._url_filter_key) + fields['count'].source = self._collection_key + '.count' + return fields + + def get_attribute(self, instance): + return instance + + +class LabelsSummarySerializer(CollectionSummarySerializer): + def __init__(self, *, model=models.Label, url_filter_key, source='get_labels', **kwargs): + super().__init__(model=model, url_filter_key=url_filter_key, source=source, **kwargs) + + +class JobsSummarySerializer(CollectionSummarySerializer): + completed = serializers.IntegerField(source='completed_jobs_count') + + def __init__(self, *, model=models.Job, url_filter_key, **kwargs): + super().__init__(model=model, url_filter_key=url_filter_key, **kwargs) + + class BasicUserSerializer(serializers.ModelSerializer): def validate(self, attrs): if hasattr(self, 'initial_data'): @@ -227,8 +264,8 @@ class JobReadSerializer(serializers.ModelSerializer): mode = serializers.ReadOnlyField(source='segment.task.mode') bug_tracker = serializers.CharField(max_length=2000, source='get_bug_tracker', allow_null=True, read_only=True) - labels = HyperlinkedEndpointSerializer('job-labels') - issues = HyperlinkedEndpointSerializer(models.Issue, filter_key='job_id') + labels = LabelsSummarySerializer(url_filter_key='job_id') + issues = CollectionSummarySerializer(models.Issue, url_filter_key='job_id') class Meta: model = models.Job @@ -560,8 +597,8 @@ class TaskReadSerializer(serializers.ModelSerializer): dimension = serializers.CharField(allow_blank=True, required=False) target_storage = StorageSerializer(required=False, allow_null=True) source_storage = StorageSerializer(required=False, allow_null=True) - labels = HyperlinkedEndpointSerializer('task-labels') - jobs = HyperlinkedEndpointSerializer(models.Job, filter_key='task_id') + jobs = JobsSummarySerializer(url_filter_key='task_id', source='segment_set') + labels = LabelsSummarySerializer(url_filter_key='task_id') class Meta: model = models.Task @@ -803,8 +840,8 @@ class ProjectReadSerializer(serializers.ModelSerializer): dimension = serializers.CharField(max_length=16, required=False, read_only=True, allow_null=True) target_storage = StorageSerializer(required=False, allow_null=True, read_only=True) source_storage = StorageSerializer(required=False, allow_null=True, read_only=True) - labels = HyperlinkedEndpointSerializer('project-labels') - tasks = HyperlinkedEndpointSerializer(models.Task, filter_key='project_id') + tasks = CollectionSummarySerializer(models.Task, url_filter_key='project_id') + labels = LabelsSummarySerializer(url_filter_key='project_id') class Meta: model = models.Project @@ -1136,12 +1173,12 @@ class IssueReadSerializer(serializers.ModelSerializer): position = serializers.ListField( child=serializers.FloatField(), allow_empty=False ) - comments = HyperlinkedEndpointSerializer(models.Comment, filter_key='issue_id') + comments = CollectionSummarySerializer(models.Comment, url_filter_key='issue_id') class Meta: model = models.Issue fields = ('id', 'frame', 'position', 'job', 'owner', 'assignee', - 'created_date', 'updated_date', 'resolved') + 'created_date', 'updated_date', 'resolved', 'comments') read_only_fields = fields extra_kwargs = { 'created_date': { 'allow_null': True }, diff --git a/cvat/apps/engine/urls.py b/cvat/apps/engine/urls.py index 67a0ceadeb93..abd3a1582f55 100644 --- a/cvat/apps/engine/urls.py +++ b/cvat/apps/engine/urls.py @@ -20,6 +20,7 @@ router.register('server', views.ServerViewSet, basename='server') router.register('issues', views.IssueViewSet) router.register('comments', views.CommentViewSet) +router.register('labels', views.LabelViewSet) router.register('cloudstorages', views.CloudStorageViewSet) urlpatterns = [ diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index a54b146844f4..28a3b167db0b 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -10,6 +10,7 @@ from django.http.request import HttpRequest from django.http.response import HttpResponse from django.utils.http import urlencode +from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.reverse import reverse as _reverse from rest_framework.serializers import Serializer @@ -84,3 +85,21 @@ def get_list_view_name(model): return '%(model_name)s-list' % { 'model_name': model._meta.object_name.lower() } + +def list_action(serializer_class: Type[Serializer], **kwargs): + params = dict( + detail=True, + methods=["GET"], + serializer_class=serializer_class, + + # Restore the default pagination + pagination_class=GenericViewSet.pagination_class, + + # Remove the regular list() parameters from the swagger schema. + # Unset, they would be taken from the enclosing class, which is wrong. + # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want + filter_fields=None, search_fields=None, ordering_fields=None, simple_filters=None + ) + params.update(kwargs) + + return action(**params) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 0614f45072ee..b7d94bdf6284 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -21,6 +21,7 @@ from django.db import IntegrityError from django.http import HttpResponse, HttpResponseNotFound, HttpResponseBadRequest from django.utils import timezone +import django.db.models as dj_models from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import ( @@ -45,7 +46,7 @@ from cvat.apps.engine.frame_provider import FrameProvider from cvat.apps.engine.media_extractors import get_mime from cvat.apps.engine.models import ( - Job, JobCommit, Task, Project, Issue, Data, + Job, JobCommit, Label, Task, Project, Issue, Data, Comment, StorageMethodChoice, StorageChoice, CloudProviderChoice, Location ) @@ -53,8 +54,8 @@ from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaReadSerializer, DataMetaWriteSerializer, DataSerializer, ExceptionSerializer, - FileInfoSerializer, FrameMetaSerializer, JobReadSerializer, JobWriteSerializer, - LabelSerializer, LabeledDataSerializer, + FileInfoSerializer, FrameMetaSerializer, JobReadSerializer, JobWriteSerializer, LabelSerializer, + LabeledDataSerializer, LogEventSerializer, ProjectReadSerializer, ProjectWriteSerializer, RqStatusSerializer, TaskReadSerializer, TaskWriteSerializer, UserSerializer, PluginsSerializer, IssueReadSerializer, @@ -63,7 +64,7 @@ ProjectFileSerializer, TaskFileSerializer) from utils.dataset_manifest import ImageManifestManager -from cvat.apps.engine.view_utils import make_paginated_response +from cvat.apps.engine.view_utils import list_action, make_paginated_response from cvat.apps.engine.utils import ( av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message ) @@ -613,20 +614,6 @@ def _get_rq_response(queue, job_id): return response - @extend_schema(description="Return a paginated list of labels", - responses=LabelSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=LabelSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None, simple_filters=None) - def labels(self, request, pk): - queryset = self.get_object().get_labels().order_by('id') - return make_paginated_response(queryset, - viewset=self, serializer_type=self.serializer_class, request=request) # from @action - - class DataChunkGetter: def __init__(self, data_type, data_num, data_quality, task_dim): possible_data_type_values = ('chunk', 'frame', 'preview', 'context_image') @@ -740,10 +727,16 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, 'data', 'assignee', 'owner', 'target_storage', 'source_storage' ).prefetch_related( + 'segment_set__job_set', 'segment_set__job_set__assignee', 'label_set__attributespec_set', 'project__label_set__attributespec_set', 'label_set__sublabels__attributespec_set', 'project__label_set__sublabels__attributespec_set' + ).annotate( + completed_jobs_count=dj_models.Count( + 'segment__job', + filter=dj_models.Q(segment__job__state=models.StateChoice.COMPLETED.value) + ) ).all() lookup_fields = { @@ -1239,9 +1232,8 @@ def metadata(self, request, pk): return Response(serializer.data) @extend_schema(summary='Returns a paginated list of frame metadata', - responses=FrameMetaSerializer(many=True)) - @action(detail=True, methods=['GET'], serializer_class=FrameMetaSerializer, - url_path='data/meta/frames') + responses=FrameMetaSerializer(many=True)) # Duplicate to still get 'list' op. name + @list_action(serializer_class=FrameMetaSerializer, url_path='data/meta/frames') def metadata_frames(self, request, pk): self.get_object() #force to call check_object_permissions db_task = models.Task.objects.prefetch_related( @@ -1259,7 +1251,7 @@ def metadata_frames(self, request, pk): 'width': item.width, 'height': item.height, 'name': item.path, - 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0 + 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0, } for item in media] serializer = FrameMetaSerializer(frame_meta, many=True) @@ -1329,20 +1321,6 @@ def preview(self, request, pk): return data_getter(request, self._object.data.start_frame, self._object.data.stop_frame, self._object.data) - @extend_schema(description="Return a paginated list of labels", - responses=LabelSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=LabelSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None, simple_filters=None) - def labels(self, request, pk): - queryset = self.get_object().get_labels().order_by('id') - return make_paginated_response(queryset, - viewset=self, serializer_type=self.serializer_class, request=request) # from @action - - @extend_schema(tags=['jobs']) @extend_schema_view( retrieve=extend_schema( @@ -1362,7 +1340,6 @@ def labels(self, request, pk): '200': JobReadSerializer, # check JobWriteSerializer.to_representation }) ) - class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin ): @@ -1720,9 +1697,8 @@ def metadata(self, request, pk): return Response(serializer.data) @extend_schema(summary='Returns a paginated list of frame metadata', - responses=FrameMetaSerializer(many=True)) - @action(detail=True, methods=['GET'], serializer_class=FrameMetaSerializer, - url_path='data/meta/frames') + responses=FrameMetaSerializer(many=True)) # Duplicate to still get 'list' op. name + @list_action(serializer_class=FrameMetaSerializer, url_path='data/meta/frames') def metadata_frames(self, request, pk): self.get_object() #force to call check_object_permissions db_job = models.Job.objects.prefetch_related( @@ -1759,13 +1735,7 @@ def metadata_frames(self, request, pk): @extend_schema(summary='The action returns the list of tracked changes for the job', responses=JobCommitSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=JobCommitSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # These non-root list endpoints do not suppose extra options, just the basic output - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None) + @list_action(serializer_class=JobCommitSerializer) def commits(self, request, pk): self.get_object() # force call of check_object_permissions() return make_paginated_response(JobCommit.objects.filter(job_id=pk).order_by('-id'), @@ -1789,20 +1759,6 @@ def preview(self, request, pk): return data_getter(request, self._object.segment.start_frame, self._object.segment.stop_frame, self._object.segment.task.data) - @extend_schema(description="Return a paginated list of labels", - responses=LabelSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=LabelSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, search_fields=None, ordering_fields=None, simple_filters=None) - def labels(self, request, pk): - queryset = self.get_object().get_labels().order_by('id') - return make_paginated_response(queryset, - viewset=self, serializer_type=self.serializer_class, request=request) # from @action - - @extend_schema(tags=['issues']) @extend_schema_view( retrieve=extend_schema( @@ -1940,6 +1896,64 @@ def get_serializer_class(self): def perform_create(self, serializer, **kwargs): super().perform_create(serializer, owner=self.request.user) + +@extend_schema(tags=['labels']) +@extend_schema_view( + retrieve=extend_schema( + summary='Method returns details of an label', + responses={ + '200': LabelSerializer, + }), + list=extend_schema( + summary='Method returns a paginated list of labels', + responses={ + '200': LabelSerializer(many=True), + }), + partial_update=extend_schema( + summary='Methods does a partial update of chosen fields in an label', + request=LabelSerializer(partial=True), + responses={ + '200': LabelSerializer, + }), + create=extend_schema( + summary='Method creates an label', + request=LabelSerializer, + responses={ + '201': LabelSerializer, + }), + destroy=extend_schema( + summary='Method deletes an label', + responses={ + '204': OpenApiResponse(description='The label has been deleted'), + }) +) +class LabelViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, + mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, + PartialUpdateModelMixin +): + queryset = Label.objects.prefetch_related('task', 'project').all() + + iam_organization_field = 'task__organization' + search_fields = ('name', 'parent') + filter_fields = list(search_fields) + ['id', 'job_id', 'task_id', 'project_id', 'parent', 'type'] + simple_filters = list(search_fields) + ['job_id', 'task_id', 'project_id', 'parent', 'type'] + ordering_fields = list(filter_fields) + lookup_fields = { + 'job_id': 'job', + 'task_id': 'task', + } + ordering = '-id' + serializer_class = LabelSerializer + + def get_queryset(self): + queryset = super().get_queryset() + if self.action == 'list': + perm = LabelPermission.create_scope_list(self.request) + queryset = perm.filter(queryset) + + return queryset + + @extend_schema(tags=['users']) @extend_schema_view( list=extend_schema( diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index ad82494bc219..b1aed1e8cb1e 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -14,7 +14,7 @@ from rest_framework.permissions import SAFE_METHODS from rest_framework.response import Response -from cvat.apps.engine.view_utils import make_paginated_response +from cvat.apps.engine.view_utils import list_action, make_paginated_response from cvat.apps.iam.permissions import WebhookPermission from .event_type import AllEvents, OrganizationEvents, ProjectEvents @@ -142,17 +142,7 @@ def events(self, request): many=True ), # Duplicate to still get 'list' op. name ) - @action( - detail=True, - methods=["GET"], - serializer_class=WebhookDeliveryReadSerializer, - pagination_class=viewsets.GenericViewSet.pagination_class, - # These non-root list endpoints do not suppose extra options, just the basic output - # Remove regular list() parameters from the swagger schema. - # Unset, they would be taken from the enclosing class, which is wrong. - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - filter_fields=None, ordering_fields=None, search_fields=None, simple_filters=None, - ) + @list_action(serializer_class=WebhookDeliveryReadSerializer) def deliveries(self, request, pk): self.get_object() # force call of check_object_permissions() queryset = WebhookDelivery.objects.filter(webhook_id=pk).order_by( From 6f932d02013d970b26808f63a448c32d03c4d755 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 3 Feb 2023 20:09:05 +0200 Subject: [PATCH 074/140] Add endpoint for labels --- cvat/apps/engine/serializers.py | 22 +- cvat/apps/engine/views.py | 17 +- cvat/apps/iam/permissions.py | 88 ++- cvat/apps/iam/rules/labels.rego | 99 +++ tests/python/rest_api/test_labels.py | 43 ++ tests/python/sdk/test_jobs.py | 6 +- tests/python/sdk/test_projects.py | 2 +- tests/python/sdk/test_tasks.py | 6 +- tests/python/shared/assets/issues.json | 25 +- tests/python/shared/assets/jobs.json | 570 ++++------------- tests/python/shared/assets/labels.json | 616 +++++++++++++++++++ tests/python/shared/assets/projects.json | 367 ++--------- tests/python/shared/assets/tasks.json | 708 +++------------------- tests/python/shared/assets/users.json | 2 +- tests/python/shared/fixtures/data.py | 6 + tests/python/shared/utils/dump_objects.py | 1 + 16 files changed, 1169 insertions(+), 1409 deletions(-) create mode 100644 cvat/apps/iam/rules/labels.rego create mode 100644 tests/python/rest_api/test_labels.py create mode 100644 tests/python/shared/assets/labels.json diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index b12c7aaaf790..4023b2273e00 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -59,7 +59,7 @@ def to_representation(self, instance): class CollectionSummarySerializer(serializers.Serializer): - count = serializers.ReadOnlyField() + count = serializers.IntegerField() def __init__(self, model, *, url_filter_key, **kwargs): super().__init__(**kwargs) @@ -171,18 +171,34 @@ class Meta: fields = ('id', 'svg',) class LabelSerializer(SublabelSerializer): - deleted = serializers.BooleanField(required=False, help_text='Delete label if value is true from proper Task/Project object') + deleted = serializers.BooleanField(required=False, write_only=True, + help_text='Delete label if value is true from proper Task/Project object') sublabels = SublabelSerializer(many=True, required=False) svg = serializers.CharField(allow_blank=True, required=False) class Meta: model = models.Label - fields = ('id', 'name', 'color', 'attributes', 'deleted', 'type', 'svg', 'sublabels', 'has_parent') + fields = ( + 'id', 'name', 'color', 'attributes', 'deleted', 'type', 'svg', + 'sublabels', 'project_id', 'task_id', 'parent_id' + ) + read_only_fields = ('id', 'type', 'svg', 'project_id', 'task_id') + extra_kwargs = { + 'project_id': { 'required': False, 'allow_null': False }, + 'task_id': { 'required': False, 'allow_null': False }, + } def to_representation(self, instance): label = super().to_representation(instance) if label['type'] == str(models.LabelType.SKELETON): label['svg'] = instance.skeleton.svg + + # Clean mutually exclusive fields + if not label.get('task_id'): + label.pop('task_id', None) + if not label.get('project_id'): + label.pop('project_id', None) + return label def validate(self, attrs): diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index b7d94bdf6284..d42e60814d26 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -75,7 +75,7 @@ from . import models, task from .log import clogger, slogger from cvat.apps.iam.permissions import (CloudStoragePermission, - CommentPermission, IssuePermission, JobPermission, ProjectPermission, + CommentPermission, IssuePermission, JobPermission, LabelPermission, ProjectPermission, TaskPermission, UserPermission) from cvat.apps.engine.cache import MediaCache @@ -1915,12 +1915,6 @@ def perform_create(self, serializer, **kwargs): responses={ '200': LabelSerializer, }), - create=extend_schema( - summary='Method creates an label', - request=LabelSerializer, - responses={ - '201': LabelSerializer, - }), destroy=extend_schema( summary='Method deletes an label', responses={ @@ -1928,19 +1922,18 @@ def perform_create(self, serializer, **kwargs): }) ) class LabelViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, - mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.DestroyModelMixin, - PartialUpdateModelMixin + mixins.RetrieveModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin ): queryset = Label.objects.prefetch_related('task', 'project').all() iam_organization_field = 'task__organization' search_fields = ('name', 'parent') - filter_fields = list(search_fields) + ['id', 'job_id', 'task_id', 'project_id', 'parent', 'type'] - simple_filters = list(search_fields) + ['job_id', 'task_id', 'project_id', 'parent', 'type'] + filter_fields = list(search_fields) + ['id', 'task_id', 'project_id', 'type', 'color', 'parent_id'] + simple_filters = list(search_fields) + ['task_id', 'project_id', 'type', 'color', 'parent_id'] ordering_fields = list(filter_fields) lookup_fields = { - 'job_id': 'job', 'task_id': 'task', + 'parent': 'parent__name', } ordering = '-id' serializer_class = LabelSerializer diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 58e4caff8025..b4012dea2959 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -19,7 +19,7 @@ from rest_framework.permissions import BasePermission from cvat.apps.organizations.models import Membership, Organization -from cvat.apps.engine.models import Project, Task, Job, Issue +from cvat.apps.engine.models import Label, Project, Task, Job, Issue from cvat.apps.iam.exceptions import LimitsReachedException from cvat.apps.limit_manager.core.limits import (CapabilityContext, LimitManager, Limits, OrgCloudStoragesContext, OrgTasksContext, ProjectWebhooksContext, @@ -1348,6 +1348,92 @@ def get_common_data(db_job): return data +class LabelPermission(OpenPolicyAgentPermission): + obj: Optional[Label] + + class Scopes(StrEnum): + LIST = 'list' + DELETE = 'delete' + UPDATE = 'update' + VIEW = 'view' + + @classmethod + def create(cls, request, view, obj): + Scopes = __class__.Scopes + + permissions = [] + if view.basename == 'labels': + for scope in cls.get_scopes(request, view, obj): + if scope in [Scopes.DELETE, Scopes.UPDATE, Scopes.VIEW]: + obj = cast(Label, obj) + + # Access rights are the same as in the owning objects + if obj.project: + if scope == Scopes.VIEW: + owning_perm_scope = ProjectPermission.Scopes.VIEW + else: + owning_perm_scope = ProjectPermission.Scopes.UPDATE_DESC + + owning_perm = ProjectPermission.create_base_perm( + request, view, scope=owning_perm_scope, obj=obj.project, + ) + else: + if scope == Scopes.VIEW: + owning_perm_scope = TaskPermission.Scopes.VIEW + else: + owning_perm_scope = TaskPermission.Scopes.UPDATE_DESC + + owning_perm = TaskPermission.create_base_perm( + request, view, scope=owning_perm_scope, obj=obj.task, + ) + + permissions.append(owning_perm) + + permissions.append(cls.create_base_perm(request, view, scope, obj)) + + return permissions + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.url = settings.IAM_OPA_DATA_URL + '/labels/allow' + + @staticmethod + def get_scopes(request, view, obj): + Scopes = __class__.Scopes + return [{ + 'list': Scopes.LIST, + 'destroy': Scopes.DELETE, + 'partial_update': Scopes.UPDATE, + 'retrieve': Scopes.VIEW, + }.get(view.action, None)] + + def get_resource(self): + data = None + + if self.obj: + if self.obj.project: + organization = self.obj.project.organization + else: + organization = self.obj.task.organization + + data = { + "id": self.obj.id, + 'organization': { + "id": getattr(organization, 'id', None) + }, + "task": { + "owner": { "id": getattr(self.obj.task.owner, 'id', None) }, + "assignee": { "id": getattr(self.obj.task.assignee, 'id', None) } + } if self.obj.project else None, + "project": { + "owner": { "id": getattr(self.obj.project.owner, 'id', None) }, + "assignee": { "id": getattr(self.obj.project.assignee, 'id', None) } + } if self.obj.project else None, + } + + return data + + class LimitPermission(OpenPolicyAgentPermission): @classmethod def create(cls, request, view, obj): diff --git a/cvat/apps/iam/rules/labels.rego b/cvat/apps/iam/rules/labels.rego new file mode 100644 index 000000000000..0a43564938ab --- /dev/null +++ b/cvat/apps/iam/rules/labels.rego @@ -0,0 +1,99 @@ +package labels + +import future.keywords.if +import future.keywords.in + +import data.utils +import data.organizations + +# input: { +# "scope": <"view"|"list"|"update"|"delete"> or null, +# "auth": { +# "user": { +# "id": , +# "privilege": <"admin"|"business"|"user"|"worker"> or null +# }, +# "organization": { +# "id": , +# "owner": { +# "id": +# }, +# "user": { +# "role": <"owner"|"maintainer"|"supervisor"|"worker"> or null +# } +# } or null, +# }, +# "resource": { +# "id": , +# "owner": { "id": }, +# "organization": { "id": } or null, +# "task": { +# "id": , +# "owner": { "id": }, +# "assignee": { "id": }, +# "organization": { "id": } or null, +# } or null, +# "project": { +# "id": , +# "owner": { "id": }, +# "assignee": { "id": }, +# "organization": { "id": } or null, +# } or null, +# } +# } + +default allow = false + +allow { + utils.is_admin +} + +allow { + input.scope == utils.LIST + utils.is_sandbox +} + +allow { + input.scope == utils.LIST + organizations.is_member +} + +filter = [] { # Django Q object to filter list of entries + utils.is_admin + utils.is_sandbox +} else = qobject { + utils.is_admin + utils.is_organization + org := input.auth.organization + qobject := [ + {"task__organization": org.id}, + {"project__organization": org.id}, "|", + ] +} else = qobject { + utils.is_sandbox + user := input.auth.user + qobject := [ + {"task__owner_id": user.id}, + {"task__assignee_id": user.id}, "|", + {"project__owner_id": user.id}, "|", + {"project__assignee_id": user.id}, "|", + ] +} else = qobject { + utils.is_organization + utils.has_perm(utils.USER) + organizations.has_perm(organizations.MAINTAINER) + org := input.auth.organization + qobject := [ + {"task_organization": org.id}, + {"project__organization": org.id}, "|", + ] +} else = qobject { + organizations.has_perm(organizations.WORKER) + user := input.auth.user + qobject := [ + {"task__owner_id": user.id}, + {"task__assignee_id": user.id}, "|", + {"project__owner_id": user.id}, "|", + {"project__assignee_id": user.id}, "|", + ] +} diff --git a/tests/python/rest_api/test_labels.py b/tests/python/rest_api/test_labels.py new file mode 100644 index 000000000000..64268a52443b --- /dev/null +++ b/tests/python/rest_api/test_labels.py @@ -0,0 +1,43 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from typing import Any, Dict, List, Tuple + +import pytest +from cvat_sdk.api_client.api_client import ApiClient, Endpoint + +from .utils import CollectionSimpleFilterTestBase + + +class TestLabelsListFilters(CollectionSimpleFilterTestBase): + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_class, admin_user, labels): + self.user = admin_user + self.samples = labels + + def _get_endpoint(self, api_client: ApiClient) -> Endpoint: + return api_client.labels_api.list_endpoint + + def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: + if field == "parent": + parent_id, gt_objects = self._get_field_samples("parent_id") + parent_name = self._get_field( + next( + filter( + lambda p: parent_id == self._get_field(p, self._map_field("id")), + self.samples, + ) + ), + self._map_field("name"), + ) + return parent_name, gt_objects + else: + return super()._get_field_samples(field) + + @pytest.mark.parametrize( + "field", + ("name", "parent", "job_id", "task_id", "project_id", "type", "color", "parent_id"), + ) + def test_can_use_simple_filter_for_object_list(self, field): + return super().test_can_use_simple_filter_for_object_list(field) diff --git a/tests/python/sdk/test_jobs.py b/tests/python/sdk/test_jobs.py index d090d5304026..59a811c7fa8f 100644 --- a/tests/python/sdk/test_jobs.py +++ b/tests/python/sdk/test_jobs.py @@ -98,7 +98,7 @@ def test_can_update_job_field_directly(self, fxt_new_task: Task): assert self.stdout.getvalue() == "" def test_can_get_labels(self, fxt_new_task: Task): - expected_labels = {'car', 'person'} + expected_labels = {"car", "person"} received_labels = fxt_new_task.get_jobs()[0].get_labels() @@ -287,9 +287,7 @@ def test_can_update_annotations(self, fxt_task_with_shapes: Task): ], ) ], - tags=[ - models.LabeledImageRequest(frame=0, label_id=labels[0].id) - ], + tags=[models.LabeledImageRequest(frame=0, label_id=labels[0].id)], ) ) diff --git a/tests/python/sdk/test_projects.py b/tests/python/sdk/test_projects.py index 429f3f14d886..74ecf5e69084 100644 --- a/tests/python/sdk/test_projects.py +++ b/tests/python/sdk/test_projects.py @@ -168,7 +168,7 @@ def test_can_get_tasks(self, fxt_project_with_shapes: Project): assert tasks[0].project_id == fxt_project_with_shapes.id def test_can_get_labels(self, fxt_project_with_shapes: Project): - expected_labels = {'car', 'person'} + expected_labels = {"car", "person"} received_labels = fxt_project_with_shapes.get_labels() diff --git a/tests/python/sdk/test_tasks.py b/tests/python/sdk/test_tasks.py index 978a60c4ae88..3a8faeddef45 100644 --- a/tests/python/sdk/test_tasks.py +++ b/tests/python/sdk/test_tasks.py @@ -395,7 +395,7 @@ def counting_do_request(uploader): assert num_requests > 1 def test_can_get_labels(self, fxt_new_task: Task): - expected_labels = {'car', 'person'} + expected_labels = {"car", "person"} received_labels = fxt_new_task.get_labels() @@ -515,9 +515,7 @@ def test_can_update_annotations(self, fxt_task_with_shapes: Task): ], ) ], - tags=[ - models.LabeledImageRequest(frame=0, label_id=labels[0].id) - ], + tags=[models.LabeledImageRequest(frame=0, label_id=labels[0].id)], ) ) diff --git a/tests/python/shared/assets/issues.json b/tests/python/shared/assets/issues.json index 3c9585cad3c1..c924eee2df48 100644 --- a/tests/python/shared/assets/issues.json +++ b/tests/python/shared/assets/issues.json @@ -5,7 +5,10 @@ "results": [ { "assignee": null, - "comments": "http://localhost:8080/api/comments?issue_id=5", + "comments": { + "count": 1, + "url": "http://localhost:8080/api/comments?issue_id=5" + }, "created_date": "2022-03-16T12:49:29.369000Z", "frame": 0, "id": 5, @@ -34,7 +37,10 @@ "url": "http://localhost:8080/api/users/2", "username": "user1" }, - "comments": "http://localhost:8080/api/comments?issue_id=4", + "comments": { + "count": 1, + "url": "http://localhost:8080/api/comments?issue_id=4" + }, "created_date": "2022-03-16T12:40:00.764000Z", "frame": 5, "id": 4, @@ -57,7 +63,10 @@ }, { "assignee": null, - "comments": "http://localhost:8080/api/comments?issue_id=3", + "comments": { + "count": 1, + "url": "http://localhost:8080/api/comments?issue_id=3" + }, "created_date": "2022-03-16T11:08:18.367000Z", "frame": 5, "id": 3, @@ -80,7 +89,10 @@ }, { "assignee": null, - "comments": "http://localhost:8080/api/comments?issue_id=2", + "comments": { + "count": 1, + "url": "http://localhost:8080/api/comments?issue_id=2" + }, "created_date": "2022-03-16T11:07:22.170000Z", "frame": 0, "id": 2, @@ -103,7 +115,10 @@ }, { "assignee": null, - "comments": "http://localhost:8080/api/comments?issue_id=1", + "comments": { + "count": 2, + "url": "http://localhost:8080/api/comments?issue_id=1" + }, "created_date": "2022-03-16T11:04:39.444000Z", "frame": 0, "id": 1, diff --git a/tests/python/shared/assets/jobs.json b/tests/python/shared/assets/jobs.json index 432c1d702087..09c5f3207c07 100644 --- a/tests/python/shared/assets/jobs.json +++ b/tests/python/shared/assets/jobs.json @@ -10,27 +10,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 19, - "issues": "http://localhost:8080/api/issues?job_id=19", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 29, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 30, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=19" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=19" + }, "mode": "interpolation", "project_id": 8, "stage": "annotation", @@ -49,186 +36,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 18, - "issues": "http://localhost:8080/api/issues?job_id=18", - "labels": [ - { - "attributes": [], - "color": "#5c5eba", - "has_parent": false, - "id": 18, - "name": "s1", - "sublabels": [ - { - "attributes": [], - "color": "#d12345", - "has_parent": true, - "id": 19, - "name": "1", - "type": "points" - }, - { - "attributes": [], - "color": "#350dea", - "has_parent": true, - "id": 20, - "name": "2", - "type": "points" - }, - { - "attributes": [], - "color": "#479ffe", - "has_parent": true, - "id": 21, - "name": "3", - "type": "points" - } - ], - "svg": "", - "type": "skeleton" - }, - { - "attributes": [], - "color": "#d12345", - "has_parent": true, - "id": 19, - "name": "1", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#350dea", - "has_parent": true, - "id": 20, - "name": "2", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#479ffe", - "has_parent": true, - "id": 21, - "name": "3", - "sublabels": [], - "type": "points" - }, - { - "attributes": [ - { - "default_value": "white", - "id": 2, - "input_type": "select", - "mutable": false, - "name": "color", - "values": [ - "white", - "black" - ] - } - ], - "color": "#0c81b5", - "has_parent": false, - "id": 22, - "name": "s2", - "sublabels": [ - { - "attributes": [], - "color": "#d53957", - "has_parent": true, - "id": 23, - "name": "1", - "type": "points" - }, - { - "attributes": [], - "color": "#4925ec", - "has_parent": true, - "id": 24, - "name": "2", - "type": "points" - }, - { - "attributes": [ - { - "default_value": "val1", - "id": 3, - "input_type": "select", - "mutable": false, - "name": "attr", - "values": [ - "val1", - "val2" - ] - } - ], - "color": "#59a8fe", - "has_parent": true, - "id": 25, - "name": "3", - "type": "points" - }, - { - "attributes": [], - "color": "#4a649f", - "has_parent": true, - "id": 26, - "name": "4", - "type": "points" - } - ], - "svg": "", - "type": "skeleton" - }, - { - "attributes": [], - "color": "#d53957", - "has_parent": true, - "id": 23, - "name": "1", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#4925ec", - "has_parent": true, - "id": 24, - "name": "2", - "sublabels": [], - "type": "points" - }, - { - "attributes": [ - { - "default_value": "val1", - "id": 3, - "input_type": "select", - "mutable": false, - "name": "attr", - "values": [ - "val1", - "val2" - ] - } - ], - "color": "#59a8fe", - "has_parent": true, - "id": 25, - "name": "3", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#4a649f", - "has_parent": true, - "id": 26, - "name": "4", - "sublabels": [], - "type": "points" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=18" + }, + "labels": { + "count": 9, + "url": "http://localhost:8080/api/labels?job_id=18" + }, "mode": "annotation", "project_id": 5, "stage": "annotation", @@ -247,27 +62,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 17, - "issues": "http://localhost:8080/api/issues?job_id=17", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 16, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 17, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=17" + }, + "labels": { + "count": 14, + "url": "http://localhost:8080/api/labels?job_id=17" + }, "mode": "annotation", "project_id": 4, "stage": "annotation", @@ -292,27 +94,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 16, - "issues": "http://localhost:8080/api/issues?job_id=16", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 7, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 8, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 1, + "url": "http://localhost:8080/api/issues?job_id=16" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=16" + }, "mode": "annotation", "project_id": 2, "stage": "annotation", @@ -331,40 +120,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 14, - "issues": "http://localhost:8080/api/issues?job_id=14", - "labels": [ - { - "attributes": [ - { - "default_value": "mazda", - "id": 1, - "input_type": "select", - "mutable": false, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ] - } - ], - "color": "#2080c0", - "has_parent": false, - "id": 5, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 6, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=14" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=14" + }, "mode": "annotation", "project_id": 1, "stage": "annotation", @@ -383,40 +146,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 13, - "issues": "http://localhost:8080/api/issues?job_id=13", - "labels": [ - { - "attributes": [ - { - "default_value": "mazda", - "id": 1, - "input_type": "select", - "mutable": false, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ] - } - ], - "color": "#2080c0", - "has_parent": false, - "id": 5, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 6, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=13" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=13" + }, "mode": "annotation", "project_id": 1, "stage": "acceptance", @@ -435,40 +172,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 12, - "issues": "http://localhost:8080/api/issues?job_id=12", - "labels": [ - { - "attributes": [ - { - "default_value": "mazda", - "id": 1, - "input_type": "select", - "mutable": false, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ] - } - ], - "color": "#2080c0", - "has_parent": false, - "id": 5, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 6, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=12" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=12" + }, "mode": "annotation", "project_id": 1, "stage": "validation", @@ -493,40 +204,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 11, - "issues": "http://localhost:8080/api/issues?job_id=11", - "labels": [ - { - "attributes": [ - { - "default_value": "mazda", - "id": 1, - "input_type": "select", - "mutable": false, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ] - } - ], - "color": "#2080c0", - "has_parent": false, - "id": 5, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 6, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 1, + "url": "http://localhost:8080/api/issues?job_id=11" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=11" + }, "mode": "annotation", "project_id": 1, "stage": "annotation", @@ -551,27 +236,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 10, - "issues": "http://localhost:8080/api/issues?job_id=10", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 13, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 14, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 1, + "url": "http://localhost:8080/api/issues?job_id=10" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=10" + }, "mode": "annotation", "project_id": null, "stage": "annotation", @@ -590,27 +262,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 9, - "issues": "http://localhost:8080/api/issues?job_id=9", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 11, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 12, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 1, + "url": "http://localhost:8080/api/issues?job_id=9" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=9" + }, "mode": "annotation", "project_id": null, "stage": "annotation", @@ -629,18 +288,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "3d", "id": 8, - "issues": "http://localhost:8080/api/issues?job_id=8", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 10, - "name": "car", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=8" + }, + "labels": { + "count": 1, + "url": "http://localhost:8080/api/labels?job_id=8" + }, "mode": "annotation", "project_id": null, "stage": "annotation", @@ -665,18 +320,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 7, - "issues": "http://localhost:8080/api/issues?job_id=7", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 9, - "name": "car", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 1, + "url": "http://localhost:8080/api/issues?job_id=7" + }, + "labels": { + "count": 1, + "url": "http://localhost:8080/api/labels?job_id=7" + }, "mode": "interpolation", "project_id": null, "stage": "annotation", @@ -701,27 +352,14 @@ "data_compressed_chunk_type": "imageset", "dimension": "2d", "id": 2, - "issues": "http://localhost:8080/api/issues?job_id=2", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 3, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 4, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "issues": { + "count": 0, + "url": "http://localhost:8080/api/issues?job_id=2" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?job_id=2" + }, "mode": "annotation", "project_id": null, "stage": "annotation", diff --git a/tests/python/shared/assets/labels.json b/tests/python/shared/assets/labels.json new file mode 100644 index 000000000000..60bcb8e85606 --- /dev/null +++ b/tests/python/shared/assets/labels.json @@ -0,0 +1,616 @@ +{ + "count": 40, + "next": null, + "previous": null, + "results": [ + { + "attributes": [], + "color": "#22b16f", + "id": 42, + "name": "11", + "parent_id": null, + "project_id": 4, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#5c73a8", + "id": 41, + "name": "4", + "parent_id": 37, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#59a8fe", + "id": 40, + "name": "3", + "parent_id": 37, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#4925ec", + "id": 39, + "name": "2", + "parent_id": 37, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#d53957", + "id": 38, + "name": "1", + "parent_id": 37, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [ + { + "default_value": "1", + "id": 4, + "input_type": "radio", + "mutable": true, + "name": "a1", + "values": [ + "1", + "23" + ] + }, + { + "default_value": "0", + "id": 5, + "input_type": "number", + "mutable": false, + "name": "a2", + "values": [ + "0", + "4", + "1" + ] + } + ], + "color": "#f3c7ed", + "id": 37, + "name": "skeleton2_all_disjoint", + "parent_id": null, + "project_id": 4, + "sublabels": [ + { + "attributes": [], + "color": "#59a8fe", + "has_parent": true, + "id": 40, + "name": "3", + "type": "points" + }, + { + "attributes": [], + "color": "#5c73a8", + "has_parent": true, + "id": 41, + "name": "4", + "type": "points" + }, + { + "attributes": [], + "color": "#d53957", + "has_parent": true, + "id": 38, + "name": "1", + "type": "points" + }, + { + "attributes": [], + "color": "#4925ec", + "has_parent": true, + "id": 39, + "name": "2", + "type": "points" + } + ], + "svg": "", + "type": "skeleton" + }, + { + "attributes": [], + "color": "#478144", + "id": 36, + "name": "5", + "parent_id": 31, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#4a649f", + "id": 35, + "name": "4", + "parent_id": 31, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#479ffe", + "id": 34, + "name": "3", + "parent_id": 31, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#350dea", + "id": 33, + "name": "2", + "parent_id": 31, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#d12345", + "id": 32, + "name": "1", + "parent_id": 31, + "project_id": 4, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#2b3145", + "id": 31, + "name": "skeleton1", + "parent_id": null, + "project_id": 4, + "sublabels": [ + { + "attributes": [], + "color": "#d12345", + "has_parent": true, + "id": 32, + "name": "1", + "type": "points" + }, + { + "attributes": [], + "color": "#350dea", + "has_parent": true, + "id": 33, + "name": "2", + "type": "points" + }, + { + "attributes": [], + "color": "#479ffe", + "has_parent": true, + "id": 34, + "name": "3", + "type": "points" + }, + { + "attributes": [], + "color": "#4a649f", + "has_parent": true, + "id": 35, + "name": "4", + "type": "points" + }, + { + "attributes": [], + "color": "#478144", + "has_parent": true, + "id": 36, + "name": "5", + "type": "points" + } + ], + "svg": "", + "type": "skeleton" + }, + { + "attributes": [], + "color": "#c06060", + "id": 30, + "name": "person", + "parent_id": null, + "project_id": 8, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#2080c0", + "id": 29, + "name": "car", + "parent_id": null, + "project_id": 8, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#bde94a", + "id": 28, + "name": "label_0", + "parent_id": null, + "project_id": 7, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#bde94a", + "id": 27, + "name": "label_0", + "parent_id": null, + "project_id": 6, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#4a649f", + "id": 26, + "name": "4", + "parent_id": 22, + "project_id": 5, + "sublabels": [], + "type": "points" + }, + { + "attributes": [ + { + "default_value": "val1", + "id": 3, + "input_type": "select", + "mutable": false, + "name": "attr", + "values": [ + "val1", + "val2" + ] + } + ], + "color": "#59a8fe", + "id": 25, + "name": "3", + "parent_id": 22, + "project_id": 5, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#4925ec", + "id": 24, + "name": "2", + "parent_id": 22, + "project_id": 5, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#d53957", + "id": 23, + "name": "1", + "parent_id": 22, + "project_id": 5, + "sublabels": [], + "type": "points" + }, + { + "attributes": [ + { + "default_value": "white", + "id": 2, + "input_type": "select", + "mutable": false, + "name": "color", + "values": [ + "white", + "black" + ] + } + ], + "color": "#0c81b5", + "id": 22, + "name": "s2", + "parent_id": null, + "project_id": 5, + "sublabels": [ + { + "attributes": [], + "color": "#d53957", + "has_parent": true, + "id": 23, + "name": "1", + "type": "points" + }, + { + "attributes": [], + "color": "#4925ec", + "has_parent": true, + "id": 24, + "name": "2", + "type": "points" + }, + { + "attributes": [ + { + "default_value": "val1", + "id": 3, + "input_type": "select", + "mutable": false, + "name": "attr", + "values": [ + "val1", + "val2" + ] + } + ], + "color": "#59a8fe", + "has_parent": true, + "id": 25, + "name": "3", + "type": "points" + }, + { + "attributes": [], + "color": "#4a649f", + "has_parent": true, + "id": 26, + "name": "4", + "type": "points" + } + ], + "svg": "", + "type": "skeleton" + }, + { + "attributes": [], + "color": "#479ffe", + "id": 21, + "name": "3", + "parent_id": 18, + "project_id": 5, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#350dea", + "id": 20, + "name": "2", + "parent_id": 18, + "project_id": 5, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#d12345", + "id": 19, + "name": "1", + "parent_id": 18, + "project_id": 5, + "sublabels": [], + "type": "points" + }, + { + "attributes": [], + "color": "#5c5eba", + "id": 18, + "name": "s1", + "parent_id": null, + "project_id": 5, + "sublabels": [ + { + "attributes": [], + "color": "#d12345", + "has_parent": true, + "id": 19, + "name": "1", + "type": "points" + }, + { + "attributes": [], + "color": "#350dea", + "has_parent": true, + "id": 20, + "name": "2", + "type": "points" + }, + { + "attributes": [], + "color": "#479ffe", + "has_parent": true, + "id": 21, + "name": "3", + "type": "points" + } + ], + "svg": "", + "type": "skeleton" + }, + { + "attributes": [], + "color": "#406040", + "id": 17, + "name": "dog", + "parent_id": null, + "project_id": 4, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#6080c0", + "id": 16, + "name": "cat", + "parent_id": null, + "project_id": 4, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#2080c0", + "id": 15, + "name": "Car", + "parent_id": null, + "sublabels": [], + "task_id": 12, + "type": "any" + }, + { + "attributes": [], + "color": "#406040", + "id": 14, + "name": "dog", + "parent_id": null, + "sublabels": [], + "task_id": 8, + "type": "any" + }, + { + "attributes": [], + "color": "#6080c0", + "id": 13, + "name": "cat", + "parent_id": null, + "sublabels": [], + "task_id": 8, + "type": "any" + }, + { + "attributes": [], + "color": "#406040", + "id": 12, + "name": "dog", + "parent_id": null, + "sublabels": [], + "task_id": 7, + "type": "any" + }, + { + "attributes": [], + "color": "#6080c0", + "id": 11, + "name": "cat", + "parent_id": null, + "sublabels": [], + "task_id": 7, + "type": "any" + }, + { + "attributes": [], + "color": "#2080c0", + "id": 10, + "name": "car", + "parent_id": null, + "sublabels": [], + "task_id": 6, + "type": "any" + }, + { + "attributes": [], + "color": "#2080c0", + "id": 9, + "name": "car", + "parent_id": null, + "sublabels": [], + "task_id": 5, + "type": "any" + }, + { + "attributes": [], + "color": "#406040", + "id": 8, + "name": "dog", + "parent_id": null, + "project_id": 2, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#6080c0", + "id": 7, + "name": "cat", + "parent_id": null, + "project_id": 2, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#c06060", + "id": 6, + "name": "person", + "parent_id": null, + "project_id": 1, + "sublabels": [], + "type": "any" + }, + { + "attributes": [ + { + "default_value": "mazda", + "id": 1, + "input_type": "select", + "mutable": false, + "name": "model", + "values": [ + "mazda", + "volvo", + "bmw" + ] + } + ], + "color": "#2080c0", + "id": 5, + "name": "car", + "parent_id": null, + "project_id": 1, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#c06060", + "id": 4, + "name": "person", + "parent_id": null, + "sublabels": [], + "task_id": 2, + "type": "any" + }, + { + "attributes": [], + "color": "#2080c0", + "id": 3, + "name": "car", + "parent_id": null, + "sublabels": [], + "task_id": 2, + "type": "any" + } + ] +} \ No newline at end of file diff --git a/tests/python/shared/assets/projects.json b/tests/python/shared/assets/projects.json index bc0c302b8abb..d280dbfd5d65 100644 --- a/tests/python/shared/assets/projects.json +++ b/tests/python/shared/assets/projects.json @@ -9,26 +9,10 @@ "created_date": "2022-12-01T12:52:42.454000Z", "dimension": "2d", "id": 8, - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 29, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 30, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?project_id=8" + }, "name": "project with video data", "organization": null, "owner": { @@ -50,7 +34,10 @@ "location": "local" }, "task_subsets": [], - "tasks": "http://localhost:8080/api/tasks?project_id=8", + "tasks": { + "count": 1, + "url": "http://localhost:8080/api/tasks?project_id=8" + }, "updated_date": "2022-12-01T12:53:34.917000Z", "url": "http://localhost:8080/api/projects/8" }, @@ -66,17 +53,10 @@ "created_date": "2022-09-28T12:26:25.296000Z", "dimension": null, "id": 7, - "labels": [ - { - "attributes": [], - "color": "#bde94a", - "has_parent": false, - "id": 28, - "name": "label_0", - "sublabels": [], - "type": "any" - } - ], + "labels": { + "count": 1, + "url": "http://localhost:8080/api/labels?project_id=7" + }, "name": "admin1_project", "organization": null, "owner": { @@ -98,7 +78,10 @@ "location": "local" }, "task_subsets": [], - "tasks": "http://localhost:8080/api/tasks?project_id=7", + "tasks": { + "count": 0, + "url": "http://localhost:8080/api/tasks?project_id=7" + }, "updated_date": "2022-09-28T12:26:29.285000Z", "url": "http://localhost:8080/api/projects/7" }, @@ -114,17 +97,10 @@ "created_date": "2022-09-28T12:15:50.768000Z", "dimension": null, "id": 6, - "labels": [ - { - "attributes": [], - "color": "#bde94a", - "has_parent": false, - "id": 27, - "name": "label_0", - "sublabels": [], - "type": "any" - } - ], + "labels": { + "count": 1, + "url": "http://localhost:8080/api/labels?project_id=6" + }, "name": "user1_project", "organization": null, "owner": { @@ -146,7 +122,10 @@ "location": "local" }, "task_subsets": [], - "tasks": "http://localhost:8080/api/tasks?project_id=6", + "tasks": { + "count": 0, + "url": "http://localhost:8080/api/tasks?project_id=6" + }, "updated_date": "2022-09-28T12:25:54.563000Z", "url": "http://localhost:8080/api/projects/6" }, @@ -156,185 +135,10 @@ "created_date": "2022-09-22T14:21:53.791000Z", "dimension": "2d", "id": 5, - "labels": [ - { - "attributes": [], - "color": "#5c5eba", - "has_parent": false, - "id": 18, - "name": "s1", - "sublabels": [ - { - "attributes": [], - "color": "#d12345", - "has_parent": true, - "id": 19, - "name": "1", - "type": "points" - }, - { - "attributes": [], - "color": "#350dea", - "has_parent": true, - "id": 20, - "name": "2", - "type": "points" - }, - { - "attributes": [], - "color": "#479ffe", - "has_parent": true, - "id": 21, - "name": "3", - "type": "points" - } - ], - "svg": "", - "type": "skeleton" - }, - { - "attributes": [], - "color": "#d12345", - "has_parent": true, - "id": 19, - "name": "1", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#350dea", - "has_parent": true, - "id": 20, - "name": "2", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#479ffe", - "has_parent": true, - "id": 21, - "name": "3", - "sublabels": [], - "type": "points" - }, - { - "attributes": [ - { - "default_value": "white", - "id": 2, - "input_type": "select", - "mutable": false, - "name": "color", - "values": [ - "white", - "black" - ] - } - ], - "color": "#0c81b5", - "has_parent": false, - "id": 22, - "name": "s2", - "sublabels": [ - { - "attributes": [], - "color": "#d53957", - "has_parent": true, - "id": 23, - "name": "1", - "type": "points" - }, - { - "attributes": [], - "color": "#4925ec", - "has_parent": true, - "id": 24, - "name": "2", - "type": "points" - }, - { - "attributes": [ - { - "default_value": "val1", - "id": 3, - "input_type": "select", - "mutable": false, - "name": "attr", - "values": [ - "val1", - "val2" - ] - } - ], - "color": "#59a8fe", - "has_parent": true, - "id": 25, - "name": "3", - "type": "points" - }, - { - "attributes": [], - "color": "#4a649f", - "has_parent": true, - "id": 26, - "name": "4", - "type": "points" - } - ], - "svg": "", - "type": "skeleton" - }, - { - "attributes": [], - "color": "#d53957", - "has_parent": true, - "id": 23, - "name": "1", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#4925ec", - "has_parent": true, - "id": 24, - "name": "2", - "sublabels": [], - "type": "points" - }, - { - "attributes": [ - { - "default_value": "val1", - "id": 3, - "input_type": "select", - "mutable": false, - "name": "attr", - "values": [ - "val1", - "val2" - ] - } - ], - "color": "#59a8fe", - "has_parent": true, - "id": 25, - "name": "3", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#4a649f", - "has_parent": true, - "id": 26, - "name": "4", - "sublabels": [], - "type": "points" - } - ], + "labels": { + "count": 9, + "url": "http://localhost:8080/api/labels?project_id=5" + }, "name": "project5", "organization": 2, "owner": { @@ -356,7 +160,10 @@ "location": "local" }, "task_subsets": [], - "tasks": "http://localhost:8080/api/tasks?project_id=5", + "tasks": { + "count": 1, + "url": "http://localhost:8080/api/tasks?project_id=5" + }, "updated_date": "2022-09-28T12:26:49.493000Z", "url": "http://localhost:8080/api/projects/5" }, @@ -366,26 +173,10 @@ "created_date": "2022-06-08T08:32:45.521000Z", "dimension": "2d", "id": 4, - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 16, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 17, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "labels": { + "count": 14, + "url": "http://localhost:8080/api/labels?project_id=4" + }, "name": "project4", "organization": 2, "owner": { @@ -399,7 +190,10 @@ "status": "annotation", "target_storage": null, "task_subsets": [], - "tasks": "http://localhost:8080/api/tasks?project_id=4", + "tasks": { + "count": 1, + "url": "http://localhost:8080/api/tasks?project_id=4" + }, "updated_date": "2022-12-05T07:47:01.518000Z", "url": "http://localhost:8080/api/projects/4" }, @@ -415,7 +209,10 @@ "created_date": "2022-03-28T13:05:24.659000Z", "dimension": null, "id": 3, - "labels": [], + "labels": { + "count": 0, + "url": "http://localhost:8080/api/labels?project_id=3" + }, "name": "project 3", "organization": 2, "owner": { @@ -429,7 +226,10 @@ "status": "annotation", "target_storage": null, "task_subsets": [], - "tasks": "http://localhost:8080/api/tasks?project_id=3", + "tasks": { + "count": 0, + "url": "http://localhost:8080/api/tasks?project_id=3" + }, "updated_date": "2022-03-28T13:06:09.283000Z", "url": "http://localhost:8080/api/projects/3" }, @@ -445,26 +245,10 @@ "created_date": "2021-12-14T19:52:37.278000Z", "dimension": "2d", "id": 2, - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 7, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 8, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?project_id=2" + }, "name": "project2", "organization": 2, "owner": { @@ -488,7 +272,10 @@ "task_subsets": [ "Train" ], - "tasks": "http://localhost:8080/api/tasks?project_id=2", + "tasks": { + "count": 1, + "url": "http://localhost:8080/api/tasks?project_id=2" + }, "updated_date": "2022-06-30T08:56:45.601000Z", "url": "http://localhost:8080/api/projects/2" }, @@ -504,39 +291,10 @@ "created_date": "2021-12-14T19:46:37.969000Z", "dimension": "2d", "id": 1, - "labels": [ - { - "attributes": [ - { - "default_value": "mazda", - "id": 1, - "input_type": "select", - "mutable": false, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ] - } - ], - "color": "#2080c0", - "has_parent": false, - "id": 5, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 6, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?project_id=1" + }, "name": "project1", "organization": null, "owner": { @@ -550,7 +308,10 @@ "status": "annotation", "target_storage": null, "task_subsets": [], - "tasks": "http://localhost:8080/api/tasks?project_id=1", + "tasks": { + "count": 1, + "url": "http://localhost:8080/api/tasks?project_id=1" + }, "updated_date": "2022-11-03T13:57:25.895000Z", "url": "http://localhost:8080/api/projects/1" } diff --git a/tests/python/shared/assets/tasks.json b/tests/python/shared/assets/tasks.json index 14386442d534..3ecc5e71b2f3 100644 --- a/tests/python/shared/assets/tasks.json +++ b/tests/python/shared/assets/tasks.json @@ -14,27 +14,15 @@ "dimension": "2d", "id": 15, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=15", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 29, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 30, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=15" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?task_id=15" + }, "mode": "interpolation", "name": "task with video data", "organization": null, @@ -48,22 +36,6 @@ }, "project_id": 8, "segment_size": 25, - "segments": [ - { - "jobs": [ - { - "assignee": null, - "id": 19, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/19" - } - ], - "start_frame": 0, - "stop_frame": 24 - } - ], "size": 25, "source_storage": { "cloud_storage_id": null, @@ -91,186 +63,15 @@ "dimension": "2d", "id": 14, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=14", - "labels": [ - { - "attributes": [], - "color": "#5c5eba", - "has_parent": false, - "id": 18, - "name": "s1", - "sublabels": [ - { - "attributes": [], - "color": "#d12345", - "has_parent": true, - "id": 19, - "name": "1", - "type": "points" - }, - { - "attributes": [], - "color": "#350dea", - "has_parent": true, - "id": 20, - "name": "2", - "type": "points" - }, - { - "attributes": [], - "color": "#479ffe", - "has_parent": true, - "id": 21, - "name": "3", - "type": "points" - } - ], - "svg": "", - "type": "skeleton" - }, - { - "attributes": [], - "color": "#d12345", - "has_parent": true, - "id": 19, - "name": "1", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#350dea", - "has_parent": true, - "id": 20, - "name": "2", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#479ffe", - "has_parent": true, - "id": 21, - "name": "3", - "sublabels": [], - "type": "points" - }, - { - "attributes": [ - { - "default_value": "white", - "id": 2, - "input_type": "select", - "mutable": false, - "name": "color", - "values": [ - "white", - "black" - ] - } - ], - "color": "#0c81b5", - "has_parent": false, - "id": 22, - "name": "s2", - "sublabels": [ - { - "attributes": [], - "color": "#d53957", - "has_parent": true, - "id": 23, - "name": "1", - "type": "points" - }, - { - "attributes": [], - "color": "#4925ec", - "has_parent": true, - "id": 24, - "name": "2", - "type": "points" - }, - { - "attributes": [ - { - "default_value": "val1", - "id": 3, - "input_type": "select", - "mutable": false, - "name": "attr", - "values": [ - "val1", - "val2" - ] - } - ], - "color": "#59a8fe", - "has_parent": true, - "id": 25, - "name": "3", - "type": "points" - }, - { - "attributes": [], - "color": "#4a649f", - "has_parent": true, - "id": 26, - "name": "4", - "type": "points" - } - ], - "svg": "", - "type": "skeleton" - }, - { - "attributes": [], - "color": "#d53957", - "has_parent": true, - "id": 23, - "name": "1", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#4925ec", - "has_parent": true, - "id": 24, - "name": "2", - "sublabels": [], - "type": "points" - }, - { - "attributes": [ - { - "default_value": "val1", - "id": 3, - "input_type": "select", - "mutable": false, - "name": "attr", - "values": [ - "val1", - "val2" - ] - } - ], - "color": "#59a8fe", - "has_parent": true, - "id": 25, - "name": "3", - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#4a649f", - "has_parent": true, - "id": 26, - "name": "4", - "sublabels": [], - "type": "points" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=14" + }, + "labels": { + "count": 9, + "url": "http://localhost:8080/api/labels?task_id=14" + }, "mode": "annotation", "name": "task1 in project5", "organization": 2, @@ -284,22 +85,6 @@ }, "project_id": 5, "segment_size": 8, - "segments": [ - { - "jobs": [ - { - "assignee": null, - "id": 18, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/18" - } - ], - "start_frame": 0, - "stop_frame": 7 - } - ], "size": 8, "source_storage": { "cloud_storage_id": null, @@ -327,27 +112,15 @@ "dimension": "2d", "id": 13, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=13", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 16, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 17, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=13" + }, + "labels": { + "count": 14, + "url": "http://localhost:8080/api/labels?task_id=13" + }, "mode": "annotation", "name": "task1_in_project4", "organization": 2, @@ -361,22 +134,6 @@ }, "project_id": 4, "segment_size": 5, - "segments": [ - { - "jobs": [ - { - "assignee": null, - "id": 17, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/17" - } - ], - "start_frame": 0, - "stop_frame": 4 - } - ], "size": 5, "source_storage": null, "status": "annotation", @@ -391,18 +148,15 @@ "created_date": "2022-03-14T13:24:05.852000Z", "dimension": "2d", "id": 12, - "jobs": "http://localhost:8080/api/jobs?task_id=12", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 15, - "name": "Car", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 0, + "url": "http://localhost:8080/api/jobs?task_id=12" + }, + "labels": { + "count": 1, + "url": "http://localhost:8080/api/labels?task_id=12" + }, "mode": "", "name": "task_without_data", "organization": null, @@ -416,7 +170,6 @@ }, "project_id": null, "segment_size": 0, - "segments": [], "source_storage": null, "status": "annotation", "subset": "", @@ -441,27 +194,15 @@ "dimension": "2d", "id": 11, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=11", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 7, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 8, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=11" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?task_id=11" + }, "mode": "annotation", "name": "task1_in_project2", "organization": 2, @@ -475,28 +216,6 @@ }, "project_id": 2, "segment_size": 11, - "segments": [ - { - "jobs": [ - { - "assignee": { - "first_name": "Worker", - "id": 7, - "last_name": "Second", - "url": "http://localhost:8080/api/users/7", - "username": "worker2" - }, - "id": 16, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/16" - } - ], - "start_frame": 0, - "stop_frame": 10 - } - ], "size": 11, "source_storage": { "cloud_storage_id": 2, @@ -530,40 +249,15 @@ "dimension": "2d", "id": 9, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=9", - "labels": [ - { - "attributes": [ - { - "default_value": "mazda", - "id": 1, - "input_type": "select", - "mutable": false, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ] - } - ], - "color": "#2080c0", - "has_parent": false, - "id": 5, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 6, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 4, + "url": "http://localhost:8080/api/jobs?task_id=9" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?task_id=9" + }, "mode": "annotation", "name": "task1_in_project1", "organization": null, @@ -577,70 +271,6 @@ }, "project_id": 1, "segment_size": 5, - "segments": [ - { - "jobs": [ - { - "assignee": { - "first_name": "Worker", - "id": 9, - "last_name": "Fourth", - "url": "http://localhost:8080/api/users/9", - "username": "worker4" - }, - "id": 11, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/11" - } - ], - "start_frame": 0, - "stop_frame": 4 - }, - { - "jobs": [ - { - "assignee": null, - "id": 12, - "stage": "validation", - "state": "new", - "status": "validation", - "url": "http://localhost:8080/api/jobs/12" - } - ], - "start_frame": 5, - "stop_frame": 9 - }, - { - "jobs": [ - { - "assignee": null, - "id": 13, - "stage": "acceptance", - "state": "new", - "status": "validation", - "url": "http://localhost:8080/api/jobs/13" - } - ], - "start_frame": 10, - "stop_frame": 14 - }, - { - "jobs": [ - { - "assignee": null, - "id": 14, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/14" - } - ], - "start_frame": 15, - "stop_frame": 19 - } - ], "size": 20, "source_storage": null, "status": "annotation", @@ -666,27 +296,15 @@ "dimension": "2d", "id": 8, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=8", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 13, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 14, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=8" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?task_id=8" + }, "mode": "annotation", "name": "task1", "organization": null, @@ -700,28 +318,6 @@ }, "project_id": null, "segment_size": 14, - "segments": [ - { - "jobs": [ - { - "assignee": { - "first_name": "Admin", - "id": 1, - "last_name": "First", - "url": "http://localhost:8080/api/users/1", - "username": "admin1" - }, - "id": 10, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/10" - } - ], - "start_frame": 0, - "stop_frame": 13 - } - ], "size": 14, "source_storage": null, "status": "annotation", @@ -747,27 +343,15 @@ "dimension": "2d", "id": 7, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=7", - "labels": [ - { - "attributes": [], - "color": "#6080c0", - "has_parent": false, - "id": 11, - "name": "cat", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "has_parent": false, - "id": 12, - "name": "dog", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=7" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?task_id=7" + }, "mode": "annotation", "name": "task_2_org2", "organization": 2, @@ -781,22 +365,6 @@ }, "project_id": null, "segment_size": 11, - "segments": [ - { - "jobs": [ - { - "assignee": null, - "id": 9, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/9" - } - ], - "start_frame": 0, - "stop_frame": 10 - } - ], "size": 11, "source_storage": null, "status": "annotation", @@ -816,18 +384,15 @@ "dimension": "3d", "id": 6, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=6", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 10, - "name": "car", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=6" + }, + "labels": { + "count": 1, + "url": "http://localhost:8080/api/labels?task_id=6" + }, "mode": "annotation", "name": "task3", "organization": null, @@ -841,22 +406,6 @@ }, "project_id": null, "segment_size": 1, - "segments": [ - { - "jobs": [ - { - "assignee": null, - "id": 8, - "stage": "annotation", - "state": "new", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/8" - } - ], - "start_frame": 0, - "stop_frame": 0 - } - ], "size": 1, "source_storage": null, "status": "annotation", @@ -882,18 +431,15 @@ "dimension": "2d", "id": 5, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=5", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 9, - "name": "car", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=5" + }, + "labels": { + "count": 1, + "url": "http://localhost:8080/api/labels?task_id=5" + }, "mode": "interpolation", "name": "task2", "organization": null, @@ -907,28 +453,6 @@ }, "project_id": null, "segment_size": 25, - "segments": [ - { - "jobs": [ - { - "assignee": { - "first_name": "Worker", - "id": 9, - "last_name": "Fourth", - "url": "http://localhost:8080/api/users/9", - "username": "worker4" - }, - "id": 7, - "stage": "annotation", - "state": "in progress", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/7" - } - ], - "start_frame": 0, - "stop_frame": 24 - } - ], "size": 25, "source_storage": null, "status": "annotation", @@ -954,27 +478,15 @@ "dimension": "2d", "id": 2, "image_quality": 70, - "jobs": "http://localhost:8080/api/jobs?task_id=2", - "labels": [ - { - "attributes": [], - "color": "#2080c0", - "has_parent": false, - "id": 3, - "name": "car", - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#c06060", - "has_parent": false, - "id": 4, - "name": "person", - "sublabels": [], - "type": "any" - } - ], + "jobs": { + "completed": 0, + "count": 1, + "url": "http://localhost:8080/api/jobs?task_id=2" + }, + "labels": { + "count": 2, + "url": "http://localhost:8080/api/labels?task_id=2" + }, "mode": "annotation", "name": "task2", "organization": 1, @@ -988,28 +500,6 @@ }, "project_id": null, "segment_size": 23, - "segments": [ - { - "jobs": [ - { - "assignee": { - "first_name": "Worker", - "id": 6, - "last_name": "First", - "url": "http://localhost:8080/api/users/6", - "username": "worker1" - }, - "id": 2, - "stage": "annotation", - "state": "new", - "status": "annotation", - "url": "http://localhost:8080/api/jobs/2" - } - ], - "start_frame": 0, - "stop_frame": 22 - } - ], "size": 23, "source_storage": null, "status": "annotation", diff --git a/tests/python/shared/assets/users.json b/tests/python/shared/assets/users.json index 8f5451532dbf..3897aba473c7 100644 --- a/tests/python/shared/assets/users.json +++ b/tests/python/shared/assets/users.json @@ -310,7 +310,7 @@ "is_active": true, "is_staff": true, "is_superuser": true, - "last_login": "2022-12-05T07:46:24.795659Z", + "last_login": "2022-12-05T07:46:24.795000Z", "last_name": "First", "url": "http://localhost:8080/api/users/1", "username": "admin1" diff --git a/tests/python/shared/fixtures/data.py b/tests/python/shared/fixtures/data.py index cdccb67497af..2da97bd63e79 100644 --- a/tests/python/shared/fixtures/data.py +++ b/tests/python/shared/fixtures/data.py @@ -106,6 +106,12 @@ def webhooks(): return Container(json.load(f)["results"]) +@pytest.fixture(scope="session") +def labels(): + with open(ASSETS_DIR / "labels.json") as f: + return Container(json.load(f)["results"]) + + @pytest.fixture(scope="session") def users_by_name(users): return {user["username"]: user for user in users} diff --git a/tests/python/shared/utils/dump_objects.py b/tests/python/shared/utils/dump_objects.py index bea25f9a461f..c4c4e5648aa9 100644 --- a/tests/python/shared/utils/dump_objects.py +++ b/tests/python/shared/utils/dump_objects.py @@ -21,6 +21,7 @@ "comment", "issue", "webhook", + "label", ]: response = get_method("admin1", f"{obj}s", page_size="all") with open(ASSETS_DIR / f"{obj}s.json", "w") as f: From 640c884812a9ee7b77b7d6cf49b7556e047743d5 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 6 Feb 2023 19:47:13 +0200 Subject: [PATCH 075/140] t --- cvat/apps/engine/serializers.py | 2 +- cvat/apps/iam/permissions.py | 2 + cvat/apps/iam/rules/tests/configs/labels.csv | 9 + .../generators/labels_test.gen.rego copy.py | 232 ++++++++++++++++++ tests/python/rest_api/test_cloud_storages.py | 15 +- tests/python/rest_api/test_invitations.py | 6 +- tests/python/rest_api/test_labels.py | 120 +++++++++ tests/python/rest_api/utils.py | 20 +- 8 files changed, 396 insertions(+), 10 deletions(-) create mode 100644 cvat/apps/iam/rules/tests/configs/labels.csv create mode 100644 cvat/apps/iam/rules/tests/generators/labels_test.gen.rego copy.py diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 4023b2273e00..340e8e660be6 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -88,7 +88,7 @@ def __init__(self, *, model=models.Label, url_filter_key, source='get_labels', * class JobsSummarySerializer(CollectionSummarySerializer): - completed = serializers.IntegerField(source='completed_jobs_count') + completed = serializers.IntegerField(source='completed_jobs_count', required=False, default=None) def __init__(self, *, model=models.Job, url_filter_key, **kwargs): super().__init__(model=model, url_filter_key=url_filter_key, **kwargs) diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 768ef34d0338..74c068551a1d 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1362,6 +1362,8 @@ def create(cls, request, view, obj): obj = cast(Label, obj) # Access rights are the same as in the owning objects + # Job assignees are not supposed to work with separate labels. + # They should only use the list operation. if obj.project: if scope == Scopes.VIEW: owning_perm_scope = ProjectPermission.Scopes.VIEW diff --git a/cvat/apps/iam/rules/tests/configs/labels.csv b/cvat/apps/iam/rules/tests/configs/labels.csv new file mode 100644 index 000000000000..543665d97c66 --- /dev/null +++ b/cvat/apps/iam/rules/tests/configs/labels.csv @@ -0,0 +1,9 @@ +Scope,Context,Membership +list,Sandbox,N/A +list,Organization,Worker +view,Sandbox,N/A +view,Organization,Worker +update,Sandbox,N/A +update,Organization,Maintainer +delete,Sandbox,N/A +delete,Organization,Maintainer diff --git a/cvat/apps/iam/rules/tests/generators/labels_test.gen.rego copy.py b/cvat/apps/iam/rules/tests/generators/labels_test.gen.rego copy.py new file mode 100644 index 000000000000..cd4afc0f5ca7 --- /dev/null +++ b/cvat/apps/iam/rules/tests/generators/labels_test.gen.rego copy.py @@ -0,0 +1,232 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import csv +import json +import os +import random +import sys +from itertools import product + +random.seed(42) + +NAME = "labels" + + +def read_rules(name): + rules = [] + with open(os.path.join(sys.argv[1], f"{name}.csv")) as f: + reader = csv.DictReader(f) + for row in reader: + row = {k.lower(): v.lower().replace("n/a", None) for k, v in row.items()} + rules.append(row) + + return rules + + +simple_rules = read_rules(NAME) + +SCOPES = list({rule["scope"] for rule in simple_rules}) +CONTEXTS = ["sandbox", "organization"] +OWNERSHIPS = [ + "project:owner", + "project:assignee", + "task:owner", + "task:assignee", + "assignee", + "none", +] +GROUPS = ["admin", "business", "user", "worker", "none"] +ORG_ROLES = ["owner", "maintainer", "supervisor", "worker", None] +SAME_ORG = [True, False] + + +def RESOURCES(scope): + if scope == "list": + return [None] + else: + return [ + { + "id": random.randrange(300, 400), + "assignee": {"id": random.randrange(500, 600)}, + "organization": {"id": random.randrange(600, 700)}, + "project": { + "id": random.randrange(300, 400), + "owner": {"id": random.randrange(700, 800)}, + "assignee": {"id": random.randrange(800, 900)}, + }, + "task": { + "id": random.randrange(300, 400), + "owner": {"id": random.randrange(900, 1000)}, + "assignee": {"id": random.randrange(1000, 1100)}, + }, + } + ] + + +def is_same_org(org1, org2): + if org1 is not None and org2 is not None: + return org1["id"] == org2["id"] + elif org1 is None and org2 is None: + return True + else: + return False + + +def eval_rule(scope, context, ownership, privilege, membership, data): + if privilege == "admin": + return True + + rules = list(filter(lambda r: scope == r["scope"], simple_rules)) + rules = list(filter(lambda r: r["context"] == "na" or context == r["context"], rules)) + rules = list(filter(lambda r: r["ownership"] == "na" or ownership == r["ownership"], rules)) + rules = list( + filter( + lambda r: r["membership"] == "na" + or ORG_ROLES.index(membership) <= ORG_ROLES.index(r["membership"]), + rules, + ) + ) + rules = list(filter(lambda r: GROUPS.index(privilege) <= GROUPS.index(r["privilege"]), rules)) + resource = data["resource"] + rules = list( + filter(lambda r: not r["limit"] or eval(r["limit"], {"resource": resource}), rules) + ) + if ( + not is_same_org(data["auth"]["organization"], data["resource"]["organization"]) + and context != "sandbox" + ): + return False + + return bool(rules) + + +def get_data(scope, context, ownership, privilege, membership, resource, same_org): + data = { + "scope": scope, + "auth": { + "user": {"id": random.randrange(0, 100), "privilege": privilege}, + "organization": { + "id": random.randrange(100, 200), + "owner": {"id": random.randrange(200, 300)}, + "user": {"role": membership}, + } + if context == "organization" + else None, + }, + "resource": resource, + } + + user_id = data["auth"]["user"]["id"] + if context == "organization": + org_id = data["auth"]["organization"]["id"] + if data["auth"]["organization"]["user"]["role"] == "owner": + data["auth"]["organization"]["owner"]["id"] = user_id + + if same_org: + data["resource"]["organization"]["id"] = org_id + + if ownership == "assignee": + data["resource"]["assignee"]["id"] = user_id + + if ownership == "project:owner": + data["resource"]["project"]["owner"]["id"] = user_id + + if ownership == "project:assignee": + data["resource"]["project"]["assignee"]["id"] = user_id + + if ownership == "task:owner": + data["resource"]["task"]["owner"]["id"] = user_id + + if ownership == "task:assignee": + data["resource"]["task"]["assignee"]["id"] = user_id + + return data + + +def _get_name(prefix, **kwargs): + name = prefix + for k, v in kwargs.items(): + if k == "resource": + continue + prefix = "_" + str(k) + if isinstance(v, dict): + if "id" in v: + v = v.copy() + v.pop("id") + if v: + name += _get_name(prefix, **v) + else: + name += "".join( + map( + lambda c: c if c.isalnum() else {"@": "_IN_"}.get(c, "_"), + f"{prefix}_{str(v).upper()}", + ) + ) + + return name + + +def get_name(scope, context, ownership, privilege, membership, resource, same_org): + return _get_name("test", **locals()) + + +def is_valid(scope, context, ownership, privilege, membership, resource, same_org): + if context == "sandbox" and membership: + return False + if scope == "list" and ownership != "None": + return False + if context == "sandbox" and same_org is False: + return False + + return True + + +def gen_test_rego(name): + with open(f"{name}_test.gen.rego", "wt") as f: + f.write(f"package {name}\n\n") + for scope, context, ownership, privilege, membership, same_org in product( + SCOPES, CONTEXTS, OWNERSHIPS, GROUPS, ORG_ROLES, SAME_ORG + ): + for resource in RESOURCES(scope): + if not is_valid( + scope, context, ownership, privilege, membership, resource, same_org + ): + continue + + data = get_data( + scope, context, ownership, privilege, membership, resource, same_org + ) + test_name = get_name( + scope, context, ownership, privilege, membership, resource, same_org + ) + result = eval_rule(scope, context, ownership, privilege, membership, data) + f.write( + "{test_name} {{\n {allow} with input as {data}\n}}\n\n".format( + test_name=test_name, + allow="allow" if result else "not allow", + data=json.dumps(data), + ) + ) + + # Write the script which is used to generate the file + with open(sys.argv[0]) as this_file: + f.write(f"\n\n# {os.path.split(sys.argv[0])[1]}\n") + for line in this_file: + if line.strip(): + f.write(f"# {line}") + else: + f.write(f"#\n") + + # Write rules which are used to generate the file + with open(os.path.join(sys.argv[1], f"{name}.csv")) as rego_file: + f.write(f"\n\n# {name}.csv\n") + for line in rego_file: + if line.strip(): + f.write(f"# {line}") + else: + f.write(f"#\n") + + +gen_test_rego(NAME) diff --git a/tests/python/rest_api/test_cloud_storages.py b/tests/python/rest_api/test_cloud_storages.py index 352a76eaa58c..a097c887528a 100644 --- a/tests/python/rest_api/test_cloud_storages.py +++ b/tests/python/rest_api/test_cloud_storages.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: MIT import io +import json +from copy import deepcopy from http import HTTPStatus from typing import List @@ -118,10 +120,17 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: def _retrieve_collection(self, **kwargs) -> List: # TODO: fix invalid serializer schema for manifests - results = super()._retrieve_collection(_parse_response=False, return_json=True, **kwargs) - for r in results: + results = super()._retrieve_collection(_parse_response=False, **kwargs) + + # validate results + fixed_results = deepcopy(results) + for r in fixed_results: r["manifests"] = [{"filename": m} for m in r["manifests"]] - return [models.CloudStorageRead._from_openapi_data(**r) for r in results] + assert not results or [ + models.CloudStorageRead._from_openapi_data(**r) for r in fixed_results + ] + + return results @pytest.mark.parametrize( "field", diff --git a/tests/python/rest_api/test_invitations.py b/tests/python/rest_api/test_invitations.py index 031a95caae95..b9bc4f91535e 100644 --- a/tests/python/rest_api/test_invitations.py +++ b/tests/python/rest_api/test_invitations.py @@ -119,8 +119,4 @@ def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ("owner",), ) def test_can_use_simple_filter_for_object_list(self, field): - value, gt_objects = self._get_field_samples(field) - - received_items = self._retrieve_collection(**{field: str(value)}) - - assert set(p["key"] for p in gt_objects) == set(p.key for p in received_items) + return super().test_can_use_simple_filter_for_object_list(field) diff --git a/tests/python/rest_api/test_labels.py b/tests/python/rest_api/test_labels.py index 64268a52443b..8086287cf025 100644 --- a/tests/python/rest_api/test_labels.py +++ b/tests/python/rest_api/test_labels.py @@ -2,10 +2,15 @@ # # SPDX-License-Identifier: MIT +import json +from http import HTTPStatus from typing import Any, Dict, List, Tuple import pytest from cvat_sdk.api_client.api_client import ApiClient, Endpoint +from deepdiff import DeepDiff + +from shared.utils.config import make_api_client from .utils import CollectionSimpleFilterTestBase @@ -41,3 +46,118 @@ def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: ) def test_can_use_simple_filter_for_object_list(self, field): return super().test_can_use_simple_filter_for_object_list(field) + + +@pytest.mark.usefixtures("restore_db_per_class") +class TestGetLabels: + def _test_get_ok(self, user, lid, data, **kwargs): + with make_api_client(user) as client: + (_, response) = client.labels_api.retrieve(lid, **kwargs) + assert response.status == HTTPStatus.OK + assert ( + DeepDiff( + data, + json.loads(response.data), + exclude_paths="root['updated_date']", + ignore_order=True, + ) + == {} + ) + + def _test_get_denied(self, user, lid, **kwargs): + with make_api_client(user) as client: + (_, response) = client.labels_api.retrieve( + lid, **kwargs, _check_status=False, _parse_response=False + ) + assert response.status == HTTPStatus.FORBIDDEN + + @pytest.mark.parametrize("source", ["task", "project"]) + @pytest.mark.parametrize("is_staff", [True, False]) + def test_admin_get_sandbox_label( + self, + labels, + tasks, + projects, + is_task_staff, + is_project_staff, + users_by_name, + admin_user, + source, + is_staff, + ): + if source == "task": + sources = tasks + is_source_staff = is_task_staff + label_source_key = "task_id" + elif source == "project": + sources = projects + is_source_staff = is_project_staff + label_source_key = "project_id" + + labels_by_source = {label.get(label_source_key): label for label in labels if label.get(label_source_key)} + sources_with_labels = [s for s in sources if labels_by_source.get(s["id"])] + + user_id = users_by_name[admin_user] + source_obj = next(filter(lambda s: is_source_staff(user_id, s["id"]) == is_staff, sources_with_labels)) + label = next(label for label in labels if label.get(label_source_key) == source_obj["id"]) + + self._test_get_ok(admin_user, label["id"], label) + + @pytest.mark.parametrize("source", ["task", "project"]) + @pytest.mark.parametrize("org_id", [1]) + def test_admin_get_org_label( + self, + labels, + tasks_by_org, + projects_by_org, + is_task_staff, + is_project_staff, + users_by_name, + admin_user, + source, + org_id, + ): + if source == "task": + sources = tasks_by_org + is_source_staff = is_task_staff + label_source_key = "task_id" + elif source == "project": + sources = projects_by_org + is_source_staff = is_project_staff + label_source_key = "project_id" + + labels_by_source = {label.get(label_source_key): label for label in labels if label.get(label_source_key)} + sources = sources[org_id] + sources_with_labels = [s for s in sources if labels_by_source.get(s["id"])] + source_obj = sources_with_labels[0] + label = next(label for label in labels if label.get(label_source_key) == source_obj["id"]) + + user_id = users_by_name[admin_user] + assert is_source_staff(user_id, source_obj["id"]) + + self._test_get_ok(admin_user, label["id"], label) + + @pytest.mark.parametrize("source", ["task", "project"]) + @pytest.mark.parametrize("is_staff", [True, False]) + def test_non_admin_get_sandbox_label( + self, labels, users, tasks, projects, is_task_staff, is_project_staff, source, is_staff + ): + if source == "task": + sources = tasks + is_source_staff = is_task_staff + label_source_key = "task_id" + elif source == "project": + sources = projects + is_source_staff = is_project_staff + label_source_key = "project_id" + + labels_by_source = {label.get(label_source_key): label for label in labels if label.get(label_source_key)} + sources_with_labels = [s for s in sources if labels_by_source.get(s["id"])] + source_obj = sources_with_labels[0] + label = labels_by_source[source_obj["id"]] + user = next(u for u in users if is_source_staff(u["id"], source_obj["id"]) == is_staff) + + if is_staff: + self._test_get_ok(user["username"], label["id"], label) + else: + self._test_get_denied(user["username"], label["id"]) diff --git a/tests/python/rest_api/utils.py b/tests/python/rest_api/utils.py index bfd1a23d27c4..9f0c68adb8f1 100644 --- a/tests/python/rest_api/utils.py +++ b/tests/python/rest_api/utils.py @@ -9,6 +9,7 @@ from cvat_sdk.api_client.api_client import ApiClient, Endpoint from cvat_sdk.core.helpers import get_paginated_collection +from deepdiff import DeepDiff from urllib3 import HTTPResponse from shared.utils.config import make_api_client @@ -39,12 +40,14 @@ class CollectionSimpleFilterTestBase(metaclass=ABCMeta): user: str samples: List[Dict[str, Any]] field_lookups: Dict[str, FieldPath] = None + cmp_ignore_keys: List[str] = ["updated_date"] @abstractmethod def _get_endpoint(self, api_client: ApiClient) -> Endpoint: ... def _retrieve_collection(self, **kwargs) -> List: + kwargs["return_json"] = True with make_api_client(self.user) as api_client: return get_paginated_collection(self._get_endpoint(api_client), **kwargs) @@ -83,9 +86,24 @@ def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: return field_value, gt_objects + def _compare_results(self, gt_objects, received_objects): + if self.cmp_ignore_keys: + ignore_keys = "root['{}']".format("|".join(self.cmp_ignore_keys)) + else: + ignore_keys = None + + diff = DeepDiff( + list(gt_objects), + received_objects, + exclude_paths=ignore_keys, + ignore_order=True, + ) + + assert diff == {}, diff + def test_can_use_simple_filter_for_object_list(self, field): value, gt_objects = self._get_field_samples(field) received_items = self._retrieve_collection(**{field: str(value)}) - assert set(p["id"] for p in gt_objects) == set(p.id for p in received_items) + self._compare_results(gt_objects, received_items) From afcc8c44ee2f9e1b014742e9d55de95653acf16d Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 7 Feb 2023 01:58:38 +0200 Subject: [PATCH 076/140] Fix tests --- cvat/apps/iam/permissions.py | 227 +-------------------------- tests/python/rest_api/test_labels.py | 55 ++++--- 2 files changed, 41 insertions(+), 241 deletions(-) diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 74c068551a1d..4ef0cc50c2c8 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1356,7 +1356,7 @@ def create(cls, request, view, obj): Scopes = __class__.Scopes permissions = [] - if view.basename == 'labels': + if view.basename == 'label': for scope in cls.get_scopes(request, view, obj): if scope in [Scopes.DELETE, Scopes.UPDATE, Scopes.VIEW]: obj = cast(Label, obj) @@ -1383,9 +1383,10 @@ def create(cls, request, view, obj): request, view, scope=owning_perm_scope, obj=obj.task, ) + # This class doesn't define its own rules for these cases permissions.append(owning_perm) - - permissions.append(cls.create_base_perm(request, view, scope, obj)) + else: + permissions.append(cls.create_base_perm(request, view, scope, obj)) return permissions @@ -1420,7 +1421,7 @@ def get_resource(self): "task": { "owner": { "id": getattr(self.obj.task.owner, 'id', None) }, "assignee": { "id": getattr(self.obj.task.assignee, 'id', None) } - } if self.obj.project else None, + } if self.obj.task else None, "project": { "owner": { "id": getattr(self.obj.project.owner, 'id', None) }, "assignee": { "id": getattr(self.obj.project.assignee, 'id', None) } @@ -1430,224 +1431,6 @@ def get_resource(self): return data -class LimitPermission(OpenPolicyAgentPermission): - @classmethod - def create(cls, request, view, obj): - return [] # There are no basic (unconditional) permissions - - @classmethod - def create_from_scopes(cls, request, view, obj, scopes: List[OpenPolicyAgentPermission]): - scope_to_caps = [ - (scope_handler, cls._prepare_capability_params(scope_handler)) - for scope_handler in scopes - ] - return [ - cls.create_base_perm(request, view, str(scope_handler.scope), obj, - scope_handler=scope_handler, capabilities=capabilities, - ) - for scope_handler, capabilities in scope_to_caps - if capabilities - ] - - def __init__(self, **kwargs): - self.url = settings.IAM_OPA_DATA_URL + '/limits/result' - self.scope_handler: OpenPolicyAgentPermission = kwargs.pop('scope_handler') - self.capabilities: Tuple[Limits, CapabilityContext] = kwargs.pop('capabilities') - super().__init__(**kwargs) - - @classmethod - def get_scopes(cls, request, view, obj): - scopes = [ - (scope_handler, cls._prepare_capability_params(scope_handler)) - for ctx in OpenPolicyAgentPermission.__subclasses__() - if not issubclass(ctx, cls) - for scope_handler in ctx.create(request, view, obj) - ] - return [ - (scope, capabilities) - for scope, capabilities in scopes - if capabilities - ] - - def get_resource(self): - data = {} - limit_manager = LimitManager() - - def _get_capability_status( - capability: Limits, context: Optional[CapabilityContext] - ) -> dict: - status = limit_manager.get_status(limit=capability, context=context) - return { 'used': status.used, 'max': status.max } - - for capability, context in self.capabilities: - data[self._get_capability_name(capability)] = _get_capability_status( - capability=capability, context=context, - ) - - return { 'limits': data } - - @classmethod - def _get_capability_name(cls, capability: Limits) -> str: - return capability.name - - @classmethod - def _prepare_capability_params(cls, scope: OpenPolicyAgentPermission - ) -> List[Tuple[Limits, CapabilityContext]]: - scope_id = (type(scope), scope.scope) - results = [] - - if scope_id in [ - (TaskPermission, TaskPermission.Scopes.CREATE), - (TaskPermission, TaskPermission.Scopes.IMPORT_BACKUP), - ]: - if getattr(scope, 'org_id') is not None: - results.append(( - Limits.ORG_TASKS, - OrgTasksContext(org_id=scope.org_id) - )) - else: - results.append(( - Limits.USER_SANDBOX_TASKS, - UserSandboxTasksContext(user_id=scope.user_id) - )) - - elif scope_id == (TaskPermission, TaskPermission.Scopes.CREATE_IN_PROJECT): - project = Project.objects.get(id=scope.project_id) - - if getattr(project, 'organization') is not None: - results.append(( - Limits.TASKS_IN_ORG_PROJECT, - TasksInOrgProjectContext( - org_id=project.organization.id, - project_id=project.id, - ) - )) - results.append(( - Limits.ORG_TASKS, - OrgTasksContext(org_id=project.organization.id) - )) - else: - results.append(( - Limits.TASKS_IN_USER_SANDBOX_PROJECT, - TasksInUserSandboxProjectContext( - user_id=project.owner.id, - project_id=project.id - ) - )) - results.append(( - Limits.USER_SANDBOX_TASKS, - UserSandboxTasksContext(user_id=project.owner.id) - )) - - elif scope_id == (TaskPermission, TaskPermission.Scopes.UPDATE_PROJECT): - task = cast(Task, scope.obj) - project = Project.objects.get(id=scope.project_id) - - class OwnerType(Enum): - org = auto() - user = auto() - - if getattr(task, 'organization', None): - old_owner = (OwnerType.org, task.organization.id) - else: - old_owner = (OwnerType.user, task.owner.id) - - if getattr(project, 'organization', None) is not None: - results.append(( - Limits.TASKS_IN_ORG_PROJECT, - TasksInOrgProjectContext( - org_id=project.organization.id, - project_id=project.id, - ) - )) - - if old_owner != (OwnerType.org, project.organization.id): - results.append(( - Limits.ORG_TASKS, - OrgTasksContext(org_id=project.organization.id) - )) - else: - results.append(( - Limits.TASKS_IN_USER_SANDBOX_PROJECT, - TasksInUserSandboxProjectContext( - user_id=project.owner.id, - project_id=project.id - ) - )) - - if old_owner != (OwnerType.user, project.owner.id): - results.append(( - Limits.USER_SANDBOX_TASKS, - UserSandboxTasksContext(user_id=project.owner.id) - )) - - elif scope_id == (TaskPermission, TaskPermission.Scopes.UPDATE_OWNER): - task = cast(Task, scope.obj) - - class OwnerType(Enum): - org = auto() - user = auto() - - if getattr(task, 'organization', None) is not None: - old_owner = (OwnerType.org, task.organization.id) - else: - old_owner = (OwnerType.user, task.owner.id) - - new_owner = getattr(scope, 'owner_id', None) - if new_owner is not None and old_owner != (OwnerType.user, new_owner): - results.append(( - Limits.USER_SANDBOX_TASKS, - UserSandboxTasksContext(user_id=new_owner) - )) - - elif scope_id in [ - (ProjectPermission, ProjectPermission.Scopes.CREATE), - (ProjectPermission, ProjectPermission.Scopes.IMPORT_BACKUP), - ]: - if getattr(scope, 'org_id') is not None: - results.append(( - Limits.ORG_PROJECTS, - OrgTasksContext(org_id=scope.org_id) - )) - else: - results.append(( - Limits.USER_SANDBOX_PROJECTS, - UserSandboxTasksContext(user_id=scope.user_id) - )) - - elif scope_id == (CloudStoragePermission, CloudStoragePermission.Scopes.CREATE): - if getattr(scope, 'org_id') is not None: - results.append(( - Limits.ORG_CLOUD_STORAGES, - OrgCloudStoragesContext(org_id=scope.org_id) - )) - else: - results.append(( - Limits.USER_SANDBOX_CLOUD_STORAGES, - UserSandboxCloudStoragesContext(user_id=scope.user_id) - )) - - elif scope_id == (OrganizationPermission, OrganizationPermission.Scopes.CREATE): - results.append(( - Limits.USER_OWNED_ORGS, - UserOrgsContext(user_id=scope.user_id) - )) - - elif scope_id == (WebhookPermission, WebhookPermission.Scopes.CREATE_IN_ORG): - results.append(( - Limits.ORG_COMMON_WEBHOOKS, - OrgCommonWebhooksContext(org_id=scope.org_id) - )) - - elif scope_id == (WebhookPermission, WebhookPermission.Scopes.CREATE_IN_PROJECT): - results.append(( - Limits.PROJECT_WEBHOOKS, - ProjectWebhooksContext(project_id=scope.project_id) - )) - - return results - - class PolicyEnforcer(BasePermission): # pylint: disable=no-self-use def check_permission(self, request, view, obj): diff --git a/tests/python/rest_api/test_labels.py b/tests/python/rest_api/test_labels.py index 8086287cf025..4d0b422e7e0f 100644 --- a/tests/python/rest_api/test_labels.py +++ b/tests/python/rest_api/test_labels.py @@ -71,8 +71,19 @@ def _test_get_denied(self, user, lid, **kwargs): ) assert response.status == HTTPStatus.FORBIDDEN + @staticmethod + def _labels_by_source(labels: List[Dict], *, source_key: str) -> Dict[int, List[Dict]]: + labels_by_source = {} + for label in labels: + label_source = label.get(source_key) + if label_source: + labels_by_source.setdefault(label_source, []).append(label) + + return labels_by_source + @pytest.mark.parametrize("source", ["task", "project"]) @pytest.mark.parametrize("is_staff", [True, False]) + @pytest.mark.parametrize("user", ["admin1"]) def test_admin_get_sandbox_label( self, labels, @@ -81,7 +92,7 @@ def test_admin_get_sandbox_label( is_task_staff, is_project_staff, users_by_name, - admin_user, + user, source, is_staff, ): @@ -94,17 +105,20 @@ def test_admin_get_sandbox_label( is_source_staff = is_project_staff label_source_key = "project_id" - labels_by_source = {label.get(label_source_key): label for label in labels if label.get(label_source_key)} + labels_by_source = self._labels_by_source(labels, source_key=label_source_key) sources_with_labels = [s for s in sources if labels_by_source.get(s["id"])] - user_id = users_by_name[admin_user] - source_obj = next(filter(lambda s: is_source_staff(user_id, s["id"]) == is_staff, sources_with_labels)) + user_id = users_by_name[user]["id"] + source_obj = next( + filter(lambda s: is_source_staff(user_id, s["id"]) == is_staff, sources_with_labels) + ) label = next(label for label in labels if label.get(label_source_key) == source_obj["id"]) - self._test_get_ok(admin_user, label["id"], label) + self._test_get_ok(user, label["id"], label) @pytest.mark.parametrize("source", ["task", "project"]) - @pytest.mark.parametrize("org_id", [1]) + @pytest.mark.parametrize("org_id", [2]) + @pytest.mark.parametrize("user", ["admin2"]) def test_admin_get_org_label( self, labels, @@ -113,7 +127,7 @@ def test_admin_get_org_label( is_task_staff, is_project_staff, users_by_name, - admin_user, + user, source, org_id, ): @@ -126,16 +140,16 @@ def test_admin_get_org_label( is_source_staff = is_project_staff label_source_key = "project_id" - labels_by_source = {label.get(label_source_key): label for label in labels if label.get(label_source_key)} sources = sources[org_id] + labels_by_source = self._labels_by_source(labels, source_key=label_source_key) sources_with_labels = [s for s in sources if labels_by_source.get(s["id"])] source_obj = sources_with_labels[0] - label = next(label for label in labels if label.get(label_source_key) == source_obj["id"]) + label = labels_by_source[source_obj["id"]][0] - user_id = users_by_name[admin_user] - assert is_source_staff(user_id, source_obj["id"]) + user_id = users_by_name[user]["id"] + assert not is_source_staff(user_id, source_obj["id"]) - self._test_get_ok(admin_user, label["id"], label) + self._test_get_ok(user, label["id"], label) @pytest.mark.parametrize("source", ["task", "project"]) @pytest.mark.parametrize("is_staff", [True, False]) @@ -144,18 +158,21 @@ def test_non_admin_get_sandbox_label( ): if source == "task": sources = tasks - is_source_staff = is_task_staff label_source_key = "task_id" elif source == "project": sources = projects - is_source_staff = is_project_staff label_source_key = "project_id" - labels_by_source = {label.get(label_source_key): label for label in labels if label.get(label_source_key)} - sources_with_labels = [s for s in sources if labels_by_source.get(s["id"])] - source_obj = sources_with_labels[0] - label = labels_by_source[source_obj["id"]] - user = next(u for u in users if is_source_staff(u["id"], source_obj["id"]) == is_staff) + simple_users = {u["id"]: u for u in users if not u["is_superuser"]} + simple_users_sources = [ + s for s in sources if s["owner"]["id"] in simple_users and s["organization"] is None + ] + labels_by_source = self._labels_by_source(labels, source_key=label_source_key) + source_obj = next(s for s in simple_users_sources if labels_by_source.get(s["id"])) + label = labels_by_source[source_obj["id"]][0] + user = next( + u for u in simple_users.values() if (u["id"] == source_obj["owner"]["id"]) == is_staff + ) if is_staff: self._test_get_ok(user["username"], label["id"], label) From f446f2772b32a130de9351fd1531ee4f6dfad794 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 7 Feb 2023 02:01:18 +0200 Subject: [PATCH 077/140] Fix merge --- tests/cypress/support/commands.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index 535fc006a6d4..b56c61646617 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -325,7 +325,6 @@ Cypress.Commands.add('pressSplitControl', () => { Cypress.Commands.add('openTaskJob', (taskName, jobID = 0, removeAnnotations = true, expectedFail = false) => { cy.openTask(taskName); - cy.get('.cvat-spinner').should('not.exist'); cy.openJob(jobID, removeAnnotations, expectedFail); }); From f988d8c41fced0fbe5853000a08d9abf7331807f Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 7 Feb 2023 19:21:23 +0200 Subject: [PATCH 078/140] Implement job_id filter --- cvat/apps/engine/views.py | 41 ++++++++++++++++++++++++---- cvat/apps/iam/permissions.py | 4 +++ tests/python/rest_api/test_labels.py | 40 ++++++++++++++++++++++----- 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index d42e60814d26..117bb4e5bead 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1906,6 +1906,10 @@ def perform_create(self, serializer, **kwargs): }), list=extend_schema( summary='Method returns a paginated list of labels', + parameters=[ + # The parameter is implemented differently from other filters + OpenApiParameter('job_id', description='A simple equality filter for job id') + ], responses={ '200': LabelSerializer(many=True), }), @@ -1924,25 +1928,52 @@ def perform_create(self, serializer, **kwargs): class LabelViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.DestroyModelMixin, PartialUpdateModelMixin ): - queryset = Label.objects.prefetch_related('task', 'project').all() + queryset = Label.objects.prefetch_related( + 'task', + 'task__owner', + 'task__assignee', + 'task__organization', + 'project', + 'project__owner', + 'project__assignee', + 'project__organization' + ).all() iam_organization_field = 'task__organization' search_fields = ('name', 'parent') - filter_fields = list(search_fields) + ['id', 'task_id', 'project_id', 'type', 'color', 'parent_id'] - simple_filters = list(search_fields) + ['task_id', 'project_id', 'type', 'color', 'parent_id'] + filter_fields = list(search_fields) + [ + 'id', 'task_id', 'project_id', 'type', 'color', 'parent_id' + ] + simple_filters = list(set(filter_fields) - {'id'}) ordering_fields = list(filter_fields) lookup_fields = { 'task_id': 'task', + 'project_id': 'project', 'parent': 'parent__name', } - ordering = '-id' + ordering = 'id' serializer_class = LabelSerializer def get_queryset(self): - queryset = super().get_queryset() if self.action == 'list': + if job_id := self.request.GET.get('job_id', None): + # NOTE: This filter is too complex to be implemented by other means + # It requires the following filter query: + # ( + # project__task__segment__job__id = job_id + # OR + # task__segment__job__id = job_id + # ) + job = Job.objects.get(id=job_id) + self.check_object_permissions(self.request, job) + queryset = job.get_labels() + else: + queryset = super().get_queryset() + perm = LabelPermission.create_scope_list(self.request) queryset = perm.filter(queryset) + else: + queryset = super().get_queryset() return queryset diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 4ef0cc50c2c8..2997509d3e8e 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1385,6 +1385,10 @@ def create(cls, request, view, obj): # This class doesn't define its own rules for these cases permissions.append(owning_perm) + elif scope == Scopes.LIST and isinstance(obj, Job): + permissions.append(JobPermission.create_base_perm( + request, view, scope=JobPermission.Scopes.VIEW, obj=obj, + )) else: permissions.append(cls.create_base_perm(request, view, scope, obj)) diff --git a/tests/python/rest_api/test_labels.py b/tests/python/rest_api/test_labels.py index 4d0b422e7e0f..62987e617fe9 100644 --- a/tests/python/rest_api/test_labels.py +++ b/tests/python/rest_api/test_labels.py @@ -17,9 +17,10 @@ class TestLabelsListFilters(CollectionSimpleFilterTestBase): @pytest.fixture(autouse=True) - def setup(self, restore_db_per_class, admin_user, labels): + def setup(self, restore_db_per_class, admin_user, labels, jobs): self.user = admin_user self.samples = labels + self.job_samples = jobs def _get_endpoint(self, api_client: ApiClient) -> Endpoint: return api_client.labels_api.list_endpoint @@ -37,6 +38,20 @@ def _get_field_samples(self, field: str) -> Tuple[Any, List[Dict[str, Any]]]: self._map_field("name"), ) return parent_name, gt_objects + elif field == "job_id": + field_path = ["id"] + field_value = self._find_valid_field_value(self.job_samples, field_path) + job_sample = next( + filter(lambda p: field_value == self._get_field(p, field_path), self.job_samples) + ) + + task_id = job_sample["task_id"] + project_id = job_sample["project_id"] + label_samples = filter( + lambda p: task_id == p.get("task_id") or project_id == p.get("project_id"), + self.samples, + ) + return field_value, label_samples else: return super()._get_field_samples(field) @@ -153,8 +168,9 @@ def test_admin_get_org_label( @pytest.mark.parametrize("source", ["task", "project"]) @pytest.mark.parametrize("is_staff", [True, False]) - def test_non_admin_get_sandbox_label( - self, labels, users, tasks, projects, is_task_staff, is_project_staff, source, is_staff + @pytest.mark.parametrize("groups", [["business"], ["user"], ["worker"], []]) + def test_regular_user_get_sandbox_label( + self, labels, users, tasks, projects, source, is_staff, groups ): if source == "task": sources = tasks @@ -163,12 +179,22 @@ def test_non_admin_get_sandbox_label( sources = projects label_source_key = "project_id" - simple_users = {u["id"]: u for u in users if not u["is_superuser"]} - simple_users_sources = [ - s for s in sources if s["owner"]["id"] in simple_users and s["organization"] is None + users = {u["id"]: u for u in users if u["groups"] == groups} + regular_users_sources = [ + s for s in sources if s["owner"]["id"] in users and s["organization"] is None ] labels_by_source = self._labels_by_source(labels, source_key=label_source_key) - source_obj = next(s for s in simple_users_sources if labels_by_source.get(s["id"])) + source_obj = next(s for s in regular_users_sources if labels_by_source.get(s["id"])) + label = labels_by_source[source_obj["id"]][0] + user = next( + u for u in users.values() if (u["id"] == source_obj["owner"]["id"]) == is_staff + ) + + if is_staff: + self._test_get_ok(user["username"], label["id"], label) + else: + self._test_get_denied(user["username"], label["id"]) + label = labels_by_source[source_obj["id"]][0] user = next( u for u in simple_users.values() if (u["id"] == source_obj["owner"]["id"]) == is_staff From d63a319a914e7a72f276d9e6d6c5212199ee6598 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 8 Feb 2023 13:50:28 +0200 Subject: [PATCH 079/140] Revert metadata changes --- cvat-sdk/cvat_sdk/core/proxies/jobs.py | 4 +- cvat-sdk/cvat_sdk/core/proxies/projects.py | 2 +- cvat-sdk/cvat_sdk/core/proxies/tasks.py | 4 +- cvat/apps/engine/serializers.py | 2 + cvat/apps/engine/views.py | 73 +++++++--------------- 5 files changed, 30 insertions(+), 55 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index b075d1cddcee..2a8d234581ed 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -152,10 +152,10 @@ def get_meta(self) -> models.IDataMetaRead: return meta def get_labels(self) -> List[models.ILabel]: - return get_paginated_collection(self.api.list_labels_endpoint, id=self.id) + return get_paginated_collection(self._client.api_client.labels_api.list_endpoint, job_id=str(self.id)) def get_frames_info(self) -> List[models.IFrameMeta]: - return get_paginated_collection(self.api.list_data_meta_frames_endpoint, id=self.id) + return self.get_meta().frames def remove_frames_by_ids(self, ids: Sequence[int]) -> None: self._client.api_client.tasks_api.jobs_partial_update_data_meta( diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index c92d5e417468..c78d1817faa6 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -133,7 +133,7 @@ def get_tasks(self) -> List[Task]: ] def get_labels(self) -> List[models.ILabel]: - return get_paginated_collection(self.api.list_labels_endpoint, id=self.id) + return get_paginated_collection(self._client.api_client.labels_api.list_endpoint, project_id=str(self.id)) def get_preview( self, diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index efe619cdcc2a..ace8fe2c4c98 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -315,10 +315,10 @@ def get_meta(self) -> models.IDataMetaRead: return meta def get_labels(self) -> List[models.ILabel]: - return get_paginated_collection(self.api.list_labels_endpoint, id=self.id) + return get_paginated_collection(self._client.api_client.labels_api.list_endpoint, task_id=str(self.id)) def get_frames_info(self) -> List[models.IFrameMeta]: - return get_paginated_collection(self.api.list_data_meta_frames_endpoint, id=self.id) + return self.get_meta().frames def remove_frames_by_ids(self, ids: Sequence[int]) -> None: self.api.partial_update_data_meta( diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 340e8e660be6..e2913aa3e2ab 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -1022,6 +1022,7 @@ class PluginsSerializer(serializers.Serializer): PREDICT = serializers.BooleanField() class DataMetaReadSerializer(serializers.ModelSerializer): + frames = FrameMetaSerializer(many=True, allow_null=True) image_quality = serializers.IntegerField(min_value=0, max_value=100) deleted_frames = serializers.ListField(child=serializers.IntegerField(min_value=0)) @@ -1034,6 +1035,7 @@ class Meta: 'start_frame', 'stop_frame', 'frame_filter', + 'frames', 'deleted_frames', ) read_only_fields = fields diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 117bb4e5bead..16ce54b46e69 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1220,7 +1220,9 @@ def _get_rq_response(queue, job_id): def metadata(self, request, pk): self.get_object() #force to call check_object_permissions db_task = models.Task.objects.prefetch_related( - Prefetch('data', queryset=models.Data.objects.select_related('video')) + Prefetch('data', queryset=models.Data.objects.select_related('video').prefetch_related( + Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) + )) ).get(pk=pk) if request.method == 'PATCH': @@ -1228,20 +1230,6 @@ def metadata(self, request, pk): if serializer.is_valid(raise_exception=True): db_task.data = serializer.save() - serializer = DataMetaReadSerializer(db_task.data) - return Response(serializer.data) - - @extend_schema(summary='Returns a paginated list of frame metadata', - responses=FrameMetaSerializer(many=True)) # Duplicate to still get 'list' op. name - @list_action(serializer_class=FrameMetaSerializer, url_path='data/meta/frames') - def metadata_frames(self, request, pk): - self.get_object() #force to call check_object_permissions - db_task = models.Task.objects.prefetch_related( - Prefetch('data', queryset=models.Data.objects.select_related('video').prefetch_related( - Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) - )) - ).get(pk=pk) - if hasattr(db_task.data, 'video'): media = [db_task.data.video] else: @@ -1251,10 +1239,13 @@ def metadata_frames(self, request, pk): 'width': item.width, 'height': item.height, 'name': item.path, - 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0, + 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0 } for item in media] - serializer = FrameMetaSerializer(frame_meta, many=True) + db_data = db_task.data + db_data.frames = frame_meta + + serializer = DataMetaReadSerializer(db_data) return Response(serializer.data) @extend_schema(summary='Export task as a dataset in a specific format', @@ -1660,7 +1651,9 @@ def metadata(self, request, pk): db_job = models.Job.objects.prefetch_related( 'segment', 'segment__task', - Prefetch('segment__task__data', queryset=models.Data.objects.select_related('video')) + Prefetch('segment__task__data', queryset=models.Data.objects.select_related('video').prefetch_related( + Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) + )) ).get(pk=pk) db_data = db_job.segment.task.data @@ -1684,6 +1677,14 @@ def metadata(self, request, pk): if db_job.segment.task.project: db_job.segment.task.project.save() + if hasattr(db_data, 'video'): + media = [db_data.video] + else: + media = list(db_data.images.filter( + frame__gte=data_start_frame, + frame__lte=data_stop_frame, + ).all()) + # Filter data with segment size # Should data.size also be cropped by segment size? db_data.deleted_frames = filter( @@ -1693,44 +1694,16 @@ def metadata(self, request, pk): db_data.start_frame = data_start_frame db_data.stop_frame = data_stop_frame - serializer = DataMetaReadSerializer(db_data) - return Response(serializer.data) - - @extend_schema(summary='Returns a paginated list of frame metadata', - responses=FrameMetaSerializer(many=True)) # Duplicate to still get 'list' op. name - @list_action(serializer_class=FrameMetaSerializer, url_path='data/meta/frames') - def metadata_frames(self, request, pk): - self.get_object() #force to call check_object_permissions - db_job = models.Job.objects.prefetch_related( - 'segment', - 'segment__task', - Prefetch('segment__task__data', queryset=models.Data.objects.select_related('video').prefetch_related( - Prefetch('images', queryset=models.Image.objects.prefetch_related('related_files').order_by('frame')) - )) - ).get(pk=pk) - - db_data = db_job.segment.task.data - start_frame = db_job.segment.start_frame - stop_frame = db_job.segment.stop_frame - data_start_frame = db_data.start_frame + start_frame * db_data.get_frame_step() - data_stop_frame = db_data.start_frame + stop_frame * db_data.get_frame_step() - - if hasattr(db_data, 'video'): - media = [db_data.video] - else: - media = list(db_data.images.filter( - frame__gte=data_start_frame, - frame__lte=data_stop_frame, - ).all()) - frame_meta = [{ 'width': item.width, 'height': item.height, 'name': item.path, - 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0, + 'related_files': item.related_files.count() if hasattr(item, 'related_files') else 0 } for item in media] - serializer = FrameMetaSerializer(frame_meta, many=True) + db_data.frames = frame_meta + + serializer = DataMetaReadSerializer(db_data) return Response(serializer.data) @extend_schema(summary='The action returns the list of tracked changes for the job', From 9678400171da02ebc69a79d73a8c036affcd2137 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 8 Feb 2023 14:01:13 +0200 Subject: [PATCH 080/140] Fix output for collection summaries --- cvat/apps/engine/serializers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index e2913aa3e2ab..4efb8ebf531e 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -59,7 +59,7 @@ def to_representation(self, instance): class CollectionSummarySerializer(serializers.Serializer): - count = serializers.IntegerField() + count = serializers.IntegerField(default=0) def __init__(self, model, *, url_filter_key, **kwargs): super().__init__(**kwargs) @@ -88,7 +88,7 @@ def __init__(self, *, model=models.Label, url_filter_key, source='get_labels', * class JobsSummarySerializer(CollectionSummarySerializer): - completed = serializers.IntegerField(source='completed_jobs_count', required=False, default=None) + completed = serializers.IntegerField(source='completed_jobs_count', default=0) def __init__(self, *, model=models.Job, url_filter_key, **kwargs): super().__init__(model=model, url_filter_key=url_filter_key, **kwargs) @@ -180,9 +180,9 @@ class Meta: model = models.Label fields = ( 'id', 'name', 'color', 'attributes', 'deleted', 'type', 'svg', - 'sublabels', 'project_id', 'task_id', 'parent_id' + 'sublabels', 'project_id', 'task_id', 'parent_id', 'has_parent' ) - read_only_fields = ('id', 'type', 'svg', 'project_id', 'task_id') + read_only_fields = ('id', 'type', 'svg', 'project_id', 'task_id', 'has_parent') extra_kwargs = { 'project_id': { 'required': False, 'allow_null': False }, 'task_id': { 'required': False, 'allow_null': False }, From 9fd98697bf915e799a8e2a18a07b459883b9217b Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 8 Feb 2023 17:29:10 +0200 Subject: [PATCH 081/140] Fix few errors --- cvat-sdk/cvat_sdk/pytorch/caching.py | 59 ++- cvat-sdk/cvat_sdk/pytorch/task_dataset.py | 4 +- cvat/apps/engine/serializers.py | 3 +- tests/python/rest_api/test_labels.py | 11 +- tests/python/sdk/test_pytorch.py | 8 +- tests/python/shared/assets/labels.json | 462 ++++++++++++---------- 6 files changed, 310 insertions(+), 237 deletions(-) diff --git a/cvat-sdk/cvat_sdk/pytorch/caching.py b/cvat-sdk/cvat_sdk/pytorch/caching.py index aba0e4b9a6e5..215ab23147cf 100644 --- a/cvat-sdk/cvat_sdk/pytorch/caching.py +++ b/cvat-sdk/cvat_sdk/pytorch/caching.py @@ -151,7 +151,7 @@ def _initialize_task_dir(self, task: Task) -> None: task_json_path = self.task_json_path(task.id) try: - saved_task = self.load_model(task_json_path, models.TaskRead) + saved_task = self.load_model(task_json_path, _OfflineTaskModel) except Exception: self._logger.info(f"Task {task.id} is not yet cached or the cache is corrupted") @@ -166,7 +166,7 @@ def _initialize_task_dir(self, task: Task) -> None: shutil.rmtree(task_dir) task_dir.mkdir(exist_ok=True, parents=True) - self.save_model(task_json_path, task._model) + self.save_model(task_json_path, _OfflineTaskModel.from_entity(task)) def ensure_task_model( self, @@ -224,7 +224,8 @@ def retrieve_project(self, project_id: int) -> Project: class _CacheManagerOffline(CacheManager): def retrieve_task(self, task_id: int) -> Task: self._logger.info(f"Retrieving task {task_id} from cache...") - return Task(self._client, self.load_model(self.task_json_path(task_id), models.TaskRead)) + cached_model = self.load_model(self.task_json_path(task_id), _OfflineTaskModel) + return _OfflineTaskProxy(self._client, cached_model, cache_manager=self) def ensure_task_model( self, @@ -249,27 +250,72 @@ def retrieve_project(self, project_id: int) -> Project: return _OfflineProjectProxy(self._client, cached_model, cache_manager=self) +@define +class _OfflineTaskModel(_CacheObjectModel): + api_model: models.ITaskRead + labels: List[models.ILabel] + + def dump(self) -> _CacheObject: + return { + "model": to_json(self.api_model), + "labels": to_json(self.labels), + } + + @classmethod + def load(cls, obj: _CacheObject): + return cls( + api_model=models.TaskRead._from_openapi_data(**obj["model"]), + labels=[models.Label._from_openapi_data(**label) for label in obj["labels"]], + ) + + @classmethod + def from_entity(cls, entity: Task): + return cls( + api_model=entity._model, + labels=entity.get_labels(), + ) + + +class _OfflineTaskProxy(Task): + def __init__( + self, client: Client, cached_model: _OfflineTaskModel, *, cache_manager: CacheManager + ) -> None: + super().__init__(client, cached_model.api_model) + self._offline_model = cached_model + self._cache_manager = cache_manager + + def get_labels(self) -> List[models.ILabel]: + return self._offline_model.labels + + @define class _OfflineProjectModel(_CacheObjectModel): api_model: models.IProjectRead task_ids: List[int] + labels: List[models.ILabel] def dump(self) -> _CacheObject: return { "model": to_json(self.api_model), "tasks": self.task_ids, + "labels": to_json(self.labels), } @classmethod def load(cls, obj: _CacheObject): return cls( - api_model=obj["model"], + api_model=models.ProjectRead._from_openapi_data(**obj["model"]), task_ids=obj["tasks"], + labels=[models.Label._from_openapi_data(**label) for label in obj["labels"]], ) @classmethod def from_entity(cls, entity: Project): - return cls(api_model=entity._model, task_ids=[t.id for t in entity.get_tasks()]) + return cls( + api_model=entity._model, + task_ids=[t.id for t in entity.get_tasks()], + labels=entity.get_labels(), + ) class _OfflineProjectProxy(Project): @@ -283,6 +329,9 @@ def __init__( def get_tasks(self) -> List[Task]: return [self._cache_manager.retrieve_task(t) for t in self._offline_model.task_ids] + def get_labels(self) -> List[models.ILabel]: + return self._offline_model.labels + _CACHE_MANAGER_CLASSES: Mapping[UpdatePolicy, Type[CacheManager]] = { UpdatePolicy.IF_MISSING_OR_STALE: _CacheManagerOnline, diff --git a/cvat-sdk/cvat_sdk/pytorch/task_dataset.py b/cvat-sdk/cvat_sdk/pytorch/task_dataset.py index aecd6b74bea4..6edd3ec24aa2 100644 --- a/cvat-sdk/cvat_sdk/pytorch/task_dataset.py +++ b/cvat-sdk/cvat_sdk/pytorch/task_dataset.py @@ -132,13 +132,13 @@ def ensure_chunk(chunk_index): { label.id: label_index for label_index, label in enumerate( - sorted(self._task.labels, key=lambda l: l.id) + sorted(self._task.get_labels(), key=lambda l: l.id) ) } ) else: self._label_id_to_index = types.MappingProxyType( - {label.id: label_name_to_index[label.name] for label in self._task.labels} + {label.id: label_name_to_index[label.name] for label in self._task.get_labels()} ) annotations = cache_manager.ensure_task_model( diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 4efb8ebf531e..10a8fbffa923 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -175,6 +175,7 @@ class LabelSerializer(SublabelSerializer): help_text='Delete label if value is true from proper Task/Project object') sublabels = SublabelSerializer(many=True, required=False) svg = serializers.CharField(allow_blank=True, required=False) + has_parent = serializers.BooleanField(read_only=True, source='has_parent_label', required=False) class Meta: model = models.Label @@ -182,7 +183,7 @@ class Meta: 'id', 'name', 'color', 'attributes', 'deleted', 'type', 'svg', 'sublabels', 'project_id', 'task_id', 'parent_id', 'has_parent' ) - read_only_fields = ('id', 'type', 'svg', 'project_id', 'task_id', 'has_parent') + read_only_fields = ('id', 'type', 'svg', 'project_id', 'task_id') extra_kwargs = { 'project_id': { 'required': False, 'allow_null': False }, 'task_id': { 'required': False, 'allow_null': False }, diff --git a/tests/python/rest_api/test_labels.py b/tests/python/rest_api/test_labels.py index 62987e617fe9..024b469d697a 100644 --- a/tests/python/rest_api/test_labels.py +++ b/tests/python/rest_api/test_labels.py @@ -168,10 +168,7 @@ def test_admin_get_org_label( @pytest.mark.parametrize("source", ["task", "project"]) @pytest.mark.parametrize("is_staff", [True, False]) - @pytest.mark.parametrize("groups", [["business"], ["user"], ["worker"], []]) - def test_regular_user_get_sandbox_label( - self, labels, users, tasks, projects, source, is_staff, groups - ): + def test_regular_user_get_sandbox_label(self, labels, users, tasks, projects, source, is_staff): if source == "task": sources = tasks label_source_key = "task_id" @@ -179,16 +176,14 @@ def test_regular_user_get_sandbox_label( sources = projects label_source_key = "project_id" - users = {u["id"]: u for u in users if u["groups"] == groups} + users = {u["id"]: u for u in users if not u["is_superuser"]} regular_users_sources = [ s for s in sources if s["owner"]["id"] in users and s["organization"] is None ] labels_by_source = self._labels_by_source(labels, source_key=label_source_key) source_obj = next(s for s in regular_users_sources if labels_by_source.get(s["id"])) label = labels_by_source[source_obj["id"]][0] - user = next( - u for u in users.values() if (u["id"] == source_obj["owner"]["id"]) == is_staff - ) + user = next(u for u in users.values() if (u["id"] == source_obj["owner"]["id"]) == is_staff) if is_staff: self._test_get_ok(user["username"], label["id"], label) diff --git a/tests/python/sdk/test_pytorch.py b/tests/python/sdk/test_pytorch.py index 90c226571921..33fb7deea3db 100644 --- a/tests/python/sdk/test_pytorch.py +++ b/tests/python/sdk/test_pytorch.py @@ -83,7 +83,7 @@ def setup( data_params={"chunk_size": 3}, ) - self.label_ids = sorted(l.id for l in self.task.labels) + self.label_ids = sorted(l.id for l in self.task.get_labels()) self.task.update_annotations( models.PatchedLabeledDataRequest( @@ -222,7 +222,7 @@ def test_transforms(self): assert isinstance(dataset[0][1], PIL.Image.Image) def test_custom_label_mapping(self): - label_name_to_id = {label.name: label.id for label in self.task.labels} + label_name_to_id = {label.name: label.id for label in self.task.get_labels()} dataset = cvatpt.TaskVisionDataset( self.client, @@ -275,7 +275,7 @@ def setup( ], ) ) - self.label_ids = sorted(l.id for l in self.project.labels) + self.label_ids = sorted(l.id for l in self.project.get_labels()) subsets = ["Train", "Test", "Val"] num_images_per_task = 3 @@ -356,7 +356,7 @@ def test_include_subsets(self): self._test_filtering(include_subsets={self.tasks[1].subset, self.tasks[2].subset}) def test_custom_label_mapping(self): - label_name_to_id = {label.name: label.id for label in self.project.labels} + label_name_to_id = {label.name: label.id for label in self.project.get_labels()} dataset = cvatpt.ProjectVisionDataset( self.client, self.project.id, label_name_to_index={"person": 123, "car": 456} diff --git a/tests/python/shared/assets/labels.json b/tests/python/shared/assets/labels.json index a6d8857b050a..9f695909ae72 100644 --- a/tests/python/shared/assets/labels.json +++ b/tests/python/shared/assets/labels.json @@ -3,94 +3,250 @@ "next": null, "previous": null, "results": [ + { + "attributes": [], + "color": "#2080c0", + "has_parent": false, + "id": 3, + "name": "car", + "parent_id": null, + "sublabels": [], + "task_id": 2, + "type": "any" + }, { "attributes": [], "color": "#c06060", - "id": 30, + "has_parent": false, + "id": 4, "name": "person", "parent_id": null, - "project_id": 8, + "sublabels": [], + "task_id": 2, + "type": "any" + }, + { + "attributes": [ + { + "default_value": "mazda", + "id": 1, + "input_type": "select", + "mutable": false, + "name": "model", + "values": [ + "mazda", + "volvo", + "bmw" + ] + } + ], + "color": "#2080c0", + "has_parent": false, + "id": 5, + "name": "car", + "parent_id": null, + "project_id": 1, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#c06060", + "has_parent": false, + "id": 6, + "name": "person", + "parent_id": null, + "project_id": 1, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#6080c0", + "has_parent": false, + "id": 7, + "name": "cat", + "parent_id": null, + "project_id": 2, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#406040", + "has_parent": false, + "id": 8, + "name": "dog", + "parent_id": null, + "project_id": 2, "sublabels": [], "type": "any" }, { "attributes": [], "color": "#2080c0", - "id": 29, + "has_parent": false, + "id": 9, "name": "car", "parent_id": null, - "project_id": 8, "sublabels": [], + "task_id": 5, "type": "any" }, { "attributes": [], - "color": "#bde94a", - "id": 28, - "name": "label_0", + "color": "#2080c0", + "has_parent": false, + "id": 10, + "name": "car", "parent_id": null, - "project_id": 7, "sublabels": [], + "task_id": 6, "type": "any" }, { "attributes": [], - "color": "#bde94a", - "id": 27, - "name": "label_0", + "color": "#6080c0", + "has_parent": false, + "id": 11, + "name": "cat", "parent_id": null, - "project_id": 6, "sublabels": [], + "task_id": 7, "type": "any" }, { "attributes": [], - "color": "#4a649f", - "id": 26, - "name": "4", - "parent_id": 22, - "project_id": 5, + "color": "#406040", + "has_parent": false, + "id": 12, + "name": "dog", + "parent_id": null, "sublabels": [], - "type": "points" + "task_id": 7, + "type": "any" }, { - "attributes": [ + "attributes": [], + "color": "#6080c0", + "has_parent": false, + "id": 13, + "name": "cat", + "parent_id": null, + "sublabels": [], + "task_id": 8, + "type": "any" + }, + { + "attributes": [], + "color": "#406040", + "has_parent": false, + "id": 14, + "name": "dog", + "parent_id": null, + "sublabels": [], + "task_id": 8, + "type": "any" + }, + { + "attributes": [], + "color": "#2080c0", + "has_parent": false, + "id": 15, + "name": "Car", + "parent_id": null, + "sublabels": [], + "task_id": 12, + "type": "any" + }, + { + "attributes": [], + "color": "#6080c0", + "has_parent": false, + "id": 16, + "name": "cat", + "parent_id": null, + "project_id": 4, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#406040", + "has_parent": false, + "id": 17, + "name": "dog", + "parent_id": null, + "project_id": 4, + "sublabels": [], + "type": "any" + }, + { + "attributes": [], + "color": "#5c5eba", + "has_parent": false, + "id": 18, + "name": "s1", + "parent_id": null, + "project_id": 5, + "sublabels": [ { - "default_value": "val1", - "id": 3, - "input_type": "select", - "mutable": false, - "name": "attr", - "values": [ - "val1", - "val2" - ] + "attributes": [], + "color": "#d12345", + "has_parent": true, + "id": 19, + "name": "1", + "type": "points" + }, + { + "attributes": [], + "color": "#350dea", + "has_parent": true, + "id": 20, + "name": "2", + "type": "points" + }, + { + "attributes": [], + "color": "#479ffe", + "has_parent": true, + "id": 21, + "name": "3", + "type": "points" } ], - "color": "#59a8fe", - "id": 25, - "name": "3", - "parent_id": 22, + "svg": "", + "type": "skeleton" + }, + { + "attributes": [], + "color": "#d12345", + "has_parent": true, + "id": 19, + "name": "1", + "parent_id": 18, "project_id": 5, "sublabels": [], "type": "points" }, { "attributes": [], - "color": "#4925ec", - "id": 24, + "color": "#350dea", + "has_parent": true, + "id": 20, "name": "2", - "parent_id": 22, + "parent_id": 18, "project_id": 5, "sublabels": [], "type": "points" }, { "attributes": [], - "color": "#d53957", - "id": 23, - "name": "1", - "parent_id": 22, + "color": "#479ffe", + "has_parent": true, + "id": 21, + "name": "3", + "parent_id": 18, "project_id": 5, "sublabels": [], "type": "points" @@ -110,6 +266,7 @@ } ], "color": "#0c81b5", + "has_parent": false, "id": 22, "name": "s2", "parent_id": null, @@ -165,231 +322,102 @@ }, { "attributes": [], - "color": "#479ffe", - "id": 21, - "name": "3", - "parent_id": 18, + "color": "#d53957", + "has_parent": true, + "id": 23, + "name": "1", + "parent_id": 22, "project_id": 5, "sublabels": [], "type": "points" }, { "attributes": [], - "color": "#350dea", - "id": 20, + "color": "#4925ec", + "has_parent": true, + "id": 24, "name": "2", - "parent_id": 18, - "project_id": 5, - "sublabels": [], - "type": "points" - }, - { - "attributes": [], - "color": "#d12345", - "id": 19, - "name": "1", - "parent_id": 18, + "parent_id": 22, "project_id": 5, "sublabels": [], "type": "points" }, { - "attributes": [], - "color": "#5c5eba", - "id": 18, - "name": "s1", - "parent_id": null, - "project_id": 5, - "sublabels": [ - { - "attributes": [], - "color": "#d12345", - "has_parent": true, - "id": 19, - "name": "1", - "type": "points" - }, - { - "attributes": [], - "color": "#350dea", - "has_parent": true, - "id": 20, - "name": "2", - "type": "points" - }, + "attributes": [ { - "attributes": [], - "color": "#479ffe", - "has_parent": true, - "id": 21, - "name": "3", - "type": "points" + "default_value": "val1", + "id": 3, + "input_type": "select", + "mutable": false, + "name": "attr", + "values": [ + "val1", + "val2" + ] } ], - "svg": "", - "type": "skeleton" - }, - { - "attributes": [], - "color": "#406040", - "id": 17, - "name": "dog", - "parent_id": null, - "project_id": 4, - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#6080c0", - "id": 16, - "name": "cat", - "parent_id": null, - "project_id": 4, - "sublabels": [], - "type": "any" - }, - { - "attributes": [], - "color": "#2080c0", - "id": 15, - "name": "Car", - "parent_id": null, - "sublabels": [], - "task_id": 12, - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "id": 14, - "name": "dog", - "parent_id": null, - "sublabels": [], - "task_id": 8, - "type": "any" - }, - { - "attributes": [], - "color": "#6080c0", - "id": 13, - "name": "cat", - "parent_id": null, - "sublabels": [], - "task_id": 8, - "type": "any" - }, - { - "attributes": [], - "color": "#406040", - "id": 12, - "name": "dog", - "parent_id": null, - "sublabels": [], - "task_id": 7, - "type": "any" - }, - { - "attributes": [], - "color": "#6080c0", - "id": 11, - "name": "cat", - "parent_id": null, - "sublabels": [], - "task_id": 7, - "type": "any" - }, - { - "attributes": [], - "color": "#2080c0", - "id": 10, - "name": "car", - "parent_id": null, + "color": "#59a8fe", + "has_parent": true, + "id": 25, + "name": "3", + "parent_id": 22, + "project_id": 5, "sublabels": [], - "task_id": 6, - "type": "any" + "type": "points" }, { "attributes": [], - "color": "#2080c0", - "id": 9, - "name": "car", - "parent_id": null, + "color": "#4a649f", + "has_parent": true, + "id": 26, + "name": "4", + "parent_id": 22, + "project_id": 5, "sublabels": [], - "task_id": 5, - "type": "any" + "type": "points" }, { "attributes": [], - "color": "#406040", - "id": 8, - "name": "dog", + "color": "#bde94a", + "has_parent": false, + "id": 27, + "name": "label_0", "parent_id": null, - "project_id": 2, + "project_id": 6, "sublabels": [], "type": "any" }, { "attributes": [], - "color": "#6080c0", - "id": 7, - "name": "cat", + "color": "#bde94a", + "has_parent": false, + "id": 28, + "name": "label_0", "parent_id": null, - "project_id": 2, + "project_id": 7, "sublabels": [], "type": "any" }, { "attributes": [], - "color": "#c06060", - "id": 6, - "name": "person", - "parent_id": null, - "project_id": 1, - "sublabels": [], - "type": "any" - }, - { - "attributes": [ - { - "default_value": "mazda", - "id": 1, - "input_type": "select", - "mutable": false, - "name": "model", - "values": [ - "mazda", - "volvo", - "bmw" - ] - } - ], "color": "#2080c0", - "id": 5, + "has_parent": false, + "id": 29, "name": "car", "parent_id": null, - "project_id": 1, + "project_id": 8, "sublabels": [], "type": "any" }, { "attributes": [], "color": "#c06060", - "id": 4, + "has_parent": false, + "id": 30, "name": "person", "parent_id": null, + "project_id": 8, "sublabels": [], - "task_id": 2, - "type": "any" - }, - { - "attributes": [], - "color": "#2080c0", - "id": 3, - "name": "car", - "parent_id": null, - "sublabels": [], - "task_id": 2, "type": "any" } ] From 1d0314d6d3a501a90c1a133c76f40ed694e5aff9 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 01:15:56 -0800 Subject: [PATCH 082/140] Fixes with labels & jobs for new API (projects & tasks) --- cvat-core/src/api-implementation.ts | 30 ++- cvat-core/src/server-proxy.ts | 228 +++++++++++++----- cvat-core/src/session.ts | 38 ++- .../src/containers/tasks-page/task-item.tsx | 2 +- .../src/containers/tasks-page/tasks-page.tsx | 2 +- 5 files changed, 203 insertions(+), 97 deletions(-) diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 7a77b3f6247c..4be45369077b 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -233,15 +233,19 @@ export default function implementAPI(cvat) { const tasksData = await serverProxy.tasks.get(searchParams); const tasks = await Promise.all(tasksData.map(async (taskItem) => { - // Temporary workaround for UI - // Fixme: too much requests on tasks page - let jobs = { results: [] }; if ('id' in filter) { - jobs = await serverProxy.jobs.get({ + // When request task by ID we also need to add labels and jobs to work with them + const labels = await serverProxy.labels.get({ task_id: taskItem.id }); + const jobs = await serverProxy.jobs.get({ filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, taskItem.id] }] }), }, true); + return new Task({ ...taskItem, jobs: jobs.results, labels: labels.results }); } - return new Task({ ...taskItem, jobs: jobs.results }); + + return new Task({ + ...taskItem, + progress: taskItem.jobs, + }); })); tasks.count = tasksData.count; @@ -260,15 +264,25 @@ export default function implementAPI(cvat) { checkExclusiveFields(filter, ['id'], ['page']); const searchParams = {}; for (const key of Object.keys(filter)) { - if (['id', 'page', 'search', 'sort', 'page', 'filter'].includes(key)) { + if (['page', 'id', 'sort', 'search', 'filter'].includes(key)) { searchParams[key] = filter[key]; } } const projectsData = await serverProxy.projects.get(searchParams); - const projects = projectsData.map((project) => new Project(project)); - projects.count = projectsData.count; + const projects = await Promise.all(projectsData.map(async (projectItem) => { + if ('id' in filter) { + // When request a project by ID we also need to add labels to work with them + const labels = await serverProxy.labels.get({ project_id: projectItem.id }); + return new Project({ ...projectItem, labels: labels.results }); + } + return new Project({ + ...projectItem, + }); + })); + + projects.count = projectsData.count; return projects; }; diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 601356475c89..ab3328a69d77 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -7,8 +7,11 @@ import FormData from 'form-data'; import store from 'store'; import Axios, { AxiosResponse } from 'axios'; import * as tus from 'tus-js-client'; +import { RawLabel } from 'labels'; import { Storage } from './storage'; -import { StorageLocation, WebhookSourceType } from './enums'; +import { + DimensionType, ProjectStatus, StorageLocation, TaskStatus, WebhookSourceType, +} from './enums'; import { isEmail } from './common'; import config from './config'; import DownloadWorker from './download.worker'; @@ -70,6 +73,66 @@ function waitFor(frequencyHz, predicate) { }); } +function fetchAll(url, filter = {}): Promise { + const pageSize = 500; + const result = { + count: 0, + results: [], + }; + return new Promise((resolve, reject) => { + Axios.get(url, { + params: { + ...filter, + page_size: pageSize, + page: 1, + }, + proxy: config.proxy, + }).then((initialData) => { + const { count, results } = initialData.data; + result.results = result.results.concat(results); + if (count <= pageSize) { + resolve(result); + return; + } + + const pages = Math.ceil(count / pageSize); + const promises = Array(pages).fill(0).map((_: number, i: number) => { + if (i) { + return Axios.get(url, { + params: { + ...filter, + page_size: pageSize, + page: i + 1, + }, + proxy: config.proxy, + }); + } + + return Promise.resolve(null); + }); + + Promise.all(promises).then((responses: AxiosResponse[]) => { + responses.forEach((resp) => { + if (resp) { + result.results = result.results.concat(resp.data.results); + } + }); + + // removing possible dublicates + const obj = result.results.reduce((acc: Record, item: any) => { + acc[item.id] = item; + return acc; + }, {}); + + result.results = Object.values(obj); + result.count = result.results.length; + + resolve(result); + }).catch((error) => reject(error)); + }).catch((error) => reject(error)); + }); +} + async function chunkUpload(file, uploadConfig) { const params = enableOrganization(); const { @@ -573,7 +636,35 @@ async function searchProjectNames(search, limit) { return response.data.results; } -async function getProjects(filter = {}) { +interface RawProjectData { + assignee: RawUserData | null; + id: number; + bug_tracker: string; + created_date: string; + updated_date: string; + dimension: DimensionType; + name: string; + organization: number | null; + owner: RawUserData; + source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + target_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + url: string; + tasks: { count: number; url: string; }; + task_subsets: string[]; + status: ProjectStatus; +} + +interface ProjectsFilter { + page?: number; + id?: number; + sort?: string; + search?: string; + filter?: string; +} + +type TasksFilter = ProjectsFilter & { ordering?: string; }; // TODO: Need to clarify how "ordering" is used + +async function getProjects(filter: ProjectsFilter = {}): Promise { const { backendAPI, proxy } = config; let response = null; @@ -583,8 +674,10 @@ async function getProjects(filter = {}) { proxy, }); const results = [response.data]; - results.count = 1; - return results; + Object.defineProperty(results, 'count', { + value: 1, + }); + return results as RawProjectData[] & { count: number }; } response = await Axios.get(`${backendAPI}/projects`, { @@ -645,7 +738,51 @@ async function createProject(projectSpec) { } } -async function getTasks(filter = {}) { +interface RawUserData { + url: string; + id: number; + username: string; + first_name: string; + last_name: string; + email?: string; + groups?: ('user' | 'business' | 'admin')[]; + is_staff?: boolean; + is_superuser?: boolean; + is_active?: boolean; + last_login?: string; + date_joined?: string; +} + +interface RawTaskData { + assignee: RawUserData | null; + bug_tracker: string; + created_date: string; + data: number; + data_chunk_size: number | null; + data_compressed_chunk_type: 'imageset' | 'video'; + data_original_chunk_type: 'imageset' | 'video'; + dimension: DimensionType; + id: number; + image_quality: number; + jobs: { count: 1; completed: 0; url: string; }; + labels: { count: number; url: string; }; + mode: 'annotation' | 'interpolation' | ''; + name: string; + organization: number | null; + overlap: number | null; + owner: RawUserData; + project_id: number | null; + segment_size: number; + size: number; + source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + target_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + status: TaskStatus; + subset: string; + updated_date: string; + url: string; +} + +async function getTasks(filter: TasksFilter = {}): Promise { const { backendAPI } = config; let response = null; @@ -655,8 +792,10 @@ async function getTasks(filter = {}) { proxy: config.proxy, }); const results = [response.data]; - results.count = 1; - return results; + Object.defineProperty(results, 'count', { + value: 1, + }); + return results as RawTaskData[] & { count: number }; } response = await Axios.get(`${backendAPI}/tasks`, { @@ -708,6 +847,17 @@ async function deleteTask(id, organizationID = null) { } } +async function getLabels(filter: { + task_id?: number, + project_id?: number, +}): Promise<{ results: RawLabel[] }> { + const { backendAPI } = config; + return fetchAll(`${backendAPI}/labels`, { + ...filter, + ...enableOrganization(), + }); +} + function exportDataset(instanceType) { return async function ( id: number, @@ -1232,66 +1382,6 @@ async function createTask(taskSpec, taskDataSpec, onUpdate) { return createdTask[0]; } -function fetchAll(url, filter = {}): Promise { - const pageSize = 500; - const result = { - count: 0, - results: [], - }; - return new Promise((resolve, reject) => { - Axios.get(url, { - params: { - ...filter, - page_size: pageSize, - page: 1, - }, - proxy: config.proxy, - }).then((initialData) => { - const { count, results } = initialData.data; - result.results = result.results.concat(results); - if (count <= pageSize) { - resolve(result); - return; - } - - const pages = Math.ceil(count / pageSize); - const promises = Array(pages).fill(0).map((_: number, i: number) => { - if (i) { - return Axios.get(url, { - params: { - ...filter, - page_size: pageSize, - page: i + 1, - }, - proxy: config.proxy, - }); - } - - return Promise.resolve(null); - }); - - Promise.all(promises).then((responses: AxiosResponse[]) => { - responses.forEach((resp) => { - if (resp) { - result.results = result.results.concat(resp.data.results); - } - }); - - // removing possible dublicates - const obj = result.results.reduce((acc: Record, item: any) => { - acc[item.id] = item; - return acc; - }, {}); - - result.results = Object.values(obj); - result.count = result.results.length; - - resolve(result); - }).catch((error) => reject(error)); - }).catch((error) => reject(error)); - }); -} - async function getJobs(filter = {}, aggregate = false) { const { backendAPI } = config; const id = filter.id || null; @@ -2574,6 +2664,10 @@ export default Object.freeze({ restore: restoreTask, }), + labels: Object.freeze({ + get: getLabels, + }), + jobs: Object.freeze({ get: getJobs, getPreview: getPreview('jobs'), diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index e776c9c2a207..7bfffb50d486 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -542,6 +542,7 @@ export class Job extends Session { export class Task extends Session { constructor(initialData) { super(); + const data = { id: undefined, name: undefined, @@ -558,22 +559,24 @@ export class Task extends Session { overlap: undefined, segment_size: undefined, image_quality: undefined, - start_frame: undefined, - stop_frame: undefined, frame_filter: undefined, data_chunk_size: undefined, data_compressed_chunk_type: undefined, data_original_chunk_type: undefined, - deleted_frames: undefined, + dimension: undefined, + source_storage: undefined, + target_storage: undefined, + organization: undefined, + progress: undefined, + labels: undefined, + jobs: undefined, + use_zip_chunks: undefined, use_cache: undefined, copy_data: undefined, - dimension: undefined, cloud_storage_id: undefined, sorting_method: undefined, - source_storage: undefined, - target_storage: undefined, - progress: undefined, + files: undefined, }; const updateTrigger = new FieldUpdateTrigger(); @@ -590,20 +593,11 @@ export class Task extends Session { data.labels = []; data.jobs = []; - // FIX ME: progress shoud come from server, not from segments - const progress = { - completedJobs: 0, - totalJobs: 0, + data.progress = { + completedJobs: initialData?.jobs?.completed || 0, + totalJobs: initialData?.jobs?.count || 0, }; - if (Array.isArray(initialData.segments)) { - for (const segment of initialData.segments) { - for (const job of segment.jobs) { - progress.totalJobs += 1; - if (job.stage === 'acceptance') progress.completedJobs += 1; - } - } - } - data.progress = progress; + data.files = Object.freeze({ server_files: [], client_files: [], @@ -625,6 +619,7 @@ export class Task extends Session { stage: job.stage, start_frame: job.start_frame, stop_frame: job.stop_frame, + // following fields also returned when doing API request /jobs/ // here we know them from task and append to constructor task_id: data.id, @@ -918,6 +913,9 @@ export class Task extends Session { sortingMethod: { get: () => data.sorting_method, }, + organization: { + get: () => data.organization, + }, sourceStorage: { get: () => ( new Storage({ diff --git a/cvat-ui/src/containers/tasks-page/task-item.tsx b/cvat-ui/src/containers/tasks-page/task-item.tsx index 992c5e761458..84a570dfebbf 100644 --- a/cvat-ui/src/containers/tasks-page/task-item.tsx +++ b/cvat-ui/src/containers/tasks-page/task-item.tsx @@ -35,7 +35,7 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { const id = own.taskID; return { - hidden: state.tasks.hideEmpty && task.progress.totalJobs === 0, + hidden: state.tasks.hideEmpty && task.size === 0, deleted: id in deletes ? deletes[id] === true : false, taskInstance: task, activeInference: state.models.inferences[id] || null, diff --git a/cvat-ui/src/containers/tasks-page/tasks-page.tsx b/cvat-ui/src/containers/tasks-page/tasks-page.tsx index 4523835062a6..ea0913a5bb33 100644 --- a/cvat-ui/src/containers/tasks-page/tasks-page.tsx +++ b/cvat-ui/src/containers/tasks-page/tasks-page.tsx @@ -23,7 +23,7 @@ function mapStateToProps(state: CombinedState): StateToProps { query: tasks.gettingQuery, count: state.tasks.count, countInvisible: tasks.hideEmpty ? - tasks.current.filter((task: Task): boolean => !task.jobs.length).length : + tasks.current.filter((task: Task): boolean => task.size === 0).length : 0, importing: state.import.tasks.backup.importing, }; From 36f8998c17fb7187534515b6973e3922ef0c665c Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 02:57:15 -0800 Subject: [PATCH 083/140] Added more types --- cvat-core/src/annotation-formats.ts | 12 +- cvat-core/src/labels.ts | 39 ++--- cvat-core/src/server-proxy.ts | 138 +++++------------- cvat-core/src/server-response-types.ts | 107 +++++++++++++- .../src/components/labels-editor/common.ts | 10 +- .../components/labels-editor/label-form.tsx | 18 +-- .../labels-editor/labels-editor.tsx | 12 +- .../components/labels-editor/raw-viewer.tsx | 14 +- cvat-ui/src/cvat-core-wrapper.ts | 7 +- 9 files changed, 183 insertions(+), 174 deletions(-) diff --git a/cvat-core/src/annotation-formats.ts b/cvat-core/src/annotation-formats.ts index d976d63922bb..584d1f89bca8 100644 --- a/cvat-core/src/annotation-formats.ts +++ b/cvat-core/src/annotation-formats.ts @@ -5,9 +5,9 @@ import { DimensionType } from 'enums'; import { - AnnotationExporterResponseBody, - AnnotationFormatsResponseBody, - AnnotationImporterResponseBody, + SerializedAnnotationExporter, + SerializedAnnotationFormats, + SerializedAnnotationImporter, } from 'server-response-types'; export class Loader { @@ -17,7 +17,7 @@ export class Loader { public enabled: boolean; public dimension: DimensionType; - constructor(initialData: AnnotationImporterResponseBody) { + constructor(initialData: SerializedAnnotationImporter) { const data = { name: initialData.name, format: initialData.ext, @@ -53,7 +53,7 @@ export class Dumper { public enabled: boolean; public dimension: DimensionType; - constructor(initialData: AnnotationExporterResponseBody) { + constructor(initialData: SerializedAnnotationExporter) { const data = { name: initialData.name, format: initialData.ext, @@ -86,7 +86,7 @@ export class AnnotationFormats { public loaders: Loader[]; public dumpers: Dumper[]; - constructor(initialData: AnnotationFormatsResponseBody) { + constructor(initialData: SerializedAnnotationFormats) { const data = { exporters: initialData.exporters.map((el) => new Dumper(el)), importers: initialData.importers.map((el) => new Loader(el)), diff --git a/cvat-core/src/labels.ts b/cvat-core/src/labels.ts index 86b41422e534..7ef23fcbf14a 100644 --- a/cvat-core/src/labels.ts +++ b/cvat-core/src/labels.ts @@ -2,20 +2,12 @@ // // SPDX-License-Identifier: MIT +import { + AttrInputType, LabelType, SerializedAttribute, SerializedLabel, +} from 'server-response-types'; import { ShapeType, AttributeType } from './enums'; import { ArgumentError } from './exceptions'; -type AttrInputType = 'select' | 'radio' | 'checkbox' | 'number' | 'text'; - -export interface RawAttribute { - name: string; - mutable: boolean; - input_type: AttrInputType; - default_value: string; - values: string[]; - id?: number; -} - export class Attribute { public id?: number; public defaultValue: string; @@ -24,7 +16,7 @@ export class Attribute { public name: string; public values: string[]; - constructor(initialData: RawAttribute) { + constructor(initialData: SerializedAttribute) { const data = { id: undefined, default_value: undefined, @@ -75,8 +67,8 @@ export class Attribute { ); } - toJSON(): RawAttribute { - const object: RawAttribute = { + toJSON(): SerializedAttribute { + const object: SerializedAttribute = { name: this.name, mutable: this.mutable, input_type: this.inputType, @@ -92,19 +84,6 @@ export class Attribute { } } -type LabelType = 'rectangle' | 'polygon' | 'polyline' | 'points' | 'ellipse' | 'cuboid' | 'skeleton' | 'mask' | 'tag' | 'any'; -export interface RawLabel { - id?: number; - name: string; - color?: string; - type: LabelType; - svg?: string; - sublabels?: RawLabel[]; - has_parent?: boolean; - deleted?: boolean; - attributes: RawAttribute[]; -} - export class Label { public name: string; public readonly id?: number; @@ -118,7 +97,7 @@ export class Label { public deleted: boolean; public readonly hasParent?: boolean; - constructor(initialData: RawLabel) { + constructor(initialData: SerializedLabel) { const data = { id: undefined, name: undefined, @@ -210,8 +189,8 @@ export class Label { ); } - toJSON(): RawLabel { - const object: RawLabel = { + toJSON(): SerializedLabel { + const object: SerializedLabel = { name: this.name, attributes: [...this.attributes.map((el) => el.toJSON())], type: this.type, diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index b1e2182ccd21..affdc6f49e96 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -7,12 +7,12 @@ import FormData from 'form-data'; import store from 'store'; import Axios, { AxiosResponse } from 'axios'; import * as tus from 'tus-js-client'; -import { RawLabel } from 'labels'; -import { AnnotationFormatsResponseBody } from 'server-response-types'; -import { Storage } from './storage'; import { - DimensionType, ProjectStatus, StorageLocation, TaskStatus, WebhookSourceType, -} from './enums'; + SerializedLabel, SerializedAnnotationFormats, ProjectsFilter, + SerializedProject, SerializedTask, TasksFilter, SerializedUser, +} from 'server-response-types'; +import { Storage } from './storage'; +import { StorageLocation, WebhookSourceType } from './enums'; import { isEmail } from './common'; import config from './config'; import DownloadWorker from './download.worker'; @@ -29,7 +29,7 @@ type Params = { action?: string, }; -function enableOrganization() { +function enableOrganization(): { org: string } { return { org: config.organizationID || '' }; } @@ -45,7 +45,7 @@ function configureStorage(storage: Storage, useDefaultLocation = false): Partial }; } -function removeToken() { +function removeToken(): void { Axios.defaults.headers.common.Authorization = ''; store.remove('token'); } @@ -56,7 +56,7 @@ function waitFor(frequencyHz, predicate) { reject(new Error(`Predicate must be a function, got ${typeof predicate}`)); } - const internalWait = () => { + const internalWait = (): void => { let result = false; try { result = predicate(); @@ -345,7 +345,7 @@ async function exception(exceptionObject) { } } -async function formats(): Promise { +async function formats(): Promise { const { backendAPI } = config; let response = null; @@ -440,7 +440,7 @@ async function loginWithSocialAccount( authParams?: string, process?: string, scope?: string, -) { +): Promise { removeToken(); const data = { code, @@ -463,7 +463,7 @@ async function loginWithSocialAccount( Axios.defaults.headers.common.Authorization = `Token ${token}`; } -async function logout() { +async function logout(): Promise { try { await Axios.post(`${config.backendAPI}/auth/logout`, { proxy: config.proxy, @@ -474,7 +474,7 @@ async function logout() { } } -async function changePassword(oldPassword, newPassword1, newPassword2) { +async function changePassword(oldPassword: string, newPassword1: string, newPassword2: string): Promise { try { const data = JSON.stringify({ old_password: oldPassword, @@ -492,7 +492,7 @@ async function changePassword(oldPassword, newPassword1, newPassword2) { } } -async function requestPasswordReset(email) { +async function requestPasswordReset(email: string): Promise { try { const data = JSON.stringify({ email, @@ -508,7 +508,7 @@ async function requestPasswordReset(email) { } } -async function resetPassword(newPassword1, newPassword2, uid, _token) { +async function resetPassword(newPassword1: string, newPassword2: string, uid: string, _token: string): Promise { try { const data = JSON.stringify({ new_password1: newPassword1, @@ -527,7 +527,7 @@ async function resetPassword(newPassword1, newPassword2, uid, _token) { } } -async function getSelf() { +async function getSelf(): Promise { const { backendAPI } = config; let response = null; @@ -542,7 +542,7 @@ async function getSelf() { return response.data; } -async function authorized() { +async function authorized(): Promise { try { await getSelf(); } catch (serverError) { @@ -561,7 +561,13 @@ async function authorized() { return true; } -async function healthCheck(maxRetries, checkPeriod, requestTimeout, progressCallback, attempt = 0) { +async function healthCheck( + maxRetries: number, + checkPeriod: number, + requestTimeout: number, + progressCallback: (status: string) => void, + attempt = 0, +): Promise { const { backendAPI } = config; const url = `${backendAPI}/server/health/?format=json`; @@ -603,7 +609,7 @@ async function healthCheck(maxRetries, checkPeriod, requestTimeout, progressCall }); } -async function serverRequest(url, data) { +async function serverRequest(url: string, data: object): Promise { try { return ( await Axios({ @@ -616,7 +622,7 @@ async function serverRequest(url, data) { } } -async function searchProjectNames(search, limit) { +async function searchProjectNames(search: string, limit: number): Promise { const { backendAPI, proxy } = config; let response = null; @@ -638,35 +644,7 @@ async function searchProjectNames(search, limit) { return response.data.results; } -interface RawProjectData { - assignee: RawUserData | null; - id: number; - bug_tracker: string; - created_date: string; - updated_date: string; - dimension: DimensionType; - name: string; - organization: number | null; - owner: RawUserData; - source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; - target_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; - url: string; - tasks: { count: number; url: string; }; - task_subsets: string[]; - status: ProjectStatus; -} - -interface ProjectsFilter { - page?: number; - id?: number; - sort?: string; - search?: string; - filter?: string; -} - -type TasksFilter = ProjectsFilter & { ordering?: string; }; // TODO: Need to clarify how "ordering" is used - -async function getProjects(filter: ProjectsFilter = {}): Promise { +async function getProjects(filter: ProjectsFilter = {}): Promise { const { backendAPI, proxy } = config; let response = null; @@ -679,7 +657,7 @@ async function getProjects(filter: ProjectsFilter = {}): Promise): Promise { const { backendAPI } = config; try { @@ -712,7 +690,7 @@ async function saveProject(id, projectData) { } } -async function deleteProject(id) { +async function deleteProject(id: number): Promise { const { backendAPI } = config; try { @@ -724,7 +702,7 @@ async function deleteProject(id) { } } -async function createProject(projectSpec) { +async function createProject(projectSpec: SerializedProject): Promise { const { backendAPI } = config; try { @@ -740,51 +718,7 @@ async function createProject(projectSpec) { } } -interface RawUserData { - url: string; - id: number; - username: string; - first_name: string; - last_name: string; - email?: string; - groups?: ('user' | 'business' | 'admin')[]; - is_staff?: boolean; - is_superuser?: boolean; - is_active?: boolean; - last_login?: string; - date_joined?: string; -} - -interface RawTaskData { - assignee: RawUserData | null; - bug_tracker: string; - created_date: string; - data: number; - data_chunk_size: number | null; - data_compressed_chunk_type: 'imageset' | 'video'; - data_original_chunk_type: 'imageset' | 'video'; - dimension: DimensionType; - id: number; - image_quality: number; - jobs: { count: 1; completed: 0; url: string; }; - labels: { count: number; url: string; }; - mode: 'annotation' | 'interpolation' | ''; - name: string; - organization: number | null; - overlap: number | null; - owner: RawUserData; - project_id: number | null; - segment_size: number; - size: number; - source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; - target_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; - status: TaskStatus; - subset: string; - updated_date: string; - url: string; -} - -async function getTasks(filter: TasksFilter = {}): Promise { +async function getTasks(filter: TasksFilter = {}): Promise { const { backendAPI } = config; let response = null; @@ -797,7 +731,7 @@ async function getTasks(filter: TasksFilter = {}): Promise { const { backendAPI } = config; let response = null; @@ -833,7 +767,7 @@ async function saveTask(id, taskData) { return response.data; } -async function deleteTask(id, organizationID = null) { +async function deleteTask(id: number, organizationID: string | null = null): Promise { const { backendAPI } = config; try { @@ -852,7 +786,7 @@ async function deleteTask(id, organizationID = null) { async function getLabels(filter: { task_id?: number, project_id?: number, -}): Promise<{ results: RawLabel[] }> { +}): Promise<{ results: SerializedLabel[] }> { const { backendAPI } = config; return fetchAll(`${backendAPI}/labels`, { ...filter, @@ -860,7 +794,7 @@ async function getLabels(filter: { }); } -function exportDataset(instanceType) { +function exportDataset(instanceType: 'projects' | 'jobs' | 'tasks') { return async function ( id: number, format: string, diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 88835113d4b9..e5a5040f9cae 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -2,10 +2,10 @@ // // SPDX-License-Identifier: MIT -import { DimensionType } from 'enums'; +import { DimensionType, ProjectStatus, TaskStatus } from 'enums'; import { SerializedModel } from 'core-types'; -export interface AnnotationImporterResponseBody { +export interface SerializedAnnotationImporter { name: string; ext: string; version: string; @@ -13,14 +13,109 @@ export interface AnnotationImporterResponseBody { dimension: DimensionType; } -export type AnnotationExporterResponseBody = AnnotationImporterResponseBody; +export type SerializedAnnotationExporter = SerializedAnnotationImporter; -export interface AnnotationFormatsResponseBody { - importers: AnnotationImporterResponseBody[]; - exporters: AnnotationExporterResponseBody[]; +export interface SerializedAnnotationFormats { + importers: SerializedAnnotationImporter[]; + exporters: SerializedAnnotationExporter[]; } export interface FunctionsResponseBody { results: SerializedModel[]; count: number; } + +export interface ProjectsFilter { + page?: number; + id?: number; + sort?: string; + search?: string; + filter?: string; +} + +export interface SerializedUser { + url: string; + id: number; + username: string; + first_name: string; + last_name: string; + email?: string; + groups?: ('user' | 'business' | 'admin')[]; + is_staff?: boolean; + is_superuser?: boolean; + is_active?: boolean; + last_login?: string; + date_joined?: string; +} + +export interface SerializedProject { + assignee: SerializedUser | null; + id: number; + bug_tracker: string; + created_date: string; + updated_date: string; + dimension: DimensionType; + name: string; + organization: number | null; + owner: SerializedUser; + source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + target_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + url: string; + tasks: { count: number; url: string; }; + task_subsets: string[]; + status: ProjectStatus; +} + +export type TasksFilter = ProjectsFilter & { ordering?: string; }; // TODO: Need to clarify how "ordering" is used + +export interface SerializedTask { + assignee: SerializedUser | null; + bug_tracker: string; + created_date: string; + data: number; + data_chunk_size: number | null; + data_compressed_chunk_type: 'imageset' | 'video'; + data_original_chunk_type: 'imageset' | 'video'; + dimension: DimensionType; + id: number; + image_quality: number; + jobs: { count: 1; completed: 0; url: string; }; + labels: { count: number; url: string; }; + mode: 'annotation' | 'interpolation' | ''; + name: string; + organization: number | null; + overlap: number | null; + owner: SerializedUser; + project_id: number | null; + segment_size: number; + size: number; + source_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + target_storage: { id: number; location: 'local' | 'cloud'; cloud_storage_id: null }; + status: TaskStatus; + subset: string; + updated_date: string; + url: string; +} + +export type AttrInputType = 'select' | 'radio' | 'checkbox' | 'number' | 'text'; +export interface SerializedAttribute { + name: string; + mutable: boolean; + input_type: AttrInputType; + default_value: string; + values: string[]; + id?: number; +} + +export type LabelType = 'rectangle' | 'polygon' | 'polyline' | 'points' | 'ellipse' | 'cuboid' | 'skeleton' | 'mask' | 'tag' | 'any'; +export interface SerializedLabel { + id?: number; + name: string; + color?: string; + type: LabelType; + svg?: string; + sublabels?: SerializedLabel[]; + has_parent?: boolean; + deleted?: boolean; + attributes: SerializedAttribute[]; +} diff --git a/cvat-ui/src/components/labels-editor/common.ts b/cvat-ui/src/components/labels-editor/common.ts index 123ad980f9f9..edf6bee7bc54 100644 --- a/cvat-ui/src/components/labels-editor/common.ts +++ b/cvat-ui/src/components/labels-editor/common.ts @@ -2,19 +2,19 @@ // // SPDX-License-Identifier: MIT -import { RawLabel, RawAttribute } from 'cvat-core-wrapper'; +import { SerializedLabel, SerializedAttribute } from 'cvat-core-wrapper'; export interface SkeletonConfiguration { type: 'skeleton'; svg: string; - sublabels: RawLabel[]; + sublabels: SerializedLabel[]; } -export type LabelOptColor = RawLabel; +export type LabelOptColor = SerializedLabel; let id = 0; -function validateParsedAttribute(attr: RawAttribute): void { +function validateParsedAttribute(attr: SerializedAttribute): void { if (typeof attr.name !== 'string') { throw new Error(`Type of attribute name must be a string. Got value ${attr.name}`); } @@ -48,7 +48,7 @@ function validateParsedAttribute(attr: RawAttribute): void { } } -export function validateParsedLabel(label: RawLabel): void { +export function validateParsedLabel(label: SerializedLabel): void { if (typeof label.name !== 'string') { throw new Error(`Type of label name must be a string. Got value ${label.name}`); } diff --git a/cvat-ui/src/components/labels-editor/label-form.tsx b/cvat-ui/src/components/labels-editor/label-form.tsx index 842547c5cd63..56a5433578de 100644 --- a/cvat-ui/src/components/labels-editor/label-form.tsx +++ b/cvat-ui/src/components/labels-editor/label-form.tsx @@ -14,7 +14,7 @@ import Form, { FormInstance } from 'antd/lib/form'; import Badge from 'antd/lib/badge'; import { Store } from 'antd/lib/form/interface'; -import { RawAttribute, LabelType } from 'cvat-core-wrapper'; +import { SerializedAttribute, LabelType } from 'cvat-core-wrapper'; import CVATTooltip from 'components/common/cvat-tooltip'; import ColorPicker from 'components/annotation-page/standard-workspace/objects-side-bar/color-picker'; import { ColorizeIcon } from 'icons'; @@ -131,7 +131,7 @@ export default class LabelForm extends React.Component { }; /* eslint-disable class-methods-use-this */ - private renderAttributeNameInput(fieldInstance: any, attr: RawAttribute | null): JSX.Element { + private renderAttributeNameInput(fieldInstance: any, attr: SerializedAttribute | null): JSX.Element { const { key } = fieldInstance; const locked = attr ? attr.id as number >= 0 : false; const value = attr ? attr.name : ''; @@ -158,7 +158,7 @@ export default class LabelForm extends React.Component { ); } - private renderAttributeTypeInput(fieldInstance: any, attr: RawAttribute | null): JSX.Element { + private renderAttributeTypeInput(fieldInstance: any, attr: SerializedAttribute | null): JSX.Element { const { key } = fieldInstance; const locked = attr ? attr.id as number >= 0 : false; const type = attr ? attr.input_type.toUpperCase() : AttributeType.SELECT; @@ -188,7 +188,7 @@ export default class LabelForm extends React.Component { ); } - private renderAttributeValuesInput(fieldInstance: any, attr: RawAttribute | null): JSX.Element { + private renderAttributeValuesInput(fieldInstance: any, attr: SerializedAttribute | null): JSX.Element { const { key } = fieldInstance; const locked = attr ? attr.id as number >= 0 : false; const existingValues = attr ? attr.values : []; @@ -259,7 +259,7 @@ export default class LabelForm extends React.Component { ); } - private renderNumberRangeInput(fieldInstance: any, attr: RawAttribute | null): JSX.Element { + private renderNumberRangeInput(fieldInstance: any, attr: SerializedAttribute | null): JSX.Element { const { key } = fieldInstance; const locked = attr ? attr.id as number >= 0 : false; const value = attr ? attr.values : ''; @@ -313,7 +313,7 @@ export default class LabelForm extends React.Component { ); } - private renderDefaultValueInput(fieldInstance: any, attr: RawAttribute | null): JSX.Element { + private renderDefaultValueInput(fieldInstance: any, attr: SerializedAttribute | null): JSX.Element { const { key } = fieldInstance; const value = attr ? attr.values[0] : ''; @@ -324,7 +324,7 @@ export default class LabelForm extends React.Component { ); } - private renderMutableAttributeInput(fieldInstance: any, attr: RawAttribute | null): JSX.Element { + private renderMutableAttributeInput(fieldInstance: any, attr: SerializedAttribute | null): JSX.Element { const { key } = fieldInstance; const locked = attr ? attr.id as number >= 0 : false; const value = attr ? attr.mutable : false; @@ -345,7 +345,7 @@ export default class LabelForm extends React.Component { ); } - private renderDeleteAttributeButton(fieldInstance: any, attr: RawAttribute | null): JSX.Element { + private renderDeleteAttributeButton(fieldInstance: any, attr: SerializedAttribute | null): JSX.Element { const { key } = fieldInstance; const locked = attr ? attr.id as number >= 0 : false; @@ -565,7 +565,7 @@ export default class LabelForm extends React.Component { const { label } = this.props; if (this.formRef.current && label && label.attributes.length) { const convertedAttributes = label.attributes.map( - (attribute: RawAttribute): Store => ({ + (attribute: SerializedAttribute): Store => ({ ...attribute, values: attribute.input_type.toUpperCase() === 'NUMBER' ? attribute.values.join(';') : attribute.values, diff --git a/cvat-ui/src/components/labels-editor/labels-editor.tsx b/cvat-ui/src/components/labels-editor/labels-editor.tsx index ef6dfcc917ac..313bae6df72f 100644 --- a/cvat-ui/src/components/labels-editor/labels-editor.tsx +++ b/cvat-ui/src/components/labels-editor/labels-editor.tsx @@ -11,7 +11,7 @@ import { EditOutlined, BuildOutlined, ExclamationCircleOutlined, } from '@ant-design/icons'; -import { RawLabel, RawAttribute } from 'cvat-core-wrapper'; +import { SerializedLabel, SerializedAttribute } from 'cvat-core-wrapper'; import RawViewer from './raw-viewer'; import ConstructorViewer from './constructor-viewer'; import ConstructorCreator from './constructor-creator'; @@ -25,7 +25,7 @@ enum ConstructorMode { } interface LabelsEditorProps { - labels: RawLabel[]; + labels: SerializedLabel[]; onSubmit: (labels: LabelOptColor[]) => void; } @@ -56,7 +56,7 @@ export default class LabelsEditor extends React.PureComponent ({ + (attr: SerializedAttribute): SerializedAttribute => ({ id: attr.id || idGenerator(), name: attr.name, input_type: attr.input_type, @@ -182,10 +182,10 @@ export default class LabelsEditor extends React.PureComponent ({ + attributes: label.attributes.map((attr: SerializedAttribute): SerializedAttribute => ({ name: attr.name, id: attr.id as number < 0 ? undefined : attr.id, - input_type: attr.input_type.toLowerCase() as RawAttribute['input_type'], + input_type: attr.input_type.toLowerCase() as SerializedAttribute['input_type'], default_value: attr.values[0], mutable: attr.mutable, values: [...attr.values], diff --git a/cvat-ui/src/components/labels-editor/raw-viewer.tsx b/cvat-ui/src/components/labels-editor/raw-viewer.tsx index 785e0dc633d5..7cea7defaaad 100644 --- a/cvat-ui/src/components/labels-editor/raw-viewer.tsx +++ b/cvat-ui/src/components/labels-editor/raw-viewer.tsx @@ -12,7 +12,7 @@ import Modal from 'antd/lib/modal'; import { Store } from 'antd/lib/form/interface'; import Paragraph from 'antd/lib/typography/Paragraph'; -import { RawLabel, RawAttribute } from 'cvat-core-wrapper'; +import { SerializedLabel, SerializedAttribute } from 'cvat-core-wrapper'; import CVATTooltip from 'components/common/cvat-tooltip'; import { validateParsedLabel, idGenerator, LabelOptColor } from './common'; @@ -57,7 +57,7 @@ function validateLabels(_: RuleObject, value: string): Promise { if (!Array.isArray(parsed)) { return Promise.reject(new Error('Field is expected to be a JSON array')); } - const labelNames = parsed.map((label: RawLabel) => label.name); + const labelNames = parsed.map((label: SerializedLabel) => label.name); if (new Set(labelNames).size !== labelNames.length) { return Promise.reject(new Error('Label names must be unique for the task')); } @@ -88,7 +88,7 @@ function convertLabels(labels: LabelOptColor[]): LabelOptColor[] { id: (label.id as number) < 0 ? undefined : label.id, svg: label.svg ? label.svg.replaceAll('"', '"') : undefined, attributes: label.attributes.map( - (attribute: any): RawAttribute => ({ + (attribute: any): SerializedAttribute => ({ ...attribute, id: attribute.id < 0 ? undefined : attribute.id, }), @@ -118,7 +118,7 @@ export default class RawViewer extends React.PureComponent { const { onSubmit, labels } = this.props; const parsed = JSON.parse( replaceTrailingCommas(values.labels), - ) as RawLabel[]; + ) as SerializedLabel[]; const labelIDs: number[] = []; const attrIDs: number[] = []; @@ -145,8 +145,8 @@ export default class RawViewer extends React.PureComponent { }); const deletedAttributes = labels - .reduce((acc: RawAttribute[], _label) => [...acc, ..._label.attributes], []) - .filter((_attr: RawAttribute) => { + .reduce((acc: SerializedAttribute[], _label) => [...acc, ..._label.attributes], []) + .filter((_attr: SerializedAttribute) => { const attrID = _attr.id as number; return attrID >= 0 && !attrIDs.includes(attrID); }); @@ -173,7 +173,7 @@ export default class RawViewer extends React.PureComponent { Following attributes are going to be removed:
- {deletedAttributes.map((_attr: RawAttribute) => ( + {deletedAttributes.map((_attr: SerializedAttribute) => ( {_attr.name} ))}
diff --git a/cvat-ui/src/cvat-core-wrapper.ts b/cvat-ui/src/cvat-core-wrapper.ts index 1e3aa6447793..21412cdc96e5 100644 --- a/cvat-ui/src/cvat-core-wrapper.ts +++ b/cvat-ui/src/cvat-core-wrapper.ts @@ -9,8 +9,9 @@ import Webhook from 'cvat-core/src/webhook'; import MLModel from 'cvat-core/src/ml-model'; import { ModelProvider } from 'cvat-core/src/lambda-manager'; import { - Label, Attribute, RawAttribute, RawLabel, + Label, Attribute, } from 'cvat-core/src/labels'; +import { SerializedAttribute, SerializedLabel } from 'cvat-core/src/server-response-types'; import { Job, Task } from 'cvat-core/src/session'; import { ShapeType, LabelType, ModelKind, ModelProviders, ModelReturnType, @@ -52,8 +53,8 @@ export { }; export type { - RawAttribute, - RawLabel, + SerializedAttribute, + SerializedLabel, StorageData, SocialAuthMethods, ModelProvider, From a237d2f3a991f1c58c1208b6f2d62c70a89379fb Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 03:12:10 -0800 Subject: [PATCH 084/140] Removed unused proxy --- cvat-core/src/config.ts | 1 - cvat-core/src/server-proxy.ts | 198 ++++++---------------------------- 2 files changed, 31 insertions(+), 168 deletions(-) diff --git a/cvat-core/src/config.ts b/cvat-core/src/config.ts index b44d136295b8..063c693a8907 100644 --- a/cvat-core/src/config.ts +++ b/cvat-core/src/config.ts @@ -5,7 +5,6 @@ const config = { backendAPI: '/api', - proxy: false, organizationID: null, origin: '', uploadChunkSize: 100, diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index affdc6f49e96..ec85c670acb1 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -50,7 +50,7 @@ function removeToken(): void { store.remove('token'); } -function waitFor(frequencyHz, predicate) { +function waitFor(frequencyHz: number, predicate: () => boolean): Promise { return new Promise((resolve, reject) => { if (typeof predicate !== 'function') { reject(new Error(`Predicate must be a function, got ${typeof predicate}`)); @@ -88,7 +88,6 @@ function fetchAll(url, filter = {}): Promise { page_size: pageSize, page: 1, }, - proxy: config.proxy, }).then((initialData) => { const { count, results } = initialData.data; result.results = result.results.concat(results); @@ -106,7 +105,6 @@ function fetchAll(url, filter = {}): Promise { page_size: pageSize, page: i + 1, }, - proxy: config.proxy, }); } @@ -304,9 +302,7 @@ async function about() { let response = null; try { - response = await Axios.get(`${backendAPI}/server/about`, { - proxy: config.proxy, - }); + response = await Axios.get(`${backendAPI}/server/about`); } catch (errorData) { throw generateError(errorData); } @@ -320,7 +316,6 @@ async function share(directoryArg) { let response = null; try { response = await Axios.get(`${backendAPI}/server/share`, { - proxy: config.proxy, params: { directory: directoryArg }, }); } catch (errorData) { @@ -335,7 +330,6 @@ async function exception(exceptionObject) { try { await Axios.post(`${backendAPI}/server/exception`, JSON.stringify(exceptionObject), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -350,9 +344,7 @@ async function formats(): Promise { let response = null; try { - response = await Axios.get(`${backendAPI}/server/annotation/formats`, { - proxy: config.proxy, - }); + response = await Axios.get(`${backendAPI}/server/annotation/formats`); } catch (errorData) { throw generateError(errorData); } @@ -365,7 +357,6 @@ async function userAgreements() { let response = null; try { response = await Axios.get(`${backendAPI}/user-agreements`, { - proxy: config.proxy, validateStatus: (status) => status === 200 || status === 404, }); @@ -392,7 +383,6 @@ async function register(username, firstName, lastName, email, password, confirma confirmations, }); response = await Axios.post(`${config.backendAPI}/auth/register`, data, { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -415,9 +405,7 @@ async function login(credential, password) { removeToken(); let authenticationResponse = null; try { - authenticationResponse = await Axios.post(`${config.backendAPI}/auth/login`, authenticationData, { - proxy: config.proxy, - }); + authenticationResponse = await Axios.post(`${config.backendAPI}/auth/login`, authenticationData); } catch (errorData) { throw generateError(errorData); } @@ -450,10 +438,7 @@ async function loginWithSocialAccount( }; let authenticationResponse = null; try { - authenticationResponse = await Axios.post(`${config.backendAPI}/auth/${provider}/login/token`, data, - { - proxy: config.proxy, - }); + authenticationResponse = await Axios.post(`${config.backendAPI}/auth/${provider}/login/token`, data); } catch (errorData) { throw generateError(errorData); } @@ -465,9 +450,7 @@ async function loginWithSocialAccount( async function logout(): Promise { try { - await Axios.post(`${config.backendAPI}/auth/logout`, { - proxy: config.proxy, - }); + await Axios.post(`${config.backendAPI}/auth/logout`); removeToken(); } catch (errorData) { throw generateError(errorData); @@ -482,7 +465,6 @@ async function changePassword(oldPassword: string, newPassword1: string, newPass new_password2: newPassword2, }); await Axios.post(`${config.backendAPI}/auth/password/change`, data, { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -498,7 +480,6 @@ async function requestPasswordReset(email: string): Promise { email, }); await Axios.post(`${config.backendAPI}/auth/password/reset`, data, { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -517,7 +498,6 @@ async function resetPassword(newPassword1: string, newPassword2: string, uid: st token: _token, }); await Axios.post(`${config.backendAPI}/auth/password/reset/confirm`, data, { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -532,9 +512,7 @@ async function getSelf(): Promise { let response = null; try { - response = await Axios.get(`${backendAPI}/users/self`, { - proxy: config.proxy, - }); + response = await Axios.get(`${backendAPI}/users/self`); } catch (errorData) { throw generateError(errorData); } @@ -576,7 +554,6 @@ async function healthCheck( } return Axios.get(url, { - proxy: config.proxy, timeout: requestTimeout, }) .then((response) => response.data) @@ -623,12 +600,11 @@ async function serverRequest(url: string, data: object): Promise { } async function searchProjectNames(search: string, limit: number): Promise { - const { backendAPI, proxy } = config; + const { backendAPI } = config; let response = null; try { response = await Axios.get(`${backendAPI}/projects`, { - proxy, params: { names_only: true, page: 1, @@ -645,14 +621,12 @@ async function searchProjectNames(search: string, limit: number): Promise { - const { backendAPI, proxy } = config; + const { backendAPI } = config; let response = null; try { if ('id' in filter) { - response = await Axios.get(`${backendAPI}/projects/${filter.id}`, { - proxy, - }); + response = await Axios.get(`${backendAPI}/projects/${filter.id}`); const results = [response.data]; Object.defineProperty(results, 'count', { value: 1, @@ -665,7 +639,6 @@ async function getProjects(filter: ProjectsFilter = {}): Promise): try { await Axios.patch(`${backendAPI}/projects/${id}`, JSON.stringify(projectData), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -694,9 +666,7 @@ async function deleteProject(id: number): Promise { const { backendAPI } = config; try { - await Axios.delete(`${backendAPI}/projects/${id}`, { - proxy: config.proxy, - }); + await Axios.delete(`${backendAPI}/projects/${id}`); } catch (errorData) { throw generateError(errorData); } @@ -707,7 +677,6 @@ async function createProject(projectSpec: SerializedProject): Promise((resolve, reject) => { async function request() { Axios.get(baseURL, { - proxy: config.proxy, params, }) .then((response) => { @@ -868,7 +831,6 @@ async function importDataset( try { const response = await Axios.get(url, { params: { ...params, action: 'import_status' }, - proxy: config.proxy, }); if (response.status === 202) { if (response.data.message) { @@ -894,7 +856,6 @@ async function importDataset( await Axios.post(url, new FormData(), { params, - proxy: config.proxy, }); } catch (errorData) { throw generateError(errorData); @@ -914,14 +875,12 @@ async function importDataset( await Axios.post(url, new FormData(), { params, - proxy: config.proxy, headers: { 'Upload-Start': true }, }); await chunkUpload(file, uploadConfig); await Axios.post(url, new FormData(), { params, - proxy: config.proxy, headers: { 'Upload-Finish': true }, }); } catch (errorData) { @@ -948,7 +907,6 @@ async function backupTask(id: number, targetStorage: Storage, useDefaultSettings async function request() { try { const response = await Axios.get(url, { - proxy: config.proxy, params, }); const isCloudStorage = targetStorage.location === StorageLocation.CLOUD_STORAGE; @@ -988,7 +946,6 @@ async function restoreTask(storage: Storage, file: File | string) { try { taskData.set('rq_id', response.data.rq_id); response = await Axios.post(url, taskData, { - proxy: config.proxy, params, }); if (response.status === 202) { @@ -1012,7 +969,6 @@ async function restoreTask(storage: Storage, file: File | string) { response = await Axios.post(url, new FormData(), { params, - proxy: config.proxy, }); } else { const uploadConfig = { @@ -1024,14 +980,12 @@ async function restoreTask(storage: Storage, file: File | string) { await Axios.post(url, new FormData(), { params, - proxy: config.proxy, headers: { 'Upload-Start': true }, }); const { filename } = await chunkUpload(file, uploadConfig); response = await Axios.post(url, new FormData(), { params: { ...params, filename }, - proxy: config.proxy, headers: { 'Upload-Finish': true }, }); } @@ -1058,7 +1012,6 @@ async function backupProject( async function request() { try { const response = await Axios.get(url, { - proxy: config.proxy, params, }); const isCloudStorage = targetStorage.location === StorageLocation.CLOUD_STORAGE; @@ -1098,7 +1051,6 @@ async function restoreProject(storage: Storage, file: File | string) { try { projectData.set('rq_id', response.data.rq_id); response = await Axios.post(`${backendAPI}/projects/backup`, projectData, { - proxy: config.proxy, params, }); if (response.status === 202) { @@ -1124,7 +1076,6 @@ async function restoreProject(storage: Storage, file: File | string) { response = await Axios.post(url, new FormData(), { params, - proxy: config.proxy, }); } else { const uploadConfig = { @@ -1136,14 +1087,12 @@ async function restoreProject(storage: Storage, file: File | string) { await Axios.post(url, new FormData(), { params, - proxy: config.proxy, headers: { 'Upload-Start': true }, }); const { filename } = await chunkUpload(file, uploadConfig); response = await Axios.post(url, new FormData(), { params: { ...params, filename }, - proxy: config.proxy, headers: { 'Upload-Finish': true }, }); } @@ -1225,7 +1174,6 @@ async function createTask(taskSpec, taskDataSpec, onUpdate) { onUpdate('The task is being created on the server..', null); try { response = await Axios.post(`${backendAPI}/tasks`, JSON.stringify(taskSpec), { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -1258,7 +1206,6 @@ async function createTask(taskSpec, taskDataSpec, onUpdate) { onUpdate('The data are being uploaded to the server', percentage); await Axios.post(`${backendAPI}/tasks/${taskId}/data`, taskData, { ...params, - proxy: config.proxy, headers: { 'Upload-Multiple': true }, }); for (let i = 0; i < fileBulks[currentChunkNumber].files.length; i++) { @@ -1273,7 +1220,6 @@ async function createTask(taskSpec, taskDataSpec, onUpdate) { await Axios.post(`${backendAPI}/tasks/${response.data.id}/data`, taskData, { ...params, - proxy: config.proxy, headers: { 'Upload-Start': true }, }); const uploadConfig = { @@ -1294,7 +1240,6 @@ async function createTask(taskSpec, taskDataSpec, onUpdate) { await Axios.post(`${backendAPI}/tasks/${response.data.id}/data`, taskData, { ...params, - proxy: config.proxy, headers: { 'Upload-Finish': true }, }); } catch (errorData) { @@ -1325,9 +1270,7 @@ async function getJobs(filter = {}, aggregate = false) { let response = null; try { if (id !== null) { - response = await Axios.get(`${backendAPI}/jobs/${id}`, { - proxy: config.proxy, - }); + response = await Axios.get(`${backendAPI}/jobs/${id}`); } else { if (aggregate) { return await fetchAll(`${backendAPI}/jobs`, { @@ -1337,7 +1280,6 @@ async function getJobs(filter = {}, aggregate = false) { } response = await Axios.get(`${backendAPI}/jobs`, { - proxy: config.proxy, params: { ...filter, page_size: 12, @@ -1395,7 +1337,6 @@ async function createComment(data) { let response = null; try { response = await Axios.post(`${backendAPI}/comments`, JSON.stringify(data), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1414,7 +1355,6 @@ async function createIssue(data) { try { const organization = enableOrganization(); response = await Axios.post(`${backendAPI}/issues`, JSON.stringify(data), { - proxy: config.proxy, params: { ...organization }, headers: { 'Content-Type': 'application/json', @@ -1440,7 +1380,6 @@ async function updateIssue(issueID, data) { let response = null; try { response = await Axios.patch(`${backendAPI}/issues/${issueID}`, JSON.stringify(data), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1468,7 +1407,6 @@ async function saveJob(id, jobData) { let response = null; try { response = await Axios.patch(`${backendAPI}/jobs/${id}`, JSON.stringify(jobData), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1486,7 +1424,6 @@ async function getUsers(filter = { page_size: 'all' }) { let response = null; try { response = await Axios.get(`${backendAPI}/users`, { - proxy: config.proxy, params: { ...filter, }, @@ -1506,7 +1443,6 @@ function getPreview(instance) { try { const url = `${backendAPI}/${instance}/${id}/preview`; response = await Axios.get(url, { - proxy: config.proxy, responseType: 'blob', }); } catch (errorData) { @@ -1529,7 +1465,6 @@ async function getImageContext(jid, frame) { type: 'context_image', number: frame, }, - proxy: config.proxy, responseType: 'arraybuffer', }); } catch (errorData) { @@ -1553,7 +1488,6 @@ async function getData(tid, jid, chunk) { type: 'chunk', number: chunk, }, - proxy: config.proxy, responseType: 'arraybuffer', }); } catch (errorData) { @@ -1591,7 +1525,6 @@ async function getMeta(session, jid): Promise { let response = null; try { response = await Axios.get(`${backendAPI}/${session}s/${jid}/data/meta`, { - proxy: config.proxy, }); } catch (errorData) { throw generateError(errorData); @@ -1605,9 +1538,7 @@ async function saveMeta(session, jid, meta) { let response = null; try { - response = await Axios.patch(`${backendAPI}/${session}s/${jid}/data/meta`, meta, { - proxy: config.proxy, - }); + response = await Axios.patch(`${backendAPI}/${session}s/${jid}/data/meta`, meta); } catch (errorData) { throw generateError(errorData); } @@ -1621,9 +1552,7 @@ async function getAnnotations(session, id) { let response = null; try { - response = await Axios.get(`${backendAPI}/${session}s/${id}/annotations`, { - proxy: config.proxy, - }); + response = await Axios.get(`${backendAPI}/${session}s/${id}/annotations`); } catch (errorData) { throw generateError(errorData); } @@ -1634,9 +1563,7 @@ async function getFunctions(): Promise { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/functions`, { - proxy: config.proxy, - }); + const response = await Axios.get(`${backendAPI}/functions`); return response.data; } catch (errorData) { if (errorData.response.status === 404) { @@ -1656,7 +1583,6 @@ async function getFunctionPreview(modelID) { try { const url = `${backendAPI}/functions/${modelID}/preview`; response = await Axios.get(url, { - proxy: config.proxy, responseType: 'blob', }); } catch (errorData) { @@ -1672,7 +1598,6 @@ async function getFunctionProviders() { try { const response = await Axios.get(`${backendAPI}/functions/info`, { - proxy: config.proxy, }); return response.data; } catch (errorData) { @@ -1688,7 +1613,6 @@ async function deleteFunction(functionId: number) { try { await Axios.delete(`${backendAPI}/functions/${functionId}`, { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1715,7 +1639,6 @@ async function updateAnnotations(session, id, data, action) { let response = null; try { response = await requestFunc(url, JSON.stringify(data), { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -1732,7 +1655,6 @@ async function runFunctionRequest(body) { try { const response = await Axios.post(`${backendAPI}/functions/requests/`, JSON.stringify(body), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1773,7 +1695,6 @@ async function uploadAnnotations( new FormData(), { params, - proxy: config.proxy, }, ); if (response.status === 202) { @@ -1795,7 +1716,6 @@ async function uploadAnnotations( await Axios.post(url, new FormData(), { params, - proxy: config.proxy, }); } catch (errorData) { throw generateError(errorData); @@ -1811,14 +1731,12 @@ async function uploadAnnotations( await Axios.post(url, new FormData(), { params, - proxy: config.proxy, headers: { 'Upload-Start': true }, }); await chunkUpload(file, uploadConfig); await Axios.post(url, new FormData(), { params, - proxy: config.proxy, headers: { 'Upload-Finish': true }, }); } catch (errorData) { @@ -1836,9 +1754,7 @@ async function getFunctionRequestStatus(requestID) { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/functions/requests/${requestID}`, { - proxy: config.proxy, - }); + const response = await Axios.get(`${backendAPI}/functions/requests/${requestID}`); return response.data; } catch (errorData) { throw generateError(errorData); @@ -1859,7 +1775,6 @@ async function dumpAnnotations(id, name, format) { return new Promise((resolve, reject) => { async function request() { Axios.get(baseURL, { - proxy: config.proxy, params, }) .then((response) => { @@ -1878,7 +1793,7 @@ async function dumpAnnotations(id, name, format) { }); } -async function cancelFunctionRequest(requestId) { +async function cancelFunctionRequest(requestId: string): Promise { const { backendAPI } = config; try { @@ -1896,7 +1811,6 @@ async function createFunction(functionData: any) { try { const response = await Axios.post(`${backendAPI}/functions`, JSON.stringify(functionData), { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -1913,7 +1827,6 @@ async function saveLogs(logs) { try { await Axios.post(`${backendAPI}/server/logs`, JSON.stringify(logs), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1928,7 +1841,6 @@ async function callFunction(funId, body) { try { const response = await Axios.post(`${backendAPI}/functions/${funId}/run`, JSON.stringify(body), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1944,10 +1856,7 @@ async function getFunctionsRequests() { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/functions/requests/`, { - proxy: config.proxy, - }); - + const response = await Axios.get(`${backendAPI}/functions/requests/`); return response.data; } catch (errorData) { if (errorData.response.status === 404) { @@ -1961,9 +1870,7 @@ async function getLambdaFunctions() { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/lambda/functions`, { - proxy: config.proxy, - }); + const response = await Axios.get(`${backendAPI}/lambda/functions`); return response.data; } catch (errorData) { if (errorData.response.status === 503) { @@ -1978,7 +1885,6 @@ async function runLambdaRequest(body) { try { const response = await Axios.post(`${backendAPI}/lambda/requests`, JSON.stringify(body), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -1995,7 +1901,6 @@ async function callLambdaFunction(funId, body) { try { const response = await Axios.post(`${backendAPI}/lambda/functions/${funId}`, JSON.stringify(body), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -2011,10 +1916,7 @@ async function getLambdaRequests() { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/lambda/requests`, { - proxy: config.proxy, - }); - + const response = await Axios.get(`${backendAPI}/lambda/requests`); return response.data; } catch (errorData) { throw generateError(errorData); @@ -2025,9 +1927,7 @@ async function getRequestStatus(requestID) { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/lambda/requests/${requestID}`, { - proxy: config.proxy, - }); + const response = await Axios.get(`${backendAPI}/lambda/requests/${requestID}`); return response.data; } catch (errorData) { throw generateError(errorData); @@ -2147,9 +2047,7 @@ predictAnnotations.latestRequest = { async function installedApps() { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/server/plugins`, { - proxy: config.proxy, - }); + const response = await Axios.get(`${backendAPI}/server/plugins`); return response.data; } catch (errorData) { throw generateError(errorData); @@ -2161,9 +2059,7 @@ async function createCloudStorage(storageDetail) { const storageDetailData = prepareData(storageDetail); try { - const response = await Axios.post(`${backendAPI}/cloudstorages`, storageDetailData, { - proxy: config.proxy, - }); + const response = await Axios.post(`${backendAPI}/cloudstorages`, storageDetailData); return response.data; } catch (errorData) { throw generateError(errorData); @@ -2175,9 +2071,7 @@ async function updateCloudStorage(id, storageDetail) { const storageDetailData = prepareData(storageDetail); try { - await Axios.patch(`${backendAPI}/cloudstorages/${id}`, storageDetailData, { - proxy: config.proxy, - }); + await Axios.patch(`${backendAPI}/cloudstorages/${id}`, storageDetailData); } catch (errorData) { throw generateError(errorData); } @@ -2189,7 +2083,6 @@ async function getCloudStorages(filter = {}) { let response = null; try { response = await Axios.get(`${backendAPI}/cloudstorages`, { - proxy: config.proxy, params: filter, page_size: 12, }); @@ -2209,9 +2102,7 @@ async function getCloudStorageContent(id, manifestPath) { const url = `${backendAPI}/cloudstorages/${id}/content${ manifestPath ? `?manifest_path=${manifestPath}` : '' }`; - response = await Axios.get(url, { - proxy: config.proxy, - }); + response = await Axios.get(url); } catch (errorData) { throw generateError(errorData); } @@ -2225,9 +2116,7 @@ async function getCloudStorageStatus(id) { let response = null; try { const url = `${backendAPI}/cloudstorages/${id}/status`; - response = await Axios.get(url, { - proxy: config.proxy, - }); + response = await Axios.get(url); } catch (errorData) { throw generateError(errorData); } @@ -2239,9 +2128,7 @@ async function deleteCloudStorage(id) { const { backendAPI } = config; try { - await Axios.delete(`${backendAPI}/cloudstorages/${id}`, { - proxy: config.proxy, - }); + await Axios.delete(`${backendAPI}/cloudstorages/${id}`); } catch (errorData) { throw generateError(errorData); } @@ -2266,7 +2153,6 @@ async function createOrganization(data) { let response = null; try { response = await Axios.post(`${backendAPI}/organizations`, JSON.stringify(data), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -2284,7 +2170,6 @@ async function updateOrganization(id, data) { let response = null; try { response = await Axios.patch(`${backendAPI}/organizations/${id}`, JSON.stringify(data), { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -2301,7 +2186,6 @@ async function deleteOrganization(id) { try { await Axios.delete(`${backendAPI}/organizations/${id}`, { - proxy: config.proxy, headers: { 'Content-Type': 'application/json', }, @@ -2317,7 +2201,6 @@ async function getOrganizationMembers(orgSlug, page, pageSize, filters = {}) { let response = null; try { response = await Axios.get(`${backendAPI}/memberships`, { - proxy: config.proxy, params: { ...filters, org: orgSlug, @@ -2341,9 +2224,6 @@ async function inviteOrganizationMembers(orgId, data) { ...data, organization: orgId, }, - { - proxy: config.proxy, - }, ); } catch (errorData) { throw generateError(errorData); @@ -2359,9 +2239,6 @@ async function updateOrganizationMembership(membershipId, data) { { ...data, }, - { - proxy: config.proxy, - }, ); } catch (errorData) { throw generateError(errorData); @@ -2374,9 +2251,7 @@ async function deleteOrganizationMembership(membershipId) { const { backendAPI } = config; try { - await Axios.delete(`${backendAPI}/memberships/${membershipId}`, { - proxy: config.proxy, - }); + await Axios.delete(`${backendAPI}/memberships/${membershipId}`); } catch (errorData) { throw generateError(errorData); } @@ -2387,9 +2262,7 @@ async function getMembershipInvitation(id) { let response = null; try { - response = await Axios.get(`${backendAPI}/invitations/${id}`, { - proxy: config.proxy, - }); + response = await Axios.get(`${backendAPI}/invitations/${id}`); return response.data; } catch (errorData) { throw generateError(errorData); @@ -2402,7 +2275,6 @@ async function getWebhookDelivery(webhookID: number, deliveryID: number): Promis try { const response = await Axios.get(`${backendAPI}/webhooks/${webhookID}/deliveries/${deliveryID}`, { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -2420,7 +2292,6 @@ async function getWebhooks(filter, pageSize = 10): Promise { try { const response = await Axios.get(`${backendAPI}/webhooks`, { - proxy: config.proxy, params: { ...params, ...filter, @@ -2444,7 +2315,6 @@ async function createWebhook(webhookData: any): Promise { try { const response = await Axios.post(`${backendAPI}/webhooks`, JSON.stringify(webhookData), { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -2463,7 +2333,6 @@ async function updateWebhook(webhookID: number, webhookData: any): Promise try { const response = await Axios .patch(`${backendAPI}/webhooks/${webhookID}`, JSON.stringify(webhookData), { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -2481,7 +2350,6 @@ async function deleteWebhook(webhookID: number): Promise { try { await Axios.delete(`${backendAPI}/webhooks/${webhookID}`, { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -2512,7 +2380,6 @@ async function pingWebhook(webhookID: number): Promise { try { const response = await Axios.post(`${backendAPI}/webhooks/${webhookID}/ping`, { - proxy: config.proxy, params, headers: { 'Content-Type': 'application/json', @@ -2532,7 +2399,6 @@ async function receiveWebhookEvents(type: WebhookSourceType): Promise try { const response = await Axios.get(`${backendAPI}/webhooks/events`, { - proxy: config.proxy, params: { type, }, @@ -2549,9 +2415,7 @@ async function receiveWebhookEvents(type: WebhookSourceType): Promise async function socialAuthentication(): Promise { const { backendAPI } = config; try { - const response = await Axios.get(`${backendAPI}/auth/social/methods`, { - proxy: config.proxy, - }); + const response = await Axios.get(`${backendAPI}/auth/social/methods`); return response.data; } catch (errorData) { throw generateError(errorData); From 784f43b26b361d1c376e0612ba8e12d411ff1ff4 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 03:40:15 -0800 Subject: [PATCH 085/140] More types --- cvat-core/src/server-proxy.ts | 31 +++++++----- cvat-core/src/server-response-types.ts | 49 ++++++++++++++++++- .../create-task-page/create-task-content.tsx | 1 + 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index ec85c670acb1..285bf7e016f1 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -5,11 +5,12 @@ import FormData from 'form-data'; import store from 'store'; -import Axios, { AxiosResponse } from 'axios'; +import Axios, { AxiosError, AxiosResponse } from 'axios'; import * as tus from 'tus-js-client'; import { SerializedLabel, SerializedAnnotationFormats, ProjectsFilter, SerializedProject, SerializedTask, TasksFilter, SerializedUser, + SerializedAbout, SerializedShare, SerializedException, SerializedUserAgreement, SerializedRegister, } from 'server-response-types'; import { Storage } from './storage'; import { StorageLocation, WebhookSourceType } from './enums'; @@ -133,7 +134,7 @@ function fetchAll(url, filter = {}): Promise { }); } -async function chunkUpload(file, uploadConfig) { +async function chunkUpload(file: File, uploadConfig) { const params = enableOrganization(); const { endpoint, chunkSize, totalSize, onUpdate, metadata, @@ -182,7 +183,7 @@ async function chunkUpload(file, uploadConfig) { }); } -function generateError(errorData) { +function generateError(errorData: AxiosError<{ message?: string }>): ServerError { if (errorData.response) { if (errorData.response.data?.message) { return new ServerError(errorData.response.data?.message, errorData.response.status); @@ -240,11 +241,11 @@ class WorkerWrappedAxios { } }; - function getRequestId() { + function getRequestId(): number { return requestId++; } - async function get(url, requestConfig) { + async function get(url: string, requestConfig) { return new Promise((resolve, reject) => { const newRequestId = getRequestId(); requests[newRequestId] = { @@ -297,7 +298,7 @@ if (token) { Axios.defaults.headers.common.Authorization = `Token ${token}`; } -async function about() { +async function about(): Promise { const { backendAPI } = config; let response = null; @@ -310,7 +311,7 @@ async function about() { return response.data; } -async function share(directoryArg) { +async function share(directoryArg: string): Promise { const { backendAPI } = config; let response = null; @@ -325,7 +326,7 @@ async function share(directoryArg) { return response.data; } -async function exception(exceptionObject) { +async function exception(exceptionObject: SerializedException): Promise { const { backendAPI } = config; try { @@ -352,7 +353,7 @@ async function formats(): Promise { return response.data; } -async function userAgreements() { +async function userAgreements(): Promise { const { backendAPI } = config; let response = null; try { @@ -370,7 +371,14 @@ async function userAgreements() { } } -async function register(username, firstName, lastName, email, password, confirmations) { +async function register( + username: string, + firstName: string, + lastName: string, + email: string, + password: string, + confirmations: Record, +): Promise { let response = null; try { const data = JSON.stringify({ @@ -394,7 +402,7 @@ async function register(username, firstName, lastName, email, password, confirma return response.data; } -async function login(credential, password) { +async function login(credential: string, password: string): Promise { const authenticationData = [ `${encodeURIComponent(isEmail(credential) ? 'email' : 'username')}=${encodeURIComponent(credential)}`, `${encodeURIComponent('password')}=${encodeURIComponent(password)}`, @@ -689,7 +697,6 @@ async function createProject(projectSpec: SerializedProject): Promise { const { backendAPI } = config; - let response = null; try { if ('id' in filter) { diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index e5a5040f9cae..46f54615a807 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -2,7 +2,10 @@ // // SPDX-License-Identifier: MIT -import { DimensionType, ProjectStatus, TaskStatus } from 'enums'; +import { + DimensionType, ProjectStatus, + ShareFileType, TaskStatus, +} from 'enums'; import { SerializedModel } from 'core-types'; export interface SerializedAnnotationImporter { @@ -119,3 +122,47 @@ export interface SerializedLabel { deleted?: boolean; attributes: SerializedAttribute[]; } + +export interface SerializedAbout { + description: string; + name: string; + version: string; +} + +export interface SerializedShare { + name: string; + type: ShareFileType; + mime_type: string; +} + +export interface SerializedException { + client: string; + client_id: string; + column: number; + filename: string; + is_active: boolean; + line: number; + message: string; + name: string; + stack: string; + system: string; + time: string; + version: string; +} + +export interface SerializedUserAgreement { + name: string; + required: boolean; + textPrefix: string; + url: string; + urlDisplayText: string; + value: boolean; +} + +export interface SerializedRegister { + email: string; + email_verification_required: boolean; + first_name: string; + last_name: string; + username: string; +} diff --git a/cvat-ui/src/components/create-task-page/create-task-content.tsx b/cvat-ui/src/components/create-task-page/create-task-content.tsx index f88889cac28a..9a45c16a3c92 100644 --- a/cvat-ui/src/components/create-task-page/create-task-content.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-content.tsx @@ -896,6 +896,7 @@ class CreateTaskContent extends React.PureComponent From 6023e2ce3b46bf81d5e0124cd1ac710f3ebe52f9 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 9 Feb 2023 14:42:16 +0200 Subject: [PATCH 086/140] Fix task id and org filtering --- cvat/apps/engine/views.py | 27 ++++++++++++++++++++------- cvat/apps/iam/permissions.py | 4 ++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 16ce54b46e69..5ed8859b12e9 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1880,8 +1880,9 @@ def perform_create(self, serializer, **kwargs): list=extend_schema( summary='Method returns a paginated list of labels', parameters=[ - # The parameter is implemented differently from other filters - OpenApiParameter('job_id', description='A simple equality filter for job id') + # These filters are implemented differently from others + OpenApiParameter('job_id', description='A simple equality filter for job id'), + OpenApiParameter('task_id', description='A simple equality filter for task id'), ], responses={ '200': LabelSerializer(many=True), @@ -1912,15 +1913,16 @@ class LabelViewSet(viewsets.GenericViewSet, mixins.ListModelMixin, 'project__organization' ).all() - iam_organization_field = 'task__organization' + # NOTE: This filter works incorrectly for this view + # it requires task__organization OR project__organization check. + # Thus, we rely on permission-based filtering + iam_organization_field = None + search_fields = ('name', 'parent') - filter_fields = list(search_fields) + [ - 'id', 'task_id', 'project_id', 'type', 'color', 'parent_id' - ] + filter_fields = list(search_fields) + ['id', 'project_id', 'type', 'color', 'parent_id'] simple_filters = list(set(filter_fields) - {'id'}) ordering_fields = list(filter_fields) lookup_fields = { - 'task_id': 'task', 'project_id': 'project', 'parent': 'parent__name', } @@ -1940,6 +1942,17 @@ def get_queryset(self): job = Job.objects.get(id=job_id) self.check_object_permissions(self.request, job) queryset = job.get_labels() + elif task_id := self.request.GET.get('task_id', None): + # NOTE: This filter is too complex to be implemented by other means + # It requires the following filter query: + # ( + # project__task__id = task_id + # OR + # task_id = task_id + # ) + task = Task.objects.get(id=task_id) + self.check_object_permissions(self.request, task) + queryset = task.get_labels() else: queryset = super().get_queryset() diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 2997509d3e8e..84c8e9c31be6 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -1389,6 +1389,10 @@ def create(cls, request, view, obj): permissions.append(JobPermission.create_base_perm( request, view, scope=JobPermission.Scopes.VIEW, obj=obj, )) + elif scope == Scopes.LIST and isinstance(obj, Task): + permissions.append(TaskPermission.create_base_perm( + request, view, scope=JobPermission.Scopes.VIEW, obj=obj, + )) else: permissions.append(cls.create_base_perm(request, view, scope, obj)) From 1323e265d7f544e27cba18e62536221e2001446f Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 05:22:26 -0800 Subject: [PATCH 087/140] Removed extra piece of code --- cvat-ui/src/components/create-task-page/create-task-content.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/cvat-ui/src/components/create-task-page/create-task-content.tsx b/cvat-ui/src/components/create-task-page/create-task-content.tsx index 9a45c16a3c92..f88889cac28a 100644 --- a/cvat-ui/src/components/create-task-page/create-task-content.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-content.tsx @@ -896,7 +896,6 @@ class CreateTaskContent extends React.PureComponent From a5875d46b6e5afa9ae0ff737c60d670f3a4426cd Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 09:11:56 -0800 Subject: [PATCH 088/140] Project page & adding, removing, patching labels for a project --- cvat-core/package.json | 2 + cvat-core/src/api-implementation.ts | 2 +- cvat-core/src/common.ts | 43 +++--- cvat-core/src/labels.ts | 12 +- cvat-core/src/project-implementation.ts | 29 +++- cvat-core/src/project.ts | 65 +++++++-- cvat-core/src/server-proxy.ts | 30 ++++- cvat-core/src/server-response-types.ts | 1 - cvat-ui/src/actions/projects-actions.ts | 30 ----- .../src/components/project-page/details.tsx | 19 ++- .../components/project-page/project-page.tsx | 125 +++++++++++++----- .../src/components/project-page/styles.scss | 4 - .../components/task-page/user-selector.tsx | 4 +- cvat-ui/src/cvat-core-wrapper.ts | 2 + cvat-ui/src/reducers/projects-reducer.ts | 31 +---- yarn.lock | 10 +- 16 files changed, 243 insertions(+), 166 deletions(-) diff --git a/cvat-core/package.json b/cvat-core/package.json index a85702d8c5c7..6612810d1861 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -25,6 +25,7 @@ "ts-jest": "26" }, "dependencies": { + "@types/lodash": "^4.14.191", "axios": "^0.27.2", "browser-or-node": "^2.0.0", "cvat-data": "link:./../cvat-data", @@ -34,6 +35,7 @@ "jest-config": "^29.0.3", "js-cookie": "^3.0.1", "json-logic-js": "^2.0.1", + "lodash": "^4.17.21", "platform": "^1.3.5", "quickhull": "^1.0.3", "store": "^2.0.12", diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 4be45369077b..1b9c15949e4a 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -214,7 +214,7 @@ export default function implementAPI(cvat) { ordering: isString, }); - checkExclusiveFields(filter, ['id', 'projectId'], ['page']); + checkExclusiveFields(filter, ['id'], ['page']); const searchParams = {}; for (const key of Object.keys(filter)) { if (['page', 'id', 'sort', 'search', 'filter', 'ordering'].includes(key)) { diff --git a/cvat-core/src/common.ts b/cvat-core/src/common.ts index ab07c5b96ca5..d05afd58b0d2 100644 --- a/cvat-core/src/common.ts +++ b/cvat-core/src/common.ts @@ -92,33 +92,22 @@ export function checkObjectType(name, value, type, instance?): boolean { } export class FieldUpdateTrigger { - constructor() { - let updatedFlags = {}; - - Object.defineProperties( - this, - Object.freeze({ - reset: { - value: () => { - updatedFlags = {}; - }, - }, - update: { - value: (name) => { - updatedFlags[name] = true; - }, - }, - getUpdated: { - value: (data, propMap = {}) => { - const result = {}; - for (const updatedField of Object.keys(updatedFlags)) { - result[propMap[updatedField] || updatedField] = data[updatedField]; - } - return result; - }, - }, - }), - ); + #updatedFlags: Record = {}; + + reset(): void { + this.#updatedFlags = {}; + } + + update(name: string): void { + this.#updatedFlags[name] = true; + } + + getUpdated(data: Record, propMap: Record = {}): Record { + const result = {}; + for (const updatedField of Object.keys(this.#updatedFlags)) { + result[propMap[updatedField] || updatedField] = data[updatedField]; + } + return result; } } diff --git a/cvat-core/src/labels.ts b/cvat-core/src/labels.ts index 7ef23fcbf14a..a95cd3d23c93 100644 --- a/cvat-core/src/labels.ts +++ b/cvat-core/src/labels.ts @@ -95,6 +95,7 @@ export class Label { svg: string; } | null; public deleted: boolean; + public patched: boolean; public readonly hasParent?: boolean; constructor(initialData: SerializedLabel) { @@ -106,6 +107,7 @@ export class Label { structure: undefined, has_parent: false, deleted: false, + patched: false, svg: undefined, elements: undefined, sublabels: undefined, @@ -182,6 +184,12 @@ export class Label { data.deleted = value; }, }, + patched: { + get: () => data.patched, + set: (value) => { + data.patched = value; + }, + }, hasParent: { get: () => data.has_parent, }, @@ -204,10 +212,6 @@ export class Label { object.id = this.id; } - if (this.deleted) { - object.deleted = this.deleted; - } - if (this.type) { object.type = this.type; } diff --git a/cvat-core/src/project-implementation.ts b/cvat-core/src/project-implementation.ts index cac2ce9232b9..cb82451af2c7 100644 --- a/cvat-core/src/project-implementation.ts +++ b/cvat-core/src/project-implementation.ts @@ -4,12 +4,12 @@ // SPDX-License-Identifier: MIT import { Storage } from './storage'; - import serverProxy from './server-proxy'; import { decodePreview } from './frames'; - import Project from './project'; import { exportDataset, importDataset } from './annotations'; +import { SerializedLabel } from './server-response-types'; +import { Label } from './labels'; export default function implementProject(projectClass) { projectClass.prototype.save.implementation = async function () { @@ -21,13 +21,28 @@ export default function implementProject(projectClass) { if (projectData.assignee_id) { projectData.assignee_id = projectData.assignee_id.id; } - if (projectData.labels) { - projectData.labels = projectData.labels.map((el) => el.toJSON()); - } - await serverProxy.projects.save(this.id, projectData); + await Promise.all(projectData.labels.map((label: Label): Promise => { + if (label.deleted) { + return serverProxy.labels.delete(label.id); + } + + if (label.patched) { + return serverProxy.labels.update(label.id, label.toJSON()); + } + + return Promise.resolve(); + })); + + // leave only new labels to create them via project PATCH request + projectData.labels = (projectData.labels || []) + .filter((label: SerializedLabel) => !Number.isInteger(label.id)).map((el) => el.toJSON()); + + const serializedProject = await serverProxy.projects.save(this.id, projectData); + const labels = await serverProxy.labels.get({ project_id: serializedProject.id }); + this._updateTrigger.reset(); - return this; + return new Project({ ...serializedProject, labels: labels.results }); } // initial creating diff --git a/cvat-core/src/project.ts b/cvat-core/src/project.ts index 6e7252ece29b..ff42011380d7 100644 --- a/cvat-core/src/project.ts +++ b/cvat-core/src/project.ts @@ -3,9 +3,10 @@ // // SPDX-License-Identifier: MIT -import { StorageLocation } from './enums'; +import _ from 'lodash'; +import { DimensionType, ProjectStatus, StorageLocation } from './enums'; import { Storage } from './storage'; - +import { SerializedLabel, SerializedProject } from './server-response-types'; import PluginRegistry from './plugins'; import { ArgumentError } from './exceptions'; import { Label } from './labels'; @@ -13,7 +14,26 @@ import User from './user'; import { FieldUpdateTrigger } from './common'; export default class Project { - constructor(initialData) { + public readonly id: number; + public name: string; + public assignee: User; + public bugTracker: string; + public readonly status: ProjectStatus; + public readonly organization: string | null; + public readonly owner: User; + public readonly createdDate: string; + public readonly updatedDate: string; + public readonly taskSubsets: string[]; + public readonly dimension: DimensionType; + public readonly sourceStorage: Storage; + public readonly targetStorage: Storage; + public labels: Label[]; + public annotations: { + exportDataset: CallableFunction; + importDataset: CallableFunction; + } + + constructor(initialData: SerializedProject & { labels?: SerializedLabel[] }) { const data = { id: undefined, name: undefined, @@ -99,24 +119,47 @@ export default class Project { }, labels: { get: () => [...data.labels], - set: (labels) => { + set: (labels: Label[]) => { if (!Array.isArray(labels)) { throw new ArgumentError('Value must be an array of Labels'); } if (!Array.isArray(labels) || labels.some((label) => !(label instanceof Label))) { throw new ArgumentError( - `Each array value must be an instance of Label. ${typeof label} was found`, + 'Each array value must be an instance of Label', ); } - const IDs = labels.map((_label) => _label.id); - const deletedLabels = data.labels.filter((_label) => !IDs.includes(_label.id)); - deletedLabels.forEach((_label) => { - _label.deleted = true; + const oldIDs = data.labels.map((_label) => _label.id); + const newIDs = labels.map((_label) => _label.id); + + // find any deleted labels and mark them + data.labels.filter((_label) => !newIDs.includes(_label.id)) + .forEach((_label) => { + // for deleted labels let's specify that they are deleted + _label.deleted = true; + }); + + // find any patched labels and mark them + labels.forEach((_label) => { + const { id } = _label; + if (oldIDs.includes(id)) { + const oldLabelIndex = data.labels.findIndex((__label) => __label.id === id); + if (oldLabelIndex !== -1) { + // replace current label by the patched one + const oldLabel = data.labels[oldLabelIndex]; + data.labels.splice(oldLabelIndex, 1, _label); + if (!_.isEqual(_label.toJSON(), oldLabel.toJSON())) { + _label.patched = true; + } + } + } }); - data.labels = [...deletedLabels, ...labels]; + // find new labels to append them to the end + const newLabels = labels.filter((_label) => !Number.isInteger(_label.id)); + data.labels = [...data.labels, ...newLabels]; + updateTrigger.update('labels'); }, }, @@ -150,7 +193,7 @@ export default class Project { // When we call a function, for example: project.annotations.get() // In the method get we lose the project context - // So, we need return it + // So, we need to bind it this.annotations = { exportDataset: Object.getPrototypeOf(this).annotations.exportDataset.bind(this), importDataset: Object.getPrototypeOf(this).annotations.importDataset.bind(this), diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 285bf7e016f1..549f08abbfbd 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -656,11 +656,12 @@ async function getProjects(filter: ProjectsFilter = {}): Promise): Promise { +async function saveProject(id: number, projectData: Partial): Promise { const { backendAPI } = config; + let response = null; try { - await Axios.patch(`${backendAPI}/projects/${id}`, JSON.stringify(projectData), { + response = await Axios.patch(`${backendAPI}/projects/${id}`, JSON.stringify(projectData), { headers: { 'Content-Type': 'application/json', }, @@ -668,6 +669,8 @@ async function saveProject(id: number, projectData: Partial): } catch (errorData) { throw generateError(errorData); } + + return response.data; } async function deleteProject(id: number): Promise { @@ -765,6 +768,27 @@ async function getLabels(filter: { }); } +async function deleteLabel(id: number): Promise { + const { backendAPI } = config; + try { + await Axios.delete(`${backendAPI}/labels/${id}`, { method: 'DELETE' }); + } catch (errorData) { + throw generateError(errorData); + } +} + +async function updateLabel(id: number, body: SerializedLabel): Promise { + const { backendAPI } = config; + let response = null; + try { + response = await Axios.patch(`${backendAPI}/labels/${id}`, body, { method: 'PATCH' }); + } catch (errorData) { + throw generateError(errorData); + } + + return response.data; +} + function exportDataset(instanceType: 'projects' | 'jobs' | 'tasks') { return async function ( id: number, @@ -2476,6 +2500,8 @@ export default Object.freeze({ labels: Object.freeze({ get: getLabels, + delete: deleteLabel, + update: updateLabel, }), jobs: Object.freeze({ diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 46f54615a807..c3bda4468837 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -119,7 +119,6 @@ export interface SerializedLabel { svg?: string; sublabels?: SerializedLabel[]; has_parent?: boolean; - deleted?: boolean; attributes: SerializedAttribute[]; } diff --git a/cvat-ui/src/actions/projects-actions.ts b/cvat-ui/src/actions/projects-actions.ts index b62523e660cc..db3cf1f396ff 100644 --- a/cvat-ui/src/actions/projects-actions.ts +++ b/cvat-ui/src/actions/projects-actions.ts @@ -24,9 +24,6 @@ export enum ProjectsActionTypes { CREATE_PROJECT = 'CREATE_PROJECT', CREATE_PROJECT_SUCCESS = 'CREATE_PROJECT_SUCCESS', CREATE_PROJECT_FAILED = 'CREATE_PROJECT_FAILED', - UPDATE_PROJECT = 'UPDATE_PROJECT', - UPDATE_PROJECT_SUCCESS = 'UPDATE_PROJECT_SUCCESS', - UPDATE_PROJECT_FAILED = 'UPDATE_PROJECT_FAILED', DELETE_PROJECT = 'DELETE_PROJECT', DELETE_PROJECT_SUCCESS = 'DELETE_PROJECT_SUCCESS', DELETE_PROJECT_FAILED = 'DELETE_PROJECT_FAILED', @@ -50,11 +47,6 @@ const projectActions = { createAction(ProjectsActionTypes.CREATE_PROJECT_SUCCESS, { projectId }) ), createProjectFailed: (error: any) => createAction(ProjectsActionTypes.CREATE_PROJECT_FAILED, { error }), - updateProject: () => createAction(ProjectsActionTypes.UPDATE_PROJECT), - updateProjectSuccess: (project: any) => createAction(ProjectsActionTypes.UPDATE_PROJECT_SUCCESS, { project }), - updateProjectFailed: (project: any, error: any) => ( - createAction(ProjectsActionTypes.UPDATE_PROJECT_FAILED, { project, error }) - ), deleteProject: (projectId: number) => createAction(ProjectsActionTypes.DELETE_PROJECT, { projectId }), deleteProjectSuccess: (projectId: number) => ( createAction(ProjectsActionTypes.DELETE_PROJECT_SUCCESS, { projectId }) @@ -143,28 +135,6 @@ export function createProjectAsync(data: any): ThunkAction { }; } -export function updateProjectAsync(projectInstance: any): ThunkAction { - return async (dispatch, getState): Promise => { - try { - const state = getState(); - dispatch(projectActions.updateProject()); - await projectInstance.save(); - const [project] = await cvat.projects.get({ id: projectInstance.id }); - dispatch(projectActions.updateProjectSuccess(project)); - dispatch(getProjectTasksAsync(state.projects.tasksGettingQuery)); - } catch (error) { - let project = null; - try { - [project] = await cvat.projects.get({ id: projectInstance.id }); - } catch (fetchError) { - dispatch(projectActions.updateProjectFailed(projectInstance, error)); - return; - } - dispatch(projectActions.updateProjectFailed(project, error)); - } - }; -} - export function deleteProjectAsync(projectInstance: any): ThunkAction { return async (dispatch: ActionCreator): Promise => { dispatch(projectActions.deleteProject(projectInstance.id)); diff --git a/cvat-ui/src/components/project-page/details.tsx b/cvat-ui/src/components/project-page/details.tsx index d89e13c86d6e..83b27271f54e 100644 --- a/cvat-ui/src/components/project-page/details.tsx +++ b/cvat-ui/src/components/project-page/details.tsx @@ -3,14 +3,12 @@ // SPDX-License-Identifier: MIT import React, { useState } from 'react'; -import { useDispatch } from 'react-redux'; import moment from 'moment'; import { Row, Col } from 'antd/lib/grid'; import Title from 'antd/lib/typography/Title'; import Text from 'antd/lib/typography/Text'; -import { getCore } from 'cvat-core-wrapper'; -import { updateProjectAsync } from 'actions/projects-actions'; +import { getCore, Project } from 'cvat-core-wrapper'; import LabelsEditor from 'components/labels-editor/labels-editor'; import BugTrackerEditor from 'components/task-page/bug-tracker-editor'; import UserSelector from 'components/task-page/user-selector'; @@ -18,13 +16,12 @@ import UserSelector from 'components/task-page/user-selector'; const core = getCore(); interface DetailsComponentProps { - project: any; + project: Project; + onUpdateProject: (project: Project) => void; } export default function DetailsComponent(props: DetailsComponentProps): JSX.Element { - const { project } = props; - - const dispatch = useDispatch(); + const { project, onUpdateProject } = props; const [projectName, setProjectName] = useState(project.name); return ( @@ -37,7 +34,7 @@ export default function DetailsComponent(props: DetailsComponentProps): JSX.Elem onChange: (value: string): void => { setProjectName(value); project.name = value; - dispatch(updateProjectAsync(project)); + onUpdateProject(project); }, }} className='cvat-text-color cvat-project-name' @@ -57,7 +54,7 @@ export default function DetailsComponent(props: DetailsComponentProps): JSX.Elem instance={project} onChange={(bugTracker): void => { project.bugTracker = bugTracker; - dispatch(updateProjectAsync(project)); + onUpdateProject(project); }} /> @@ -67,7 +64,7 @@ export default function DetailsComponent(props: DetailsComponentProps): JSX.Elem value={project.assignee} onSelect={(user) => { project.assignee = user; - dispatch(updateProjectAsync(project)); + onUpdateProject(project); }} /> @@ -76,7 +73,7 @@ export default function DetailsComponent(props: DetailsComponentProps): JSX.Elem labels={project.labels.map((label: any): string => label.toJSON())} onSubmit={(labels: any[]): void => { project.labels = labels.map((labelData): any => new core.classes.Label(labelData)); - dispatch(updateProjectAsync(project)); + onUpdateProject(project); }} />
diff --git a/cvat-ui/src/components/project-page/project-page.tsx b/cvat-ui/src/components/project-page/project-page.tsx index 5b46fe47c179..afffc495d440 100644 --- a/cvat-ui/src/components/project-page/project-page.tsx +++ b/cvat-ui/src/components/project-page/project-page.tsx @@ -4,7 +4,7 @@ // SPDX-License-Identifier: MIT import './styles.scss'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { useSelector, useDispatch } from 'react-redux'; import { useHistory, useParams } from 'react-router'; import Spin from 'antd/lib/spin'; @@ -18,9 +18,11 @@ import { MultiPlusIcon } from 'icons'; import { PlusOutlined } from '@ant-design/icons'; import Empty from 'antd/lib/empty'; import Input from 'antd/lib/input'; +import notification from 'antd/lib/notification'; -import { CombinedState, Task, Indexable } from 'reducers'; -import { getProjectsAsync, getProjectTasksAsync } from 'actions/projects-actions'; +import { getCore, Project, Task } from 'cvat-core-wrapper'; +import { CombinedState, Indexable } from 'reducers'; +import { getProjectTasksAsync } from 'actions/projects-actions'; import { cancelInferenceAsync } from 'actions/models-actions'; import TaskItem from 'components/tasks-page/task-item'; import MoveTaskModal from 'components/move-task-modal/move-task-modal'; @@ -36,6 +38,8 @@ import { localStorageRecentKeyword, localStorageRecentCapacity, predefinedFilterValues, config, } from './project-tasks-filter-configuration'; +const core = getCore(); + const FilteringComponent = ResourceFilterHOC( config, localStorageRecentKeyword, localStorageRecentCapacity, predefinedFilterValues, ); @@ -48,8 +52,12 @@ export default function ProjectPageComponent(): JSX.Element { const id = +useParams().id; const dispatch = useDispatch(); const history = useHistory(); - const projects = useSelector((state: CombinedState) => state.projects.current); - const projectsFetching = useSelector((state: CombinedState) => state.projects.fetching); + + const [projectInstance, setProjectInstance] = useState(null); + const [fechingProject, setFetchingProject] = useState(true); + const [updatingProject, setUpdatingProject] = useState(false); + const mounted = useRef(false); + const deletes = useSelector((state: CombinedState) => state.projects.activities.deletes); const taskDeletes = useSelector((state: CombinedState) => state.tasks.activities.deletes); const tasksActiveInferences = useSelector((state: CombinedState) => state.models.inferences); @@ -57,7 +65,6 @@ export default function ProjectPageComponent(): JSX.Element { const tasksCount = useSelector((state: CombinedState) => state.tasks.count); const tasksQuery = useSelector((state: CombinedState) => state.projects.tasksGettingQuery); const tasksFetching = useSelector((state: CombinedState) => state.tasks.fetching); - const [isMounted, setIsMounted] = useState(false); const [visibility, setVisibility] = useState(defaultVisibility); const queryParams = new URLSearchParams(history.location.search); @@ -70,58 +77,76 @@ export default function ProjectPageComponent(): JSX.Element { } useEffect(() => { - dispatch(getProjectTasksAsync({ ...updatedQuery, projectId: id })); - setIsMounted(true); - }, []); - - const [project] = projects.filter((_project) => _project.id === id); - const projectSubsets: Array = []; - for (const task of tasks) { - if (!projectSubsets.includes(task.subset)) projectSubsets.push(task.subset); - } - - useEffect(() => { - if (!project) { - dispatch(getProjectsAsync({ id }, updatedQuery)); + if (Number.isInteger(id)) { + core.projects.get({ id }) + .then(([project]: Project[]) => { + if (project && mounted.current) { + dispatch(getProjectTasksAsync({ ...updatedQuery, projectId: id })); + setProjectInstance(project); + } + }).catch((error: Error) => { + if (mounted.current) { + notification.error({ + message: 'Could not receive the requested project from the server', + description: error.toString(), + }); + } + }).finally(() => { + if (mounted.current) { + setFetchingProject(false); + } + }); + } else { + notification.error({ + message: 'Could not receive the requested project from the server', + description: `Requested project id "${id}" is not valid`, + }); + setFetchingProject(false); } + + mounted.current = true; + return () => { + mounted.current = false; + }; }, []); useEffect(() => { - if (isMounted) { - history.replace({ - search: updateHistoryFromQuery(tasksQuery), - }); - } + history.replace({ + search: updateHistoryFromQuery(tasksQuery), + }); }, [tasksQuery]); useEffect(() => { - if (project && id in deletes && deletes[id]) { + if (projectInstance && id in deletes && deletes[id]) { history.push('/projects'); } }, [deletes]); - if (projectsFetching) { + if (fechingProject) { return ; } - if (!project) { + if (!projectInstance) { return ( ); } + const subsets = Array.from( + new Set(tasks.map((task: Task) => task.subset).filter((subset: string) => !!subset)), + ); const content = tasksCount ? ( <> - {projectSubsets.map((subset: string) => ( + {subsets.map((subset: string) => ( {subset && {subset}} {tasks - .filter((task) => task.projectId === project.id && task.subset === subset) + .filter((task) => task.projectId === projectInstance.id && task.subset === subset) .map((task: Task) => ( - - - + { updatingProject && } + + + { + setUpdatingProject(true); + project.save().then((updatedProject: Project) => { + if (mounted.current) { + dispatch(getProjectTasksAsync({ ...updatedQuery, projectId: id })); + setProjectInstance(updatedProject); + } + }).catch((error: Error) => { + if (mounted.current) { + notification.error({ + message: 'Could not update the project', + description: error.toString(), + }); + } + }).finally(() => { + if (mounted.current) { + setUpdatingProject(false); + } + }); + }} + project={projectInstance} + />
diff --git a/cvat-ui/src/components/project-page/styles.scss b/cvat-ui/src/components/project-page/styles.scss index 0721ba215d07..bdbe30228030 100644 --- a/cvat-ui/src/components/project-page/styles.scss +++ b/cvat-ui/src/components/project-page/styles.scss @@ -7,10 +7,6 @@ .cvat-project-page { overflow-y: auto; height: 100%; - - .cvat-spinner { - position: relative; - } } .cvat-project-page-tasks-bar { diff --git a/cvat-ui/src/components/task-page/user-selector.tsx b/cvat-ui/src/components/task-page/user-selector.tsx index 38595d7ded16..624fdb3348e1 100644 --- a/cvat-ui/src/components/task-page/user-selector.tsx +++ b/cvat-ui/src/components/task-page/user-selector.tsx @@ -84,7 +84,9 @@ export default function UserSelector(props: Props): JSX.Element { const potentialUsers = users.filter((_user) => _user.username.includes(searchPhrase)); if (potentialUsers.length === 1) { setSearchPhrase(potentialUsers[0].username); - onSelect(potentialUsers[0]); + if (value?.id !== potentialUsers[0].id) { + onSelect(potentialUsers[0]); + } } else { setSearchPhrase(value?.username || ''); } diff --git a/cvat-ui/src/cvat-core-wrapper.ts b/cvat-ui/src/cvat-core-wrapper.ts index 21412cdc96e5..5770e4f06634 100644 --- a/cvat-ui/src/cvat-core-wrapper.ts +++ b/cvat-ui/src/cvat-core-wrapper.ts @@ -13,6 +13,7 @@ import { } from 'cvat-core/src/labels'; import { SerializedAttribute, SerializedLabel } from 'cvat-core/src/server-response-types'; import { Job, Task } from 'cvat-core/src/session'; +import Project from 'cvat-core/src/project'; import { ShapeType, LabelType, ModelKind, ModelProviders, ModelReturnType, } from 'cvat-core/src/enums'; @@ -38,6 +39,7 @@ export { Label, Job, Task, + Project, Attribute, ShapeType, LabelType, diff --git a/cvat-ui/src/reducers/projects-reducer.ts b/cvat-ui/src/reducers/projects-reducer.ts index 56d3d189b4b7..c87574ebd3ee 100644 --- a/cvat-ui/src/reducers/projects-reducer.ts +++ b/cvat-ui/src/reducers/projects-reducer.ts @@ -4,11 +4,11 @@ // SPDX-License-Identifier: MIT import { AnyAction } from 'redux'; + import { ProjectsActionTypes } from 'actions/projects-actions'; import { BoundariesActionTypes } from 'actions/boundaries-actions'; import { AuthActionTypes } from 'actions/auth-actions'; - -import { Project, ProjectsState } from '.'; +import { ProjectsState } from '.'; const defaultState: ProjectsState = { initialized: false, @@ -115,33 +115,6 @@ export default (state: ProjectsState = defaultState, action: AnyAction): Project }, }; } - case ProjectsActionTypes.UPDATE_PROJECT: { - return { - ...state, - }; - } - case ProjectsActionTypes.UPDATE_PROJECT_SUCCESS: { - return { - ...state, - current: state.current.map( - (project): Project => ( - project.id === action.payload.project.id ? - action.payload.project : - project - ), - ), - }; - } - case ProjectsActionTypes.UPDATE_PROJECT_FAILED: { - return { - ...state, - current: state.current.map( - (project): Project => (project.id === action.payload.project.id ? - action.payload.project : - project), - ), - }; - } case ProjectsActionTypes.DELETE_PROJECT: { const { projectId } = action.payload; const { deletes } = state.activities; diff --git a/yarn.lock b/yarn.lock index 785ca46d3fc1..0bd42203931b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1813,7 +1813,7 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== -"@types/lodash@^4.14.172": +"@types/lodash@^4.14.172", "@types/lodash@^4.14.191": version "4.14.191" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa" integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ== @@ -1952,9 +1952,9 @@ "@types/react" "*" "@types/react@*", "@types/react@^16.14.15", "@types/react@^17.0.30": - version "17.0.52" - resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.52.tgz#10d8b907b5c563ac014a541f289ae8eaa9bf2e9b" - integrity sha512-vwk8QqVODi0VaZZpDXQCmEmiOuyjEFPY7Ttaw5vjM112LOq37yz1CDJGrRJwA1fYEq4Iitd5rnjd1yWAc/bT+A== + version "17.0.53" + resolved "https://registry.yarnpkg.com/@types/react/-/react-17.0.53.tgz#10d4d5999b8af3d6bc6a9369d7eb953da82442ab" + integrity sha512-1yIpQR2zdYu1Z/dc1OxC+MA6GR240u3gcnP4l6mvj/PJiVaqHsQPmWttsvHsfnhfPbU2FuGmo0wSITPygjBmsw== dependencies: "@types/prop-types" "*" "@types/scheduler" "*" @@ -4117,6 +4117,7 @@ custom-error-instance@2.1.1: "cvat-core@link:./cvat-core": version "8.1.0" dependencies: + "@types/lodash" "^4.14.191" axios "^0.27.2" browser-or-node "^2.0.0" cvat-data "link:./cvat-data" @@ -4126,6 +4127,7 @@ custom-error-instance@2.1.1: jest-config "^29.0.3" js-cookie "^3.0.1" json-logic-js "^2.0.1" + lodash "^4.17.21" platform "^1.3.5" quickhull "^1.0.3" store "^2.0.12" From 438a26bcb0bb6175ba422797bc09b6042d3eba56 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 09:19:33 -0800 Subject: [PATCH 089/140] Small fix --- cvat-core/src/common.ts | 2 +- cvat-core/src/project-implementation.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat-core/src/common.ts b/cvat-core/src/common.ts index d05afd58b0d2..d03550f2bab5 100644 --- a/cvat-core/src/common.ts +++ b/cvat-core/src/common.ts @@ -102,7 +102,7 @@ export class FieldUpdateTrigger { this.#updatedFlags[name] = true; } - getUpdated(data: Record, propMap: Record = {}): Record { + getUpdated(data: object, propMap: Record = {}): Record { const result = {}; for (const updatedField of Object.keys(this.#updatedFlags)) { result[propMap[updatedField] || updatedField] = data[updatedField]; diff --git a/cvat-core/src/project-implementation.ts b/cvat-core/src/project-implementation.ts index cb82451af2c7..99b1002134d4 100644 --- a/cvat-core/src/project-implementation.ts +++ b/cvat-core/src/project-implementation.ts @@ -22,7 +22,7 @@ export default function implementProject(projectClass) { projectData.assignee_id = projectData.assignee_id.id; } - await Promise.all(projectData.labels.map((label: Label): Promise => { + await Promise.all((projectData.labels || []).map((label: Label): Promise => { if (label.deleted) { return serverProxy.labels.delete(label.id); } From 07f1fc237685daa9bca5c34c8278acabee5693eb Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 9 Feb 2023 23:36:00 -0800 Subject: [PATCH 090/140] Task page, updating, creating and deleting labels, updating jobs --- cvat-core/src/annotations-history.ts | 1 + cvat-core/src/api-implementation.ts | 4 +- cvat-core/src/enums.ts | 5 + cvat-core/src/project-implementation.ts | 4 +- cvat-core/src/server-proxy.ts | 128 -------- cvat-core/src/server-response-types.ts | 9 +- cvat-core/src/session-implementation.ts | 146 ++++----- cvat-core/src/session.ts | 277 ++++++++++-------- cvat-ui/src/actions/annotation-actions.ts | 123 -------- cvat-ui/src/actions/tasks-actions.ts | 112 +------ .../components/annotation-page/styles.scss | 46 --- .../annotation-page/top-bar/right-group.tsx | 108 +------ .../annotation-page/top-bar/top-bar.tsx | 13 +- .../create-project-content.tsx | 73 +---- .../create-project-page.tsx | 51 +--- .../create-project.context.ts | 31 -- cvat-ui/src/components/cvat-app.tsx | 4 +- .../move-task-modal/move-task-modal.tsx | 147 +++++++--- .../components/project-page/project-page.tsx | 5 +- cvat-ui/src/components/task-page/details.tsx | 109 ++++--- cvat-ui/src/components/task-page/job-list.tsx | 24 +- .../src/components/task-page/task-page.tsx | 203 +++++++++---- .../annotation-page/top-bar/top-bar.tsx | 22 -- cvat-ui/src/containers/task-page/details.tsx | 78 ----- cvat-ui/src/containers/task-page/job-list.tsx | 33 --- .../src/containers/task-page/task-page.tsx | 75 ----- cvat-ui/src/reducers/annotation-reducer.ts | 53 ---- cvat-ui/src/reducers/index.ts | 22 -- cvat-ui/src/reducers/notifications-reducer.ts | 69 ----- cvat-ui/src/reducers/plugins-reducer.ts | 1 - cvat-ui/src/reducers/tasks-reducer.ts | 85 +----- 31 files changed, 578 insertions(+), 1483 deletions(-) delete mode 100644 cvat-ui/src/components/create-project-page/create-project.context.ts delete mode 100644 cvat-ui/src/containers/task-page/details.tsx delete mode 100644 cvat-ui/src/containers/task-page/job-list.tsx delete mode 100644 cvat-ui/src/containers/task-page/task-page.tsx diff --git a/cvat-core/src/annotations-history.ts b/cvat-core/src/annotations-history.ts index 5d7fecf26be7..748d55bcf93d 100644 --- a/cvat-core/src/annotations-history.ts +++ b/cvat-core/src/annotations-history.ts @@ -1,4 +1,5 @@ // Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts index 1b9c15949e4a..a118deae289a 100644 --- a/cvat-core/src/api-implementation.ts +++ b/cvat-core/src/api-implementation.ts @@ -239,7 +239,9 @@ export default function implementAPI(cvat) { const jobs = await serverProxy.jobs.get({ filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, taskItem.id] }] }), }, true); - return new Task({ ...taskItem, jobs: jobs.results, labels: labels.results }); + return new Task({ + ...taskItem, progress: taskItem.jobs, jobs: jobs.results, labels: labels.results, + }); } return new Task({ diff --git a/cvat-core/src/enums.ts b/cvat-core/src/enums.ts index f41e48f15c40..1ab24b6c9902 100644 --- a/cvat-core/src/enums.ts +++ b/cvat-core/src/enums.ts @@ -8,6 +8,11 @@ export enum ShareFileType { REG = 'REG', } +export enum ChunkType { + IMAGESET = 'imageset', + VIDEO = 'video', +} + export enum TaskStatus { ANNOTATION = 'annotation', VALIDATION = 'validation', diff --git a/cvat-core/src/project-implementation.ts b/cvat-core/src/project-implementation.ts index 99b1002134d4..74934f9298c6 100644 --- a/cvat-core/src/project-implementation.ts +++ b/cvat-core/src/project-implementation.ts @@ -18,6 +18,7 @@ export default function implementProject(projectClass) { bugTracker: 'bug_tracker', assignee: 'assignee_id', }); + if (projectData.assignee_id) { projectData.assignee_id = projectData.assignee_id.id; } @@ -64,7 +65,8 @@ export default function implementProject(projectClass) { } const project = await serverProxy.projects.create(projectSpec); - return new Project(project); + const labels = await serverProxy.labels.get({ project_id: project.id }); + return new Project({ ...project, labels: labels.results }); }; projectClass.prototype.delete.implementation = async function () { diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 549f08abbfbd..70d0feec5803 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -51,31 +51,6 @@ function removeToken(): void { store.remove('token'); } -function waitFor(frequencyHz: number, predicate: () => boolean): Promise { - return new Promise((resolve, reject) => { - if (typeof predicate !== 'function') { - reject(new Error(`Predicate must be a function, got ${typeof predicate}`)); - } - - const internalWait = (): void => { - let result = false; - try { - result = predicate(); - } catch (error) { - reject(error); - } - - if (result) { - resolve(); - } else { - setTimeout(internalWait, 1000 / frequencyHz); - } - }; - - setTimeout(internalWait); - }); -} - function fetchAll(url, filter = {}): Promise { const pageSize = 500; const result = { @@ -1977,104 +1952,6 @@ async function cancelLambdaRequest(requestId) { } } -function predictorStatus(projectId) { - const { backendAPI } = config; - - return new Promise((resolve, reject) => { - async function request() { - try { - const response = await Axios.get(`${backendAPI}/predict/status`, { - params: { - project: projectId, - }, - }); - return response.data; - } catch (errorData) { - throw generateError(errorData); - } - } - - const timeoutCallback = async () => { - let data = null; - try { - data = await request(); - if (data.status === 'queued') { - setTimeout(timeoutCallback, 1000); - } else if (data.status === 'done') { - resolve(data); - } else { - throw new Error(`Unknown status was received "${data.status}"`); - } - } catch (error) { - reject(error); - } - }; - - setTimeout(timeoutCallback); - }); -} - -function predictAnnotations(taskId, frame) { - return new Promise((resolve, reject) => { - const { backendAPI } = config; - - async function request() { - try { - const response = await Axios.get(`${backendAPI}/predict/frame`, { - params: { - task: taskId, - frame, - }, - }); - return response.data; - } catch (errorData) { - throw generateError(errorData); - } - } - - const timeoutCallback = async () => { - let data = null; - try { - data = await request(); - if (data.status === 'queued') { - setTimeout(timeoutCallback, 1000); - } else if (data.status === 'done') { - predictAnnotations.latestRequest.fetching = false; - resolve(data.annotation); - } else { - throw new Error(`Unknown status was received "${data.status}"`); - } - } catch (error) { - predictAnnotations.latestRequest.fetching = false; - reject(error); - } - }; - - const closureId = Date.now(); - predictAnnotations.latestRequest.id = closureId; - const predicate = () => !predictAnnotations.latestRequest.fetching || - predictAnnotations.latestRequest.id !== closureId; - if (predictAnnotations.latestRequest.fetching) { - waitFor(5, predicate).then(() => { - if (predictAnnotations.latestRequest.id !== closureId) { - resolve(null); - } else { - predictAnnotations.latestRequest.fetching = true; - setTimeout(timeoutCallback); - } - }); - } else { - predictAnnotations.latestRequest.fetching = true; - setTimeout(timeoutCallback); - } - }); -} - -predictAnnotations.latestRequest = { - fetching: false, - id: null, -}; - async function installedApps() { const { backendAPI } = config; try { @@ -2568,11 +2445,6 @@ export default Object.freeze({ create: createComment, }), - predictor: Object.freeze({ - status: predictorStatus, - predict: predictAnnotations, - }), - cloudStorages: Object.freeze({ get: getCloudStorages, getContent: getCloudStorageContent, diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index c3bda4468837..b14e4617266f 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -3,8 +3,9 @@ // SPDX-License-Identifier: MIT import { + ChunkType, DimensionType, ProjectStatus, - ShareFileType, TaskStatus, + ShareFileType, TaskMode, TaskStatus, } from 'enums'; import { SerializedModel } from 'core-types'; @@ -77,14 +78,14 @@ export interface SerializedTask { created_date: string; data: number; data_chunk_size: number | null; - data_compressed_chunk_type: 'imageset' | 'video'; - data_original_chunk_type: 'imageset' | 'video'; + data_compressed_chunk_type: ChunkType + data_original_chunk_type: ChunkType; dimension: DimensionType; id: number; image_quality: number; jobs: { count: 1; completed: 0; url: string; }; labels: { count: number; url: string; }; - mode: 'annotation' | 'interpolation' | ''; + mode: TaskMode | ''; name: string; organization: number | null; overlap: number | null; diff --git a/cvat-core/src/session-implementation.ts b/cvat-core/src/session-implementation.ts index e05b90067d56..ccd77801cc22 100644 --- a/cvat-core/src/session-implementation.ts +++ b/cvat-core/src/session-implementation.ts @@ -1,5 +1,11 @@ -import { ArgumentError, DataError } from './exceptions'; +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ArgumentError } from './exceptions'; import { HistoryActions } from './enums'; +import { Storage } from './storage'; import loggerStorage from './logger-storage'; import serverProxy from './server-proxy'; import { @@ -15,6 +21,8 @@ import { decodePreview, } from './frames'; import Issue from './issue'; +import { Label } from './labels'; +import { SerializedLabel } from './server-response-types'; import { checkObjectType } from './common'; import { getAnnotations, putAnnotations, saveAnnotations, @@ -348,39 +356,6 @@ export function implementJob(Job) { return result; }; - Job.prototype.predictor.status.implementation = async function () { - if (!Number.isInteger(this.projectId)) { - throw new DataError('The job must belong to a project to use the feature'); - } - - const result = await serverProxy.predictor.status(this.projectId); - return { - message: result.message, - progress: result.progress, - projectScore: result.score, - timeRemaining: result.time_remaining, - mediaAmount: result.media_amount, - annotationAmount: result.annotation_amount, - }; - }; - - Job.prototype.predictor.predict.implementation = async function (frame) { - if (!Number.isInteger(frame) || frame < 0) { - throw new ArgumentError(`Frame must be a positive integer. Got: "${frame}"`); - } - - if (frame < this.startFrame || frame > this.stopFrame) { - throw new ArgumentError(`The frame with number ${frame} is out of the job`); - } - - if (!Number.isInteger(this.projectId)) { - throw new DataError('The job must belong to a project to use the feature'); - } - - const result = await serverProxy.predictor.predict(this.taskId, frame); - return result; - }; - Job.prototype.close.implementation = function closeTask() { clearFrames(this.id); clearCache(this); @@ -402,7 +377,6 @@ export function implementTask(Task) { }; Task.prototype.save.implementation = async function (onUpdate) { - // TODO: Add ability to change an owner and an assignee if (typeof this.id !== 'undefined') { // If the task has been already created, we update it const taskData = this._updateTrigger.getUpdated(this, { @@ -410,21 +384,40 @@ export function implementTask(Task) { projectId: 'project_id', assignee: 'assignee_id', }); + if (taskData.assignee_id) { taskData.assignee_id = taskData.assignee_id.id; } - if (taskData.labels) { - taskData.labels = this._internalData.labels; - taskData.labels = taskData.labels.map((el) => el.toJSON()); - } - const data = await serverProxy.tasks.save(this.id, taskData); - // Temporary workaround for UI + await Promise.all((taskData.labels || []).map((label: Label): Promise => { + if (label.deleted) { + return serverProxy.labels.delete(label.id); + } + + if (label.patched) { + return serverProxy.labels.update(label.id, label.toJSON()); + } + + return Promise.resolve(); + })); + + // leave only new labels to create them via project PATCH request + taskData.labels = (taskData.labels || []) + .filter((label: SerializedLabel) => !Number.isInteger(label.id)).map((el) => el.toJSON()); + + const serializedTask = await serverProxy.tasks.save(this.id, taskData); + const labels = await serverProxy.labels.get({ task_id: serializedTask.id }); const jobs = await serverProxy.jobs.get({ - filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, data.id] }] }), + filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, serializedTask.id] }] }), }, true); + this._updateTrigger.reset(); - return new Task({ ...data, jobs: jobs.results }); + return new Task({ + ...serializedTask, + progress: serializedTask.jobs, + jobs: jobs.results, + labels: labels.results, + }); } const taskSpec: any = { @@ -464,33 +457,24 @@ export function implementTask(Task) { use_zip_chunks: this.useZipChunks, use_cache: this.useCache, sorting_method: this.sortingMethod, + ...(typeof this.frameFilter !== 'undefined' ? { frame_filter: this.frameFilter } : {}), + ...(typeof this.dataChunkSize !== 'undefined' ? { chunk_size: this.dataChunkSize } : {}), + ...(typeof this.copyData !== 'undefined' ? { copy_data: this.copyData } : {}), + ...(typeof this.cloudStorageId !== 'undefined' ? { cloud_storage_id: this.cloudStorageId } : {}), }; - if (typeof this.startFrame !== 'undefined') { - taskDataSpec.start_frame = this.startFrame; - } - if (typeof this.stopFrame !== 'undefined') { - taskDataSpec.stop_frame = this.stopFrame; - } - if (typeof this.frameFilter !== 'undefined') { - taskDataSpec.frame_filter = this.frameFilter; - } - if (typeof this.dataChunkSize !== 'undefined') { - taskDataSpec.chunk_size = this.dataChunkSize; - } - if (typeof this.copyData !== 'undefined') { - taskDataSpec.copy_data = this.copyData; - } - if (typeof this.cloudStorageId !== 'undefined') { - taskDataSpec.cloud_storage_id = this.cloudStorageId; - } - const task = await serverProxy.tasks.create(taskSpec, taskDataSpec, onUpdate); - // Temporary workaround for UI + const labels = await serverProxy.labels.get({ task_id: task.id }); const jobs = await serverProxy.jobs.get({ filter: JSON.stringify({ and: [{ '==': [{ var: 'task_id' }, task.id] }] }), }, true); - return new Task({ ...task, jobs: jobs.results }); + + return new Task({ + ...task, + progress: task.jobs, + jobs: jobs.results, + labels: labels.results, + }); }; Task.prototype.delete.implementation = async function () { @@ -534,6 +518,7 @@ export function implementTask(Task) { job.stopFrame, isPlaying, step, + this.dimension, ); return result; }; @@ -799,38 +784,5 @@ export function implementTask(Task) { return result; }; - Task.prototype.predictor.status.implementation = async function () { - if (!Number.isInteger(this.projectId)) { - throw new DataError('The task must belong to a project to use the feature'); - } - - const result = await serverProxy.predictor.status(this.projectId); - return { - message: result.message, - progress: result.progress, - projectScore: result.score, - timeRemaining: result.time_remaining, - mediaAmount: result.media_amount, - annotationAmount: result.annotation_amount, - }; - }; - - Task.prototype.predictor.predict.implementation = async function (frame) { - if (!Number.isInteger(frame) || frame < 0) { - throw new ArgumentError(`Frame must be a positive integer. Got: "${frame}"`); - } - - if (frame >= this.size) { - throw new ArgumentError(`The frame with number ${frame} is out of the task`); - } - - if (!Number.isInteger(this.projectId)) { - throw new DataError('The task must belong to a project to use the feature'); - } - - const result = await serverProxy.predictor.predict(this.id, frame); - return result; - }; - return Task; } diff --git a/cvat-core/src/session.ts b/cvat-core/src/session.ts index 7bfffb50d486..e0d6e8be8e37 100644 --- a/cvat-core/src/session.ts +++ b/cvat-core/src/session.ts @@ -3,7 +3,11 @@ // // SPDX-License-Identifier: MIT -import { JobStage, JobState, StorageLocation } from './enums'; +import _ from 'lodash'; +import { + ChunkType, DimensionType, JobStage, + JobState, StorageLocation, TaskMode, TaskStatus, +} from './enums'; import { Storage } from './storage'; import PluginRegistry from './plugins'; @@ -295,25 +299,54 @@ function buildDuplicatedAPI(prototype) { }, writable: true, }), - predictor: Object.freeze({ - value: { - async status() { - const result = await PluginRegistry.apiWrapper.call(this, prototype.predictor.status); - return result; - }, - async predict(frame) { - const result = await PluginRegistry.apiWrapper.call(this, prototype.predictor.predict, frame); - return result; - }, - }, - writable: true, - }), }); } export class Session {} export class Job extends Session { + public annotations: { + get: CallableFunction; + put: CallableFunction; + save: CallableFunction; + merge: CallableFunction; + split: CallableFunction; + group: CallableFunction; + clear: CallableFunction; + search: CallableFunction; + searchEmpty: CallableFunction; + upload: CallableFunction; + select: CallableFunction; + import: CallableFunction; + export: CallableFunction; + statistics: CallableFunction; + hasUnsavedChanges: CallableFunction; + exportDataset: CallableFunction; + }; + + public actions: { + undo: CallableFunction; + redo: CallableFunction; + freeze: CallableFunction; + clear: CallableFunction; + get: CallableFunction; + }; + + public frames: { + get: CallableFunction; + delete: CallableFunction; + restore: CallableFunction; + save: CallableFunction; + ranges: CallableFunction; + preview: CallableFunction; + contextImage: CallableFunction; + search: CallableFunction; + }; + + public logger: { + log: CallableFunction; + }; + constructor(initialData) { super(); const data = { @@ -435,7 +468,7 @@ export class Job extends Session { get: () => data.task_id, }, labels: { - get: () => data.labels.filter((_label) => !_label.deleted), + get: () => [...data.labels], }, dimension: { get: () => data.dimension, @@ -511,11 +544,6 @@ export class Job extends Session { this.logger = { log: Object.getPrototypeOf(this).logger.log.bind(this), }; - - this.predictor = { - status: Object.getPrototypeOf(this).predictor.status.bind(this), - predict: Object.getPrototypeOf(this).predictor.predict.bind(this), - }; } async save() { @@ -540,6 +568,81 @@ export class Job extends Session { } export class Task extends Session { + public name: string; + public projectId: number | null; + public assignee: User | null; + public bugTracker: string; + public subset: string; + public labels: Label[]; + public readonly id: number; + public readonly status: TaskStatus; + public readonly size: number; + public readonly mode: TaskMode; + public readonly owner: User; + public readonly createdDate: string; + public readonly updatedDate: string; + public readonly overlap: number | null; + public readonly segmentSize: number; + public readonly imageQuality: number; + public readonly dataChunkSize: number; + public readonly dataCompressedChunkType: ChunkType; + public readonly dataOriginalChunkType: ChunkType; + public readonly dimension: DimensionType; + public readonly sourceStorage: Storage; + public readonly targetStorage: Storage; + public readonly organization: number | null; + public readonly progress: { count: number; completed: number }; + public readonly jobs: Job[]; + + public readonly frameFilter: string; + public readonly useZipChunks: boolean; + public readonly useCache: boolean; + public readonly copyData: boolean; + public readonly cloudStorageID: number; + public readonly sortingMethod: string; + + public annotations: { + get: CallableFunction; + put: CallableFunction; + save: CallableFunction; + merge: CallableFunction; + split: CallableFunction; + group: CallableFunction; + clear: CallableFunction; + search: CallableFunction; + searchEmpty: CallableFunction; + upload: CallableFunction; + select: CallableFunction; + import: CallableFunction; + export: CallableFunction; + statistics: CallableFunction; + hasUnsavedChanges: CallableFunction; + exportDataset: CallableFunction; + }; + + public actions: { + undo: CallableFunction; + redo: CallableFunction; + freeze: CallableFunction; + clear: CallableFunction; + get: CallableFunction; + }; + + public frames: { + get: CallableFunction; + delete: CallableFunction; + restore: CallableFunction; + save: CallableFunction; + ranges: CallableFunction; + preview: CallableFunction; + contextImage: CallableFunction; + search: CallableFunction; + }; + + public logger: { + log: CallableFunction; + }; + constructor(initialData) { super(); @@ -559,7 +662,6 @@ export class Task extends Session { overlap: undefined, segment_size: undefined, image_quality: undefined, - frame_filter: undefined, data_chunk_size: undefined, data_compressed_chunk_type: undefined, data_original_chunk_type: undefined, @@ -571,6 +673,7 @@ export class Task extends Session { labels: undefined, jobs: undefined, + frame_filter: undefined, use_zip_chunks: undefined, use_cache: undefined, copy_data: undefined, @@ -719,85 +822,70 @@ export class Task extends Session { }, overlap: { get: () => data.overlap, - set: (overlap) => { - if (!Number.isInteger(overlap) || overlap < 0) { - throw new ArgumentError('Value must be a non negative integer'); - } - data.overlap = overlap; - }, }, segmentSize: { get: () => data.segment_size, - set: (segment) => { - if (!Number.isInteger(segment) || segment < 0) { - throw new ArgumentError('Value must be a positive integer'); - } - data.segment_size = segment; - }, }, imageQuality: { get: () => data.image_quality, - set: (quality) => { - if (!Number.isInteger(quality) || quality < 0) { - throw new ArgumentError('Value must be a positive integer'); - } - data.image_quality = quality; - }, }, useZipChunks: { get: () => data.use_zip_chunks, - set: (useZipChunks) => { - if (typeof useZipChunks !== 'boolean') { - throw new ArgumentError('Value must be a boolean'); - } - data.use_zip_chunks = useZipChunks; - }, }, useCache: { get: () => data.use_cache, - set: (useCache) => { - if (typeof useCache !== 'boolean') { - throw new ArgumentError('Value must be a boolean'); - } - data.use_cache = useCache; - }, }, copyData: { get: () => data.copy_data, - set: (copyData) => { - if (typeof copyData !== 'boolean') { - throw new ArgumentError('Value must be a boolean'); - } - data.copy_data = copyData; - }, }, labels: { - get: () => data.labels.filter((_label) => !_label.deleted), - set: (labels) => { + get: () => [...data.labels], + set: (labels: Label[]) => { if (!Array.isArray(labels)) { throw new ArgumentError('Value must be an array of Labels'); } - for (const label of labels) { - if (!(label instanceof Label)) { - throw new ArgumentError( - `Each array value must be an instance of Label. ${typeof label} was found`, - ); - } + if (!Array.isArray(labels) || labels.some((label) => !(label instanceof Label))) { + throw new ArgumentError( + 'Each array value must be an instance of Label', + ); } - const IDs = labels.map((_label) => _label.id); - const deletedLabels = data.labels.filter((_label) => !IDs.includes(_label.id)); - deletedLabels.forEach((_label) => { - _label.deleted = true; + const oldIDs = data.labels.map((_label) => _label.id); + const newIDs = labels.map((_label) => _label.id); + + // find any deleted labels and mark them + data.labels.filter((_label) => !newIDs.includes(_label.id)) + .forEach((_label) => { + // for deleted labels let's specify that they are deleted + _label.deleted = true; + }); + + // find any patched labels and mark them + labels.forEach((_label) => { + const { id } = _label; + if (oldIDs.includes(id)) { + const oldLabelIndex = data.labels.findIndex((__label) => __label.id === id); + if (oldLabelIndex !== -1) { + // replace current label by the patched one + const oldLabel = data.labels[oldLabelIndex]; + data.labels.splice(oldLabelIndex, 1, _label); + if (!_.isEqual(_label.toJSON(), oldLabel.toJSON())) { + _label.patched = true; + } + } + } }); + // find new labels to append them to the end + const newLabels = labels.filter((_label) => !Number.isInteger(_label.id)); + data.labels = [...data.labels, ...newLabels]; + updateTrigger.update('labels'); - data.labels = [...deletedLabels, ...labels]; }, }, jobs: { - get: () => [...data.jobs], + get: () => [...(data.jobs || [])], }, serverFiles: { get: () => [...data.files.server_files], @@ -859,47 +947,11 @@ export class Task extends Session { Array.prototype.push.apply(data.files.remote_files, remoteFiles); }, }, - startFrame: { - get: () => data.start_frame, - set: (frame) => { - if (!Number.isInteger(frame) || frame < 0) { - throw new ArgumentError('Value must be a not negative integer'); - } - data.start_frame = frame; - }, - }, - stopFrame: { - get: () => data.stop_frame, - set: (frame) => { - if (!Number.isInteger(frame) || frame < 0) { - throw new ArgumentError('Value must be a not negative integer'); - } - data.stop_frame = frame; - }, - }, frameFilter: { get: () => data.frame_filter, - set: (filter) => { - if (typeof filter !== 'string') { - throw new ArgumentError( - `Filter value must be a string. But ${typeof filter} has been got.`, - ); - } - - data.frame_filter = filter; - }, }, dataChunkSize: { get: () => data.data_chunk_size, - set: (chunkSize) => { - if (typeof chunkSize !== 'number' || chunkSize < 1) { - throw new ArgumentError( - `Chunk size value must be a positive number. But value ${chunkSize} has been got.`, - ); - } - - data.data_chunk_size = chunkSize; - }, }, dataChunkType: { get: () => data.data_compressed_chunk_type, @@ -988,24 +1040,19 @@ export class Task extends Session { this.logger = { log: Object.getPrototypeOf(this).logger.log.bind(this), }; - - this.predictor = { - status: Object.getPrototypeOf(this).predictor.status.bind(this), - predict: Object.getPrototypeOf(this).predictor.predict.bind(this), - }; } - async close() { + async close(): Promise { const result = await PluginRegistry.apiWrapper.call(this, Task.prototype.close); return result; } - async save(onUpdate = () => {}) { + async save(onUpdate = () => {}): Promise { const result = await PluginRegistry.apiWrapper.call(this, Task.prototype.save, onUpdate); return result; } - async delete() { + async delete(): Promise { const result = await PluginRegistry.apiWrapper.call(this, Task.prototype.delete); return result; } diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 3118999ad6f7..254e4c9bc8d4 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -190,10 +190,6 @@ export enum AnnotationActionTypes { INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS', GET_DATA_FAILED = 'GET_DATA_FAILED', SET_FORCE_EXIT_ANNOTATION_PAGE_FLAG = 'SET_FORCE_EXIT_ANNOTATION_PAGE_FLAG', - UPDATE_PREDICTOR_STATE = 'UPDATE_PREDICTOR_STATE', - GET_PREDICTIONS = 'GET_PREDICTIONS', - GET_PREDICTIONS_FAILED = 'GET_PREDICTIONS_FAILED', - GET_PREDICTIONS_SUCCESS = 'GET_PREDICTIONS_SUCCESS', SWITCH_NAVIGATION_BLOCKED = 'SWITCH_NAVIGATION_BLOCKED', DELETE_FRAME = 'DELETE_FRAME', DELETE_FRAME_SUCCESS = 'DELETE_FRAME_SUCCESS', @@ -575,86 +571,6 @@ export function switchPlay(playing: boolean): AnyAction { }; } -export function getPredictionsAsync(): ThunkAction { - return async (dispatch: ActionCreator): Promise => { - const { - annotations: { - states: currentStates, - zLayer: { cur: curZOrder }, - }, - predictor: { enabled, annotatedFrames }, - } = getStore().getState().annotation; - - const { - filters, frame, showAllInterpolationTracks, jobInstance: job, - } = receiveAnnotationsParameters(); - if (!enabled || currentStates.length || annotatedFrames.includes(frame)) return; - - dispatch({ - type: AnnotationActionTypes.GET_PREDICTIONS, - payload: {}, - }); - - let annotations = []; - try { - annotations = await job.predictor.predict(frame); - // current frame could be changed during a request above, need to fetch it from store again - const { number: currentFrame } = getStore().getState().annotation.player.frame; - if (frame !== currentFrame || annotations === null) { - // another request has already been sent or user went to another frame - // we do not need dispatch predictions success action - return; - } - annotations = annotations.map( - (data: any): any => new cvat.classes.ObjectState({ - shapeType: data.type, - label: job.labels.filter((label: any): boolean => label.id === data.label)[0], - points: data.points, - objectType: ObjectType.SHAPE, - frame, - occluded: false, - source: 'auto', - attributes: {}, - zOrder: curZOrder, - }), - ); - - dispatch({ - type: AnnotationActionTypes.GET_PREDICTIONS_SUCCESS, - payload: { frame }, - }); - } catch (error) { - dispatch({ - type: AnnotationActionTypes.GET_PREDICTIONS_FAILED, - payload: { - error, - }, - }); - } - - try { - await job.annotations.put(annotations); - const states = await job.annotations.get(frame, showAllInterpolationTracks, filters); - const history = await job.actions.get(); - - dispatch({ - type: AnnotationActionTypes.CREATE_ANNOTATIONS_SUCCESS, - payload: { - states, - history, - }, - }); - } catch (error) { - dispatch({ - type: AnnotationActionTypes.CREATE_ANNOTATIONS_FAILED, - payload: { - error, - }, - }); - } - }; -} - export function confirmCanvasReady(): AnyAction { return { type: AnnotationActionTypes.CONFIRM_CANVAS_READY, @@ -770,7 +686,6 @@ export function changeFrameAsync( delay, }, }); - dispatch(getPredictionsAsync()); } catch (error) { if (error !== 'not needed') { dispatch({ @@ -1054,35 +969,6 @@ export function getJobAsync( dispatch(changeWorkspace(workspace)); } - const updatePredictorStatus = async (): Promise => { - // get current job - const currentState: CombinedState = getState(); - const { openTime: currentOpenTime, instance: currentJob } = currentState.annotation.job; - if (currentJob === null || currentJob.id !== job.id || currentOpenTime !== openTime) { - // the job was closed, changed or reopened - return; - } - - try { - const status = await job.predictor.status(); - dispatch({ - type: AnnotationActionTypes.UPDATE_PREDICTOR_STATE, - payload: status, - }); - setTimeout(updatePredictorStatus, 60 * 1000); - } catch (error) { - dispatch({ - type: AnnotationActionTypes.UPDATE_PREDICTOR_STATE, - payload: { error }, - }); - setTimeout(updatePredictorStatus, 20 * 1000); - } - }; - - if (state.plugins.list.PREDICT && job.projectId !== null) { - updatePredictorStatus(); - } - dispatch(changeFrameAsync(frameNumber, false)); } catch (error) { dispatch({ @@ -1642,15 +1528,6 @@ export function setForceExitAnnotationFlag(forceExit: boolean): AnyAction { }; } -export function switchPredictor(predictorEnabled: boolean): AnyAction { - return { - type: AnnotationActionTypes.UPDATE_PREDICTOR_STATE, - payload: { - enabled: predictorEnabled, - }, - }; -} - export function switchNavigationBlocked(navigationBlocked: boolean): AnyAction { return { type: AnnotationActionTypes.SWITCH_NAVIGATION_BLOCKED, diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index ee6d8d805a1c..4a28ae1de936 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -5,9 +5,7 @@ import { AnyAction, Dispatch, ActionCreator } from 'redux'; import { ThunkAction } from 'redux-thunk'; -import { - TasksQuery, CombinedState, StorageLocation, -} from 'reducers'; +import { TasksQuery, StorageLocation } from 'reducers'; import { getCore, Storage } from 'cvat-core-wrapper'; import { filterNull } from 'utils/filter-null'; import { getInferenceStatusAsync } from './models-actions'; @@ -22,11 +20,6 @@ export enum TasksActionTypes { DELETE_TASK_SUCCESS = 'DELETE_TASK_SUCCESS', DELETE_TASK_FAILED = 'DELETE_TASK_FAILED', CREATE_TASK_FAILED = 'CREATE_TASK_FAILED', - UPDATE_TASK = 'UPDATE_TASK', - UPDATE_TASK_SUCCESS = 'UPDATE_TASK_SUCCESS', - UPDATE_TASK_FAILED = 'UPDATE_TASK_FAILED', - UPDATE_JOB = 'UPDATE_JOB', - UPDATE_JOB_SUCCESS = 'UPDATE_JOB_SUCCESS', UPDATE_JOB_FAILED = 'UPDATE_JOB_FAILED', HIDE_EMPTY_TASKS = 'HIDE_EMPTY_TASKS', SWITCH_MOVE_TASK_MODAL_VISIBLE = 'SWITCH_MOVE_TASK_MODAL_VISIBLE', @@ -233,51 +226,6 @@ ThunkAction, {}, {}, AnyAction> { }; } -function updateTask(): AnyAction { - const action = { - type: TasksActionTypes.UPDATE_TASK, - payload: {}, - }; - - return action; -} - -export function updateTaskSuccess(task: any, taskID: number): AnyAction { - const action = { - type: TasksActionTypes.UPDATE_TASK_SUCCESS, - payload: { task, taskID }, - }; - - return action; -} - -function updateTaskFailed(error: any, task: any): AnyAction { - const action = { - type: TasksActionTypes.UPDATE_TASK_FAILED, - payload: { error, task }, - }; - - return action; -} - -function updateJob(jobID: number): AnyAction { - const action = { - type: TasksActionTypes.UPDATE_JOB, - payload: { jobID }, - }; - - return action; -} - -function updateJobSuccess(jobInstance: any, jobID: number): AnyAction { - const action = { - type: TasksActionTypes.UPDATE_JOB_SUCCESS, - payload: { jobID, jobInstance }, - }; - - return action; -} - function updateJobFailed(jobID: number, error: any): AnyAction { const action = { type: TasksActionTypes.UPDATE_JOB_FAILED, @@ -287,35 +235,10 @@ function updateJobFailed(jobID: number, error: any): AnyAction { return action; } -export function updateTaskAsync(taskInstance: any): ThunkAction, CombinedState, {}, AnyAction> { - return async (dispatch: ActionCreator): Promise => { - try { - dispatch(updateTask()); - const task = await taskInstance.save(); - dispatch(updateTaskSuccess(task, taskInstance.id)); - } catch (error) { - // try abort all changes - let task = null; - try { - [task] = await cvat.tasks.get({ id: taskInstance.id }); - } catch (fetchError) { - dispatch(updateTaskFailed(error, taskInstance)); - return; - } - - dispatch(updateTaskFailed(error, task)); - } - }; -} - -// a job is a part of a task, so for simplify we consider -// updating the job as updating a task export function updateJobAsync(jobInstance: any): ThunkAction, {}, {}, AnyAction> { return async (dispatch: ActionCreator): Promise => { try { - dispatch(updateJob(jobInstance.id)); - const newJob = await jobInstance.save(); - dispatch(updateJobSuccess(newJob, newJob.id)); + await jobInstance.save(); } catch (error) { dispatch(updateJobFailed(jobInstance.id, error)); } @@ -345,37 +268,6 @@ export function switchMoveTaskModalVisible(visible: boolean, taskId: number | nu return action; } -interface LabelMap { - label_id: number; - new_label_name: string | null; - clear_attributes: boolean; -} - -export function moveTaskToProjectAsync( - taskInstance: any, - projectId: any, - labelMap: LabelMap[], -): ThunkAction, {}, {}, AnyAction> { - return async (dispatch: ActionCreator): Promise => { - dispatch(updateTask()); - try { - // eslint-disable-next-line no-param-reassign - taskInstance.labels = labelMap.map((mapper) => { - const [label] = taskInstance.labels.filter((_label: any) => mapper.label_id === _label.id); - label.name = mapper.new_label_name; - return label; - }); - // eslint-disable-next-line no-param-reassign - taskInstance.projectId = projectId; - await taskInstance.save(); - const [task] = await cvat.tasks.get({ id: taskInstance.id }); - dispatch(updateTaskSuccess(task, task.id)); - } catch (error) { - dispatch(updateTaskFailed(error, taskInstance)); - } - }; -} - function getTaskPreview(taskID: number): AnyAction { const action = { type: TasksActionTypes.GET_TASK_PREVIEW, diff --git a/cvat-ui/src/components/annotation-page/styles.scss b/cvat-ui/src/components/annotation-page/styles.scss index f776f43bd766..f88d90fa6912 100644 --- a/cvat-ui/src/components/annotation-page/styles.scss +++ b/cvat-ui/src/components/annotation-page/styles.scss @@ -77,52 +77,6 @@ } } -button.cvat-predictor-button { - &.cvat-predictor-inprogress { - > span { - > svg { - fill: $inprogress-progress-color; - } - } - } - - &.cvat-predictor-fetching { - > span { - > svg { - animation-duration: 500ms; - animation-name: predictorBlinking; - animation-iteration-count: infinite; - - @keyframes predictorBlinking { - 0% { - fill: $inprogress-progress-color; - } - - 50% { - fill: $completed-progress-color; - } - - 100% { - fill: $inprogress-progress-color; - } - } - } - } - } - - &.cvat-predictor-disabled { - opacity: 0.5; - - &:active { - pointer-events: none; - } - - > span[role='img'] { - transform: scale(0.8) !important; - } - } -} - .cvat-annotation-disabled-header-button { @extend .cvat-annotation-header-button; diff --git a/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx b/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx index de0363b1581b..b730677916d5 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/right-group.tsx @@ -7,29 +7,18 @@ import { Col } from 'antd/lib/grid'; import Icon from '@ant-design/icons'; import Select from 'antd/lib/select'; import Button from 'antd/lib/button'; -import Text from 'antd/lib/typography/Text'; -import Tooltip from 'antd/lib/tooltip'; -import Moment from 'react-moment'; - -import moment from 'moment'; import { useSelector } from 'react-redux'; +import { FilterIcon, FullscreenIcon, InfoIcon } from 'icons'; import { - FilterIcon, FullscreenIcon, InfoIcon, BrainIcon, -} from 'icons'; -import { - CombinedState, DimensionType, Workspace, PredictorState, + CombinedState, DimensionType, Workspace, } from 'reducers'; interface Props { workspace: Workspace; - predictor: PredictorState; - isTrainingActive: boolean; showStatistics(): void; - switchPredictor(predictorEnabled: boolean): void; showFilters(): void; changeWorkspace(workspace: Workspace): void; - jobInstance: any; } @@ -37,108 +26,15 @@ function RightGroup(props: Props): JSX.Element { const { showStatistics, changeWorkspace, - switchPredictor, workspace, - predictor, jobInstance, - isTrainingActive, showFilters, } = props; - const annotationAmount = predictor.annotationAmount || 0; - const mediaAmount = predictor.mediaAmount || 0; - const formattedScore = `${(predictor.projectScore * 100).toFixed(0)}%`; - const predictorTooltip = ( -
- Adaptive auto annotation is - {predictor.enabled ? ( - - {' active'} - - ) : ( - - {' inactive'} - - )} -
- - Annotations amount: - {annotationAmount} - -
- - Media amount: - {mediaAmount} - -
- {annotationAmount > 0 ? ( - - Model mAP is - {' '} - {formattedScore} -
- - ) : null} - {predictor.error ? ( - - {predictor.error.toString()} -
- - ) : null} - {predictor.message ? ( - - Status: - {' '} - {predictor.message} -
- - ) : null} - {predictor.timeRemaining > 0 ? ( - - Time Remaining: - {' '} - -
- - ) : null} - {predictor.progress > 0 ? ( - - Progress: - {predictor.progress.toFixed(1)} - {' '} - % - - ) : null} -
- ); - - let predictorClassName = 'cvat-annotation-header-button cvat-predictor-button'; - if (!!predictor.error || !predictor.projectScore) { - predictorClassName += ' cvat-predictor-disabled'; - } else if (predictor.enabled) { - if (predictor.fetching) { - predictorClassName += ' cvat-predictor-fetching'; - } - predictorClassName += ' cvat-predictor-inprogress'; - } const filters = useSelector((state: CombinedState) => state.annotation.annotations.filters); return ( - {isTrainingActive && ( - - )}