diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c8d0d829803..f7534e58d33b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Shortcut to change color of an activated shape in new UI (Enter) () - Shortcut to switch split mode () - Built-in search for labels when create an object or change a label () +- Better validation of labels and attributes in raw viewer () +- ClamAV antivirus integration () - Polygon and polylines interpolation () - Ability to redraw shape from scratch (Shift + N) for an activated shape () - Highlights for the first point of a polygon/polyline and direction () @@ -38,6 +40,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Wrong rexex for account name validation () - Wrong description on register view for the username field () - Wrong resolution for resizing a shape () +- React warning because of not unique keys in labels viewer () + ### Security - SQL injection in Django `CVE-2020-9402` () diff --git a/Dockerfile b/Dockerfile index 9c00bf8f0494..e6eb2911652b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -129,6 +129,19 @@ RUN if [ "$WITH_DEXTR" = "yes" ]; then \ 7z e ${DEXTR_MODEL_DIR}/dextr.zip -o${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \ fi +ARG CLAM_AV +ENV CLAM_AV=${CLAM_AV} +RUN if [ "$CLAM_AV" = "yes" ]; then \ + apt-get update && \ + apt-get --no-install-recommends install -yq \ + clamav \ + libclamunrar9 && \ + sed -i 's/ReceiveTimeout 30/ReceiveTimeout 300/g' /etc/clamav/freshclam.conf && \ + freshclam && \ + chown -R ${USER}:${USER} /var/lib/clamav && \ + rm -rf /var/lib/apt/lists/*; \ + fi + COPY ssh ${HOME}/.ssh COPY utils ${HOME}/utils COPY cvat/ ${HOME}/cvat diff --git a/README.md b/README.md index c1c9ac2ee52d..f1c74934d826 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Try it online [cvat.org](https://cvat.org). - [Command line interface](utils/cli/) - [XML annotation format](cvat/apps/documentation/xml_format.md) - [AWS Deployment Guide](cvat/apps/documentation/AWS-Deployment-Guide.md) +- [Frequently asked questions](cvat/apps/documentation/faq.md) - [Questions](#questions) ## Screencasts diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index d429a17f89d9..d7d6dc4ddb92 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.3.0", + "version": "1.3.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 2a9f9705c084..e15f89230fee 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.3.0", + "version": "1.3.1", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/components/labels-editor/common.ts b/cvat-ui/src/components/labels-editor/common.ts index 5ea21f53c5e1..9627be41e3cb 100644 --- a/cvat-ui/src/components/labels-editor/common.ts +++ b/cvat-ui/src/components/labels-editor/common.ts @@ -18,6 +18,59 @@ export interface Label { let id = 0; +function validateParsedAttribute(attr: Attribute): void { + if (typeof (attr.name) !== 'string') { + throw new Error(`Type of attribute name must be a string. Got value ${attr.name}`); + } + + if (!['number', 'undefined'].includes(typeof (attr.id))) { + throw new Error(`Attribute: "${attr.name}". ` + + `Type of attribute id must be a number or undefined. Got value ${attr.id}`); + } + + if (!['checkbox', 'number', 'text', 'radio', 'select'].includes((attr.input_type || '').toLowerCase())) { + throw new Error(`Attribute: "${attr.name}". ` + + `Unknown input type: ${attr.input_type}`); + } + + if (typeof (attr.mutable) !== 'boolean') { + throw new Error(`Attribute: "${attr.name}". ` + + `Mutable flag must be a boolean value. Got value ${attr.mutable}`); + } + + if (!Array.isArray(attr.values)) { + throw new Error(`Attribute: "${attr.name}". ` + + `Attribute values must be an array. Got type ${typeof (attr.values)}`); + } + + for (const value of attr.values) { + if (typeof (value) !== 'string') { + throw new Error(`Attribute: "${attr.name}". ` + + `Each value must be a string. Got value ${value}`); + } + } +} + +export function validateParsedLabel(label: Label): void { + if (typeof (label.name) !== 'string') { + throw new Error(`Type of label name must be a string. Got value ${label.name}`); + } + + if (!['number', 'undefined'].includes(typeof (label.id))) { + throw new Error(`Label "${label.name}". ` + + `Type of label id must be only a number or undefined. Got value ${label.id}`); + } + + if (!Array.isArray(label.attributes)) { + throw new Error(`Label "${label.name}". ` + + `attributes must be an array. Got type ${typeof (label.attributes)}`); + } + + for (const attr of label.attributes) { + validateParsedAttribute(attr); + } +} + export function idGenerator(): number { return --id; } diff --git a/cvat-ui/src/components/labels-editor/raw-viewer.tsx b/cvat-ui/src/components/labels-editor/raw-viewer.tsx index a19fef874796..532046f42bbf 100644 --- a/cvat-ui/src/components/labels-editor/raw-viewer.tsx +++ b/cvat-ui/src/components/labels-editor/raw-viewer.tsx @@ -12,6 +12,8 @@ import Form, { FormComponentProps } from 'antd/lib/form/Form'; import { Label, Attribute, + validateParsedLabel, + idGenerator, } from './common'; type Props = FormComponentProps & { @@ -22,7 +24,18 @@ type Props = FormComponentProps & { class RawViewer extends React.PureComponent { private validateLabels = (_: any, value: string, callback: any): void => { try { - JSON.parse(value); + const parsed = JSON.parse(value); + if (!Array.isArray(parsed)) { + callback('Field is expected to be a JSON array'); + } + + for (const label of parsed) { + try { + validateParsedLabel(label); + } catch (error) { + callback(error.toString()); + } + } } catch (error) { callback(error.toString()); } @@ -39,7 +52,14 @@ class RawViewer extends React.PureComponent { e.preventDefault(); form.validateFields((error, values): void => { if (!error) { - onSubmit(JSON.parse(values.labels)); + const parsed = JSON.parse(values.labels); + for (const label of parsed) { + label.id = label.id || idGenerator(); + for (const attr of label.attributes) { + attr.id = attr.id || idGenerator(); + } + } + onSubmit(parsed); } }); }; diff --git a/cvat/apps/auto_annotation/model_manager.py b/cvat/apps/auto_annotation/model_manager.py index 37f6cc059bcc..7bac221a0e69 100644 --- a/cvat/apps/auto_annotation/model_manager.py +++ b/cvat/apps/auto_annotation/model_manager.py @@ -19,6 +19,7 @@ from cvat.apps.engine.serializers import LabeledDataSerializer from cvat.apps.dataset_manager.task import put_task_data, patch_task_data from cvat.apps.engine.frame_provider import FrameProvider +from cvat.apps.engine.utils import av_scan_paths from .models import AnnotationModel, FrameworkChoice from .model_loader import load_labelmap @@ -139,6 +140,7 @@ def save_file_as_tmp(data): tmp_file.write(chunk) os.close(fd) return filename + is_create_request = dl_model_id is None if is_create_request: dl_model_id = create_empty(owner=owner) @@ -155,6 +157,17 @@ def save_file_as_tmp(data): labelmap_file = save_file_as_tmp(labelmap_file) interpretation_file = save_file_as_tmp(interpretation_file) + files_to_scan = [] + if model_file: + files_to_scan.append(model_file) + if weights_file: + files_to_scan.append(weights_file) + if labelmap_file: + files_to_scan.append(labelmap_file) + if interpretation_file: + files_to_scan.append(interpretation_file) + av_scan_paths(*files_to_scan) + if owner: restricted = not has_admin_role(owner) else: diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 7c8de445e458..71c955f3a4e8 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -11,6 +11,7 @@ import datumaro.components.extractor as datumaro from cvat.apps.engine.frame_provider import FrameProvider from cvat.apps.engine.models import AttributeType, ShapeType +from datumaro.util import cast from datumaro.util.image import Image from .annotation import AnnotationManager, TrackManager @@ -422,8 +423,9 @@ def __init__(self, task_data, include_images=False): size=(frame_data.height, frame_data.width) ) dm_anno = self._read_cvat_anno(frame_data, task_data) - dm_item = datumaro.DatasetItem(id=frame_data.frame, - annotations=dm_anno, image=dm_image) + dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0], + annotations=dm_anno, image=dm_image, + attributes={'frame': frame_data.frame}) dm_items.append(dm_item) self._items = dm_items @@ -533,23 +535,21 @@ def match_frame(item, task_data): is_video = task_data.meta['task']['mode'] == 'interpolation' frame_number = None - if frame_number is None: - try: - frame_number = task_data.match_frame(item.id) - except Exception: - pass if frame_number is None and item.has_image: try: - frame_number = task_data.match_frame(item.image.filename) + frame_number = task_data.match_frame(item.image.path) except Exception: pass if frame_number is None: try: - frame_number = int(item.id) + frame_number = task_data.match_frame(item.id) except Exception: pass - if frame_number is None and is_video and item.id.startswith('frame_'): - frame_number = int(item.id[len('frame_'):]) + if frame_number is None: + frame_number = cast(item.attributes.get('frame', item.id), int) + if frame_number is None and is_video: + frame_number = cast(osp.basename(item.id)[len('frame_'):], int) + if not frame_number in task_data.frame_info: raise Exception("Could not match item id: '%s' with any task frame" % item.id) diff --git a/cvat/apps/dataset_manager/formats/labelme.py b/cvat/apps/dataset_manager/formats/labelme.py index 9ea1a76b800c..31ee2cbde3ad 100644 --- a/cvat/apps/dataset_manager/formats/labelme.py +++ b/cvat/apps/dataset_manager/formats/labelme.py @@ -17,8 +17,6 @@ @exporter(name='LabelMe', ext='ZIP', version='3.0') def _export(dst_file, task_data, save_images=False): extractor = CvatTaskDataExtractor(task_data, include_images=save_images) - envt = dm_env.transforms - extractor = extractor.transform(envt.get('id_from_image_name')) extractor = Dataset.from_extractors(extractor) # apply lazy transforms with TemporaryDirectory() as temp_dir: converter = dm_env.make_converter('label_me', save_images=save_images) diff --git a/cvat/apps/dataset_manager/formats/mask.py b/cvat/apps/dataset_manager/formats/mask.py index 56693532831d..f0cb361f615e 100644 --- a/cvat/apps/dataset_manager/formats/mask.py +++ b/cvat/apps/dataset_manager/formats/mask.py @@ -24,7 +24,6 @@ def _export(dst_file, task_data, save_images=False): extractor = extractor.transform(envt.get('polygons_to_masks')) extractor = extractor.transform(envt.get('boxes_to_masks')) extractor = extractor.transform(envt.get('merge_instance_segments')) - extractor = extractor.transform(envt.get('id_from_image_name')) extractor = Dataset.from_extractors(extractor) # apply lazy transforms with TemporaryDirectory() as temp_dir: converter = dm_env.make_converter('voc_segmentation', diff --git a/cvat/apps/dataset_manager/formats/mot.py b/cvat/apps/dataset_manager/formats/mot.py index 7cc7c0ae43ee..f9e7a02c4cf8 100644 --- a/cvat/apps/dataset_manager/formats/mot.py +++ b/cvat/apps/dataset_manager/formats/mot.py @@ -7,8 +7,7 @@ from pyunpack import Archive import datumaro.components.extractor as datumaro -from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, - match_frame) +from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor from cvat.apps.dataset_manager.util import make_zip_archive from datumaro.components.project import Dataset @@ -18,8 +17,6 @@ @exporter(name='MOT', ext='ZIP', version='1.1') def _export(dst_file, task_data, save_images=False): extractor = CvatTaskDataExtractor(task_data, include_images=save_images) - envt = dm_env.transforms - extractor = extractor.transform(envt.get('id_from_image_name')) extractor = Dataset.from_extractors(extractor) # apply lazy transforms with TemporaryDirectory() as temp_dir: converter = dm_env.make_converter('mot_seq_gt', @@ -39,8 +36,8 @@ def _import(src_file, task_data): label_cat = dataset.categories()[datumaro.AnnotationType.label] for item in dataset: - item = item.wrap(id=int(item.id) - 1) # NOTE: MOT frames start from 1 - frame_number = task_data.abs_frame_id(match_frame(item, task_data)) + frame_number = int(item.id) - 1 # NOTE: MOT frames start from 1 + frame_number = task_data.abs_frame_id(frame_number) for ann in item.annotations: if ann.type != datumaro.AnnotationType.bbox: diff --git a/cvat/apps/dataset_manager/formats/pascal_voc.py b/cvat/apps/dataset_manager/formats/pascal_voc.py index ea53ad730be3..0973fe4e0b64 100644 --- a/cvat/apps/dataset_manager/formats/pascal_voc.py +++ b/cvat/apps/dataset_manager/formats/pascal_voc.py @@ -22,8 +22,6 @@ @exporter(name='PASCAL VOC', ext='ZIP', version='1.1') def _export(dst_file, task_data, save_images=False): extractor = CvatTaskDataExtractor(task_data, include_images=save_images) - envt = dm_env.transforms - extractor = extractor.transform(envt.get('id_from_image_name')) extractor = Dataset.from_extractors(extractor) # apply lazy transforms with TemporaryDirectory() as temp_dir: converter = dm_env.make_converter('voc', label_map='source', diff --git a/cvat/apps/documentation/faq.md b/cvat/apps/documentation/faq.md new file mode 100644 index 000000000000..3bdea808992b --- /dev/null +++ b/cvat/apps/documentation/faq.md @@ -0,0 +1,81 @@ +# Frequently asked questions +- [How to update CVAT](#how-to-update-cvat) +- [Kibana app works, but no logs are displayed](#kibana-app-works-but-no-logs-are-displayed) +- [How to change default CVAT hostname or port](#how-to-change-default-cvat-hostname-or-port) +- [How to configure connected share folder on Windows](#how-to-configure-connected-share-folder-on-windows) +- [How to make unassigned tasks not visible to all users](#how-to-make-unassigned-tasks-not-visible-to-all-users) +- [Can Nvidia GPU be used to run inference with my own model](#can-nvidia-gpu-be-used-to-run-inference-with-my-own-model) + +## How to update CVAT +Before upgrading, please follow the official docker +[manual](https://docs.docker.com/storage/volumes/#backup-restore-or-migrate-data-volumes) and backup all CVAT volumes. + +To update CVAT, you should clone or download the new version of CVAT and rebuild the CVAT docker images as usual. +```sh +docker-compose build +``` +and run containers: +```sh +docker-compose up -d +``` + +Sometimes the update process takes a lot of time due to changes in the database schema and data. +You can check the current status with `docker logs cvat`. +Please do not terminate the migration and wait till the process is complete. + +## Kibana app works, but no logs are displayed +Make sure there aren't error messages from Elasticsearch: +```sh +docker logs cvat_elasticsearch +``` +If you see errors like this: +```sh +lood stage disk watermark [95%] exceeded on [uMg9WI30QIOJxxJNDiIPgQ][uMg9WI3][/usr/share/elasticsearch/data/nodes/0] free: 116.5gb[4%], all indices on this node will be marked read-only +``` +You should free up disk space or change the threshold, to do so check: [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/disk-allocator.html). + +## How to change default CVAT hostname or port +The best way to do that is to create docker-compose.override.yml and override the host and port settings here. + +version: "2.3" +```yaml +services: + cvat_proxy: + environment: + CVAT_HOST: example.com + ports: + - "80:80" +``` + +Please don't forget to include this file in docker-compose commands +using the `-f` option (in some cases it can be omitted). + +## How to configure connected share folder on Windows +Follow the Docker manual and configure the directory that you want to use as a shared directory: +- [Docker toolbox manual](https://docs.docker.com/toolbox/toolbox_install_windows/#optional-add-shared-directories) +- [Docker for windows (see FILE SHARING section)](https://docs.docker.com/docker-for-windows/#resources) + +After that, it should be possible to use this directory as a CVAT share: +```yaml +version: "2.3" + +services: + cvat: + volumes: + - cvat_share:/home/django/share:ro + +volumes: + cvat_share: + driver_opts: + type: none + device: /d/my_cvat_share + o: bind +``` + +## How to make unassigned tasks not visible to all users +Set [reduce_task_visibility](../../settings/base.py#L424) variable to `True`. + +## Can Nvidia GPU be used to run inference with my own model +Nvidia GPU can be used to accelerate inference of [tf_annotation](../../../components/tf_annotation/README.md) and [auto_segmentation](../../../components/auto_segmentation/README.md) models. + +OpenVino doesn't support Nvidia cards, so you can run your own models only on CPU. diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index baf9f81e4a99..998bc4af829b 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -1,5 +1,5 @@ -# Copyright (C) 2018 Intel Corporation +# Copyright (C) 2018-2020 Intel Corporation # # SPDX-License-Identifier: MIT @@ -15,6 +15,7 @@ from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter from cvat.apps.engine.models import DataChoice +from cvat.apps.engine.utils import av_scan_paths import django_rq from django.conf import settings @@ -223,6 +224,8 @@ def _create_thread(tid, data): if data['server_files']: _copy_data_from_share(data['server_files'], upload_dir) + av_scan_paths(upload_dir) + job = rq.get_current_job() job.meta['status'] = 'Media files are being extracted...' job.save_meta() diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index e3b4954d0b30..dd4a083d7c75 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -1,8 +1,16 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT + import ast from collections import namedtuple import importlib import sys import traceback +import subprocess +import os + +from django.core.exceptions import ValidationError Import = namedtuple("Import", ["module", "name", "alias"]) @@ -58,3 +66,11 @@ def execute_python_code(source_code, global_vars=None, local_vars=None): _, _, tb = sys.exc_info() line_number = traceback.extract_tb(tb)[-1][1] raise InterpreterError("{} at line {}: {}".format(error_class, line_number, details)) + +def av_scan_paths(*paths): + if 'yes' == os.environ.get('CLAM_AV'): + command = ['clamscan', '--no-summary', '-i', '-o'] + command.extend(paths) + res = subprocess.run(command, capture_output=True) + if res.returncode: + raise ValidationError(res.stdout) \ No newline at end of file diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 735d77c43370..120125ea1605 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2019 Intel Corporation +# Copyright (C) 2018-2020 Intel Corporation # # SPDX-License-Identifier: MIT @@ -45,6 +45,7 @@ LogEventSerializer, PluginSerializer, ProjectSerializer, RqStatusSerializer, TaskSerializer, UserSerializer) from cvat.settings.base import CSS_3RDPARTY, JS_3RDPARTY +from cvat.apps.engine.utils import av_scan_paths from . import models, task from .log import clogger, slogger @@ -821,6 +822,8 @@ def _import_annotations(request, rq_id, rq_func, pk, format_name): with open(filename, 'wb+') as f: for chunk in anno_file.chunks(): f.write(chunk) + + av_scan_paths(filename) rq_job = queue.enqueue_call( func=rq_func, args=(pk, filename, format_name), diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 7c5be7d9d7f3..7b177d37ecd3 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -1,9 +1,9 @@ click==6.7 Django==2.2.13 -django-appconf==1.0.2 +django-appconf==1.0.4 django-auth-ldap==1.4.0 django-cacheops==5.0 -django-compressor==2.2 +django-compressor==2.4 django-rq==2.0.0 EasyProcess==0.2.3 Pillow==7.1.2 diff --git a/datumaro/datumaro/components/extractor.py b/datumaro/datumaro/components/extractor.py index 7bb99a4ff328..fe4d897b6cc9 100644 --- a/datumaro/datumaro/components/extractor.py +++ b/datumaro/datumaro/components/extractor.py @@ -578,7 +578,7 @@ class DatasetItem: def __init__(self, id=None, annotations=None, subset=None, path=None, image=None, attributes=None): assert id is not None - self._id = str(id) + self._id = str(id).replace('\\', '/') if subset is None: subset = '' diff --git a/datumaro/datumaro/plugins/coco_format/converter.py b/datumaro/datumaro/plugins/coco_format/converter.py index 9d1d7289ecb4..d5247bdbd766 100644 --- a/datumaro/datumaro/plugins/coco_format/converter.py +++ b/datumaro/datumaro/plugins/coco_format/converter.py @@ -496,25 +496,19 @@ def _make_task_converters(self): def _get_image_id(self, item): image_id = self._image_ids.get(item.id) if image_id is None: - image_id = cast(item.id, int, len(self._image_ids) + 1) + image_id = cast(item.attributes.get('id'), int, + len(self._image_ids) + 1) self._image_ids[item.id] = image_id return image_id - def _save_image(self, item): + def _save_image(self, item, filename): image = item.image.data if image is None: log.warning("Item '%s' has no image" % item.id) return '' - filename = item.image.filename - if filename: - filename = osp.splitext(filename)[0] - else: - filename = item.id - filename += CocoPath.IMAGE_EXT - path = osp.join(self._images_dir, filename) - save_image(path, image) - return path + save_image(osp.join(self._images_dir, filename), image, + create_dir=True) def convert(self): self._make_dirs() @@ -534,12 +528,10 @@ def convert(self): for task_conv in task_converters.values(): task_conv.save_categories(subset) for item in subset: - filename = '' - if item.has_image: - filename = item.image.path + filename = item.id + CocoPath.IMAGE_EXT if self._save_images: if item.has_image: - filename = self._save_image(item) + self._save_image(item, filename) else: log.debug("Item '%s' has no image info" % item.id) for task_conv in task_converters.values(): diff --git a/datumaro/datumaro/plugins/coco_format/extractor.py b/datumaro/datumaro/plugins/coco_format/extractor.py index a4f52f814048..8ba0d87d2b36 100644 --- a/datumaro/datumaro/plugins/coco_format/extractor.py +++ b/datumaro/datumaro/plugins/coco_format/extractor.py @@ -24,7 +24,8 @@ class _CocoExtractor(SourceExtractor): def __init__(self, path, task, merge_instance_polygons=False): assert osp.isfile(path), path - subset = osp.splitext(osp.basename(path))[0].rsplit('_', maxsplit=1)[1] + subset = osp.splitext(osp.basename(path))[0].rsplit('_', maxsplit=1) + subset = subset[1] if len(subset) == 2 else None super().__init__(subset=subset) rootpath = '' @@ -125,8 +126,10 @@ def _load_items(self, loader): anns = loader.loadAnns(anns) anns = sum((self._load_annotations(a, image_info) for a in anns), []) - items[img_id] = DatasetItem(id=img_id, subset=self._subset, - image=image, annotations=anns) + items[img_id] = DatasetItem( + id=osp.splitext(image_info['file_name'])[0], + subset=self._subset, image=image, annotations=anns, + attributes={'id': img_id}) return items diff --git a/datumaro/datumaro/plugins/cvat_format/converter.py b/datumaro/datumaro/plugins/cvat_format/converter.py index 8249bd0d1e45..36588e2cfe22 100644 --- a/datumaro/datumaro/plugins/cvat_format/converter.py +++ b/datumaro/datumaro/plugins/cvat_format/converter.py @@ -163,26 +163,21 @@ def write(self): self._writer.close_root() - def _save_image(self, item): + def _save_image(self, item, filename): image = item.image.data if image is None: log.warning("Item '%s' has no image" % item.id) return '' - filename = item.image.filename - if filename: - filename = osp.splitext(filename)[0] - else: - filename = item.id - filename += CvatPath.IMAGE_EXT - image_path = osp.join(self._context._images_dir, filename) - save_image(image_path, image) - return filename + save_image(osp.join(self._context._images_dir, filename), image, + create_dir=True) def _write_item(self, item, index): image_info = OrderedDict([ - ("id", str(cast(item.id, int, index))), + ("id", str(cast(item.attributes.get('frame'), int, index))), ]) + filename = item.id + CvatPath.IMAGE_EXT + image_info["name"] = filename if item.has_image: size = item.image.size if size: @@ -190,10 +185,8 @@ def _write_item(self, item, index): image_info["width"] = str(w) image_info["height"] = str(h) - filename = item.image.filename if self._context._save_images: - filename = self._save_image(item) - image_info["name"] = filename + self._save_image(item, filename) else: log.debug("Item '%s' has no image info" % item.id) self._writer.open_image(image_info) diff --git a/datumaro/datumaro/plugins/cvat_format/extractor.py b/datumaro/datumaro/plugins/cvat_format/extractor.py index 0478cf05c444..2c63bd2eb1ae 100644 --- a/datumaro/datumaro/plugins/cvat_format/extractor.py +++ b/datumaro/datumaro/plugins/cvat_format/extractor.py @@ -303,17 +303,14 @@ def _parse_tag_ann(cls, ann, categories): def _load_items(self, parsed): for frame_id, item_desc in parsed.items(): - path = item_desc.get('name', 'frame_%06d.png' % int(frame_id)) + name = item_desc.get('name', 'frame_%06d.png' % int(frame_id)) + image = osp.join(self._images_dir, name) image_size = (item_desc.get('height'), item_desc.get('width')) if all(image_size): - image_size = (int(image_size[0]), int(image_size[1])) - else: - image_size = None - image = None - if path: - image = Image(path=osp.join(self._images_dir, path), - size=image_size) - - parsed[frame_id] = DatasetItem(id=frame_id, subset=self._subset, - image=image, annotations=item_desc.get('annotations')) + image = Image(path=image, size=tuple(map(int, image_size))) + + parsed[frame_id] = DatasetItem(id=osp.splitext(name)[0], + subset=self._subset, image=image, + annotations=item_desc.get('annotations'), + attributes={'frame': int(frame_id)}) return parsed diff --git a/datumaro/datumaro/plugins/datumaro_format/converter.py b/datumaro/datumaro/plugins/datumaro_format/converter.py index b1067b698773..7130dc97f07d 100644 --- a/datumaro/datumaro/plugins/datumaro_format/converter.py +++ b/datumaro/datumaro/plugins/datumaro_format/converter.py @@ -253,12 +253,7 @@ def _save_image(self, item): if image is None: return '' - filename = item.image.filename - if filename: - filename = osp.splitext(filename)[0] - else: - filename = item.id - filename += DatumaroPath.IMAGE_EXT + filename = item.id + DatumaroPath.IMAGE_EXT image_path = osp.join(self._images_dir, filename) save_image(image_path, image, create_dir=True) return filename diff --git a/datumaro/datumaro/plugins/datumaro_format/extractor.py b/datumaro/datumaro/plugins/datumaro_format/extractor.py index ed8813e63df0..71eb68562f67 100644 --- a/datumaro/datumaro/plugins/datumaro_format/extractor.py +++ b/datumaro/datumaro/plugins/datumaro_format/extractor.py @@ -82,10 +82,11 @@ def _load_items(self, parsed): item_id = item_desc['id'] image = None - image_info = item_desc.get('image', {}) + image_info = item_desc.get('image') if image_info: - image_path = osp.join(self._images_dir, - image_info.get('path', '')) # relative or absolute fits + image_path = image_info.get('path') or \ + item_id + DatumaroPath.IMAGE_EXT + image_path = osp.join(self._images_dir, image_path) image = Image(path=image_path, size=image_info.get('size')) annotations = self._load_annotations(item_desc) diff --git a/datumaro/datumaro/plugins/image_dir.py b/datumaro/datumaro/plugins/image_dir.py index 0ba68ee3c325..b6c25d60b5ac 100644 --- a/datumaro/datumaro/plugins/image_dir.py +++ b/datumaro/datumaro/plugins/image_dir.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: MIT -from collections import OrderedDict import os import os.path as osp @@ -41,29 +40,24 @@ def __init__(self, url): assert osp.isdir(url), url items = [] - for name in os.listdir(url): - path = osp.join(url, name) - if self._is_image(path): - item_id = osp.splitext(name)[0] - item = DatasetItem(id=item_id, image=path) - items.append((item.id, item)) - - items = sorted(items, key=lambda e: e[0]) - items = OrderedDict(items) + for dirpath, _, filenames in os.walk(url): + for name in filenames: + path = osp.join(dirpath, name) + if not self._is_image(path): + continue + + item_id = osp.relpath(osp.splitext(path)[0], url) + items.append(DatasetItem(id=item_id, image=path)) + self._items = items def __iter__(self): - for item in self._items.values(): + for item in self._items: yield item def __len__(self): return len(self._items) - def get(self, item_id, subset=None, path=None): - if path or subset: - raise KeyError() - return self._items[item_id] - def _is_image(self, path): if not osp.isfile(path): return False @@ -79,11 +73,5 @@ def __call__(self, extractor, save_dir): for item in extractor: if item.has_image and item.image.has_data: - filename = item.image.filename - if filename: - filename = osp.splitext(filename)[0] - else: - filename = item.id - filename += '.jpg' - save_image(osp.join(save_dir, filename), item.image.data, - create_dir=True) \ No newline at end of file + save_image(osp.join(save_dir, item.id + '.jpg'), + item.image.data, create_dir=True) diff --git a/datumaro/datumaro/plugins/labelme_format.py b/datumaro/datumaro/plugins/labelme_format.py index 96bdf3f041e8..ac998cbf654e 100644 --- a/datumaro/datumaro/plugins/labelme_format.py +++ b/datumaro/datumaro/plugins/labelme_format.py @@ -331,16 +331,13 @@ def _save_item(self, item, subset_dir): log.debug("Converting item '%s'", item.id) - image_filename = '' - if item.has_image: - image_filename = item.image.filename + if '/' in item.id: + raise Exception("Can't export item '%s': " + "LabelMe format only supports flat image layout" % item.id) + + image_filename = item.id + LabelMePath.IMAGE_EXT if self._save_images: if item.has_image and item.image.has_data: - if image_filename: - image_filename = osp.splitext(image_filename)[0] - else: - image_filename = item.id - image_filename += LabelMePath.IMAGE_EXT save_image(osp.join(subset_dir, image_filename), item.image.data, create_dir=True) else: diff --git a/datumaro/datumaro/plugins/mot_format.py b/datumaro/datumaro/plugins/mot_format.py index b39c607052c8..6406d47b5af1 100644 --- a/datumaro/datumaro/plugins/mot_format.py +++ b/datumaro/datumaro/plugins/mot_format.py @@ -311,7 +311,9 @@ def __call__(self, extractor, save_dir): if self._save_images: if item.has_image and item.image.has_data: - self._save_image(item, index=frame_id) + save_image(osp.join(self._images_dir, + '%06d%s' % (frame_id, MotPath.IMAGE_EXT)), + item.image.data) else: log.debug("Item '%s' has no image" % item.id) @@ -320,13 +322,3 @@ def __call__(self, extractor, save_dir): f.write('\n'.join(l.name for l in extractor.categories()[AnnotationType.label].items) ) - - def _save_image(self, item, index): - if item.image.filename: - frame_id = osp.splitext(item.image.filename)[0] - else: - frame_id = item.id - frame_id = cast(frame_id, int, index) - image_filename = '%06d%s' % (frame_id, MotPath.IMAGE_EXT) - save_image(osp.join(self._images_dir, image_filename), - item.image.data) \ No newline at end of file diff --git a/datumaro/datumaro/plugins/tf_detection_api_format/converter.py b/datumaro/datumaro/plugins/tf_detection_api_format/converter.py index 01e2bd0ea45d..1273b7b221eb 100644 --- a/datumaro/datumaro/plugins/tf_detection_api_format/converter.py +++ b/datumaro/datumaro/plugins/tf_detection_api_format/converter.py @@ -162,14 +162,12 @@ def _export_instances(self, instances, width, height): def _make_tf_example(self, item): features = { - 'image/source_id': bytes_feature(str(item.id).encode('utf-8')), + 'image/source_id': bytes_feature( + str(item.attributes.get('source_id') or '').encode('utf-8') + ), } - filename = '' - if item.has_image: - filename = item.image.filename - if not filename: - filename = item.id + DetectionApiPath.IMAGE_EXT + filename = item.id + DetectionApiPath.IMAGE_EXT features['image/filename'] = bytes_feature(filename.encode('utf-8')) if not item.has_image: diff --git a/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py b/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py index 0f4c474b406c..139588950299 100644 --- a/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py +++ b/datumaro/datumaro/plugins/tf_detection_api_format/extractor.py @@ -145,9 +145,7 @@ def _parse_tfrecord_file(cls, filepath, subset, images_dir): continue dataset_labels[label] = label_id - 1 - item_id = frame_id - if not item_id: - item_id = osp.splitext(frame_filename)[0] + item_id = osp.splitext(frame_filename)[0] annotations = [] for shape_id, shape in enumerate( @@ -188,6 +186,7 @@ def _parse_tfrecord_file(cls, filepath, subset, images_dir): image = Image(**image_params, size=image_size) dataset_items.append(DatasetItem(id=item_id, subset=subset, - image=image, annotations=annotations)) + image=image, annotations=annotations, + attributes={'source_id': frame_id})) return dataset_items, dataset_labels diff --git a/datumaro/datumaro/plugins/transforms.py b/datumaro/datumaro/plugins/transforms.py index d37e284a9853..9d1baedafc16 100644 --- a/datumaro/datumaro/plugins/transforms.py +++ b/datumaro/datumaro/plugins/transforms.py @@ -364,10 +364,13 @@ def __iter__(self): class IdFromImageName(Transform, CliPlugin): def transform_item(self, item): - name = item.id - if item.has_image and item.image.filename: - name = osp.splitext(item.image.filename)[0] - return self.wrap_item(item, id=name) + if item.has_image and item.image.path: + name = osp.splitext(osp.basename(item.image.path))[0] + return self.wrap_item(item, id=name) + else: + log.debug("Can't change item id for item '%s': " + "item has no image info" % item.id) + return item class RemapLabels(Transform, CliPlugin): DefaultAction = Enum('DefaultAction', ['keep', 'delete']) diff --git a/datumaro/datumaro/plugins/voc_format/converter.py b/datumaro/datumaro/plugins/voc_format/converter.py index 15b4086553c5..42dfaffcc938 100644 --- a/datumaro/datumaro/plugins/voc_format/converter.py +++ b/datumaro/datumaro/plugins/voc_format/converter.py @@ -135,16 +135,9 @@ def save_subsets(self): for item in subset: log.debug("Converting item '%s'", item.id) - image_filename = '' - if item.has_image: - image_filename = item.image.filename + image_filename = item.id + VocPath.IMAGE_EXT if self._save_images: if item.has_image and item.image.has_data: - if image_filename: - image_filename = osp.splitext(image_filename)[0] - else: - image_filename = item.id - image_filename += VocPath.IMAGE_EXT save_image(osp.join(self._images_dir, image_filename), item.image.data, create_dir=True) else: diff --git a/datumaro/datumaro/plugins/voc_format/extractor.py b/datumaro/datumaro/plugins/voc_format/extractor.py index a14682958d3b..fb724fbd3e2b 100644 --- a/datumaro/datumaro/plugins/voc_format/extractor.py +++ b/datumaro/datumaro/plugins/voc_format/extractor.py @@ -108,8 +108,8 @@ def __iter__(self): for item_id in self._items: log.debug("Reading item '%s'" % item_id) - image = osp.join(self._dataset_dir, VocPath.IMAGES_DIR, - item_id + VocPath.IMAGE_EXT) + image = item_id + VocPath.IMAGE_EXT + height, width = 0, 0 anns = [] ann_file = osp.join(anno_dir, item_id + '.xml') @@ -121,11 +121,15 @@ def __iter__(self): width = root_elem.find('size/width') if width is not None: width = int(width.text) - if height and width: - image = Image(path=image, size=(height, width)) - + filename_elem = root_elem.find('filename') + if filename_elem is not None: + image = filename_elem.text anns = self._parse_annotations(root_elem) + image = osp.join(self._dataset_dir, VocPath.IMAGES_DIR, image) + if height and width: + image = Image(path=image, size=(height, width)) + yield DatasetItem(id=item_id, subset=self._subset, image=image, annotations=anns) diff --git a/datumaro/datumaro/plugins/yolo_format/converter.py b/datumaro/datumaro/plugins/yolo_format/converter.py index a4fe3316188b..2d14a06368a7 100644 --- a/datumaro/datumaro/plugins/yolo_format/converter.py +++ b/datumaro/datumaro/plugins/yolo_format/converter.py @@ -80,13 +80,9 @@ def __call__(self, extractor, save_dir): "item has no image info" % item.id) height, width = item.image.size - image_name = item.image.filename - item_name = osp.splitext(item.image.filename)[0] + image_name = item.id + '.jpg' if self._save_images: if item.has_image and item.image.has_data: - if not item_name: - item_name = item.id - image_name = item_name + '.jpg' save_image(osp.join(subset_dir, image_name), item.image.data, create_dir=True) else: @@ -105,7 +101,8 @@ def __call__(self, extractor, save_dir): yolo_bb = ' '.join('%.6f' % p for p in yolo_bb) yolo_annotation += '%s %s\n' % (bbox.label, yolo_bb) - annotation_path = osp.join(subset_dir, '%s.txt' % item_name) + annotation_path = osp.join(subset_dir, '%s.txt' % item.id) + os.makedirs(osp.dirname(annotation_path), exist_ok=True) with open(annotation_path, 'w') as f: f.write(yolo_annotation) diff --git a/datumaro/datumaro/plugins/yolo_format/extractor.py b/datumaro/datumaro/plugins/yolo_format/extractor.py index 135f6f8f206c..9e34508c4d98 100644 --- a/datumaro/datumaro/plugins/yolo_format/extractor.py +++ b/datumaro/datumaro/plugins/yolo_format/extractor.py @@ -10,6 +10,7 @@ from datumaro.components.extractor import (SourceExtractor, Extractor, DatasetItem, AnnotationType, Bbox, LabelCategories ) +from datumaro.util import split_path from datumaro.util.image import Image from .format import YoloPath @@ -83,14 +84,14 @@ def __init__(self, config_path, image_info=None): config_path) for subset_name, list_path in subsets.items(): - list_path = self._make_local_path(list_path) + list_path = osp.join(self._path, self.localize_path(list_path)) if not osp.isfile(list_path): raise Exception("Not found '%s' subset list file" % subset_name) subset = YoloExtractor.Subset(subset_name, self) with open(list_path, 'r') as f: subset.items = OrderedDict( - (osp.splitext(osp.basename(p.strip()))[0], p.strip()) + (self.name_from_path(p), self.localize_path(p)) for p in f ) subsets[subset_name] = subset @@ -99,25 +100,38 @@ def __init__(self, config_path, image_info=None): self._categories = { AnnotationType.label: - self._load_categories(self._make_local_path(names_path)) + self._load_categories( + osp.join(self._path, self.localize_path(names_path))) } - def _make_local_path(self, path): + @staticmethod + def localize_path(path): + path = path.strip() default_base = osp.join('data', '') if path.startswith(default_base): # default path path = path[len(default_base) : ] - return osp.join(self._path, path) # relative or absolute path + return path + + @classmethod + def name_from_path(cls, path): + path = cls.localize_path(path) + parts = split_path(path) + if 1 < len(parts) and not osp.isabs(path): + # NOTE: when path is like [data/]/ + # drop everything but + # can be , so no just basename() + path = osp.join(*parts[1:]) + return osp.splitext(path)[0] def _get(self, item_id, subset_name): subset = self._subsets[subset_name] item = subset.items[item_id] if isinstance(item, str): - image_path = self._make_local_path(item) image_size = self._image_info.get(item_id) - image = Image(path=image_path, size=image_size) + image = Image(path=osp.join(self._path, item), size=image_size) - anno_path = osp.splitext(image_path)[0] + '.txt' + anno_path = osp.splitext(image.path)[0] + '.txt' annotations = self._parse_annotations(anno_path, image) item = DatasetItem(id=item_id, subset=subset_name, @@ -137,8 +151,10 @@ def _parse_annotations(anno_path, image): annotations = [] if lines: - # use image info as late as possible - image_height, image_width = image.size + size = image.size # use image info as late as possible + if size is None: + raise Exception("Can't find image info for '%s'" % image.path) + image_height, image_width = size for line in lines: label_id, xc, yc, w, h = line.split() label_id = int(label_id) diff --git a/datumaro/datumaro/util/image.py b/datumaro/datumaro/util/image.py index 47d5fff1fabc..fc6a113c530c 100644 --- a/datumaro/datumaro/util/image.py +++ b/datumaro/datumaro/util/image.py @@ -215,10 +215,6 @@ def __init__(self, data=None, path=None, loader=None, cache=None, def path(self): return self._path - @property - def filename(self): - return osp.basename(self._path) - @property def data(self): if callable(self._data): diff --git a/datumaro/datumaro/util/test_utils.py b/datumaro/datumaro/util/test_utils.py index 2fb25c54f2a3..f9ce03690b88 100644 --- a/datumaro/datumaro/util/test_utils.py +++ b/datumaro/datumaro/util/test_utils.py @@ -87,6 +87,7 @@ def compare_datasets(test, expected, actual): item_b = find(actual, lambda x: x.id == item_a.id and \ x.subset == item_a.subset) test.assertFalse(item_b is None, item_a.id) + test.assertEqual(item_a.attributes, item_b.attributes) test.assertEqual(len(item_a.annotations), len(item_b.annotations)) for ann_a in item_a.annotations: # We might find few corresponding items, so check them all diff --git a/datumaro/datumaro/util/tf_util.py b/datumaro/datumaro/util/tf_util.py index 841fc53faf1f..f5d70090da92 100644 --- a/datumaro/datumaro/util/tf_util.py +++ b/datumaro/datumaro/util/tf_util.py @@ -35,8 +35,11 @@ def check_import(): def import_tf(check=True): import sys - tf = sys.modules.get('tensorflow', None) - if tf is not None: + not_found = object() + tf = sys.modules.get('tensorflow', not_found) + if tf is None: + import tensorflow as tf # emit default error + elif tf is not not_found: return tf # Reduce output noise, https://stackoverflow.com/questions/38073432/how-to-suppress-verbose-tensorflow-logging @@ -44,7 +47,11 @@ def import_tf(check=True): os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' if check: - check_import() + try: + check_import() + except Exception: + sys.modules['tensorflow'] = None # prevent further import + raise import tensorflow as tf diff --git a/datumaro/tests/assets/coco_dataset/annotations/instances_val.json b/datumaro/tests/assets/coco_dataset/annotations/instances_val.json index a7f566ccd7e1..b5d9bd8697b7 100644 --- a/datumaro/tests/assets/coco_dataset/annotations/instances_val.json +++ b/datumaro/tests/assets/coco_dataset/annotations/instances_val.json @@ -56,4 +56,4 @@ "iscrowd": 1 } ] - } \ No newline at end of file + } diff --git a/datumaro/tests/assets/voc_dataset/Annotations/2007_000001.xml b/datumaro/tests/assets/voc_dataset/Annotations/2007_000001.xml index e6de2513ce23..4f1e25a2112f 100644 --- a/datumaro/tests/assets/voc_dataset/Annotations/2007_000001.xml +++ b/datumaro/tests/assets/voc_dataset/Annotations/2007_000001.xml @@ -51,4 +51,4 @@ 1 - \ No newline at end of file + diff --git a/datumaro/tests/test_coco_format.py b/datumaro/tests/test_coco_format.py index 8da66717c891..7a31180bea44 100644 --- a/datumaro/tests/test_coco_format.py +++ b/datumaro/tests/test_coco_format.py @@ -28,7 +28,8 @@ def test_can_import(self): class DstExtractor(Extractor): def __iter__(self): return iter([ - DatasetItem(id=1, image=np.ones((10, 5, 3)), subset='val', + DatasetItem(id='000000000001', image=np.ones((10, 5, 3)), + subset='val', attributes={'id': 1}, annotations=[ Polygon([0, 0, 1, 0, 1, 2, 0, 2], label=0, id=1, group=1, attributes={'is_crowd': False}), @@ -76,16 +77,16 @@ def __iter__(self): annotations=[ Caption('hello', id=1, group=1), Caption('world', id=2, group=2), - ]), + ], attributes={'id': 1}), DatasetItem(id=2, subset='train', annotations=[ Caption('test', id=3, group=3), - ]), + ], attributes={'id': 2}), DatasetItem(id=3, subset='val', annotations=[ Caption('word', id=1, group=1), - ] + ], attributes={'id': 1} ), ]) @@ -111,7 +112,7 @@ def __iter__(self): Polygon([0, 1, 2, 1, 2, 3, 0, 3], attributes={ 'is_crowd': False }, label=2, group=1, id=1), - ]), + ], attributes={'id': 1}), DatasetItem(id=2, subset='train', image=np.ones((4, 4, 3)), annotations=[ # Mask + bbox @@ -125,7 +126,7 @@ def __iter__(self): label=4, group=3, id=3), Bbox(1, 0, 2, 2, label=4, group=3, id=3, attributes={ 'is_crowd': True }), - ]), + ], attributes={'id': 2}), DatasetItem(id=3, subset='val', image=np.ones((4, 4, 3)), annotations=[ @@ -140,7 +141,7 @@ def __iter__(self): ), attributes={ 'is_crowd': True }, label=4, group=3, id=3), - ]), + ], attributes={'id': 1}), ]) def categories(self): @@ -154,7 +155,7 @@ def __iter__(self): Polygon([0, 1, 2, 1, 2, 3, 0, 3], attributes={ 'is_crowd': False }, label=2, group=1, id=1), - ]), + ], attributes={'id': 1}), DatasetItem(id=2, subset='train', image=np.ones((4, 4, 3)), annotations=[ Mask(np.array([ @@ -165,7 +166,7 @@ def __iter__(self): ), attributes={ 'is_crowd': True }, label=4, group=3, id=3), - ]), + ], attributes={'id': 2}), DatasetItem(id=3, subset='val', image=np.ones((4, 4, 3)), annotations=[ @@ -177,7 +178,7 @@ def __iter__(self): ), attributes={ 'is_crowd': True }, label=4, group=3, id=3), - ]), + ], attributes={'id': 1}), ]) def categories(self): @@ -227,7 +228,7 @@ def __iter__(self): ), label=3, id=4, group=4, attributes={ 'is_crowd': False }), - ] + ], attributes={'id': 1} ), ]) @@ -285,7 +286,7 @@ def __iter__(self): Polygon([1, 1, 4, 1, 4, 4, 1, 4], label=1, id=2, group=2, attributes={ 'is_crowd': False }), - ] + ], attributes={'id': 1} ), ]) @@ -335,7 +336,7 @@ def __iter__(self): ), attributes={ 'is_crowd': True }, label=3, id=4, group=4), - ] + ], attributes={'id': 1} ), ]) @@ -385,7 +386,7 @@ def __iter__(self): [5.0, 3.5, 4.5, 0.0, 8.0, 0.0, 5.0, 3.5], label=3, id=4, group=4, attributes={ 'is_crowd': False }), - ] + ], attributes={'id': 1} ), ]) @@ -401,14 +402,14 @@ def test_can_save_and_load_images(self): class TestExtractor(Extractor): def __iter__(self): return iter([ - DatasetItem(id=1, subset='train'), - DatasetItem(id=2, subset='train'), + DatasetItem(id=1, subset='train', attributes={'id': 1}), + DatasetItem(id=2, subset='train', attributes={'id': 2}), - DatasetItem(id=2, subset='val'), - DatasetItem(id=3, subset='val'), - DatasetItem(id=4, subset='val'), + DatasetItem(id=2, subset='val', attributes={'id': 2}), + DatasetItem(id=3, subset='val', attributes={'id': 3}), + DatasetItem(id=4, subset='val', attributes={'id': 4}), - DatasetItem(id=5, subset='test'), + DatasetItem(id=5, subset='test', attributes={'id': 1}), ]) with TestDir() as test_dir: @@ -423,7 +424,7 @@ def __iter__(self): annotations=[ Label(4, id=1, group=1), Label(9, id=2, group=2), - ] + ], attributes={'id': 1} ), ]) @@ -511,7 +512,7 @@ def __iter__(self): Polygon([1, 2, 3, 2, 3, 4, 1, 4], group=5, id=5, attributes={'is_crowd': False}), - ]), + ], attributes={'id': 1}), ]) with TestDir() as test_dir: @@ -523,8 +524,8 @@ def test_can_save_dataset_with_no_subsets(self): class TestExtractor(Extractor): def __iter__(self): return iter([ - DatasetItem(id=1), - DatasetItem(id=2), + DatasetItem(id=1, attributes={'id': 1}), + DatasetItem(id=2, attributes={'id': 2}), ]) def categories(self): @@ -538,9 +539,38 @@ def test_can_save_dataset_with_image_info(self): class TestExtractor(Extractor): def __iter__(self): return iter([ - DatasetItem(id=1, image=Image(path='1.jpg', size=(10, 15))), + DatasetItem(id=1, image=Image(path='1.jpg', size=(10, 15)), + attributes={'id': 1}), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoConverter(tasks='image_info'), test_dir) + + def test_relative_paths(self): + class TestExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='1', image=np.ones((4, 2, 3)), + attributes={'id': 1}), + DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3)), + attributes={'id': 2}), + DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3)), + attributes={'id': 3}), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoConverter(tasks='image_info', save_images=True), test_dir) + + def test_preserve_coco_ids(self): + class TestExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), + attributes={'id': 40}), ]) with TestDir() as test_dir: self._test_save_and_load(TestExtractor(), - CocoConverter(tasks='image_info'), test_dir) \ No newline at end of file + CocoConverter(tasks='image_info', save_images=True), test_dir) diff --git a/datumaro/tests/test_cvat_format.py b/datumaro/tests/test_cvat_format.py index 76c2e434e363..463c662d9f30 100644 --- a/datumaro/tests/test_cvat_format.py +++ b/datumaro/tests/test_cvat_format.py @@ -30,7 +30,7 @@ def test_can_load_image(self): class DstExtractor(Extractor): def __iter__(self): return iter([ - DatasetItem(id=0, subset='train', + DatasetItem(id='img0', subset='train', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2, label=0, z_order=1, @@ -40,15 +40,15 @@ def __iter__(self): }), PolyLine([1, 2, 3, 4, 5, 6, 7, 8], attributes={'occluded': False}), - ]), - DatasetItem(id=1, subset='train', + ], attributes={'frame': 0}), + DatasetItem(id='img1', subset='train', image=np.ones((10, 10, 3)), annotations=[ Polygon([1, 2, 3, 4, 6, 5], z_order=1, attributes={'occluded': False}), Points([1, 2, 3, 4, 5, 6], label=1, z_order=2, attributes={'occluded': False}), - ]), + ], attributes={'frame': 1}), ]) def categories(self): @@ -65,7 +65,7 @@ def test_can_load_video(self): class DstExtractor(Extractor): def __iter__(self): return iter([ - DatasetItem(id=10, subset='annotations', + DatasetItem(id='frame_000010', subset='annotations', image=np.ones((20, 25, 3)), annotations=[ Bbox(3, 4, 7, 1, label=2, @@ -83,8 +83,8 @@ def __iter__(self): 'outside': False, 'keyframe': True, 'track_id': 1, 'hgl': 'hgkf', }), - ]), - DatasetItem(id=13, subset='annotations', + ], attributes={'frame': 10}), + DatasetItem(id='frame_000013', subset='annotations', image=np.ones((20, 25, 3)), annotations=[ Bbox(7, 6, 7, 2, label=2, @@ -110,10 +110,9 @@ def __iter__(self): 'outside': False, 'keyframe': True, 'track_id': 2, }), - ]), - DatasetItem(id=16, subset='annotations', - image=Image(path='frame_0000016.png', - size=(20, 25)), # no image in the dataset files + ], attributes={'frame': 13}), + DatasetItem(id='frame_000016', subset='annotations', + image=Image(path='frame_0000016.png', size=(20, 25)), annotations=[ Bbox(8, 7, 6, 10, label=2, id=0, @@ -130,7 +129,7 @@ def __iter__(self): 'outside': True, 'keyframe': True, 'track_id': 2, }), - ]), + ], attributes={'frame': 16}), ]) def categories(self): @@ -220,7 +219,7 @@ def __iter__(self): 'a1': 'x', 'a2': 42 }), Label(1), Label(2, attributes={ 'a1': 'y', 'a2': 44 }), - ] + ], attributes={'frame': 0} ), DatasetItem(id=1, subset='s1', annotations=[ @@ -230,7 +229,7 @@ def __iter__(self): Bbox(5, 0, 1, 9, label=3, group=4, attributes={ 'occluded': False }), - ] + ], attributes={'frame': 1} ), DatasetItem(id=2, subset='s2', image=np.ones((5, 10, 3)), @@ -238,11 +237,12 @@ def __iter__(self): Polygon([0, 0, 4, 0, 4, 4], z_order=1, label=3, group=4, attributes={ 'occluded': False }), - ] + ], attributes={'frame': 0} ), DatasetItem(id=3, subset='s3', image=Image( - path='3.jpg', size=(2, 4))), + path='3.jpg', size=(2, 4)), + attributes={'frame': 0}), ]) def categories(self): @@ -252,3 +252,49 @@ def categories(self): self._test_save_and_load(SrcExtractor(), CvatConverter(save_images=True), test_dir, target_dataset=DstExtractor()) + + def test_relative_paths(self): + class SrcExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='1', image=np.ones((4, 2, 3))), + DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3))), + DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3))), + ]) + + def categories(self): + return { AnnotationType.label: LabelCategories() } + + class DstExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='1', image=np.ones((4, 2, 3)), + attributes={'frame': 0}), + DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3)), + attributes={'frame': 1}), + DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3)), + attributes={'frame': 2}), + ]) + + def categories(self): + return { AnnotationType.label: LabelCategories() } + + with TestDir() as test_dir: + self._test_save_and_load(SrcExtractor(), + CvatConverter(save_images=True), test_dir, + target_dataset=DstExtractor()) + + def test_preserve_frame_ids(self): + class TestExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), + attributes={'frame': 40}), + ]) + + def categories(self): + return { AnnotationType.label: LabelCategories() } + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CvatConverter(save_images=True), test_dir) diff --git a/datumaro/tests/test_datumaro_format.py b/datumaro/tests/test_datumaro_format.py index f617465e0842..8d5b723a7808 100644 --- a/datumaro/tests/test_datumaro_format.py +++ b/datumaro/tests/test_datumaro_format.py @@ -14,6 +14,7 @@ from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets_strict + class DatumaroConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, target_dataset=None, importer_args=None): @@ -96,3 +97,16 @@ def test_can_detect(self): DatumaroConverter()(self.TestExtractor(), save_dir=test_dir) self.assertTrue(DatumaroImporter.detect(test_dir)) + + def test_relative_paths(self): + class TestExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='1', image=np.ones((4, 2, 3))), + DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3))), + DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3))), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + DatumaroConverter(save_images=True), test_dir) diff --git a/datumaro/tests/test_image_dir_format.py b/datumaro/tests/test_image_dir_format.py index 67302fef5a9b..bcf50fd42885 100644 --- a/datumaro/tests/test_image_dir_format.py +++ b/datumaro/tests/test_image_dir_format.py @@ -26,3 +26,23 @@ def __iter__(self): parsed_dataset = project.make_dataset() compare_datasets(self, source_dataset, parsed_dataset) + + def test_relative_paths(self): + class TestExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='1', image=np.ones((4, 2, 3))), + DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3))), + DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3))), + ]) + + with TestDir() as test_dir: + source_dataset = TestExtractor() + + ImageDirConverter()(source_dataset, save_dir=test_dir) + + project = Project.import_from(test_dir, 'image_dir') + parsed_dataset = project.make_dataset() + + compare_datasets(self, source_dataset, parsed_dataset) + diff --git a/datumaro/tests/test_labelme_format.py b/datumaro/tests/test_labelme_format.py index 1b82e0666776..f6c4596417a3 100644 --- a/datumaro/tests/test_labelme_format.py +++ b/datumaro/tests/test_labelme_format.py @@ -108,6 +108,21 @@ def categories(self): SrcExtractor(), LabelMeConverter(save_images=True), test_dir, target_dataset=DstExtractor()) + def test_cant_save_dataset_with_relative_paths(self): + class SrcExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='dir/1', image=np.ones((2, 6, 3))), + ]) + + def categories(self): + return { AnnotationType.label: LabelCategories() } + + with self.assertRaisesRegex(Exception, r'only supports flat'): + with TestDir() as test_dir: + self._test_save_and_load(SrcExtractor(), + LabelMeConverter(save_images=True), test_dir) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'labelme_dataset') diff --git a/datumaro/tests/test_tfrecord_format.py b/datumaro/tests/test_tfrecord_format.py index 403f9517345f..b7110c7fc904 100644 --- a/datumaro/tests/test_tfrecord_format.py +++ b/datumaro/tests/test_tfrecord_format.py @@ -56,7 +56,7 @@ def __iter__(self): Bbox(0, 4, 4, 8, label=2), Bbox(0, 4, 4, 4, label=3), Bbox(2, 4, 4, 4), - ] + ], attributes={'source_id': ''} ), ]) @@ -85,7 +85,8 @@ def __iter__(self): [0, 1, 1, 0], [1, 0, 0, 1], ]), label=1), - ] + ], + attributes={'source_id': ''} ), ]) @@ -111,18 +112,21 @@ def __iter__(self): annotations=[ Bbox(2, 1, 4, 4, label=2), Bbox(4, 2, 8, 4, label=3), - ] + ], + attributes={'source_id': ''} ), DatasetItem(id=2, image=np.ones((8, 8, 3)) * 2, annotations=[ Bbox(4, 4, 4, 4, label=3), - ] + ], + attributes={'source_id': ''} ), DatasetItem(id=3, image=np.ones((8, 4, 3)) * 3, + attributes={'source_id': ''} ), ]) @@ -143,7 +147,10 @@ def test_can_save_dataset_with_image_info(self): class TestExtractor(Extractor): def __iter__(self): return iter([ - DatasetItem(id=1, image=Image(path='1/q.e', size=(10, 15))), + DatasetItem(id='1/q.e', + image=Image(path='1/q.e', size=(10, 15)), + attributes={'source_id': ''} + ) ]) def categories(self): @@ -199,6 +206,7 @@ def __iter__(self): Bbox(0, 4, 4, 4, label=3), Bbox(2, 4, 4, 4), ], + attributes={'source_id': '1'} ), DatasetItem(id=2, subset='val', @@ -206,10 +214,12 @@ def __iter__(self): annotations=[ Bbox(1, 2, 4, 2, label=3), ], + attributes={'source_id': '2'} ), DatasetItem(id=3, subset='test', image=np.ones((5, 4, 3)) * 3, + attributes={'source_id': '3'} ), ]) diff --git a/datumaro/tests/test_voc_format.py b/datumaro/tests/test_voc_format.py index 92b63e6090e5..f401ff620740 100644 --- a/datumaro/tests/test_voc_format.py +++ b/datumaro/tests/test_voc_format.py @@ -145,7 +145,7 @@ def test_can_save_voc_cls(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=0, subset='a', annotations=[ + DatasetItem(id='a/0', subset='a', annotations=[ Label(1), Label(2), Label(3), @@ -164,7 +164,7 @@ def test_can_save_voc_det(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, subset='a', annotations=[ + DatasetItem(id='a/1', subset='a', annotations=[ Bbox(2, 3, 4, 5, label=2, attributes={ 'occluded': True } ), @@ -183,7 +183,7 @@ def __iter__(self): class DstExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, subset='a', annotations=[ + DatasetItem(id='a/1', subset='a', annotations=[ Bbox(2, 3, 4, 5, label=2, id=1, group=1, attributes={ 'truncated': False, @@ -220,7 +220,7 @@ def test_can_save_voc_segm(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, subset='a', annotations=[ + DatasetItem(id='a/b/1', subset='a', annotations=[ # overlapping masks, the first should be truncated # the second and third are different instances Mask(image=np.array([[0, 0, 0, 1, 0]]), label=3, @@ -235,7 +235,7 @@ def __iter__(self): class DstExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, subset='a', annotations=[ + DatasetItem(id='a/b/1', subset='a', annotations=[ Mask(image=np.array([[0, 0, 1, 0, 0]]), label=4, group=1), Mask(image=np.array([[1, 1, 0, 0, 0]]), label=3, @@ -323,7 +323,7 @@ def test_can_save_voc_layout(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, subset='a', annotations=[ + DatasetItem(id='a/b/1', subset='a', annotations=[ Bbox(2, 3, 4, 5, label=2, id=1, group=1, attributes={ 'pose': VOC.VocPose(1).name, @@ -347,7 +347,7 @@ def test_can_save_voc_action(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, subset='a', annotations=[ + DatasetItem(id='a/b/1', subset='a', annotations=[ Bbox(2, 3, 4, 5, label=2, attributes={ 'truncated': True, @@ -368,7 +368,7 @@ def __iter__(self): class DstExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, subset='a', annotations=[ + DatasetItem(id='a/b/1', subset='a', annotations=[ Bbox(2, 3, 4, 5, label=2, id=1, group=1, attributes={ 'truncated': True, @@ -666,3 +666,16 @@ def __iter__(self): with TestDir() as test_dir: self._test_save_and_load(TestExtractor(), VocConverter(label_map='voc'), test_dir) + + def test_relative_paths(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='1', image=np.ones((4, 2, 3))), + DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3))), + DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3))), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + VocConverter(label_map='voc', save_images=True), test_dir) diff --git a/datumaro/tests/test_yolo_format.py b/datumaro/tests/test_yolo_format.py index df71f5f02a28..05fc2322e560 100644 --- a/datumaro/tests/test_yolo_format.py +++ b/datumaro/tests/test_yolo_format.py @@ -116,6 +116,32 @@ def categories(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_relative_paths(self): + class TestExtractor(Extractor): + def __iter__(self): + return iter([ + DatasetItem(id='1', subset='train', + image=np.ones((4, 2, 3))), + DatasetItem(id='subdir1/1', subset='train', + image=np.ones((2, 6, 3))), + DatasetItem(id='subdir2/1', subset='train', + image=np.ones((5, 4, 3))), + ]) + + def categories(self): + return { AnnotationType.label: LabelCategories() } + + for save_images in {True, False}: + with self.subTest(save_images=save_images): + with TestDir() as test_dir: + source_dataset = TestExtractor() + + YoloConverter(save_images=save_images)( + source_dataset, test_dir) + parsed_dataset = YoloImporter()(test_dir).make_dataset() + + compare_datasets(self, source_dataset, parsed_dataset) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'yolo_dataset') diff --git a/docker-compose.yml b/docker-compose.yml index d615ef5e8590..0b8aebfbd6b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: DJANGO_CONFIGURATION: "production" TZ: "Etc/UTC" OPENVINO_TOOLKIT: "no" + CLAM_AV: "no" environment: DJANGO_MODWSGI_EXTRA_ARGS: "" ALLOWED_HOSTS: '*' diff --git a/supervisord.conf b/supervisord.conf index dde373cf157a..a991b2b9ddba 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -47,6 +47,11 @@ command=%(ENV_HOME)s/wait-for-it.sh %(ENV_CVAT_REDIS_HOST)s:6379 -t 0 -- bash -i environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 +[program:clamav_update] +command=bash -c "if [ \"${CLAM_AV}\" = 'yes' ]; then /usr/bin/freshclam -d \ + -l %(ENV_HOME)s/logs/freshclam.log --foreground=true; fi" +numprocs=1 + [program:runserver] ; Here need to run a couple of commands to initialize DB and copy static files. ; We cannot initialize DB on build because the DB should be online. Also some