Merged develop

CHANGELOG.md

@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Shortcut to change color of an activated shape in new UI (Enter) (<https://github.com/opencv/cvat/pull/1683>)
 - Shortcut to switch split mode (<https://github.com/opencv/cvat/pull/1683>)
 - Built-in search for labels when create an object or change a label (<https://github.com/opencv/cvat/pull/1683>)
+- Better validation of labels and attributes in raw viewer (<https://github.com/opencv/cvat/pull/1727>)
+- ClamAV antivirus integration (<https://github.com/opencv/cvat/pull/1712>)
 - Polygon and polylines interpolation (<https://github.com/opencv/cvat/pull/1571>)
 - Ability to redraw shape from scratch (Shift + N) for an activated shape (<https://github.com/opencv/cvat/pull/1571>)
 - Highlights for the first point of a polygon/polyline and direction (<https://github.com/opencv/cvat/pull/1571>)
@@ -38,6 +40,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Wrong rexex for account name validation (<https://github.com/opencv/cvat/pull/1667>)
 - Wrong description on register view for the username field (<https://github.com/opencv/cvat/pull/1667>)
 - Wrong resolution for resizing a shape (<https://github.com/opencv/cvat/pull/1667>)
+- React warning because of not unique keys in labels viewer (<https://github.com/opencv/cvat/pull/1727>)
+
 
 ### Security
 - SQL injection in Django `CVE-2020-9402` (<https://github.com/opencv/cvat/pull/1657>)

Dockerfile

@@ -129,6 +129,19 @@ RUN if [ "$WITH_DEXTR" = "yes" ]; then \
         7z e ${DEXTR_MODEL_DIR}/dextr.zip -o${DEXTR_MODEL_DIR} && rm ${DEXTR_MODEL_DIR}/dextr.zip; \
     fi
 
+ARG CLAM_AV
+ENV CLAM_AV=${CLAM_AV}
+RUN if [ "$CLAM_AV" = "yes" ]; then \
+        apt-get update && \
+        apt-get --no-install-recommends install -yq \
+            clamav \
+            libclamunrar9 && \
+        sed -i 's/ReceiveTimeout 30/ReceiveTimeout 300/g' /etc/clamav/freshclam.conf && \
+        freshclam && \
+        chown -R ${USER}:${USER} /var/lib/clamav && \
+        rm -rf /var/lib/apt/lists/*; \
+    fi
+
 COPY ssh ${HOME}/.ssh
 COPY utils ${HOME}/utils
 COPY cvat/ ${HOME}/cvat

README.md

@@ -24,6 +24,7 @@ Try it online [cvat.org](https://cvat.org).
 - [Command line interface](utils/cli/)
 - [XML annotation format](cvat/apps/documentation/xml_format.md)
 - [AWS Deployment Guide](cvat/apps/documentation/AWS-Deployment-Guide.md)
+- [Frequently asked questions](cvat/apps/documentation/faq.md)
 - [Questions](#questions)
 
 ## Screencasts

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/components/labels-editor/common.ts

@@ -18,6 +18,59 @@ export interface Label {
 
 let id = 0;
 
+function validateParsedAttribute(attr: Attribute): void {
+    if (typeof (attr.name) !== 'string') {
+        throw new Error(`Type of attribute name must be a string. Got value ${attr.name}`);
+    }
+
+    if (!['number', 'undefined'].includes(typeof (attr.id))) {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Type of attribute id must be a number or undefined. Got value ${attr.id}`);
+    }
+
+    if (!['checkbox', 'number', 'text', 'radio', 'select'].includes((attr.input_type || '').toLowerCase())) {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Unknown input type: ${attr.input_type}`);
+    }
+
+    if (typeof (attr.mutable) !== 'boolean') {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Mutable flag must be a boolean value. Got value ${attr.mutable}`);
+    }
+
+    if (!Array.isArray(attr.values)) {
+        throw new Error(`Attribute: "${attr.name}". `
+        + `Attribute values must be an array. Got type ${typeof (attr.values)}`);
+    }
+
+    for (const value of attr.values) {
+        if (typeof (value) !== 'string') {
+            throw new Error(`Attribute: "${attr.name}". `
+            + `Each value must be a string. Got value ${value}`);
+        }
+    }
+}
+
+export function validateParsedLabel(label: Label): void {
+    if (typeof (label.name) !== 'string') {
+        throw new Error(`Type of label name must be a string. Got value ${label.name}`);
+    }
+
+    if (!['number', 'undefined'].includes(typeof (label.id))) {
+        throw new Error(`Label "${label.name}". `
+        + `Type of label id must be only a number or undefined. Got value ${label.id}`);
+    }
+
+    if (!Array.isArray(label.attributes)) {
+        throw new Error(`Label "${label.name}". `
+        + `attributes must be an array. Got type ${typeof (label.attributes)}`);
+    }
+
+    for (const attr of label.attributes) {
+        validateParsedAttribute(attr);
+    }
+}
+
 export function idGenerator(): number {
     return --id;
 }

cvat-ui/src/components/labels-editor/raw-viewer.tsx

@@ -12,6 +12,8 @@ import Form, { FormComponentProps } from 'antd/lib/form/Form';
 import {
     Label,
     Attribute,
+    validateParsedLabel,
+    idGenerator,
 } from './common';
 
 type Props = FormComponentProps & {
@@ -22,7 +24,18 @@ type Props = FormComponentProps & {
 class RawViewer extends React.PureComponent<Props> {
     private validateLabels = (_: any, value: string, callback: any): void => {
         try {
-            JSON.parse(value);
+            const parsed = JSON.parse(value);
+            if (!Array.isArray(parsed)) {
+                callback('Field is expected to be a JSON array');
+            }
+
+            for (const label of parsed) {
+                try {
+                    validateParsedLabel(label);
+                } catch (error) {
+                    callback(error.toString());
+                }
+            }
         } catch (error) {
             callback(error.toString());
         }
@@ -39,7 +52,14 @@ class RawViewer extends React.PureComponent<Props> {
         e.preventDefault();
         form.validateFields((error, values): void => {
             if (!error) {
-                onSubmit(JSON.parse(values.labels));
+                const parsed = JSON.parse(values.labels);
+                for (const label of parsed) {
+                    label.id = label.id || idGenerator();
+                    for (const attr of label.attributes) {
+                        attr.id = attr.id || idGenerator();
+                    }
+                }
+                onSubmit(parsed);
             }
         });
     };

cvat/apps/auto_annotation/model_manager.py

@@ -19,6 +19,7 @@
 from cvat.apps.engine.serializers import LabeledDataSerializer
 from cvat.apps.dataset_manager.task import put_task_data, patch_task_data
 from cvat.apps.engine.frame_provider import FrameProvider
+from cvat.apps.engine.utils import av_scan_paths
 
 from .models import AnnotationModel, FrameworkChoice
 from .model_loader import load_labelmap
@@ -139,6 +140,7 @@ def save_file_as_tmp(data):
                 tmp_file.write(chunk)
         os.close(fd)
         return filename
+
     is_create_request = dl_model_id is None
     if is_create_request:
         dl_model_id = create_empty(owner=owner)
@@ -155,6 +157,17 @@ def save_file_as_tmp(data):
         labelmap_file = save_file_as_tmp(labelmap_file)
         interpretation_file = save_file_as_tmp(interpretation_file)
 
+    files_to_scan = []
+    if model_file:
+        files_to_scan.append(model_file)
+    if weights_file:
+        files_to_scan.append(weights_file)
+    if labelmap_file:
+        files_to_scan.append(labelmap_file)
+    if interpretation_file:
+        files_to_scan.append(interpretation_file)
+    av_scan_paths(*files_to_scan)
+
     if owner:
         restricted = not has_admin_role(owner)
     else:

cvat/apps/dataset_manager/bindings.py

@@ -11,6 +11,7 @@
 import datumaro.components.extractor as datumaro
 from cvat.apps.engine.frame_provider import FrameProvider
 from cvat.apps.engine.models import AttributeType, ShapeType
+from datumaro.util import cast
 from datumaro.util.image import Image
 
 from .annotation import AnnotationManager, TrackManager
@@ -422,8 +423,9 @@ def __init__(self, task_data, include_images=False):
                 size=(frame_data.height, frame_data.width)
             )
             dm_anno = self._read_cvat_anno(frame_data, task_data)
-            dm_item = datumaro.DatasetItem(id=frame_data.frame,
-                annotations=dm_anno, image=dm_image)
+            dm_item = datumaro.DatasetItem(id=osp.splitext(frame_data.name)[0],
+                annotations=dm_anno, image=dm_image,
+                attributes={'frame': frame_data.frame})
             dm_items.append(dm_item)
 
         self._items = dm_items
@@ -533,23 +535,21 @@ def match_frame(item, task_data):
     is_video = task_data.meta['task']['mode'] == 'interpolation'
 
     frame_number = None
-    if frame_number is None:
-        try:
-            frame_number = task_data.match_frame(item.id)
-        except Exception:
-            pass
     if frame_number is None and item.has_image:
         try:
-            frame_number = task_data.match_frame(item.image.filename)
+            frame_number = task_data.match_frame(item.image.path)
         except Exception:
             pass
     if frame_number is None:
         try:
-            frame_number = int(item.id)
+            frame_number = task_data.match_frame(item.id)
         except Exception:
             pass
-    if frame_number is None and is_video and item.id.startswith('frame_'):
-        frame_number = int(item.id[len('frame_'):])
+    if frame_number is None:
+        frame_number = cast(item.attributes.get('frame', item.id), int)
+    if frame_number is None and is_video:
+        frame_number = cast(osp.basename(item.id)[len('frame_'):], int)
+
     if not frame_number in task_data.frame_info:
         raise Exception("Could not match item id: '%s' with any task frame" %
             item.id)

cvat/apps/dataset_manager/formats/labelme.py

@@ -17,8 +17,6 @@
 @exporter(name='LabelMe', ext='ZIP', version='3.0')
 def _export(dst_file, task_data, save_images=False):
     extractor = CvatTaskDataExtractor(task_data, include_images=save_images)
-    envt = dm_env.transforms
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('label_me', save_images=save_images)

cvat/apps/dataset_manager/formats/mask.py

@@ -24,7 +24,6 @@ def _export(dst_file, task_data, save_images=False):
     extractor = extractor.transform(envt.get('polygons_to_masks'))
     extractor = extractor.transform(envt.get('boxes_to_masks'))
     extractor = extractor.transform(envt.get('merge_instance_segments'))
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('voc_segmentation',

cvat/apps/dataset_manager/formats/mot.py

@@ -7,8 +7,7 @@
 from pyunpack import Archive
 
 import datumaro.components.extractor as datumaro
-from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
-    match_frame)
+from cvat.apps.dataset_manager.bindings import CvatTaskDataExtractor
 from cvat.apps.dataset_manager.util import make_zip_archive
 from datumaro.components.project import Dataset
 
@@ -18,8 +17,6 @@
 @exporter(name='MOT', ext='ZIP', version='1.1')
 def _export(dst_file, task_data, save_images=False):
     extractor = CvatTaskDataExtractor(task_data, include_images=save_images)
-    envt = dm_env.transforms
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('mot_seq_gt',
@@ -39,8 +36,8 @@ def _import(src_file, task_data):
         label_cat = dataset.categories()[datumaro.AnnotationType.label]
 
         for item in dataset:
-            item = item.wrap(id=int(item.id) - 1) # NOTE: MOT frames start from 1
-            frame_number = task_data.abs_frame_id(match_frame(item, task_data))
+            frame_number = int(item.id) - 1 # NOTE: MOT frames start from 1
+            frame_number = task_data.abs_frame_id(frame_number)
 
             for ann in item.annotations:
                 if ann.type != datumaro.AnnotationType.bbox:

cvat/apps/dataset_manager/formats/pascal_voc.py

@@ -22,8 +22,6 @@
 @exporter(name='PASCAL VOC', ext='ZIP', version='1.1')
 def _export(dst_file, task_data, save_images=False):
     extractor = CvatTaskDataExtractor(task_data, include_images=save_images)
-    envt = dm_env.transforms
-    extractor = extractor.transform(envt.get('id_from_image_name'))
     extractor = Dataset.from_extractors(extractor) # apply lazy transforms
     with TemporaryDirectory() as temp_dir:
         converter = dm_env.make_converter('voc', label_map='source',

cvat/apps/documentation/faq.md

@@ -0,0 +1,81 @@
+# Frequently asked questions
+- [How to update CVAT](#how-to-update-cvat)
+- [Kibana app works, but no logs are displayed](#kibana-app-works-but-no-logs-are-displayed)
+- [How to change default CVAT hostname or port](#how-to-change-default-cvat-hostname-or-port)
+- [How to configure connected share folder on Windows](#how-to-configure-connected-share-folder-on-windows)
+- [How to make unassigned tasks not visible to all users](#how-to-make-unassigned-tasks-not-visible-to-all-users)
+- [Can Nvidia GPU be used to run inference with my own model](#can-nvidia-gpu-be-used-to-run-inference-with-my-own-model)
+
+## How to update CVAT
+Before upgrading, please follow the official docker
+[manual](https://docs.docker.com/storage/volumes/#backup-restore-or-migrate-data-volumes) and backup all CVAT volumes.
+
+To update CVAT, you should clone or download the new version of CVAT and rebuild the CVAT docker images as usual.
+```sh
+docker-compose build
+```
+and run containers:
+```sh
+docker-compose up -d
+```
+
+Sometimes the update process takes a lot of time due to changes in the database schema and data.
+You can check the current status with `docker logs cvat`.
+Please do not terminate the migration and wait till the process is complete.
+
+## Kibana app works, but no logs are displayed
+Make sure there aren't error messages from Elasticsearch:
+```sh
+docker logs cvat_elasticsearch
+```
+If you see errors like this:
+```sh
+lood stage disk watermark [95%] exceeded on [uMg9WI30QIOJxxJNDiIPgQ][uMg9WI3][/usr/share/elasticsearch/data/nodes/0] free: 116.5gb[4%], all indices on this node will be marked read-only
+```
+You should free up disk space or change the threshold, to do so check: [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/disk-allocator.html).
+
+## How to change default CVAT hostname or port
+The best way to do that is to create docker-compose.override.yml and override the host and port settings here.
+
+version: "2.3"
+```yaml
+services:
+  cvat_proxy:
+    environment:
+      CVAT_HOST: example.com
+    ports:
+      - "80:80"
+```
+
+Please don't forget to include this file in docker-compose commands
+using the `-f` option (in some cases it can be omitted).
+
+## How to configure connected share folder on Windows
+Follow the Docker manual and configure the directory that you want to use as a shared directory:
+- [Docker toolbox manual](https://docs.docker.com/toolbox/toolbox_install_windows/#optional-add-shared-directories)
+- [Docker for windows (see FILE SHARING section)](https://docs.docker.com/docker-for-windows/#resources)
+
+After that, it should be possible to use this directory as a CVAT share:
+```yaml
+version: "2.3"
+
+services:
+  cvat:
+    volumes:
+      - cvat_share:/home/django/share:ro
+
+volumes:
+  cvat_share:
+    driver_opts:
+      type: none
+      device: /d/my_cvat_share
+      o: bind
+```
+
+## How to make unassigned tasks not visible to all users
+Set [reduce_task_visibility](../../settings/base.py#L424) variable to `True`.
+
+## Can Nvidia GPU be used to run inference with my own model
+Nvidia GPU can be used to accelerate inference of [tf_annotation](../../../components/tf_annotation/README.md) and [auto_segmentation](../../../components/auto_segmentation/README.md) models.
+
+OpenVino doesn't support Nvidia cards, so you can run your own models only on CPU.