From ca56b7688d10d8b5a3154986d3a8aa44a104d7ff Mon Sep 17 00:00:00 2001
From: Zhiltsov Max <zhiltsov.max35@gmail.com>
Date: Tue, 23 Jun 2020 12:10:04 +0300
Subject: [PATCH 1/6] Add voc attributes
---
.../datumaro/plugins/voc_format/converter.py | 28 ++++++++++++---
.../datumaro/plugins/voc_format/extractor.py | 6 ++++
datumaro/tests/test_voc_format.py | 35 +++++++++++++++++--
3 files changed, 63 insertions(+), 6 deletions(-)
diff --git a/datumaro/datumaro/plugins/voc_format/converter.py b/datumaro/datumaro/plugins/voc_format/converter.py
index 42dfaffcc938..9478c190928e 100644
--- a/datumaro/datumaro/plugins/voc_format/converter.py
+++ b/datumaro/datumaro/plugins/voc_format/converter.py
@@ -53,7 +53,8 @@ def _write_xml_bbox(bbox, parent_elem):
class _Converter:
def __init__(self, extractor, save_dir,
- tasks=None, apply_colormap=True, save_images=False, label_map=None):
+ tasks=None, apply_colormap=True, save_images=False, label_map=None,
+ allow_attributes=True):
assert tasks is None or isinstance(tasks, (VocTask, list, set))
if tasks is None:
tasks = set(VocTask)
@@ -66,6 +67,7 @@ def __init__(self, extractor, save_dir,
self._extractor = extractor
self._save_dir = save_dir
self._apply_colormap = apply_colormap
+ self._allow_attributes = allow_attributes
self._save_images = save_images
self._load_categories(label_map)
@@ -252,6 +254,21 @@ def save_subsets(self):
if len(actions_elem) != 0:
obj_elem.append(actions_elem)
+ if self._allow_attributes:
+ native_attrs = {'difficult', 'pose',
+ 'truncated', 'occluded' }
+ native_attrs.update(label_actions)
+
+ attrs_elem = ET.Element('attributes')
+ for k, v in attr.items():
+ if k in native_attrs:
+ continue
+ attr_elem = ET.SubElement(attrs_elem, 'attribute')
+ ET.SubElement(attr_elem, 'name').text = str(k)
+ ET.SubElement(attr_elem, 'value').text = str(v)
+ if len(attrs_elem):
+ obj_elem.append(attrs_elem)
+
if self._tasks & {None,
VocTask.detection,
VocTask.person_layout,
@@ -565,15 +582,17 @@ def build_cmdline_parser(cls, **kwargs):
parser.add_argument('--label-map', type=cls._get_labelmap, default=None,
help="Labelmap file path or one of %s" % \
', '.join(t.name for t in LabelmapType))
+ parser.add_argument('--allow-attributes',
+ type=str_to_bool, default=True,
+ help="Allow export of attributes (default: %(default)s)")
parser.add_argument('--tasks', type=cls._split_tasks_string,
- default=None,
help="VOC task filter, comma-separated list of {%s} "
- "(default: all)" % ', '.join([t.name for t in VocTask]))
+ "(default: all)" % ', '.join(t.name for t in VocTask))
return parser
def __init__(self, tasks=None, save_images=False,
- apply_colormap=False, label_map=None):
+ apply_colormap=False, label_map=None, allow_attributes=True):
super().__init__()
self._options = {
@@ -581,6 +600,7 @@ def __init__(self, tasks=None, save_images=False,
'save_images': save_images,
'apply_colormap': apply_colormap,
'label_map': label_map,
+ 'allow_attributes': allow_attributes,
}
def __call__(self, extractor, save_dir):
diff --git a/datumaro/datumaro/plugins/voc_format/extractor.py b/datumaro/datumaro/plugins/voc_format/extractor.py
index fb724fbd3e2b..669d78104646 100644
--- a/datumaro/datumaro/plugins/voc_format/extractor.py
+++ b/datumaro/datumaro/plugins/voc_format/extractor.py
@@ -198,6 +198,12 @@ def _parse_annotations(self, root_elem):
item_annotations.append(Bbox(*part_bbox, label=part_label_id,
group=group))
+ attributes_elem = object_elem.find('attributes')
+ if attributes_elem is not None:
+ for attr_elem in attributes_elem.iter('attribute'):
+ attributes[attr_elem.find('name').text] = \
+ attr_elem.find('value').text
+
if self._task is VocTask.person_layout and not has_parts:
continue
if self._task is VocTask.action_classification and not actions:
diff --git a/datumaro/tests/test_voc_format.py b/datumaro/tests/test_voc_format.py
index f401ff620740..d66b6db01018 100644
--- a/datumaro/tests/test_voc_format.py
+++ b/datumaro/tests/test_voc_format.py
@@ -395,8 +395,8 @@ def __iter__(self):
with TestDir() as test_dir:
self._test_save_and_load(TestExtractor(),
- VocActionConverter(label_map='voc'), test_dir,
- target_dataset=DstExtractor())
+ VocActionConverter(label_map='voc', allow_attributes=False),
+ test_dir, target_dataset=DstExtractor())
def test_can_save_dataset_with_no_subsets(self):
class TestExtractor(TestExtractorBase):
@@ -679,3 +679,34 @@ def __iter__(self):
with TestDir() as test_dir:
self._test_save_and_load(TestExtractor(),
VocConverter(label_map='voc', save_images=True), test_dir)
+
+ def test_can_save_attributes(self):
+ class TestExtractor(TestExtractorBase):
+ def __iter__(self):
+ return iter([
+ DatasetItem(id='a', annotations=[
+ Bbox(2, 3, 4, 5, label=2,
+ attributes={ 'occluded': True, 'x': 1, 'y': '2' }
+ ),
+ ]),
+ ])
+
+ class DstExtractor(TestExtractorBase):
+ def __iter__(self):
+ return iter([
+ DatasetItem(id='a', annotations=[
+ Bbox(2, 3, 4, 5, label=2, id=1, group=1,
+ attributes={
+ 'truncated': False,
+ 'difficult': False,
+ 'occluded': True,
+ 'x': '1', 'y': '2', # can only read strings
+ }
+ ),
+ ]),
+ ])
+
+ with TestDir() as test_dir:
+ self._test_save_and_load(TestExtractor(),
+ VocDetectionConverter(label_map='voc'), test_dir,
+ target_dataset=DstExtractor())
\ No newline at end of file
From b0baa6a236f2fc396b717ebf611f189deea140d6 Mon Sep 17 00:00:00 2001
From: Zhiltsov Max <zhiltsov.max35@gmail.com>
Date: Tue, 23 Jun 2020 12:17:36 +0300
Subject: [PATCH 2/6] Allow any values for voc pose
---
datumaro/datumaro/plugins/voc_format/converter.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/datumaro/datumaro/plugins/voc_format/converter.py b/datumaro/datumaro/plugins/voc_format/converter.py
index 9478c190928e..1465a06ac314 100644
--- a/datumaro/datumaro/plugins/voc_format/converter.py
+++ b/datumaro/datumaro/plugins/voc_format/converter.py
@@ -207,9 +207,8 @@ def save_subsets(self):
ET.SubElement(obj_elem, 'name').text = obj_label
if 'pose' in attr:
- pose = _convert_attr('pose', attr,
- lambda v: VocPose[v], VocPose.Unspecified)
- ET.SubElement(obj_elem, 'pose').text = pose.name
+ ET.SubElement(obj_elem, 'pose').text = \
+ str(attr['pose'])
if 'truncated' in attr:
truncated = _convert_attr('truncated', attr, int, 0)
From fb0f8b6a660069252313daf65d5beb41e886fb3e Mon Sep 17 00:00:00 2001
From: Zhiltsov Max <zhiltsov.max35@gmail.com>
Date: Tue, 23 Jun 2020 12:19:55 +0300
Subject: [PATCH 3/6] update changelog
---
CHANGELOG.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11e82a54f957..4204cb85a098 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- ClamAV antivirus integration (<https://github.com/opencv/cvat/pull/1712>)
- Supported import and export or single boxes in MOT format (https://github.com/opencv/cvat/pull/1764)
- [Datumaro] Added `stats` command, which shows some dataset statistics like image mean and std (https://github.com/opencv/cvat/pull/1734)
-- Add option to upload annotations upon task creation on CLI
+- Add option to upload annotations upon task creation on CLI
- Polygon and polylines interpolation (<https://github.com/opencv/cvat/pull/1571>)
- Ability to redraw shape from scratch (Shift + N) for an activated shape (<https://github.com/opencv/cvat/pull/1571>)
- Highlights for the first point of a polygon/polyline and direction (<https://github.com/opencv/cvat/pull/1571>)
@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Update https install manual. Makes it easier and more robust. Includes automatic renewing of lets encrypt certificates.
- Implemented import and export of annotations with relative image paths (<https://github.com/opencv/cvat/pull/1463>)
- Using only single click to start editing or remove a point (<https://github.com/opencv/cvat/pull/1571>)
+- Added support for attributes in VOC XML format (https://github.com/opencv/cvat/pull/1792)
### Deprecated
-
From 2d8e1420ebfd1feeba26385d33d73e1120c0031e Mon Sep 17 00:00:00 2001
From: Zhiltsov Max <zhiltsov.max35@gmail.com>
Date: Wed, 24 Jun 2020 19:19:42 +0300
Subject: [PATCH 4/6] Add attribute conversion
---
cvat/apps/dataset_manager/bindings.py | 32 ++++++++++++++++++++++-----
1 file changed, 26 insertions(+), 6 deletions(-)
diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py
index 534a23e742ed..0cb1efa5dcd4 100644
--- a/cvat/apps/dataset_manager/bindings.py
+++ b/cvat/apps/dataset_manager/bindings.py
@@ -11,7 +11,7 @@
import datumaro.components.extractor as datumaro
from cvat.apps.engine.frame_provider import FrameProvider
-from cvat.apps.engine.models import AttributeType, ShapeType
+from cvat.apps.engine.models import AttributeType, ShapeType, AttributeSpec
from datumaro.util import cast
from datumaro.util.image import Image
@@ -49,7 +49,8 @@ def __init__(self, annotation_ir, db_task, host='', create_callback=None):
(db_label.id, db_label) for db_label in db_labels)
self._attribute_mapping = {db_label.id: {
- 'mutable': {}, 'immutable': {}} for db_label in db_labels}
+ 'mutable': {}, 'immutable': {}, 'spec': {}}
+ for db_label in db_labels}
for db_label in db_labels:
for db_attribute in db_label.attributespec_set.all():
@@ -57,6 +58,7 @@ def __init__(self, annotation_ir, db_task, host='', create_callback=None):
self._attribute_mapping[db_label.id]['mutable'][db_attribute.id] = db_attribute.name
else:
self._attribute_mapping[db_label.id]['immutable'][db_attribute.id] = db_attribute.name
+ self._attribute_mapping[db_label.id]['spec'][db_attribute.id] = db_attribute
self._attribute_mapping_merged = {}
for label_id, attr_mapping in self._attribute_mapping.items():
@@ -317,10 +319,28 @@ def _import_tag(self, tag):
return _tag
def _import_attribute(self, label_id, attribute):
- return {
- 'spec_id': self._get_attribute_id(label_id, attribute.name),
- 'value': attribute.value,
- }
+ spec_id = self._get_attribute_id(label_id, attribute.name)
+ value = attribute.value
+
+ if spec_id:
+ spec = self._attribute_mapping[label_id]['spec'][spec_id]
+
+ try:
+ if spec.input_type == AttributeType.NUMBER:
+ pass # no extra processing required
+ elif spec.input_type == AttributeType.CHECKBOX:
+ if isinstance(value, str):
+ value = value.lower()
+ assert value in {'true', 'false'}
+ elif isinstance(value, (bool, int, float)):
+ value = 'true' if value else 'false'
+ else:
+ raise ValueError("Unexpected attribute value")
+ except Exception as e:
+ raise Exception("Failed to convert attribute '%s'='%s': %s" %
+ (self._get_label_name(label_id), value, e))
+
+ return { 'spec_id': spec_id, 'value': value }
def _import_shape(self, shape):
_shape = shape._asdict()
From f8847cdc35741ef38b6bf552a77a12e87b5a1b8b Mon Sep 17 00:00:00 2001
From: Zhiltsov Max <zhiltsov.max35@gmail.com>
Date: Wed, 24 Jun 2020 19:24:09 +0300
Subject: [PATCH 5/6] linter
---
cvat/apps/dataset_manager/bindings.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py
index 0cb1efa5dcd4..d7c92ed4f724 100644
--- a/cvat/apps/dataset_manager/bindings.py
+++ b/cvat/apps/dataset_manager/bindings.py
@@ -11,7 +11,7 @@
import datumaro.components.extractor as datumaro
from cvat.apps.engine.frame_provider import FrameProvider
-from cvat.apps.engine.models import AttributeType, ShapeType, AttributeSpec
+from cvat.apps.engine.models import AttributeType, ShapeType
from datumaro.util import cast
from datumaro.util.image import Image
From 046af64ba26b80b0d2e6b8201a874970d0f7e1aa Mon Sep 17 00:00:00 2001
From: Zhiltsov Max <zhiltsov.max35@gmail.com>
Date: Thu, 25 Jun 2020 11:08:08 +0300
Subject: [PATCH 6/6] fix tests
---
cvat/apps/engine/tests/_test_rest_api.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cvat/apps/engine/tests/_test_rest_api.py b/cvat/apps/engine/tests/_test_rest_api.py
index aa9e0e77341d..8046efbc4b6e 100644
--- a/cvat/apps/engine/tests/_test_rest_api.py
+++ b/cvat/apps/engine/tests/_test_rest_api.py
@@ -1988,7 +1988,7 @@ def _create_task(self, owner, assignee):
"name": "parked",
"mutable": True,
"input_type": "checkbox",
- "default_value": False
+ "default_value": "false"
},
]
},