From 5335bfee3856d3f800e841620fc5cab8bb82281d Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 2 Feb 2021 14:51:35 +0300 Subject: [PATCH 01/29] Added support for manifest file --- cvat/apps/engine/cache.py | 27 +- cvat/apps/engine/media_extractors.py | 85 ++++++ cvat/apps/engine/models.py | 9 +- cvat/apps/engine/prepare.py | 393 ++++++++++++++++++--------- cvat/apps/engine/task.py | 114 +++++--- cvat/apps/engine/utils.py | 7 + 6 files changed, 456 insertions(+), 179 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 5ea9a1e87ccd..65cffcfca7d1 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -9,9 +9,9 @@ from django.conf import settings from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, - Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter) + Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter, + MImagesReader, MVideoReader) from cvat.apps.engine.models import DataChoice, StorageChoice -from cvat.apps.engine.prepare import PrepareInfo from cvat.apps.engine.models import DimensionType class CacheInteraction: @@ -51,17 +51,24 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number): StorageChoice.LOCAL: db_data.get_upload_dirname(), StorageChoice.SHARE: settings.SHARE_ROOT }[db_data.storage] - if os.path.exists(db_data.get_meta_path()): + if hasattr(db_data, 'video'): source_path = os.path.join(upload_dir, db_data.video.path) - meta = PrepareInfo(source_path=source_path, meta_path=db_data.get_meta_path()) - for frame in meta.decode_needed_frames(chunk_number, db_data): - images.append(frame) - writer.save_as_chunk([(image, source_path, None) for image in images], buff) + reader = MVideoReader(manifest_path=db_data.get_manifest_path(), + source_path=source_path, chunk_number=chunk_number, + chunk_size=db_data.chunk_size, start=db_data.start_frame, + stop=db_data.stop_frame,step=db_data.get_frame_step()) + for frame in reader: + images.append((frame, source_path, None)) else: - with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: - images = [os.path.join(upload_dir, line.strip()) for line in dummy_file] - writer.save_as_chunk([(image, image, None) for image in images], buff) + reader = MImagesReader(manifest_path=db_data.get_manifest_path(), + chunk_number=chunk_number,chunk_size=db_data.chunk_size, + start=db_data.start_frame, stop=db_data.stop_frame, + step=db_data.get_frame_step()) + for item in reader: + source_path = os.path.join(upload_dir, f"{item['name']}{item['extension']}") + images.append((source_path, source_path, None)) + writer.save_as_chunk(images, buff) buff.seek(0) return buff, mime_type diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index b72bf0cda297..657fea819938 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -25,6 +25,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True from cvat.apps.engine.mime_types import mimetypes +from cvat.apps.engine.prepare import VManifestManager, IManifestManager, WorkWithVideo def get_mime(name): for type_name, type_def in MEDIA_TYPES.items(): @@ -311,6 +312,90 @@ def get_image_size(self, i): image = (next(iter(self)))[0] return image.width, image.height +class FragmentMediaReader: + def __init__(self, chunk_number, chunk_size, start, stop, step=1, *args, **kwargs): + self._start = start + self._stop = stop + 1 # up to the last inclusive + self._step = step + self._chunk_number = chunk_number + self._chunk_size = chunk_size + self._start_chunk_frame_number = self._start + self._chunk_number * self._chunk_size * self._step + self._end_chunk_frame_number = min(self._start_chunk_frame_number + (self._chunk_size - 1) * self._step + 1, self._stop) + self._frame_range = self._get_frame_range() + + @property + def frame_range(self): + return self._frame_range + + def _get_frame_range(self): + frame_range = [] + for idx in range(self._start, self._stop, self._step): + if idx < self._start_chunk_frame_number: + continue + elif idx < self._end_chunk_frame_number and not ((idx - self._start_chunk_frame_number) % self._step): + frame_range.append(idx) + elif (idx - self._start_chunk_frame_number) % self._step: + continue + else: + break + return frame_range + +class MImagesReader(FragmentMediaReader): + def __init__(self, manifest_path, **kwargs): + super().__init__(**kwargs) + self._manifest = IManifestManager(manifest_path) + self._manifest.init_index() + + def __iter__(self): + for idx in self._frame_range: + yield self._manifest[idx] + +class MVideoReader(WorkWithVideo, FragmentMediaReader): + def __init__(self, manifest_path, **kwargs): + WorkWithVideo.__init__(self, **kwargs) + FragmentMediaReader.__init__(self, **kwargs) + self._manifest = VManifestManager(manifest_path) + self._manifest.init_index() + + def _get_nearest_left_key_frame(self): + start_decode_frame_number = 0 + start_decode_timestamp = 0 + for _, frame in self._manifest: + frame_number, timestamp = frame.get('number'), frame.get('pts') + if int(frame_number) <= self._start_chunk_frame_number: + start_decode_frame_number = frame_number + start_decode_timestamp = timestamp + else: + break + return int(start_decode_frame_number), int(start_decode_timestamp) + + def __iter__(self): + start_decode_frame_number, start_decode_timestamp = self._get_nearest_left_key_frame() + container = self._open_video_container(self.source_path, mode='r') + video_stream = self._get_video_stream(container) + container.seek(offset=start_decode_timestamp, stream=video_stream) + + frame_number = start_decode_frame_number - 1 + for packet in container.demux(video_stream): + for frame in packet.decode(): + frame_number += 1 + if frame_number in self._frame_range: + if video_stream.metadata.get('rotate'): + frame = av.VideoFrame().from_ndarray( + rotate_image( + frame.to_ndarray(format='bgr24'), + 360 - int(container.streams.video[0].metadata.get('rotate')) + ), + format ='bgr24' + ) + yield frame + elif frame_number < self._frame_range[-1]: + continue + else: + self._close_video_container(container) + return + self._close_video_container(container) + class IChunkWriter(ABC): def __init__(self, quality, dimension=DimensionType.DIM_2D): self._image_quality = quality diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index fc25a6e485dc..c6e78a45f743 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -138,11 +138,10 @@ def get_compressed_chunk_path(self, chunk_number): def get_preview_path(self): return os.path.join(self.get_data_dirname(), 'preview.jpeg') - def get_meta_path(self): - return os.path.join(self.get_upload_dirname(), 'meta_info.txt') - - def get_dummy_chunk_path(self, chunk_number): - return os.path.join(self.get_upload_dirname(), 'dummy_{}.txt'.format(chunk_number)) + def get_manifest_path(self): + return os.path.join(self.get_upload_dirname(), 'manifest.jsonl') + def get_index_path(self): + return os.path.join(self.get_upload_dirname(), 'index') class Video(models.Model): data = models.OneToOneField(Data, on_delete=models.CASCADE, related_name="video", null=True) diff --git a/cvat/apps/engine/prepare.py b/cvat/apps/engine/prepare.py index 4cedf4ab0175..8e9586b5f7ef 100644 --- a/cvat/apps/engine/prepare.py +++ b/cvat/apps/engine/prepare.py @@ -3,10 +3,14 @@ # SPDX-License-Identifier: MIT import av -from collections import OrderedDict -import hashlib +import json +import marshal import os +from abc import ABC, abstractmethod +from collections import OrderedDict +from PIL import Image from cvat.apps.engine.utils import rotate_image +from .utils import md5_hash class WorkWithVideo: def __init__(self, **kwargs): @@ -74,19 +78,30 @@ def check_video_timestamps_sequences(self): frame_pts, frame_dts = frame.pts, frame.dts self._close_video_container(container) -def md5_hash(frame): - return hashlib.md5(frame.to_image().tobytes()).hexdigest() +class IPrepareInfo: + def __init__(self, sources, is_sorted=True, *args, **kwargs): + self._sources = sources if is_sorted else sorted(sources) + self._content = [] + self._data_dir = kwargs.get('data_dir', None) -class PrepareInfo(WorkWithVideo): + def __iter__(self): + for image in self._sources: + img = Image.open(image, mode='r') + img_name = os.path.relpath(image, self._data_dir) if self._data_dir else os.path.basename(image) + yield (img_name, img.width, img.height, md5_hash(img)) - def __init__(self, **kwargs): - super().__init__(**kwargs) + def create(self): + for item in self: + self._content.append(item) - if not kwargs.get('meta_path'): - raise Exception('No meta path') + @property + def content(self): + return self._content - self.meta_path = kwargs.get('meta_path') - self.key_frames = {} +class VPrepareInfo(WorkWithVideo): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._key_frames = OrderedDict() self.frames = 0 container = self._open_video_container(self.source_path, 'r') @@ -100,25 +115,25 @@ def get_task_size(self): def frame_sizes(self): return (self.width, self.height) - def check_key_frame(self, container, video_stream, key_frame): + def validate_key_frame(self, container, video_stream, key_frame): for packet in container.demux(video_stream): for frame in packet.decode(): if md5_hash(frame) != key_frame[1]['md5'] or frame.pts != key_frame[1]['pts']: - self.key_frames.pop(key_frame[0]) + self._key_frames.pop(key_frame[0]) return - def check_seek_key_frames(self): + def validate_seek_key_frames(self): container = self._open_video_container(self.source_path, mode='r') video_stream = self._get_video_stream(container) - key_frames_copy = self.key_frames.copy() + key_frames_copy = self._key_frames.copy() for key_frame in key_frames_copy.items(): container.seek(offset=key_frame[1]['pts'], stream=video_stream) - self.check_key_frame(container, video_stream, key_frame) + self.validate_key_frame(container, video_stream, key_frame) - def check_frames_ratio(self, chunk_size): - return (len(self.key_frames) and (self.frames // len(self.key_frames)) <= 2 * chunk_size) + def validate_frames_ratio(self, chunk_size): + return (len(self._key_frames) and (self.frames // len(self._key_frames)) <= 2 * chunk_size) def save_key_frames(self): container = self._open_video_container(self.source_path, mode='r') @@ -128,7 +143,7 @@ def save_key_frames(self): for packet in container.demux(video_stream): for frame in packet.decode(): if frame.key_frame: - self.key_frames[frame_number] = { + self._key_frames[frame_number] = { 'pts': frame.pts, 'md5': md5_hash(frame), } @@ -137,141 +152,267 @@ def save_key_frames(self): self.frames = frame_number self._close_video_container(container) - def save_meta_info(self): - with open(self.meta_path, 'w') as meta_file: - for index, frame in self.key_frames.items(): - meta_file.write('{} {}\n'.format(index, frame['pts'])) + @property + def key_frames(self): + return self._key_frames - def get_nearest_left_key_frame(self, start_chunk_frame_number): - start_decode_frame_number = 0 - start_decode_timestamp = 0 + def __len__(self): + return len(self._key_frames) - with open(self.meta_path, 'r') as file: - for line in file: - frame_number, timestamp = line.strip().split(' ') + def __iter__(self): + for idx, key_frame in self._key_frames.items(): + yield (idx, key_frame['pts'], key_frame['md5']) - if int(frame_number) <= start_chunk_frame_number: - start_decode_frame_number = frame_number - start_decode_timestamp = timestamp - else: - break +def _prepare_video_meta(media_file, upload_dir=None, chunk_size=None): + source_path = os.path.join(upload_dir, media_file) if upload_dir else media_file + analyzer = AnalyzeVideo(source_path=source_path) + analyzer.check_type_first_frame() + analyzer.check_video_timestamps_sequences() - return int(start_decode_frame_number), int(start_decode_timestamp) + meta_info = VPrepareInfo(source_path=source_path) + meta_info.save_key_frames() + meta_info.validate_seek_key_frames() + smooth_decoding = meta_info.validate_frames_ratio(chunk_size) if chunk_size else None + return (meta_info, smooth_decoding) - def decode_needed_frames(self, chunk_number, db_data): - step = db_data.get_frame_step() - start_chunk_frame_number = db_data.start_frame + chunk_number * db_data.chunk_size * step - end_chunk_frame_number = min(start_chunk_frame_number + (db_data.chunk_size - 1) * step + 1, db_data.stop_frame + 1) - start_decode_frame_number, start_decode_timestamp = self.get_nearest_left_key_frame(start_chunk_frame_number) - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) - container.seek(offset=start_decode_timestamp, stream=video_stream) +def _prepare_images_meta(sources, **kwargs): + meta_info = IPrepareInfo(sources=sources, **kwargs) + meta_info.create() + return meta_info - frame_number = start_decode_frame_number - 1 - for packet in container.demux(video_stream): - for frame in packet.decode(): - frame_number += 1 - if frame_number < start_chunk_frame_number: - continue - elif frame_number < end_chunk_frame_number and not ((frame_number - start_chunk_frame_number) % step): - if video_stream.metadata.get('rotate'): - frame = av.VideoFrame().from_ndarray( - rotate_image( - frame.to_ndarray(format='bgr24'), - 360 - int(container.streams.video[0].metadata.get('rotate')) - ), - format ='bgr24' - ) - yield frame - elif (frame_number - start_chunk_frame_number) % step: - continue - else: - self._close_video_container(container) - return +def prepare_meta(data_type, **kwargs): + assert data_type in ('video', 'images'), 'prepare_meta: Unknown data type' + actions = { + 'video': _prepare_video_meta, + 'images': _prepare_images_meta, + } + return actions[data_type](**kwargs) - self._close_video_container(container) +class _Manifest: + FILE_NAME = 'manifest.jsonl' + VERSION = '1.0' -class UploadedMeta(PrepareInfo): - def __init__(self, **kwargs): - super().__init__(**kwargs) - uploaded_meta = kwargs.get('uploaded_meta') - assert uploaded_meta is not None , 'No uploaded meta path' + def __init__(self, path, is_created=False): + assert path, 'A path to manifest file not found' + self._path = os.path.join(path, self.FILE_NAME) if os.path.isdir(path) else path + self._is_created = is_created + + @property + def path(self): + return self._path + + @property + def is_created(self): + return self._is_created - with open(uploaded_meta, 'r') as meta_file: - lines = meta_file.read().strip().split('\n') - self.frames = int(lines.pop()) + @is_created.setter + def is_created(self, value): + assert isinstance(value, bool) + self._is_created = value - key_frames = {int(line.split()[0]): int(line.split()[1]) for line in lines} - self.key_frames = OrderedDict(sorted(key_frames.items(), key=lambda x: x[0])) +# Needed for faster iteration over the manifest file, will be generated to work inside CVAT +# and will not be generated when manually creating a manifest +class _Index: + FILE_NAME = 'index' + + def __init__(self, path): + assert path and os.path.isdir(path), 'No index directory path' + self._path = os.path.join(path, self.FILE_NAME) + self._index = {} @property - def frame_sizes(self): - container = self._open_video_container(self.source_path, 'r') - video_stream = self._get_video_stream(container) - container.seek(offset=next(iter(self.key_frames.values())), stream=video_stream) - for packet in container.demux(video_stream): - for frame in packet.decode(): - if video_stream.metadata.get('rotate'): - frame = av.VideoFrame().from_ndarray( - rotate_image( - frame.to_ndarray(format='bgr24'), - 360 - int(container.streams.video[0].metadata.get('rotate')) - ), - format ='bgr24' - ) - self._close_video_container(container) - return (frame.width, frame.height) + def path(self): + return self._path + + def dump(self): + with open(self._path, 'wb') as index_file: + marshal.dump(self._index, index_file, 4) + + def load(self): + with open(self._path, 'rb') as index_file: + self._index = marshal.load(index_file) + + def create(self, manifest, skip): + assert os.path.exists(manifest), 'A manifest file not exists, index cannot be created' + with open(manifest, 'r+') as manifest_file: + while skip: + manifest_file.readline() + skip -= 1 + image_number = 0 + self._index[image_number] = manifest_file.tell() + while (line := manifest_file.readline()): + if line.strip(): + image_number += 1 + self._index[image_number] = manifest_file.tell() + + def partial_update(self, manifest, number): + with open(manifest, 'r+') as manifest_file: + manifest_file.seek(self._index[number]) + while (line := manifest_file.readline()): + if line.strip(): + index[number] = manifest_file.tell() + number += 1 + + def __getitem__(self, number): + assert 0 <= number < len(self), 'A invalid index number' + return self._index[number] + + def __len__(self): + return len(self._index) + +class _ManifestManager(ABC): + BASE_INFORMATION = { + 'version' : 1, + 'type': 2, + } + def __init__(self, path, *args, **kwargs): + self._manifest = _Manifest(path) + + def _parse_line(self, line): + """ Getting a random line from the manifest file """ + with open(self._manifest.path, 'r') as manifest_file: + if isinstance(line, str): + assert line in self.BASE_INFORMATION.keys() + for _ in range(self.BASE_INFORMATION[line]): + fline = manifest_file.readline() + return json.loads(fline)[line] + else: + assert self._index, 'No prepared index' + offset = self._index[line] + manifest_file.seek(offset) + properties = manifest_file.readline() + return json.loads(properties) + + def init_index(self): + self._index = _Index(os.path.dirname(self._manifest.path)) + if os.path.exists(self._index.path): + self._index.load() + else: + self._index.create(self._manifest.path, 3 if self._manifest.TYPE == 'video' else 2) + self._index.dump() + + @abstractmethod + def create(self, content, **kwargs): + pass + + @abstractmethod + def partial_update(self, number, properties): + pass + + def __iter__(self): + with open(self._manifest.path, 'r') as manifest_file: + manifest_file.seek(self._index[0]) + image_number = 0 + while (line := manifest_file.readline()): + if not line.strip(): + continue + yield (image_number, json.loads(line)) + image_number += 1 + + @property + def manifest(self): + return self._manifest + + def __len__(self): + if hasattr(self, '_index'): + return len(self._index) + else: + return None - def save_meta_info(self): - with open(self.meta_path, 'w') as meta_file: - for index, pts in self.key_frames.items(): - meta_file.write('{} {}\n'.format(index, pts)) + def __getitem__(self, item): + return self._parse_line(item) + + @property + def index(self): + return self._index + +class VManifestManager(_ManifestManager): + #TODO: + #NUMBER_ADDITIONAL_LINES = 3 + def __init__(self, manifest_path, *args, **kwargs): + super().__init__(manifest_path) + setattr(self._manifest, 'TYPE', 'video') + self.BASE_INFORMATION['properties'] = 3 + + def create(self, content, **kwargs): + """ Creating and saving a manifest file """ + with open(self._manifest.path, 'w') as manifest_file: + manifest_file.write(f"{json.dumps({'version':self._manifest.VERSION})}\n") + manifest_file.write(f"{json.dumps({'type':self._manifest.TYPE})}\n") + manifest_file.write(f"{json.dumps({'properties':{'name':os.path.basename(content.source_path),'resolution': content.frame_sizes, 'length': content.get_task_size()}})}\n") + for item in content: + json_item = json.dumps({'number': item[0], 'pts': item[1], 'checksum': item[2]}, separators=(',', ':')) + manifest_file.write(f"{json_item}\n") + self._manifest.is_created = True + + def partial_update(self, number, properties): + """ Updating a part of a manifest file """ + pass + +#TODO: +class ManifestValidator: + def validate_base_info(self): + with open(self._manifest.path, 'r') as manifest_file: + assert self._manifest.VERSION != json.loads(manifest_file.readline())['version'] + assert self._manifest.TYPE != json.loads(manifest_file.readline())['type'] + +class VManifestValidator(VManifestManager, WorkWithVideo): + def __init__(self, **kwargs): + WorkWithVideo.__init__(self, **kwargs) + VManifestManager.__init__(self, **kwargs) - def check_key_frame(self, container, video_stream, key_frame): + def validate_key_frame(self, container, video_stream, key_frame): for packet in container.demux(video_stream): for frame in packet.decode(): - assert frame.pts == key_frame[1], "Uploaded meta information does not match the video" + assert frame.pts == key_frame['pts'], "The uploaded manifest does not match the video" return - def check_seek_key_frames(self): + def validate_seek_key_frames(self): container = self._open_video_container(self.source_path, mode='r') video_stream = self._get_video_stream(container) + last_key_frame = None - for key_frame in self.key_frames.items(): - container.seek(offset=key_frame[1], stream=video_stream) - self.check_key_frame(container, video_stream, key_frame) + for _, key_frame in self: + # chack that key frames sequence sorted + if last_key_frame and last_key_frame['number'] >= key_frame['number']: + raise AssertionError('Invalid saved key frames sequence in manifest file') + container.seek(offset=key_frame['pts'], stream=video_stream) + self.validate_key_frame(container, video_stream, key_frame) + last_key_frame = key_frame self._close_video_container(container) - def check_frames_numbers(self): + def validate_frames_numbers(self): container = self._open_video_container(self.source_path, mode='r') video_stream = self._get_video_stream(container) # not all videos contain information about numbers of frames - if video_stream.frames: + frames = video_stream.frames + if frames: self._close_video_container(container) - assert video_stream.frames == self.frames, "Uploaded meta information does not match the video" + assert frames == self['properties']['length'], "The uploaded manifest does not match the video" return self._close_video_container(container) -def prepare_meta(media_file, upload_dir=None, meta_dir=None, chunk_size=None): - paths = { - 'source_path': os.path.join(upload_dir, media_file) if upload_dir else media_file, - 'meta_path': os.path.join(meta_dir, 'meta_info.txt') if meta_dir else os.path.join(upload_dir, 'meta_info.txt'), - } - analyzer = AnalyzeVideo(source_path=paths.get('source_path')) - analyzer.check_type_first_frame() - analyzer.check_video_timestamps_sequences() - - meta_info = PrepareInfo(source_path=paths.get('source_path'), - meta_path=paths.get('meta_path')) - meta_info.save_key_frames() - meta_info.check_seek_key_frames() - meta_info.save_meta_info() - smooth_decoding = meta_info.check_frames_ratio(chunk_size) if chunk_size else None - return (meta_info, smooth_decoding) - -def prepare_meta_for_upload(func, *args): - meta_info, smooth_decoding = func(*args) - with open(meta_info.meta_path, 'a') as meta_file: - meta_file.write(str(meta_info.get_task_size())) - return smooth_decoding +class IManifestManager(_ManifestManager): + #NUMBER_ADDITIONAL_LINES = 2 + def __init__(self, manifest_path): + super().__init__(manifest_path) + setattr(self._manifest, 'TYPE', 'images') + + def create(self, content, **kwargs): + """ Creating and saving a manifest file""" + with open(self._manifest.path, 'w') as manifest_file: + manifest_file.write(f"{json.dumps({'version': self._manifest.VERSION})}\n") + manifest_file.write(f"{json.dumps({'type': self._manifest.TYPE})}\n") + + for item in content: + name, ext = os.path.splitext(item[0]) + json_item = json.dumps({'name': name, 'extension': ext, + 'width': item[1], 'height': item[2], + 'checksum': item[3]}, separators=(',', ':')) + manifest_file.write(f"{json_item}\n") + self._manifest.is_created = True + + def partial_update(self, number, properties): + """ Updating a part of a manifest file """ + pass \ No newline at end of file diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 46a4bf9accdd..c94652b1c666 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -17,7 +17,7 @@ from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter, ValidateDimension from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile from cvat.apps.engine.utils import av_scan_paths -from cvat.apps.engine.prepare import prepare_meta +from cvat.apps.engine.prepare import prepare_meta, IManifestManager, VManifestManager from cvat.apps.engine.models import DimensionType import django_rq @@ -107,7 +107,7 @@ def _save_task_to_db(db_task): db_task.data.save() db_task.save() -def _count_files(data, meta_info_file=None): +def _count_files(data, manifest_file=None): share_root = settings.SHARE_ROOT server_files = [] @@ -134,8 +134,8 @@ def count_files(file_mapping, counter): mime = get_mime(full_path) if mime in counter: counter[mime].append(rel_path) - elif findall('meta_info.txt$', rel_path): - meta_info_file.append(rel_path) + elif findall('manifest.jsonl$', rel_path): + manifest_file.append(rel_path) else: slogger.glob.warn("Skip '{}' file (its mime type doesn't " "correspond to a video or an image file)".format(full_path)) @@ -154,7 +154,7 @@ def count_files(file_mapping, counter): return counter -def _validate_data(counter, meta_info_file=None): +def _validate_data(counter, manifest_file=None): unique_entries = 0 multiple_entries = 0 for media_type, media_config in MEDIA_TYPES.items(): @@ -164,8 +164,8 @@ def _validate_data(counter, meta_info_file=None): else: multiple_entries += len(counter[media_type]) - if meta_info_file and media_type != 'video': - raise Exception('File with meta information can only be uploaded with video file') + if manifest_file and media_type not in ('video', 'image'): + raise Exception('File with meta information can only be uploaded with video/images ') if unique_entries == 1 and multiple_entries > 0 or unique_entries > 1: unique_types = ', '.join([k for k, v in MEDIA_TYPES.items() if v['unique']]) @@ -225,10 +225,10 @@ def _create_thread(tid, data): if data['remote_files']: data['remote_files'] = _download_data(data['remote_files'], upload_dir) - meta_info_file = [] - media = _count_files(data, meta_info_file) - media, task_mode = _validate_data(media, meta_info_file) - if meta_info_file: + manifest_file = [] + media = _count_files(data, manifest_file) + media, task_mode = _validate_data(media, manifest_file) + if manifest_file: assert settings.USE_CACHE and db_data.storage_method == StorageMethodChoice.CACHE, \ "File with meta information can be uploaded if 'Use cache' option is also selected" @@ -319,68 +319,106 @@ def update_progress(progress): video_path = "" video_size = (0, 0) + def _update_status(msg): + job.meta['status'] = msg + job.save_meta() + if settings.USE_CACHE and db_data.storage_method == StorageMethodChoice.CACHE: for media_type, media_files in media.items(): if not media_files: continue + # replace manifest file (e.g was uploaded 'subdir/manifest.jsonl') + if manifest_file and not os.path.exists(db_data.get_manifest_path()): + shutil.copyfile(os.path.join(upload_dir, manifest_file[0]), + db_data.get_manifest_path()) + if upload_dir != settings.SHARE_ROOT: + os.remove(os.path.join(upload_dir, manifest_file[0])) + if task_mode == MEDIA_TYPES['video']['mode']: try: - if meta_info_file: + if manifest_file: try: - from cvat.apps.engine.prepare import UploadedMeta - meta_info = UploadedMeta(source_path=os.path.join(upload_dir, media_files[0]), - meta_path=db_data.get_meta_path(), - uploaded_meta=os.path.join(upload_dir, meta_info_file[0])) - meta_info.check_seek_key_frames() - meta_info.check_frames_numbers() - meta_info.save_meta_info() - assert len(meta_info.key_frames) > 0, 'No key frames.' + from cvat.apps.engine.prepare import VManifestValidator + manifest = VManifestValidator(source_path=os.path.join(upload_dir, media_files[0]), + manifest_path=db_data.get_manifest_path()) + manifest.init_index() + manifest.validate_seek_key_frames() + manifest.validate_frames_numbers() + assert len(manifest) > 0, 'No key frames.' + + all_frames = manifest['properties']['length'] + video_size = manifest['properties']['resolution'] except Exception as ex: base_msg = str(ex) if isinstance(ex, AssertionError) else \ 'Invalid meta information was upload.' - job.meta['status'] = '{} Start prepare valid meta information.'.format(base_msg) - job.save_meta() + _update_status('{} Start prepare valid meta information.'.format(base_msg)) meta_info, smooth_decoding = prepare_meta( + data_type='video', media_file=media_files[0], upload_dir=upload_dir, - meta_dir=os.path.dirname(db_data.get_meta_path()), chunk_size=db_data.chunk_size ) assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' + _update_status('Start prepare a manifest file') + manifest = VManifestManager(db_data.get_manifest_path()) + manifest.create(meta_info) + manifest.init_index() + _update_status('A manifest had been created') + + all_frames = meta_info.get_task_size() + video_size = meta_info.frame_sizes else: meta_info, smooth_decoding = prepare_meta( + data_type='video', media_file=media_files[0], upload_dir=upload_dir, - meta_dir=os.path.dirname(db_data.get_meta_path()), chunk_size=db_data.chunk_size ) - assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' + assert smooth_decoding, 'Too few keyframes for smooth video decoding.' + _update_status('Start prepare a manifest file') + manifest = VManifestManager(db_data.get_manifest_path()) + manifest.create(meta_info) + manifest.init_index() + _update_status('A manifest had been created') - all_frames = meta_info.get_task_size() - video_size = meta_info.frame_sizes + all_frames = meta_info.get_task_size() + video_size = meta_info.frame_sizes db_data.size = len(range(db_data.start_frame, min(data['stop_frame'] + 1 if data['stop_frame'] else all_frames, all_frames), db_data.get_frame_step())) video_path = os.path.join(upload_dir, media_files[0]) except Exception as ex: db_data.storage_method = StorageMethodChoice.FILE_SYSTEM - if os.path.exists(db_data.get_meta_path()): - os.remove(db_data.get_meta_path()) + if os.path.exists(db_data.get_manifest_path()): + os.remove(db_data.get_manifest_path()) + if os.path.exists(db_data.get_index_path()): + os.remove(db_data.get_index_path()) base_msg = str(ex) if isinstance(ex, AssertionError) else "Uploaded video does not support a quick way of task creating." - job.meta['status'] = "{} The task will be created using the old method".format(base_msg) - job.save_meta() - else:#images,archive + _update_status("{} The task will be created using the old method".format(base_msg)) + else:# images, archive, pdf db_data.size = len(extractor) - + manifest = IManifestManager(db_data.get_manifest_path()) + if not manifest_file: + # redefine paths due to archives, pdf + sources = [] + for _, path, _ in extractor: + sources.append(path) + meta_info = prepare_meta( + data_type='images', + sources=sources, + data_dir=upload_dir + ) + manifest.create(meta_info.content) + manifest.init_index() counter = itertools.count() - for chunk_number, chunk_frames in itertools.groupby(extractor.frame_range, lambda x: next(counter) // db_data.chunk_size): + for _, chunk_frames in itertools.groupby(extractor.frame_range, lambda x: next(counter) // db_data.chunk_size): chunk_paths = [(extractor.get_path(i), i) for i in chunk_frames] img_sizes = [] - with open(db_data.get_dummy_chunk_path(chunk_number), 'w') as dummy_chunk: - for path, frame_id in chunk_paths: - dummy_chunk.write(os.path.relpath(path, upload_dir) + '\n') - img_sizes.append(extractor.get_image_size(frame_id)) + + for _, frame_id in chunk_paths: + properties = manifest[frame_id] + img_sizes.append((properties['width'], properties['height'])) db_images.extend([ models.Image(data=db_data, diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index 854393cfa75f..a6d49530351d 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -5,11 +5,13 @@ import ast import cv2 as cv from collections import namedtuple +import hashlib import importlib import sys import traceback import subprocess import os +from av import VideoFrame from django.core.exceptions import ValidationError @@ -88,3 +90,8 @@ def rotate_image(image, angle): matrix[1, 2] += bound_h/2 - image_center[1] matrix = cv.warpAffine(image, matrix, (bound_w, bound_h)) return matrix + +def md5_hash(frame): + if isinstance(frame, VideoFrame): + frame = frame.to_image() + return hashlib.md5(frame.tobytes()).hexdigest() \ No newline at end of file From 417e82f751bfef60f802673bf41444680ab8c407 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 2 Feb 2021 14:53:41 +0300 Subject: [PATCH 02/29] Added data migration --- .../migrations/0037_auto_20210127_1354.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 cvat/apps/engine/migrations/0037_auto_20210127_1354.py diff --git a/cvat/apps/engine/migrations/0037_auto_20210127_1354.py b/cvat/apps/engine/migrations/0037_auto_20210127_1354.py new file mode 100644 index 000000000000..bb6260bac30a --- /dev/null +++ b/cvat/apps/engine/migrations/0037_auto_20210127_1354.py @@ -0,0 +1,46 @@ +# Generated by Django 3.1.1 on 2021-01-29 14:39 + +from django.db import migrations +from cvat.apps.engine.models import StorageMethodChoice, StorageChoice +from cvat.apps.engine.prepare import prepare_meta, VManifestManager, IManifestManager +from django.conf import settings +import glob +import os + +def migrate_data(apps, shema_editor): + Data = apps.get_model("engine", "Data") + query_set = Data.objects.filter(storage_method=StorageMethodChoice.CACHE) + for db_data in query_set: + upload_dir = '{}/{}/raw'.format(settings.MEDIA_DATA_ROOT, db_data.id) + data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT + if hasattr(db_data, 'video'): + media_file = os.path.join(data_dir, db_data.video.path) + meta_info, _ = prepare_meta( + data_type='video', + media_file=media_file, + ) + manifest = VManifestManager(manifest_path=upload_dir) + manifest.create(meta_info) + manifest.init_index() + if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')): + os.remove(os.path.join(upload_dir, 'meta_info.txt')) + else: + sources = [os.path.join(data_dir, db_image.path) for db_image in db_data.images.all().order_by('frame')] + # or better to get all possible needed info from db? + meta_info = prepare_meta(data_type='images', sources=sources, data_dir=data_dir) + manifest = IManifestManager(manifest_path=upload_dir) + manifest.create(meta_info.content) + manifest.init_index() + for path in glob.glob(f'{upload_dir}/dummy_*.txt'): + os.remove(path) + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0036_auto_20201216_0943'), + ] + + operations = [ + migrations.RunPython(migrate_data) + ] From eca3465a6ba8af0d26f0219850a2ae2ad67014d8 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 2 Feb 2021 14:54:40 +0300 Subject: [PATCH 03/29] Updated tests --- cvat/apps/engine/tests/test_rest_api.py | 77 ++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index d9d2c182a8e0..031b76c4acc8 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -32,7 +32,7 @@ from cvat.apps.engine.models import (AttributeType, Data, Job, Project, Segment, StatusChoice, Task, Label, StorageMethodChoice, StorageChoice) -from cvat.apps.engine.prepare import prepare_meta, prepare_meta_for_upload +from cvat.apps.engine.prepare import prepare_meta, IManifestManager, VManifestManager from cvat.apps.engine.media_extractors import ValidateDimension from cvat.apps.engine.models import DimensionType @@ -1763,6 +1763,29 @@ def generate_pdf_file(filename, page_count=1): file_buf.seek(0) return image_sizes, file_buf +def generate_manifest_file(data_type, manifest_path, sources): + kwargs = { + 'images': { + 'sources': sources, + 'is_sorted': False, + }, + 'video': { + 'media_file': sources[0], + 'upload_dir': os.path.dirname(sources[0]) + } + } + + prepared_meta = prepare_meta( + data_type=data_type, + **kwargs[data_type] + ) + if data_type == 'video': + manifest = VManifestManager(manifest_path) + manifest.create(prepared_meta[0]) + else: + manifest = IManifestManager(manifest_path) + manifest.create(prepared_meta) + class TaskDataAPITestCase(APITestCase): _image_sizes = {} @@ -1885,6 +1908,12 @@ def setUpClass(cls): shutil.rmtree(root_path) cls._image_sizes[filename] = image_sizes + generate_manifest_file(data_type='video', manifest_path=os.path.join(settings.SHARE_ROOT, 'videos', 'manifest.jsonl'), + sources=[os.path.join(settings.SHARE_ROOT, 'videos', 'test_video_1.mp4')]) + + generate_manifest_file(data_type='images', manifest_path=os.path.join(settings.SHARE_ROOT, 'manifest.jsonl'), + sources=[os.path.join(settings.SHARE_ROOT, f'test_{i}.jpg') for i in range(1,4)]) + @classmethod def tearDownClass(cls): super().tearDownClass() @@ -1906,7 +1935,10 @@ def tearDownClass(cls): path = os.path.join(settings.SHARE_ROOT, "videos", "test_video_1.mp4") os.remove(path) - path = os.path.join(settings.SHARE_ROOT, "videos", "meta_info.txt") + path = os.path.join(settings.SHARE_ROOT, "videos", "manifest.jsonl") + os.remove(path) + + path = os.path.join(settings.SHARE_ROOT, "manifest.jsonl") os.remove(path) def _run_api_v1_tasks_id_data_post(self, tid, user, data): @@ -2049,7 +2081,7 @@ def _test_api_v1_tasks_id_data_spec(self, user, spec, data, expected_compressed_ self.assertEqual(len(images), min(task["data_chunk_size"], len(image_sizes))) if task["data_original_chunk_type"] == self.ChunkType.IMAGESET: - server_files = [img for key, img in data.items() if key.startswith("server_files")] + server_files = [img for key, img in data.items() if key.startswith("server_files") and not img.endswith("manifest.jsonl")] client_files = [img for key, img in data.items() if key.startswith("client_files")] if server_files: @@ -2387,11 +2419,6 @@ def _test_api_v1_tasks_id_data(self, user): self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes) - prepare_meta_for_upload( - prepare_meta, - os.path.join(settings.SHARE_ROOT, "videos", "test_video_1.mp4"), - os.path.join(settings.SHARE_ROOT, "videos") - ) task_spec = { "name": "my video with meta info task without copying #22", "overlap": 0, @@ -2403,7 +2430,7 @@ def _test_api_v1_tasks_id_data(self, user): } task_data = { "server_files[0]": os.path.join("videos", "test_video_1.mp4"), - "server_files[1]": os.path.join("videos", "meta_info.txt"), + "server_files[1]": os.path.join("videos", "manifest.jsonl"), "image_quality": 70, "use_cache": True } @@ -2515,6 +2542,38 @@ def _test_api_v1_tasks_id_data(self, user): self.ChunkType.IMAGESET, image_sizes, dimension=DimensionType.DIM_3D) + task_spec = { + "name": "my images+manifest without copying #26", + "overlap": 0, + "segment_size": 0, + "labels": [ + {"name": "car"}, + {"name": "person"}, + ] + } + + task_data = { + "server_files[0]": "test_1.jpg", + "server_files[1]": "test_2.jpg", + "server_files[2]": "test_3.jpg", + "server_files[3]": "manifest.jsonl", + "image_quality": 70, + "use_cache": True + } + image_sizes = [ + self._image_sizes[task_data["server_files[0]"]], + self._image_sizes[task_data["server_files[1]"]], + self._image_sizes[task_data["server_files[2]"]], + ] + + self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, + image_sizes, StorageMethodChoice.CACHE, StorageChoice.SHARE) + + task_spec.update([('name', 'my images+manifest #27')]) + task_data.update([('copy_data', True)]) + self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, + image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL) + def test_api_v1_tasks_id_data_admin(self): self._test_api_v1_tasks_id_data(self.admin) From 23792f17234bb9eca409b0dc14fd15d290d3ef19 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 2 Feb 2021 14:56:24 +0300 Subject: [PATCH 04/29] Changed script for manually preparing --- .../README.md | 4 +- utils/prepare_manifest_file/prepare.py | 41 +++++++++++++++++++ utils/prepare_meta_information/prepare.py | 37 ----------------- 3 files changed, 43 insertions(+), 39 deletions(-) rename utils/{prepare_meta_information => prepare_manifest_file}/README.md (88%) create mode 100644 utils/prepare_manifest_file/prepare.py delete mode 100644 utils/prepare_meta_information/prepare.py diff --git a/utils/prepare_meta_information/README.md b/utils/prepare_manifest_file/README.md similarity index 88% rename from utils/prepare_meta_information/README.md rename to utils/prepare_manifest_file/README.md index 67f6fff7146f..e757f1384124 100644 --- a/utils/prepare_meta_information/README.md +++ b/utils/prepare_manifest_file/README.md @@ -1,9 +1,9 @@ -# Simple command line for prepare meta information for video data +# Simple command line for prepare dataset manifest file **Usage** ```bash -usage: prepare.py [-h] [-chunk_size CHUNK_SIZE] video_file meta_directory +usage: prepare.py [-h] [-chunk_size CHUNK_SIZE] video_file manifest_directory positional arguments: video_file Path to video file diff --git a/utils/prepare_manifest_file/prepare.py b/utils/prepare_manifest_file/prepare.py new file mode 100644 index 000000000000..2f5c13ea9cd3 --- /dev/null +++ b/utils/prepare_manifest_file/prepare.py @@ -0,0 +1,41 @@ +# Copyright (C) 2020 Intel Corporation +# +# SPDX-License-Identifier: MIT +import argparse +import sys +import os + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--type', type=str, choices=('video', 'images'), help='Type of datset data', required=True) + parser.add_argument('manifest_directory',type=str, help='Directory where the manifest file will be saved') + parser.add_argument('--chunk_size', type=int, + help='Chunk size that will be specified when creating the task with specified video and generated manifest file') + parser.add_argument('sources', nargs='+', help='Source paths') + return parser.parse_args() + +def main(): + args = get_args() + + if args.type == 'video': + try: + assert len(args.sources) == 1, 'Unsupporting prepare manifest file for several video files' + meta_info, smooth_decoding = prepare_meta( + data_type=args.type, media_file=args.sources[0], chunk_size=args.chunk_size + ) + manifest = VManifestManager(manifest_path=args.manifest_directory) + manifest.create(meta_info) + if smooth_decoding != None and not smooth_decoding: + print('NOTE: prepared manifest file contains too few key frames for smooth decoding.') + print('A manifest file had been prepared ') + except Exception as ex: + print(ex) + else: + meta_info = prepare_meta(data_type=args.type, sources=args.sources, is_sorted=False) + manifest = IManifestManager(manifest_path=args.manifest_directory) + manifest.create(meta_info) +if __name__ == "__main__": + base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + sys.path.append(base_dir) + from cvat.apps.engine.prepare import prepare_meta, IManifestManager, VManifestManager + main() \ No newline at end of file diff --git a/utils/prepare_meta_information/prepare.py b/utils/prepare_meta_information/prepare.py deleted file mode 100644 index 0cd200a0c866..000000000000 --- a/utils/prepare_meta_information/prepare.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 2020 Intel Corporation -# -# SPDX-License-Identifier: MIT -import argparse -import sys -import os - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('video_file', - type=str, - help='Path to video file') - parser.add_argument('meta_directory', - type=str, - help='Directory where the file with meta information will be saved') - parser.add_argument('-chunk_size', - type=int, - help='Chunk size that will be specified when creating the task with specified video and generated meta information') - - return parser.parse_args() - -def main(): - args = get_args() - try: - smooth_decoding = prepare_meta_for_upload(prepare_meta, args.video_file, None, args.meta_directory, args.chunk_size) - print('Meta information for video has been prepared') - - if smooth_decoding != None and not smooth_decoding: - print('NOTE: prepared meta information contains too few key frames for smooth decoding.') - except Exception: - print('Impossible to prepare meta information') - -if __name__ == "__main__": - base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - sys.path.append(base_dir) - from cvat.apps.engine.prepare import prepare_meta, prepare_meta_for_upload - main() \ No newline at end of file From 971ebfe0796e55fb5259f0bd545afe40d3698f99 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 5 Feb 2021 10:38:05 +0300 Subject: [PATCH 05/29] Fixs --- cvat/apps/engine/cache.py | 6 ++-- cvat/apps/engine/media_extractors.py | 11 ++++-- .../migrations/0037_auto_20210127_1354.py | 2 +- cvat/apps/engine/task.py | 10 ++---- cvat/apps/engine/tests/test_rest_api.py | 2 +- utils/dataset_manifest/README.md | 36 +++++++++++++++++++ utils/dataset_manifest/__init__.py | 1 + .../dataset_manifest/core.py | 22 ++++++------ .../prepare.py => dataset_manifest/create.py} | 6 ++-- utils/dataset_manifest/utils.py | 21 +++++++++++ utils/prepare_manifest_file/README.md | 30 ---------------- 11 files changed, 89 insertions(+), 58 deletions(-) create mode 100644 utils/dataset_manifest/README.md create mode 100644 utils/dataset_manifest/__init__.py rename cvat/apps/engine/prepare.py => utils/dataset_manifest/core.py (96%) rename utils/{prepare_manifest_file/prepare.py => dataset_manifest/create.py} (90%) create mode 100644 utils/dataset_manifest/utils.py delete mode 100644 utils/prepare_manifest_file/README.md diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 65cffcfca7d1..4b3fe42a6d81 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -10,7 +10,7 @@ from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter, - MImagesReader, MVideoReader) + IDatasetManifestReader, VDatasetManifestReader) from cvat.apps.engine.models import DataChoice, StorageChoice from cvat.apps.engine.models import DimensionType @@ -53,14 +53,14 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number): }[db_data.storage] if hasattr(db_data, 'video'): source_path = os.path.join(upload_dir, db_data.video.path) - reader = MVideoReader(manifest_path=db_data.get_manifest_path(), + reader = VDatasetManifestReader(manifest_path=db_data.get_manifest_path(), source_path=source_path, chunk_number=chunk_number, chunk_size=db_data.chunk_size, start=db_data.start_frame, stop=db_data.stop_frame,step=db_data.get_frame_step()) for frame in reader: images.append((frame, source_path, None)) else: - reader = MImagesReader(manifest_path=db_data.get_manifest_path(), + reader = IDatasetManifestReader(manifest_path=db_data.get_manifest_path(), chunk_number=chunk_number,chunk_size=db_data.chunk_size, start=db_data.start_frame, stop=db_data.stop_frame, step=db_data.get_frame_step()) diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 657fea819938..273a38b1db09 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -25,7 +25,8 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True from cvat.apps.engine.mime_types import mimetypes -from cvat.apps.engine.prepare import VManifestManager, IManifestManager, WorkWithVideo +from utils.dataset_manifest import VManifestManager, IManifestManager +from utils.dataset_manifest.core import WorkWithVideo def get_mime(name): for type_name, type_def in MEDIA_TYPES.items(): @@ -122,6 +123,10 @@ def get_image_size(self, i): img = Image.open(self._source_path[i]) return img.width, img.height + @property + def absolute_source_paths(self): + return [self.get_path(idx) for idx, _ in enumerate(self._source_path)] + class DirectoryReader(ImageListReader): def __init__(self, source_path, step=1, start=0, stop=None): image_paths = [] @@ -340,7 +345,7 @@ def _get_frame_range(self): break return frame_range -class MImagesReader(FragmentMediaReader): +class IDatasetManifestReader(FragmentMediaReader): def __init__(self, manifest_path, **kwargs): super().__init__(**kwargs) self._manifest = IManifestManager(manifest_path) @@ -350,7 +355,7 @@ def __iter__(self): for idx in self._frame_range: yield self._manifest[idx] -class MVideoReader(WorkWithVideo, FragmentMediaReader): +class VDatasetManifestReader(WorkWithVideo, FragmentMediaReader): def __init__(self, manifest_path, **kwargs): WorkWithVideo.__init__(self, **kwargs) FragmentMediaReader.__init__(self, **kwargs) diff --git a/cvat/apps/engine/migrations/0037_auto_20210127_1354.py b/cvat/apps/engine/migrations/0037_auto_20210127_1354.py index bb6260bac30a..28acc9567103 100644 --- a/cvat/apps/engine/migrations/0037_auto_20210127_1354.py +++ b/cvat/apps/engine/migrations/0037_auto_20210127_1354.py @@ -2,8 +2,8 @@ from django.db import migrations from cvat.apps.engine.models import StorageMethodChoice, StorageChoice -from cvat.apps.engine.prepare import prepare_meta, VManifestManager, IManifestManager from django.conf import settings +from utils.dataset_manifest import prepare_meta, VManifestManager, IManifestManager import glob import os diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index c94652b1c666..06f1b67bfb44 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -17,8 +17,8 @@ from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter, ValidateDimension from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile from cvat.apps.engine.utils import av_scan_paths -from cvat.apps.engine.prepare import prepare_meta, IManifestManager, VManifestManager from cvat.apps.engine.models import DimensionType +from utils.dataset_manifest import prepare_meta, IManifestManager, VManifestManager import django_rq from django.conf import settings @@ -340,7 +340,7 @@ def _update_status(msg): try: if manifest_file: try: - from cvat.apps.engine.prepare import VManifestValidator + from utils.dataset_manifest.core import VManifestValidator manifest = VManifestValidator(source_path=os.path.join(upload_dir, media_files[0]), manifest_path=db_data.get_manifest_path()) manifest.init_index() @@ -400,13 +400,9 @@ def _update_status(msg): db_data.size = len(extractor) manifest = IManifestManager(db_data.get_manifest_path()) if not manifest_file: - # redefine paths due to archives, pdf - sources = [] - for _, path, _ in extractor: - sources.append(path) meta_info = prepare_meta( data_type='images', - sources=sources, + sources=extractor.absolute_source_paths, data_dir=upload_dir ) manifest.create(meta_info.content) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 031b76c4acc8..5fea586fb923 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -32,9 +32,9 @@ from cvat.apps.engine.models import (AttributeType, Data, Job, Project, Segment, StatusChoice, Task, Label, StorageMethodChoice, StorageChoice) -from cvat.apps.engine.prepare import prepare_meta, IManifestManager, VManifestManager from cvat.apps.engine.media_extractors import ValidateDimension from cvat.apps.engine.models import DimensionType +from utils.dataset_manifest import prepare_meta, IManifestManager, VManifestManager def create_db_users(cls): (group_admin, _) = Group.objects.get_or_create(name="admin") diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md new file mode 100644 index 000000000000..634ca596e649 --- /dev/null +++ b/utils/dataset_manifest/README.md @@ -0,0 +1,36 @@ +## Simple command line for prepare dataset manifest file + +### **Usage** + +```bash +usage: create.py [-h] --type {video,images} [--chunk_size CHUNK_SIZE] manifest_directory sources [sources ...] + +positional arguments: + manifest_directory Directory where the manifest file will be saved + sources Source paths + +optional arguments: + -h, --help show this help message and exit + --type {video,images} + Type of datset data + --chunk_size CHUNK_SIZE + Chunk size that will be specified when creating the task with specified video and generated manifest file +``` + +**NOTE**: If ratio of number of frames to number of key frames is small compared to the `chunk size`, +then when creating a task with prepared meta information, you should expect that the waiting time for some chunks +will be longer than the waiting time for other chunks. (At the first iteration, when there is no chunk in the cache) + +### **Examples** + +Create a dataset manifest with video: + +```bash +python create.py --type video ~/Documents ~/Documents/video.mp4 +``` + +Create a dataset manifest with images: + +```bash +python create.py --type images ~/Documents ~/Documents/image1.jpg ~/Documents/image2.jpg ~/Documents/image3.jpg +``` diff --git a/utils/dataset_manifest/__init__.py b/utils/dataset_manifest/__init__.py new file mode 100644 index 000000000000..c7b358faa5eb --- /dev/null +++ b/utils/dataset_manifest/__init__.py @@ -0,0 +1 @@ +from .core import prepare_meta, VManifestManager, IManifestManager \ No newline at end of file diff --git a/cvat/apps/engine/prepare.py b/utils/dataset_manifest/core.py similarity index 96% rename from cvat/apps/engine/prepare.py rename to utils/dataset_manifest/core.py index 8e9586b5f7ef..4f13e84057f5 100644 --- a/cvat/apps/engine/prepare.py +++ b/utils/dataset_manifest/core.py @@ -9,8 +9,7 @@ from abc import ABC, abstractmethod from collections import OrderedDict from PIL import Image -from cvat.apps.engine.utils import rotate_image -from .utils import md5_hash +from .utils import md5_hash, rotate_image class WorkWithVideo: def __init__(self, **kwargs): @@ -240,18 +239,22 @@ def create(self, manifest, skip): skip -= 1 image_number = 0 self._index[image_number] = manifest_file.tell() - while (line := manifest_file.readline()): + line = manifest_file.readline() + while line: if line.strip(): image_number += 1 self._index[image_number] = manifest_file.tell() + line = manifest_file.readline() def partial_update(self, manifest, number): with open(manifest, 'r+') as manifest_file: manifest_file.seek(self._index[number]) - while (line := manifest_file.readline()): + line = manifest_file.readline() + while line: if line.strip(): - index[number] = manifest_file.tell() + self._index[number] = manifest_file.tell() number += 1 + line = manifest_file.readline() def __getitem__(self, number): assert 0 <= number < len(self), 'A invalid index number' @@ -303,11 +306,13 @@ def __iter__(self): with open(self._manifest.path, 'r') as manifest_file: manifest_file.seek(self._index[0]) image_number = 0 - while (line := manifest_file.readline()): + line = manifest_file.readline() + while line: if not line.strip(): continue yield (image_number, json.loads(line)) image_number += 1 + line = manifest_file.readline() @property def manifest(self): @@ -327,8 +332,6 @@ def index(self): return self._index class VManifestManager(_ManifestManager): - #TODO: - #NUMBER_ADDITIONAL_LINES = 3 def __init__(self, manifest_path, *args, **kwargs): super().__init__(manifest_path) setattr(self._manifest, 'TYPE', 'video') @@ -373,7 +376,7 @@ def validate_seek_key_frames(self): last_key_frame = None for _, key_frame in self: - # chack that key frames sequence sorted + # check that key frames sequence sorted if last_key_frame and last_key_frame['number'] >= key_frame['number']: raise AssertionError('Invalid saved key frames sequence in manifest file') container.seek(offset=key_frame['pts'], stream=video_stream) @@ -394,7 +397,6 @@ def validate_frames_numbers(self): self._close_video_container(container) class IManifestManager(_ManifestManager): - #NUMBER_ADDITIONAL_LINES = 2 def __init__(self, manifest_path): super().__init__(manifest_path) setattr(self._manifest, 'TYPE', 'images') diff --git a/utils/prepare_manifest_file/prepare.py b/utils/dataset_manifest/create.py similarity index 90% rename from utils/prepare_manifest_file/prepare.py rename to utils/dataset_manifest/create.py index 2f5c13ea9cd3..9bbe5dbf8a70 100644 --- a/utils/prepare_manifest_file/prepare.py +++ b/utils/dataset_manifest/create.py @@ -2,8 +2,8 @@ # # SPDX-License-Identifier: MIT import argparse -import sys import os +import sys def get_args(): parser = argparse.ArgumentParser() @@ -35,7 +35,7 @@ def main(): manifest = IManifestManager(manifest_path=args.manifest_directory) manifest.create(meta_info) if __name__ == "__main__": - base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) - from cvat.apps.engine.prepare import prepare_meta, IManifestManager, VManifestManager + from dataset_manifest.core import prepare_meta, VManifestManager, IManifestManager main() \ No newline at end of file diff --git a/utils/dataset_manifest/utils.py b/utils/dataset_manifest/utils.py new file mode 100644 index 000000000000..cc221f1ae8de --- /dev/null +++ b/utils/dataset_manifest/utils.py @@ -0,0 +1,21 @@ +import hashlib +import cv2 as cv +from av import VideoFrame + +def rotate_image(image, angle): + height, width = image.shape[:2] + image_center = (width/2, height/2) + matrix = cv.getRotationMatrix2D(image_center, angle, 1.) + abs_cos = abs(matrix[0,0]) + abs_sin = abs(matrix[0,1]) + bound_w = int(height * abs_sin + width * abs_cos) + bound_h = int(height * abs_cos + width * abs_sin) + matrix[0, 2] += bound_w/2 - image_center[0] + matrix[1, 2] += bound_h/2 - image_center[1] + matrix = cv.warpAffine(image, matrix, (bound_w, bound_h)) + return matrix + +def md5_hash(frame): + if isinstance(frame, VideoFrame): + frame = frame.to_image() + return hashlib.md5(frame.tobytes()).hexdigest() \ No newline at end of file diff --git a/utils/prepare_manifest_file/README.md b/utils/prepare_manifest_file/README.md deleted file mode 100644 index e757f1384124..000000000000 --- a/utils/prepare_manifest_file/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Simple command line for prepare dataset manifest file - -**Usage** - -```bash -usage: prepare.py [-h] [-chunk_size CHUNK_SIZE] video_file manifest_directory - -positional arguments: - video_file Path to video file - meta_directory Directory where the file with meta information will be saved - -optional arguments: - -h, --help show this help message and exit - -chunk_size CHUNK_SIZE - Chunk size that will be specified when creating the task with specified video and generated meta information -``` - -**NOTE**: For smooth video decoding, the `chunk size` must be greater than or equal to the ratio of number of frames -to a number of key frames. -You can understand the approximate `chunk size` by preparing and looking at the file with meta information. - -**NOTE**: If ratio of number of frames to number of key frames is small compared to the `chunk size`, -then when creating a task with prepared meta information, you should expect that the waiting time for some chunks -will be longer than the waiting time for other chunks. (At the first iteration, when there is no chunk in the cache) - -**Examples** - -```bash -python prepare.py ~/Documents/some_video.mp4 ~/Documents -``` From d8afb6e8577e7f617e1f36e5aa721d5439916f6e Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 5 Feb 2021 15:58:21 +0300 Subject: [PATCH 06/29] Fixed paths --- cvat/apps/engine/task.py | 4 +++- cvat/apps/engine/tests/test_rest_api.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 06f1b67bfb44..acda35cbb5eb 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -252,8 +252,10 @@ def _create_thread(tid, data): if extractor is not None: raise Exception('Combined data types are not supported') source_paths=[os.path.join(upload_dir, f) for f in media_files] - if media_type in ('archive', 'zip') and db_data.storage == StorageChoice.SHARE: + if media_type in ('archive', 'zip') and db_data.storage == StorageChoice.SHARE: source_paths.append(db_data.get_upload_dirname()) + upload_dir = db_data.get_upload_dirname() + db_data.storage = StorageChoice.LOCAL extractor = MEDIA_TYPES[media_type]['extractor']( source_path=source_paths, step=db_data.get_frame_step(), diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 5fea586fb923..36007a17bc16 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -2270,7 +2270,7 @@ def _test_api_v1_tasks_id_data(self, user): image_sizes = self._image_sizes[task_data["server_files[0]"]] self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, self.ChunkType.IMAGESET, image_sizes, - expected_uploaded_data_location=StorageChoice.SHARE) + expected_uploaded_data_location=StorageChoice.LOCAL) task_spec.update([('name', 'my archive task #12')]) task_data.update([('copy_data', True)]) @@ -2370,7 +2370,7 @@ def _test_api_v1_tasks_id_data(self, user): image_sizes = self._image_sizes[task_data["server_files[0]"]] self._test_api_v1_tasks_id_data_spec(user, task_spec, task_data, self.ChunkType.IMAGESET, - self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE, StorageChoice.SHARE) + self.ChunkType.IMAGESET, image_sizes, StorageMethodChoice.CACHE, StorageChoice.LOCAL) task_spec.update([('name', 'my cached zip archive task #19')]) task_data.update([('copy_data', True)]) From 6ead5c7c60e743a72f1570d2c6725dbc74865f77 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 5 Feb 2021 18:27:47 +0300 Subject: [PATCH 07/29] some fix & licence headers --- cvat/apps/engine/cache.py | 2 +- cvat/apps/engine/task.py | 2 +- cvat/apps/engine/utils.py | 2 +- utils/dataset_manifest/__init__.py | 3 +++ utils/dataset_manifest/core.py | 10 ++++++---- utils/dataset_manifest/create.py | 6 +++--- utils/dataset_manifest/utils.py | 3 +++ 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 4b3fe42a6d81..60982cc46025 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index acda35cbb5eb..94619529d8e7 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -1,5 +1,5 @@ -# Copyright (C) 2018-2020 Intel Corporation +# Copyright (C) 2018-2021 Intel Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index a6d49530351d..ef4d5ee104df 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT diff --git a/utils/dataset_manifest/__init__.py b/utils/dataset_manifest/__init__.py index c7b358faa5eb..bfdfd2673e7e 100644 --- a/utils/dataset_manifest/__init__.py +++ b/utils/dataset_manifest/__init__.py @@ -1 +1,4 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT from .core import prepare_meta, VManifestManager, IManifestManager \ No newline at end of file diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 4f13e84057f5..9568e7129cbb 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT @@ -247,6 +247,7 @@ def create(self, manifest, skip): line = manifest_file.readline() def partial_update(self, manifest, number): + assert os.path.exists(manifest), 'A manifest file not exists, index cannot be updated' with open(manifest, 'r+') as manifest_file: manifest_file.seek(self._index[number]) line = manifest_file.readline() @@ -275,7 +276,8 @@ def _parse_line(self, line): """ Getting a random line from the manifest file """ with open(self._manifest.path, 'r') as manifest_file: if isinstance(line, str): - assert line in self.BASE_INFORMATION.keys() + assert line in self.BASE_INFORMATION.keys(), \ + 'An attempt to get non-existent information from the manifest' for _ in range(self.BASE_INFORMATION[line]): fline = manifest_file.readline() return json.loads(fline)[line] @@ -340,8 +342,8 @@ def __init__(self, manifest_path, *args, **kwargs): def create(self, content, **kwargs): """ Creating and saving a manifest file """ with open(self._manifest.path, 'w') as manifest_file: - manifest_file.write(f"{json.dumps({'version':self._manifest.VERSION})}\n") - manifest_file.write(f"{json.dumps({'type':self._manifest.TYPE})}\n") + manifest_file.write(f"{json.dumps({'version': self._manifest.VERSION}, separators=(',', ':'))}\n") + manifest_file.write(f"{json.dumps({'type': self._manifest.TYPE}, separators=(',', ':'))}\n") manifest_file.write(f"{json.dumps({'properties':{'name':os.path.basename(content.source_path),'resolution': content.frame_sizes, 'length': content.get_task_size()}})}\n") for item in content: json_item = json.dumps({'number': item[0], 'pts': item[1], 'checksum': item[2]}, separators=(',', ':')) diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index 9bbe5dbf8a70..d45fff95272c 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 Intel Corporation +# Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT import argparse @@ -25,15 +25,15 @@ def main(): ) manifest = VManifestManager(manifest_path=args.manifest_directory) manifest.create(meta_info) - if smooth_decoding != None and not smooth_decoding: + if smooth_decoding is not None and not smooth_decoding: print('NOTE: prepared manifest file contains too few key frames for smooth decoding.') - print('A manifest file had been prepared ') except Exception as ex: print(ex) else: meta_info = prepare_meta(data_type=args.type, sources=args.sources, is_sorted=False) manifest = IManifestManager(manifest_path=args.manifest_directory) manifest.create(meta_info) + print('A manifest file had been prepared ') if __name__ == "__main__": base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) diff --git a/utils/dataset_manifest/utils.py b/utils/dataset_manifest/utils.py index cc221f1ae8de..0bbbb8752fbb 100644 --- a/utils/dataset_manifest/utils.py +++ b/utils/dataset_manifest/utils.py @@ -1,3 +1,6 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT import hashlib import cv2 as cv from av import VideoFrame From 2cd29727562fcc7c2869e1e8f977b9f8b2185e55 Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 17 Feb 2021 18:41:52 +0300 Subject: [PATCH 08/29] Fixes --- cvat/apps/engine/cache.py | 10 +- cvat/apps/engine/media_extractors.py | 96 ++++---- .../migrations/0037_auto_20210127_1354.py | 6 +- cvat/apps/engine/task.py | 21 +- cvat/apps/engine/tests/test_rest_api.py | 6 +- utils/dataset_manifest/__init__.py | 2 +- utils/dataset_manifest/core.py | 222 +++++++++--------- utils/dataset_manifest/create.py | 9 +- 8 files changed, 196 insertions(+), 176 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 60982cc46025..9fc61a3a3f88 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -10,7 +10,7 @@ from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter, - IDatasetManifestReader, VDatasetManifestReader) + ImageDatasetManifestReader, VideoDatasetManifestReader) from cvat.apps.engine.models import DataChoice, StorageChoice from cvat.apps.engine.models import DimensionType @@ -53,15 +53,15 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number): }[db_data.storage] if hasattr(db_data, 'video'): source_path = os.path.join(upload_dir, db_data.video.path) - reader = VDatasetManifestReader(manifest_path=db_data.get_manifest_path(), + reader = VideoDatasetManifestReader(manifest_path=db_data.get_manifest_path(), source_path=source_path, chunk_number=chunk_number, chunk_size=db_data.chunk_size, start=db_data.start_frame, - stop=db_data.stop_frame,step=db_data.get_frame_step()) + stop=db_data.stop_frame, step=db_data.get_frame_step()) for frame in reader: images.append((frame, source_path, None)) else: - reader = IDatasetManifestReader(manifest_path=db_data.get_manifest_path(), - chunk_number=chunk_number,chunk_size=db_data.chunk_size, + reader = ImageDatasetManifestReader(manifest_path=db_data.get_manifest_path(), + chunk_number=chunk_number, chunk_size=db_data.chunk_size, start=db_data.start_frame, stop=db_data.stop_frame, step=db_data.get_frame_step()) for item in reader: diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 273a38b1db09..8bc185c6f548 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -11,6 +11,7 @@ import struct import re from abc import ABC, abstractmethod +from contextlib import closing import av import numpy as np @@ -25,7 +26,7 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True from cvat.apps.engine.mime_types import mimetypes -from utils.dataset_manifest import VManifestManager, IManifestManager +from utils.dataset_manifest import VideoManifestManager, ImageManifestManager from utils.dataset_manifest.core import WorkWithVideo def get_mime(name): @@ -318,14 +319,18 @@ def get_image_size(self, i): return image.width, image.height class FragmentMediaReader: - def __init__(self, chunk_number, chunk_size, start, stop, step=1, *args, **kwargs): + def __init__(self, chunk_number, chunk_size, start, stop, step=1): self._start = start self._stop = stop + 1 # up to the last inclusive self._step = step self._chunk_number = chunk_number self._chunk_size = chunk_size - self._start_chunk_frame_number = self._start + self._chunk_number * self._chunk_size * self._step - self._end_chunk_frame_number = min(self._start_chunk_frame_number + (self._chunk_size - 1) * self._step + 1, self._stop) + self._start_chunk_frame_number = (self._start + + self._chunk_number + * self._chunk_size + * self._step) + self._end_chunk_frame_number = min(self._start_chunk_frame_number \ + + (self._chunk_size - 1) * self._step + 1, self._stop) self._frame_range = self._get_frame_range() @property @@ -337,7 +342,8 @@ def _get_frame_range(self): for idx in range(self._start, self._stop, self._step): if idx < self._start_chunk_frame_number: continue - elif idx < self._end_chunk_frame_number and not ((idx - self._start_chunk_frame_number) % self._step): + elif idx < self._end_chunk_frame_number and \ + not ((idx - self._start_chunk_frame_number) % self._step): frame_range.append(idx) elif (idx - self._start_chunk_frame_number) % self._step: continue @@ -345,61 +351,65 @@ def _get_frame_range(self): break return frame_range -class IDatasetManifestReader(FragmentMediaReader): +class ImageDatasetManifestReader(FragmentMediaReader): def __init__(self, manifest_path, **kwargs): super().__init__(**kwargs) - self._manifest = IManifestManager(manifest_path) + self._manifest = ImageManifestManager(manifest_path) self._manifest.init_index() def __iter__(self): for idx in self._frame_range: yield self._manifest[idx] -class VDatasetManifestReader(WorkWithVideo, FragmentMediaReader): +class VideoDatasetManifestReader(WorkWithVideo, FragmentMediaReader): def __init__(self, manifest_path, **kwargs): - WorkWithVideo.__init__(self, **kwargs) + WorkWithVideo.__init__(self, kwargs.pop('source_path')) FragmentMediaReader.__init__(self, **kwargs) - self._manifest = VManifestManager(manifest_path) + self._manifest = VideoManifestManager(manifest_path) self._manifest.init_index() def _get_nearest_left_key_frame(self): - start_decode_frame_number = 0 - start_decode_timestamp = 0 - for _, frame in self._manifest: - frame_number, timestamp = frame.get('number'), frame.get('pts') - if int(frame_number) <= self._start_chunk_frame_number: - start_decode_frame_number = frame_number - start_decode_timestamp = timestamp + left_border = 0 + delta = len(self._manifest) + while delta: + step = delta // 2 + cur_position = left_border + step + if self._manifest[cur_position].get('number') < self._start_chunk_frame_number: + cur_position += 1 + left_border = cur_position + delta -= step + 1 else: - break - return int(start_decode_frame_number), int(start_decode_timestamp) + delta = step + if self._manifest[cur_position].get('number') > self._start_chunk_frame_number: + left_border -= 1 + frame_number = self._manifest[left_border].get('number') + timestamp = self._manifest[left_border].get('pts') + return frame_number, timestamp def __iter__(self): start_decode_frame_number, start_decode_timestamp = self._get_nearest_left_key_frame() - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) - container.seek(offset=start_decode_timestamp, stream=video_stream) - - frame_number = start_decode_frame_number - 1 - for packet in container.demux(video_stream): - for frame in packet.decode(): - frame_number += 1 - if frame_number in self._frame_range: - if video_stream.metadata.get('rotate'): - frame = av.VideoFrame().from_ndarray( - rotate_image( - frame.to_ndarray(format='bgr24'), - 360 - int(container.streams.video[0].metadata.get('rotate')) - ), - format ='bgr24' - ) - yield frame - elif frame_number < self._frame_range[-1]: - continue - else: - self._close_video_container(container) - return - self._close_video_container(container) + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) + container.seek(offset=start_decode_timestamp, stream=video_stream) + + frame_number = start_decode_frame_number - 1 + for packet in container.demux(video_stream): + for frame in packet.decode(): + frame_number += 1 + if frame_number in self._frame_range: + if video_stream.metadata.get('rotate'): + frame = av.VideoFrame().from_ndarray( + rotate_image( + frame.to_ndarray(format='bgr24'), + 360 - int(container.streams.video[0].metadata.get('rotate')) + ), + format ='bgr24' + ) + yield frame + elif frame_number < self._frame_range[-1]: + continue + else: + return class IChunkWriter(ABC): def __init__(self, quality, dimension=DimensionType.DIM_2D): diff --git a/cvat/apps/engine/migrations/0037_auto_20210127_1354.py b/cvat/apps/engine/migrations/0037_auto_20210127_1354.py index 28acc9567103..89ec8122ff5f 100644 --- a/cvat/apps/engine/migrations/0037_auto_20210127_1354.py +++ b/cvat/apps/engine/migrations/0037_auto_20210127_1354.py @@ -3,7 +3,7 @@ from django.db import migrations from cvat.apps.engine.models import StorageMethodChoice, StorageChoice from django.conf import settings -from utils.dataset_manifest import prepare_meta, VManifestManager, IManifestManager +from utils.dataset_manifest import prepare_meta, VideoManifestManager, ImageManifestManager import glob import os @@ -19,7 +19,7 @@ def migrate_data(apps, shema_editor): data_type='video', media_file=media_file, ) - manifest = VManifestManager(manifest_path=upload_dir) + manifest = VideoManifestManager(manifest_path=upload_dir) manifest.create(meta_info) manifest.init_index() if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')): @@ -28,7 +28,7 @@ def migrate_data(apps, shema_editor): sources = [os.path.join(data_dir, db_image.path) for db_image in db_data.images.all().order_by('frame')] # or better to get all possible needed info from db? meta_info = prepare_meta(data_type='images', sources=sources, data_dir=data_dir) - manifest = IManifestManager(manifest_path=upload_dir) + manifest = ImageManifestManager(manifest_path=upload_dir) manifest.create(meta_info.content) manifest.init_index() for path in glob.glob(f'{upload_dir}/dummy_*.txt'): diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 94619529d8e7..0125dde04860 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -6,7 +6,6 @@ import itertools import os import sys -from re import findall import rq import shutil from traceback import print_exception @@ -18,7 +17,8 @@ from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile from cvat.apps.engine.utils import av_scan_paths from cvat.apps.engine.models import DimensionType -from utils.dataset_manifest import prepare_meta, IManifestManager, VManifestManager +from utils.dataset_manifest import prepare_meta, ImageManifestManager, VideoManifestManager +from utils.dataset_manifest.core import VideoManifestValidator import django_rq from django.conf import settings @@ -134,7 +134,7 @@ def count_files(file_mapping, counter): mime = get_mime(full_path) if mime in counter: counter[mime].append(rel_path) - elif findall('manifest.jsonl$', rel_path): + elif 'manifest.jsonl' == os.path.basename(rel_path): manifest_file.append(rel_path) else: slogger.glob.warn("Skip '{}' file (its mime type doesn't " @@ -252,7 +252,7 @@ def _create_thread(tid, data): if extractor is not None: raise Exception('Combined data types are not supported') source_paths=[os.path.join(upload_dir, f) for f in media_files] - if media_type in ('archive', 'zip') and db_data.storage == StorageChoice.SHARE: + if media_type in {'archive', 'zip'} and db_data.storage == StorageChoice.SHARE: source_paths.append(db_data.get_upload_dirname()) upload_dir = db_data.get_upload_dirname() db_data.storage = StorageChoice.LOCAL @@ -342,12 +342,11 @@ def _update_status(msg): try: if manifest_file: try: - from utils.dataset_manifest.core import VManifestValidator - manifest = VManifestValidator(source_path=os.path.join(upload_dir, media_files[0]), - manifest_path=db_data.get_manifest_path()) + manifest = VideoManifestValidator(source_path=os.path.join(upload_dir, media_files[0]), + manifest_path=db_data.get_manifest_path()) manifest.init_index() manifest.validate_seek_key_frames() - manifest.validate_frames_numbers() + manifest.validate_frame_numbers() assert len(manifest) > 0, 'No key frames.' all_frames = manifest['properties']['length'] @@ -364,7 +363,7 @@ def _update_status(msg): ) assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' _update_status('Start prepare a manifest file') - manifest = VManifestManager(db_data.get_manifest_path()) + manifest = VideoManifestManager(db_data.get_manifest_path()) manifest.create(meta_info) manifest.init_index() _update_status('A manifest had been created') @@ -380,7 +379,7 @@ def _update_status(msg): ) assert smooth_decoding, 'Too few keyframes for smooth video decoding.' _update_status('Start prepare a manifest file') - manifest = VManifestManager(db_data.get_manifest_path()) + manifest = VideoManifestManager(db_data.get_manifest_path()) manifest.create(meta_info) manifest.init_index() _update_status('A manifest had been created') @@ -400,7 +399,7 @@ def _update_status(msg): _update_status("{} The task will be created using the old method".format(base_msg)) else:# images, archive, pdf db_data.size = len(extractor) - manifest = IManifestManager(db_data.get_manifest_path()) + manifest = ImageManifestManager(db_data.get_manifest_path()) if not manifest_file: meta_info = prepare_meta( data_type='images', diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 36007a17bc16..2acd8e8d4aef 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -34,7 +34,7 @@ Segment, StatusChoice, Task, Label, StorageMethodChoice, StorageChoice) from cvat.apps.engine.media_extractors import ValidateDimension from cvat.apps.engine.models import DimensionType -from utils.dataset_manifest import prepare_meta, IManifestManager, VManifestManager +from utils.dataset_manifest import prepare_meta, ImageManifestManager, VideoManifestManager def create_db_users(cls): (group_admin, _) = Group.objects.get_or_create(name="admin") @@ -1780,10 +1780,10 @@ def generate_manifest_file(data_type, manifest_path, sources): **kwargs[data_type] ) if data_type == 'video': - manifest = VManifestManager(manifest_path) + manifest = VideoManifestManager(manifest_path) manifest.create(prepared_meta[0]) else: - manifest = IManifestManager(manifest_path) + manifest = ImageManifestManager(manifest_path) manifest.create(prepared_meta) class TaskDataAPITestCase(APITestCase): diff --git a/utils/dataset_manifest/__init__.py b/utils/dataset_manifest/__init__.py index bfdfd2673e7e..17485bd87517 100644 --- a/utils/dataset_manifest/__init__.py +++ b/utils/dataset_manifest/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT -from .core import prepare_meta, VManifestManager, IManifestManager \ No newline at end of file +from .core import prepare_meta, VideoManifestManager, ImageManifestManager \ No newline at end of file diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 9568e7129cbb..4a12956f4b42 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -8,22 +8,13 @@ import os from abc import ABC, abstractmethod from collections import OrderedDict +from contextlib import contextmanager, closing from PIL import Image from .utils import md5_hash, rotate_image class WorkWithVideo: - def __init__(self, **kwargs): - if not kwargs.get('source_path'): - raise Exception('No sourse path') - self.source_path = kwargs.get('source_path') - - @staticmethod - def _open_video_container(sourse_path, mode, options=None): - return av.open(sourse_path, mode=mode, options=options) - - @staticmethod - def _close_video_container(container): - container.close() + def __init__(self, source_path): + self.source_path = source_path @staticmethod def _get_video_stream(container): @@ -48,46 +39,53 @@ def _get_frame_size(container): class AnalyzeVideo(WorkWithVideo): def check_type_first_frame(self): - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) - for packet in container.demux(video_stream): - for frame in packet.decode(): - self._close_video_container(container) - assert frame.pict_type.name == 'I', 'First frame is not key frame' - return + for packet in container.demux(video_stream): + for frame in packet.decode(): + assert frame.pict_type.name == 'I', 'First frame is not key frame' + return def check_video_timestamps_sequences(self): - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) - frame_pts = -1 - frame_dts = -1 - for packet in container.demux(video_stream): - for frame in packet.decode(): + frame_pts = -1 + frame_dts = -1 + for packet in container.demux(video_stream): + for frame in packet.decode(): - if None not in [frame.pts, frame_pts] and frame.pts <= frame_pts: - self._close_video_container(container) - raise Exception('Invalid pts sequences') + if None not in {frame.pts, frame_pts} and frame.pts <= frame_pts: + raise Exception('Invalid pts sequences') - if None not in [frame.dts, frame_dts] and frame.dts <= frame_dts: - self._close_video_container(container) - raise Exception('Invalid dts sequences') + if None not in {frame.dts, frame_dts} and frame.dts <= frame_dts: + raise Exception('Invalid dts sequences') - frame_pts, frame_dts = frame.pts, frame.dts - self._close_video_container(container) + frame_pts, frame_dts = frame.pts, frame.dts -class IPrepareInfo: - def __init__(self, sources, is_sorted=True, *args, **kwargs): +class PrepareImageInfo: + def __init__(self, sources, is_sorted=True, use_image_hash=False, *args, **kwargs): self._sources = sources if is_sorted else sorted(sources) self._content = [] self._data_dir = kwargs.get('data_dir', None) + self._use_image_hash = use_image_hash def __iter__(self): for image in self._sources: img = Image.open(image, mode='r') - img_name = os.path.relpath(image, self._data_dir) if self._data_dir else os.path.basename(image) - yield (img_name, img.width, img.height, md5_hash(img)) + img_name = os.path.relpath(image, self._data_dir) if self._data_dir \ + else os.path.basename(image) + name, extension = os.path.splitext(img_name) + image_properties = { + 'name': name, + 'extension': extension, + 'width': img.width, + 'height': img.height, + } + if self._use_image_hash: + image_properties['checksum'] = md5_hash(img) + yield image_properties def create(self): for item in self: @@ -97,15 +95,14 @@ def create(self): def content(self): return self._content -class VPrepareInfo(WorkWithVideo): +class PrepareVideoInfo(WorkWithVideo): def __init__(self, **kwargs): super().__init__(**kwargs) self._key_frames = OrderedDict() self.frames = 0 - container = self._open_video_container(self.source_path, 'r') - self.width, self.height = self._get_frame_size(container) - self._close_video_container(container) + with closing(av.open(self.source_path, mode='r')) as container: + self.width, self.height = self._get_frame_size(container) def get_task_size(self): return self.frames @@ -122,34 +119,32 @@ def validate_key_frame(self, container, video_stream, key_frame): return def validate_seek_key_frames(self): - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) - key_frames_copy = self._key_frames.copy() + key_frames_copy = self._key_frames.copy() - for key_frame in key_frames_copy.items(): - container.seek(offset=key_frame[1]['pts'], stream=video_stream) - self.validate_key_frame(container, video_stream, key_frame) + for key_frame in key_frames_copy.items(): + container.seek(offset=key_frame[1]['pts'], stream=video_stream) + self.validate_key_frame(container, video_stream, key_frame) def validate_frames_ratio(self, chunk_size): return (len(self._key_frames) and (self.frames // len(self._key_frames)) <= 2 * chunk_size) def save_key_frames(self): - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) - frame_number = 0 - - for packet in container.demux(video_stream): - for frame in packet.decode(): - if frame.key_frame: - self._key_frames[frame_number] = { - 'pts': frame.pts, - 'md5': md5_hash(frame), - } - frame_number += 1 - - self.frames = frame_number - self._close_video_container(container) + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) + frame_number = 0 + + for packet in container.demux(video_stream): + for frame in packet.decode(): + if frame.key_frame: + self._key_frames[frame_number] = { + 'pts': frame.pts, + 'md5': md5_hash(frame), + } + frame_number += 1 + self.frames = frame_number @property def key_frames(self): @@ -168,14 +163,14 @@ def _prepare_video_meta(media_file, upload_dir=None, chunk_size=None): analyzer.check_type_first_frame() analyzer.check_video_timestamps_sequences() - meta_info = VPrepareInfo(source_path=source_path) + meta_info = PrepareVideoInfo(source_path=source_path) meta_info.save_key_frames() meta_info.validate_seek_key_frames() smooth_decoding = meta_info.validate_frames_ratio(chunk_size) if chunk_size else None return (meta_info, smooth_decoding) def _prepare_images_meta(sources, **kwargs): - meta_info = IPrepareInfo(sources=sources, **kwargs) + meta_info = PrepareImageInfo(sources=sources, **kwargs) meta_info.create() return meta_info @@ -238,12 +233,13 @@ def create(self, manifest, skip): manifest_file.readline() skip -= 1 image_number = 0 - self._index[image_number] = manifest_file.tell() + position = manifest_file.tell() line = manifest_file.readline() while line: if line.strip(): + self._index[image_number] = position image_number += 1 - self._index[image_number] = manifest_file.tell() + position = manifest_file.tell() line = manifest_file.readline() def partial_update(self, manifest, number): @@ -333,7 +329,7 @@ def __getitem__(self, item): def index(self): return self._index -class VManifestManager(_ManifestManager): +class VideoManifestManager(_ManifestManager): def __init__(self, manifest_path, *args, **kwargs): super().__init__(manifest_path) setattr(self._manifest, 'TYPE', 'video') @@ -342,11 +338,25 @@ def __init__(self, manifest_path, *args, **kwargs): def create(self, content, **kwargs): """ Creating and saving a manifest file """ with open(self._manifest.path, 'w') as manifest_file: - manifest_file.write(f"{json.dumps({'version': self._manifest.VERSION}, separators=(',', ':'))}\n") - manifest_file.write(f"{json.dumps({'type': self._manifest.TYPE}, separators=(',', ':'))}\n") - manifest_file.write(f"{json.dumps({'properties':{'name':os.path.basename(content.source_path),'resolution': content.frame_sizes, 'length': content.get_task_size()}})}\n") + base_info = { + 'version': self._manifest.VERSION, + 'type': self._manifest.TYPE, + 'properties': { + 'name': os.path.basename(content.source_path), + 'resolution': content.frame_sizes, + 'length': content.get_task_size(), + }, + } + for key, value in base_info.items(): + json_item = json.dumps({key: value}, separators=(',', ':')) + manifest_file.write(f'{json_item}\n') + for item in content: - json_item = json.dumps({'number': item[0], 'pts': item[1], 'checksum': item[2]}, separators=(',', ':')) + json_item = json.dumps({ + 'number': item[0], + 'pts': item[1], + 'checksum': item[2] + }, separators=(',', ':')) manifest_file.write(f"{json_item}\n") self._manifest.is_created = True @@ -361,10 +371,10 @@ def validate_base_info(self): assert self._manifest.VERSION != json.loads(manifest_file.readline())['version'] assert self._manifest.TYPE != json.loads(manifest_file.readline())['type'] -class VManifestValidator(VManifestManager, WorkWithVideo): +class VideoManifestValidator(VideoManifestManager, WorkWithVideo): def __init__(self, **kwargs): - WorkWithVideo.__init__(self, **kwargs) - VManifestManager.__init__(self, **kwargs) + WorkWithVideo.__init__(self, kwargs.pop('source_path')) + VideoManifestManager.__init__(self, **kwargs) def validate_key_frame(self, container, video_stream, key_frame): for packet in container.demux(video_stream): @@ -373,32 +383,28 @@ def validate_key_frame(self, container, video_stream, key_frame): return def validate_seek_key_frames(self): - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) - last_key_frame = None - - for _, key_frame in self: - # check that key frames sequence sorted - if last_key_frame and last_key_frame['number'] >= key_frame['number']: - raise AssertionError('Invalid saved key frames sequence in manifest file') - container.seek(offset=key_frame['pts'], stream=video_stream) - self.validate_key_frame(container, video_stream, key_frame) - last_key_frame = key_frame - - self._close_video_container(container) - - def validate_frames_numbers(self): - container = self._open_video_container(self.source_path, mode='r') - video_stream = self._get_video_stream(container) - # not all videos contain information about numbers of frames - frames = video_stream.frames - if frames: - self._close_video_container(container) - assert frames == self['properties']['length'], "The uploaded manifest does not match the video" - return - self._close_video_container(container) - -class IManifestManager(_ManifestManager): + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) + last_key_frame = None + + for _, key_frame in self: + # check that key frames sequence sorted + if last_key_frame and last_key_frame['number'] >= key_frame['number']: + raise AssertionError('Invalid saved key frames sequence in manifest file') + container.seek(offset=key_frame['pts'], stream=video_stream) + self.validate_key_frame(container, video_stream, key_frame) + last_key_frame = key_frame + + def validate_frame_numbers(self): + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) + # not all videos contain information about numbers of frames + frames = video_stream.frames + if frames: + assert frames == self['properties']['length'], "The uploaded manifest does not match the video" + return + +class ImageManifestManager(_ManifestManager): def __init__(self, manifest_path): super().__init__(manifest_path) setattr(self._manifest, 'TYPE', 'images') @@ -406,14 +412,18 @@ def __init__(self, manifest_path): def create(self, content, **kwargs): """ Creating and saving a manifest file""" with open(self._manifest.path, 'w') as manifest_file: - manifest_file.write(f"{json.dumps({'version': self._manifest.VERSION})}\n") - manifest_file.write(f"{json.dumps({'type': self._manifest.TYPE})}\n") + base_info = { + 'version': self._manifest.VERSION, + 'type': self._manifest.TYPE, + } + for key, value in base_info.items(): + json_item = json.dumps({key: value}, separators=(',', ':')) + manifest_file.write(f'{json_item}\n') for item in content: - name, ext = os.path.splitext(item[0]) - json_item = json.dumps({'name': name, 'extension': ext, - 'width': item[1], 'height': item[2], - 'checksum': item[3]}, separators=(',', ':')) + json_item = json.dumps({ + key: value for key, value in item.items() + }, separators=(',', ':')) manifest_file.write(f"{json_item}\n") self._manifest.is_created = True diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index d45fff95272c..0f4acb962fd5 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -23,19 +23,20 @@ def main(): meta_info, smooth_decoding = prepare_meta( data_type=args.type, media_file=args.sources[0], chunk_size=args.chunk_size ) - manifest = VManifestManager(manifest_path=args.manifest_directory) + manifest = VideoManifestManager(manifest_path=args.manifest_directory) manifest.create(meta_info) if smooth_decoding is not None and not smooth_decoding: print('NOTE: prepared manifest file contains too few key frames for smooth decoding.') except Exception as ex: print(ex) else: - meta_info = prepare_meta(data_type=args.type, sources=args.sources, is_sorted=False) - manifest = IManifestManager(manifest_path=args.manifest_directory) + meta_info = prepare_meta(data_type=args.type, sources=args.sources, + is_sorted=False, use_image_hash=True) + manifest = ImageManifestManager(manifest_path=args.manifest_directory) manifest.create(meta_info) print('A manifest file had been prepared ') if __name__ == "__main__": base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) - from dataset_manifest.core import prepare_meta, VManifestManager, IManifestManager + from dataset_manifest.core import prepare_meta, VideoManifestManager, ImageManifestManager main() \ No newline at end of file From 79d7a36be5f80a194f0d8d6111361a1ab8cda691 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 19 Feb 2021 10:28:37 +0300 Subject: [PATCH 09/29] Fix stop_frame saving --- cvat/apps/engine/task.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 0125dde04860..406510b3c588 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -485,6 +485,10 @@ def _update_status(msg): if db_data.stop_frame == 0: db_data.stop_frame = db_data.start_frame + (db_data.size - 1) * db_data.get_frame_step() + else: + # validate stop_frame + db_data.stop_frame = min(db_data.stop_frame, \ + db_data.start_frame + (db_data.size - 1) * db_data.get_frame_step()) preview = extractor.get_preview() preview.save(db_data.get_preview_path()) From 97c1746cc44bdf1e566b8d589bb565f51ed2edc7 Mon Sep 17 00:00:00 2001 From: Maya Date: Sat, 20 Feb 2021 11:39:24 +0300 Subject: [PATCH 10/29] Update migration --- .../{0037_auto_20210127_1354.py => 0038_manifest.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename cvat/apps/engine/migrations/{0037_auto_20210127_1354.py => 0038_manifest.py} (95%) diff --git a/cvat/apps/engine/migrations/0037_auto_20210127_1354.py b/cvat/apps/engine/migrations/0038_manifest.py similarity index 95% rename from cvat/apps/engine/migrations/0037_auto_20210127_1354.py rename to cvat/apps/engine/migrations/0038_manifest.py index 89ec8122ff5f..eeecd8e774f3 100644 --- a/cvat/apps/engine/migrations/0037_auto_20210127_1354.py +++ b/cvat/apps/engine/migrations/0038_manifest.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1.1 on 2021-01-29 14:39 +# Generated by Django 3.1.1 on 2021-02-20 08:36 from django.db import migrations from cvat.apps.engine.models import StorageMethodChoice, StorageChoice @@ -38,7 +38,7 @@ def migrate_data(apps, shema_editor): class Migration(migrations.Migration): dependencies = [ - ('engine', '0036_auto_20201216_0943'), + ('engine', '0037_task_subset'), ] operations = [ From cbe10667b11d20ca6751a4c65211eb16e81a8676 Mon Sep 17 00:00:00 2001 From: Maya Date: Sat, 20 Feb 2021 11:46:16 +0300 Subject: [PATCH 11/29] Fix codacy --- utils/dataset_manifest/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 4a12956f4b42..eed4b815da61 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -8,7 +8,7 @@ import os from abc import ABC, abstractmethod from collections import OrderedDict -from contextlib import contextmanager, closing +from contextlib import closing from PIL import Image from .utils import md5_hash, rotate_image From 01c940dfd373b77f31ee5d2bedffd609eb7c7507 Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 24 Feb 2021 19:23:07 +0300 Subject: [PATCH 12/29] Bandit issue & json instead marshal --- cvat/apps/engine/cache.py | 2 +- cvat/apps/engine/task.py | 2 +- cvat/apps/engine/utils.py | 6 +++--- utils/dataset_manifest/core.py | 12 ++++++------ utils/dataset_manifest/utils.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 9fc61a3a3f88..077e6ef14fe9 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 406510b3c588..5b9c6058c688 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -197,7 +197,7 @@ def _download_data(urls, upload_dir): req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) try: - with urlrequest.urlopen(req) as fp, open(os.path.join(upload_dir, name), 'wb') as tfp: + with urlrequest.urlopen(req) as fp, open(os.path.join(upload_dir, name), 'wb') as tfp: # nosec while True: block = fp.read(8192) if not block: diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index ef4d5ee104df..d5a42e18169c 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2020-2021 Intel Corporation # # SPDX-License-Identifier: MIT @@ -74,7 +74,7 @@ def av_scan_paths(*paths): if 'yes' == os.environ.get('CLAM_AV'): command = ['clamscan', '--no-summary', '-i', '-o'] command.extend(paths) - res = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + res = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # nosec if res.returncode: raise ValidationError(res.stdout) @@ -94,4 +94,4 @@ def rotate_image(image, angle): def md5_hash(frame): if isinstance(frame, VideoFrame): frame = frame.to_image() - return hashlib.md5(frame.tobytes()).hexdigest() \ No newline at end of file + return hashlib.md5(frame.tobytes()).hexdigest() # nosec \ No newline at end of file diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index eed4b815da61..8b94defe1bd1 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -4,7 +4,6 @@ import av import json -import marshal import os from abc import ABC, abstractmethod from collections import OrderedDict @@ -207,7 +206,7 @@ def is_created(self, value): # Needed for faster iteration over the manifest file, will be generated to work inside CVAT # and will not be generated when manually creating a manifest class _Index: - FILE_NAME = 'index' + FILE_NAME = 'index.json' def __init__(self, path): assert path and os.path.isdir(path), 'No index directory path' @@ -219,12 +218,13 @@ def path(self): return self._path def dump(self): - with open(self._path, 'wb') as index_file: - marshal.dump(self._index, index_file, 4) + with open(self._path, 'w') as index_file: + json.dump(self._index, index_file, separators=(',', ':')) def load(self): - with open(self._path, 'rb') as index_file: - self._index = marshal.load(index_file) + with open(self._path, 'r') as index_file: + self._index = json.load(index_file, + object_hook=lambda d: {int(k): v for k, v in d.items()}) def create(self, manifest, skip): assert os.path.exists(manifest), 'A manifest file not exists, index cannot be created' diff --git a/utils/dataset_manifest/utils.py b/utils/dataset_manifest/utils.py index 0bbbb8752fbb..c5e9feeac1d1 100644 --- a/utils/dataset_manifest/utils.py +++ b/utils/dataset_manifest/utils.py @@ -21,4 +21,4 @@ def rotate_image(image, angle): def md5_hash(frame): if isinstance(frame, VideoFrame): frame = frame.to_image() - return hashlib.md5(frame.tobytes()).hexdigest() \ No newline at end of file + return hashlib.md5(frame.tobytes()).hexdigest() # nosec \ No newline at end of file From 7081a967b20b79dbeeb9f3d1fa717c12ddd6f342 Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 24 Feb 2021 20:05:09 +0300 Subject: [PATCH 13/29] f --- cvat/apps/engine/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index db8da95201aa..d9fcda7743e8 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -141,7 +141,7 @@ def get_preview_path(self): def get_manifest_path(self): return os.path.join(self.get_upload_dirname(), 'manifest.jsonl') def get_index_path(self): - return os.path.join(self.get_upload_dirname(), 'index') + return os.path.join(self.get_upload_dirname(), 'index.json') class Video(models.Model): data = models.OneToOneField(Data, on_delete=models.CASCADE, related_name="video", null=True) From 83c01ed5436ec2be6db5abf6eba90fddbb36d877 Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 25 Feb 2021 16:20:37 +0300 Subject: [PATCH 14/29] pylint fixes --- cvat/apps/engine/tests/test_rest_api.py | 4 +--- cvat/apps/engine/utils.py | 1 + utils/dataset_manifest/core.py | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 2acd8e8d4aef..704c50e84fc2 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -16,8 +16,6 @@ from glob import glob from io import BytesIO from unittest import mock -import open3d as o3d -import struct import av import numpy as np @@ -739,7 +737,7 @@ def _run_api_v1_users(self, user): return response - def _check_response(self, user, response, is_full): + def _check_response(self, user, response, is_full=True): self.assertEqual(response.status_code, status.HTTP_200_OK) for user_info in response.data['results']: db_user = getattr(self, user_info['username']) diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index d5a42e18169c..f37440731281 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -53,6 +53,7 @@ class InterpreterError(Exception): def execute_python_code(source_code, global_vars=None, local_vars=None): try: + # pylint: disable=exec-used exec(source_code, global_vars, local_vars) except SyntaxError as err: error_class = err.__class__.__name__ diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 8b94defe1bd1..e5d00260c7f2 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -361,7 +361,6 @@ def create(self, content, **kwargs): self._manifest.is_created = True def partial_update(self, number, properties): - """ Updating a part of a manifest file """ pass #TODO: @@ -428,5 +427,4 @@ def create(self, content, **kwargs): self._manifest.is_created = True def partial_update(self, number, properties): - """ Updating a part of a manifest file """ pass \ No newline at end of file From 934c63ed586c9f0f4cef245669d2541984c7e4be Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 25 Feb 2021 16:32:36 +0300 Subject: [PATCH 15/29] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e4b8bcd6ac7..f55ab03a2274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub () - Project task subsets () +- Ability of upload manifest for dataset with images () ### Changed @@ -29,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - All methods for interative segmentation accept negative points as well - Persistent queue added to logstash () - Improved maintanance of popups visibility () +- Using manifest support instead video meta information and dummy chunks () ### Deprecated From 7b835171196f0285e8edf8c880f8d77caedad450 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 2 Mar 2021 17:15:01 +0300 Subject: [PATCH 16/29] Some fixes --- .coveragerc | 1 + cvat/apps/engine/media_extractors.py | 36 +++++++++++----------- cvat/apps/engine/task.py | 40 +++++++++++-------------- utils/dataset_manifest/README.md | 13 +++++++- utils/dataset_manifest/core.py | 3 +- utils/dataset_manifest/create.py | 6 ++-- utils/dataset_manifest/requirements.txt | 3 ++ 7 files changed, 59 insertions(+), 43 deletions(-) create mode 100644 utils/dataset_manifest/requirements.txt diff --git a/.coveragerc b/.coveragerc index 6e95757cd008..c669baf71266 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,7 @@ branch = true source = cvat/apps/ utils/cli/ + utils/dataset_manifest omit = cvat/settings/* diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 8bc185c6f548..4cf55479c9da 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -325,10 +325,8 @@ def __init__(self, chunk_number, chunk_size, start, stop, step=1): self._step = step self._chunk_number = chunk_number self._chunk_size = chunk_size - self._start_chunk_frame_number = (self._start - + self._chunk_number - * self._chunk_size - * self._step) + self._start_chunk_frame_number = \ + self._start + self._chunk_number * self._chunk_size * self._step self._end_chunk_frame_number = min(self._start_chunk_frame_number \ + (self._chunk_size - 1) * self._step + 1, self._stop) self._frame_range = self._get_frame_range() @@ -369,19 +367,23 @@ def __init__(self, manifest_path, **kwargs): self._manifest.init_index() def _get_nearest_left_key_frame(self): - left_border = 0 - delta = len(self._manifest) - while delta: - step = delta // 2 - cur_position = left_border + step - if self._manifest[cur_position].get('number') < self._start_chunk_frame_number: - cur_position += 1 - left_border = cur_position - delta -= step + 1 - else: - delta = step - if self._manifest[cur_position].get('number') > self._start_chunk_frame_number: - left_border -= 1 + if self._start_chunk_frame_number >= \ + self._manifest[len(self._manifest) - 1].get('number'): + left_border = len(self._manifest) - 1 + else: + left_border = 0 + delta = len(self._manifest) + while delta: + step = delta // 2 + cur_position = left_border + step + if self._manifest[cur_position].get('number') < self._start_chunk_frame_number: + cur_position += 1 + left_border = cur_position + delta -= step + 1 + else: + delta = step + if self._manifest[cur_position].get('number') > self._start_chunk_frame_number: + left_border -= 1 frame_number = self._manifest[left_border].get('number') timestamp = self._manifest[left_border].get('pts') return frame_number, timestamp diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 5b9c6058c688..e6505d6f5ac0 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -340,6 +340,7 @@ def _update_status(msg): if task_mode == MEDIA_TYPES['video']['mode']: try: + manifest_is_prepared = False if manifest_file: try: manifest = VideoManifestValidator(source_path=os.path.join(upload_dir, media_files[0]), @@ -351,33 +352,25 @@ def _update_status(msg): all_frames = manifest['properties']['length'] video_size = manifest['properties']['resolution'] + manifest_is_prepared = True except Exception as ex: - base_msg = str(ex) if isinstance(ex, AssertionError) else \ - 'Invalid meta information was upload.' - _update_status('{} Start prepare valid meta information.'.format(base_msg)) - meta_info, smooth_decoding = prepare_meta( - data_type='video', - media_file=media_files[0], - upload_dir=upload_dir, - chunk_size=db_data.chunk_size - ) - assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' - _update_status('Start prepare a manifest file') - manifest = VideoManifestManager(db_data.get_manifest_path()) - manifest.create(meta_info) - manifest.init_index() - _update_status('A manifest had been created') - - all_frames = meta_info.get_task_size() - video_size = meta_info.frame_sizes - else: + if os.path.exists(db_data.get_index_path()): + os.remove(db_data.get_index_path()) + if isinstance(ex, AssertionError): + base_msg = str(ex) + else: + base_msg = 'Invalid manifest file was upload.' + slogger.glob.warning(str(ex)) + _update_status('{} Start prepare a valid manifest file.'.format(base_msg)) + + if not manifest_is_prepared: meta_info, smooth_decoding = prepare_meta( data_type='video', media_file=media_files[0], upload_dir=upload_dir, chunk_size=db_data.chunk_size ) - assert smooth_decoding, 'Too few keyframes for smooth video decoding.' + assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' _update_status('Start prepare a manifest file') manifest = VideoManifestManager(db_data.get_manifest_path()) manifest.create(meta_info) @@ -386,8 +379,10 @@ def _update_status(msg): all_frames = meta_info.get_task_size() video_size = meta_info.frame_sizes + manifest_is_prepared = True - db_data.size = len(range(db_data.start_frame, min(data['stop_frame'] + 1 if data['stop_frame'] else all_frames, all_frames), db_data.get_frame_step())) + db_data.size = len(range(db_data.start_frame, min(data['stop_frame'] + 1 \ + if data['stop_frame'] else all_frames, all_frames), db_data.get_frame_step())) video_path = os.path.join(upload_dir, media_files[0]) except Exception as ex: db_data.storage_method = StorageMethodChoice.FILE_SYSTEM @@ -395,7 +390,8 @@ def _update_status(msg): os.remove(db_data.get_manifest_path()) if os.path.exists(db_data.get_index_path()): os.remove(db_data.get_index_path()) - base_msg = str(ex) if isinstance(ex, AssertionError) else "Uploaded video does not support a quick way of task creating." + base_msg = str(ex) if isinstance(ex, AssertionError) \ + else "Uploaded video does not support a quick way of task creating." _update_status("{} The task will be created using the old method".format(base_msg)) else:# images, archive, pdf db_data.size = len(extractor) diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md index 634ca596e649..acfc3457df63 100644 --- a/utils/dataset_manifest/README.md +++ b/utils/dataset_manifest/README.md @@ -1,5 +1,16 @@ ## Simple command line for prepare dataset manifest file +### Steps bufore using + +When used separately from Computer Vision Annotation Tool(CVAT), the required modules must be installed + +```bash +python3 -m venv .env +. .env/bin/activate +pip install -U pip +pip install -r requirements.txt +``` + ### **Usage** ```bash @@ -18,7 +29,7 @@ optional arguments: ``` **NOTE**: If ratio of number of frames to number of key frames is small compared to the `chunk size`, -then when creating a task with prepared meta information, you should expect that the waiting time for some chunks +then when creating a task with prepared manifest file, you should expect that the waiting time for some chunks will be longer than the waiting time for other chunks. (At the first iteration, when there is no chunk in the cache) ### **Examples** diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index e5d00260c7f2..5f58c5df202d 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -254,7 +254,8 @@ def partial_update(self, manifest, number): line = manifest_file.readline() def __getitem__(self, number): - assert 0 <= number < len(self), 'A invalid index number' + assert 0 <= number < len(self), \ + 'A invalid index number: {}\nMax: {}'.format(number, len(self)) return self._index[number] def __len__(self): diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index 0f4acb962fd5..81621a01e7c2 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -17,13 +17,15 @@ def get_args(): def main(): args = get_args() + manifest_directory = os.path.abspath(args.manifest_directory) + os.makedirs(manifest_directory, exist_ok=True) if args.type == 'video': try: assert len(args.sources) == 1, 'Unsupporting prepare manifest file for several video files' meta_info, smooth_decoding = prepare_meta( data_type=args.type, media_file=args.sources[0], chunk_size=args.chunk_size ) - manifest = VideoManifestManager(manifest_path=args.manifest_directory) + manifest = VideoManifestManager(manifest_path=manifest_directory) manifest.create(meta_info) if smooth_decoding is not None and not smooth_decoding: print('NOTE: prepared manifest file contains too few key frames for smooth decoding.') @@ -32,7 +34,7 @@ def main(): else: meta_info = prepare_meta(data_type=args.type, sources=args.sources, is_sorted=False, use_image_hash=True) - manifest = ImageManifestManager(manifest_path=args.manifest_directory) + manifest = ImageManifestManager(manifest_path=manifest_directory) manifest.create(meta_info) print('A manifest file had been prepared ') if __name__ == "__main__": diff --git a/utils/dataset_manifest/requirements.txt b/utils/dataset_manifest/requirements.txt new file mode 100644 index 000000000000..1089a5f0a331 --- /dev/null +++ b/utils/dataset_manifest/requirements.txt @@ -0,0 +1,3 @@ +av==8.0.2 --no-binary=av +opencv-python-headless==4.4.0.42 +Pillow==7.2.0 \ No newline at end of file From d8c860600566eac5d630ab2489bccf5e1dbaf2e0 Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 3 Mar 2021 12:32:15 +0300 Subject: [PATCH 17/29] Update manifest documentation --- cvat/apps/documentation/data_on_fly.md | 24 ++++--------------- cvat/apps/documentation/user_guide.md | 6 ++--- utils/dataset_manifest/README.md | 32 ++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/cvat/apps/documentation/data_on_fly.md b/cvat/apps/documentation/data_on_fly.md index 5963bdf30133..e82dce1530ee 100644 --- a/cvat/apps/documentation/data_on_fly.md +++ b/cvat/apps/documentation/data_on_fly.md @@ -16,26 +16,10 @@ This method of working with data allows: - reduce the task creation time. - store data in a cache of limited size with a policy of evicting less popular items. -## Prepare meta information - -Different meta information is collected for different types of uploaded data. - -### Video - -For video, this is a valid mapping of key frame numbers and their timestamps. This information is saved to `meta_info.txt`. - -Unfortunately, this method will not work for all videos with valid meta information. +Unfortunately, this method will not work for all videos with valid manifest file. If there are not enough keyframes in the video for smooth video decoding, the task will be created in the old way. -#### Uploading meta information along with data - -When creating a task, you can upload a file with meta information along with the video, -which will further reduce the time for creating a task. -You can see how to prepare meta information [here](/utils/prepare_meta_information/README.md). - -It is worth noting that the generated file also contains information about the number of frames in the video at the end. - -### Images +#### Uploading a manifest with data -Mapping of chunk number and paths to images that should enter the chunk -is saved at the time of creating a task in a files `dummy_{chunk_number}.txt` +When creating a task, you can upload a `manifest.jsonl` file along with the video or dataset with images. +You can see how to prepare it [here](/utils/dataset_manifest/README.md). diff --git a/cvat/apps/documentation/user_guide.md b/cvat/apps/documentation/user_guide.md index 3904b0158328..77b334f14a91 100644 --- a/cvat/apps/documentation/user_guide.md +++ b/cvat/apps/documentation/user_guide.md @@ -153,8 +153,8 @@ Go to the [Django administration panel](http://localhost:8080/admin). There you **Select files**. Press tab `My computer` to choose some files for annotation from your PC. If you select tab `Connected file share` you can choose files for annotation from your network. If you select ` Remote source` , you'll see a field where you can enter a list of URLs (one URL per line). - If you upload a video data and select `Use cache` option, you can along with the video file attach a file with meta information. - You can find how to prepare it [here](/utils/prepare_meta_information/README.md). + If you upload a video or dataset with images and select `Use cache` option, you can attach a `manifest.jsonl` file. + You can find how to prepare it [here](/utils/dataset_manifest/README.md). ![](static/documentation/images/image127.jpg) @@ -1157,8 +1157,6 @@ Intelligent scissors is an CV method of creating a polygon by placing points wit The distance between the adjacent points is limited by the threshold of action, displayed as a red square which is tied to the cursor. - - - First, select the label and then click on the `intelligent scissors` button. ![](static/documentation/images/image199.jpg) diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md index acfc3457df63..92db9ab383d7 100644 --- a/utils/dataset_manifest/README.md +++ b/utils/dataset_manifest/README.md @@ -1,6 +1,6 @@ -## Simple command line for prepare dataset manifest file +## Simple command line to prepare dataset manifest file -### Steps bufore using +### Steps before use When used separately from Computer Vision Annotation Tool(CVAT), the required modules must be installed @@ -11,7 +11,7 @@ pip install -U pip pip install -r requirements.txt ``` -### **Usage** +### Using ```bash usage: create.py [-h] --type {video,images} [--chunk_size CHUNK_SIZE] manifest_directory sources [sources ...] @@ -32,7 +32,7 @@ optional arguments: then when creating a task with prepared manifest file, you should expect that the waiting time for some chunks will be longer than the waiting time for other chunks. (At the first iteration, when there is no chunk in the cache) -### **Examples** +### Examples of using Create a dataset manifest with video: @@ -45,3 +45,27 @@ Create a dataset manifest with images: ```bash python create.py --type images ~/Documents ~/Documents/image1.jpg ~/Documents/image2.jpg ~/Documents/image3.jpg ``` + +### Example of generated `manifest.jsonl` for video + +```json +{"version":"1.0"} +{"type":"video"} +{"properties":{"name":"video.mp4","resolution":[1280,720],"length":778}} +{"number":0,"pts":0,"checksum":"17bb40d76887b56fe8213c6fded3d540"} +{"number":135,"pts":486000,"checksum":"9da9b4d42c1206d71bf17a7070a05847"} +{"number":270,"pts":972000,"checksum":"a1c3a61814f9b58b00a795fa18bb6d3e"} +{"number":405,"pts":1458000,"checksum":"18c0803b3cc1aa62ac75b112439d2b62"} +{"number":540,"pts":1944000,"checksum":"4551ecea0f80e95a6c32c32e70cac59e"} +{"number":675,"pts":2430000,"checksum":"0e72faf67e5218c70b506445ac91cdd7"} +``` + +### Example of generated `manifest.jsonl` for dataset with images + +```json +{"version":"1.0"} +{"type":"images"} +{"name":"image1","extension":".jpg","width":720,"height":405,"checksum":"548918ec4b56132a5cff1d4acabe9947"} +{"name":"image2","extension":".jpg","width":183,"height":275,"checksum":"4b4eefd03cc6a45c1c068b98477fb639"} +{"name":"image3","extension":".jpg","width":301,"height":167,"checksum":"0e454a6f4a13d56c82890c98be063663"} +``` From a6438539a7607bfaf485f5cfe5182f306ca81404 Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 3 Mar 2021 15:53:17 +0300 Subject: [PATCH 18/29] Fix create.py --- utils/dataset_manifest/README.md | 2 +- utils/dataset_manifest/create.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md index 92db9ab383d7..43370dc98ae9 100644 --- a/utils/dataset_manifest/README.md +++ b/utils/dataset_manifest/README.md @@ -14,7 +14,7 @@ pip install -r requirements.txt ### Using ```bash -usage: create.py [-h] --type {video,images} [--chunk_size CHUNK_SIZE] manifest_directory sources [sources ...] +usage: python create.py [-h] --type {video,images} [--chunk_size CHUNK_SIZE] manifest_directory sources [sources ...] positional arguments: manifest_directory Directory where the manifest file will be saved diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index 81621a01e7c2..dec29987d818 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -19,6 +19,11 @@ def main(): manifest_directory = os.path.abspath(args.manifest_directory) os.makedirs(manifest_directory, exist_ok=True) + try: + for source in args.sources: + assert os.path.exists(source), 'A file {} not found'.format(source) + except AssertionError as ex: + sys.exit(str(ex)) if args.type == 'video': try: assert len(args.sources) == 1, 'Unsupporting prepare manifest file for several video files' @@ -36,7 +41,7 @@ def main(): is_sorted=False, use_image_hash=True) manifest = ImageManifestManager(manifest_path=manifest_directory) manifest.create(meta_info) - print('A manifest file had been prepared ') + print('A manifest file has been prepared ') if __name__ == "__main__": base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) From eb88ed0d22daaa98343fd24c49b54778a1fde2ff Mon Sep 17 00:00:00 2001 From: Maya Date: Thu, 4 Mar 2021 13:26:31 +0300 Subject: [PATCH 19/29] Fix case with 3d data --- cvat/apps/engine/migrations/0038_manifest.py | 58 ++++++++++++-------- cvat/apps/engine/task.py | 28 +++++++--- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/cvat/apps/engine/migrations/0038_manifest.py b/cvat/apps/engine/migrations/0038_manifest.py index eeecd8e774f3..98d88dd29049 100644 --- a/cvat/apps/engine/migrations/0038_manifest.py +++ b/cvat/apps/engine/migrations/0038_manifest.py @@ -1,7 +1,7 @@ # Generated by Django 3.1.1 on 2021-02-20 08:36 from django.db import migrations -from cvat.apps.engine.models import StorageMethodChoice, StorageChoice +from cvat.apps.engine.models import StorageMethodChoice, StorageChoice, DimensionType from django.conf import settings from utils.dataset_manifest import prepare_meta, VideoManifestManager, ImageManifestManager import glob @@ -11,28 +11,40 @@ def migrate_data(apps, shema_editor): Data = apps.get_model("engine", "Data") query_set = Data.objects.filter(storage_method=StorageMethodChoice.CACHE) for db_data in query_set: - upload_dir = '{}/{}/raw'.format(settings.MEDIA_DATA_ROOT, db_data.id) - data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT - if hasattr(db_data, 'video'): - media_file = os.path.join(data_dir, db_data.video.path) - meta_info, _ = prepare_meta( - data_type='video', - media_file=media_file, - ) - manifest = VideoManifestManager(manifest_path=upload_dir) - manifest.create(meta_info) - manifest.init_index() - if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')): - os.remove(os.path.join(upload_dir, 'meta_info.txt')) - else: - sources = [os.path.join(data_dir, db_image.path) for db_image in db_data.images.all().order_by('frame')] - # or better to get all possible needed info from db? - meta_info = prepare_meta(data_type='images', sources=sources, data_dir=data_dir) - manifest = ImageManifestManager(manifest_path=upload_dir) - manifest.create(meta_info.content) - manifest.init_index() - for path in glob.glob(f'{upload_dir}/dummy_*.txt'): - os.remove(path) + try: + upload_dir = '{}/{}/raw'.format(settings.MEDIA_DATA_ROOT, db_data.id) + data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT + if hasattr(db_data, 'video'): + media_file = os.path.join(data_dir, db_data.video.path) + meta_info, _ = prepare_meta( + data_type='video', + media_file=media_file, + ) + manifest = VideoManifestManager(manifest_path=upload_dir) + manifest.create(meta_info) + manifest.init_index() + if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')): + os.remove(os.path.join(upload_dir, 'meta_info.txt')) + else: + sources = [os.path.join(data_dir, db_image.path) for db_image in db_data.images.all().order_by('frame')] + if any(list(filter(lambda x: x.dimension==DimensionType.DIM_3D, db_data.tasks.all()))): + content = [] + for source in sources: + name, ext = os.path.splitext(os.path.relpath(source, upload_dir)) + content.append({ + 'name': name, + 'extension': ext + }) + else: + meta_info = prepare_meta(data_type='images', sources=sources, data_dir=data_dir) + content = meta_info.content + manifest = ImageManifestManager(manifest_path=upload_dir) + manifest.create(content) + manifest.init_index() + for path in glob.glob(f'{upload_dir}/dummy_*.txt'): + os.remove(path) + except Exception as ex: + print(str(ex)) class Migration(migrations.Migration): diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index e6505d6f5ac0..066f3be3fac7 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -397,12 +397,22 @@ def _update_status(msg): db_data.size = len(extractor) manifest = ImageManifestManager(db_data.get_manifest_path()) if not manifest_file: - meta_info = prepare_meta( - data_type='images', - sources=extractor.absolute_source_paths, - data_dir=upload_dir - ) - manifest.create(meta_info.content) + if db_task.dimension == DimensionType.DIM_2D: + meta_info = prepare_meta( + data_type='images', + sources=extractor.absolute_source_paths, + data_dir=upload_dir + ) + content = meta_info.content + else: + content = [] + for source in extractor.absolute_source_paths: + name, ext = os.path.splitext(os.path.relpath(source, upload_dir)) + content.append({ + 'name': name, + 'extension': ext + }) + manifest.create(content) manifest.init_index() counter = itertools.count() for _, chunk_frames in itertools.groupby(extractor.frame_range, lambda x: next(counter) // db_data.chunk_size): @@ -411,7 +421,11 @@ def _update_status(msg): for _, frame_id in chunk_paths: properties = manifest[frame_id] - img_sizes.append((properties['width'], properties['height'])) + if db_task.dimension == DimensionType.DIM_2D: + resolution = (properties['width'], properties['height']) + else: + resolution = extractor.get_image_size(frame_id) + img_sizes.append(resolution) db_images.extend([ models.Image(data=db_data, From 4284cac67acf4d8424fc713b9b9c390db8494582 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 9 Mar 2021 12:17:42 +0300 Subject: [PATCH 20/29] Modify migration --- cvat/apps/engine/migrations/0038_manifest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cvat/apps/engine/migrations/0038_manifest.py b/cvat/apps/engine/migrations/0038_manifest.py index 98d88dd29049..3ca0853771b8 100644 --- a/cvat/apps/engine/migrations/0038_manifest.py +++ b/cvat/apps/engine/migrations/0038_manifest.py @@ -13,6 +13,14 @@ def migrate_data(apps, shema_editor): for db_data in query_set: try: upload_dir = '{}/{}/raw'.format(settings.MEDIA_DATA_ROOT, db_data.id) + if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')): + os.remove(os.path.join(upload_dir, 'meta_info.txt')) + else: + for path in glob.glob(f'{upload_dir}/dummy_*.txt'): + os.remove(path) + # it's necessary for case with long data migration + if os.path.exists(os.path.join(upload_dir, 'manifest.jsonl')): + continue data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT if hasattr(db_data, 'video'): media_file = os.path.join(data_dir, db_data.video.path) @@ -23,8 +31,6 @@ def migrate_data(apps, shema_editor): manifest = VideoManifestManager(manifest_path=upload_dir) manifest.create(meta_info) manifest.init_index() - if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')): - os.remove(os.path.join(upload_dir, 'meta_info.txt')) else: sources = [os.path.join(data_dir, db_image.path) for db_image in db_data.images.all().order_by('frame')] if any(list(filter(lambda x: x.dimension==DimensionType.DIM_3D, db_data.tasks.all()))): @@ -41,8 +47,6 @@ def migrate_data(apps, shema_editor): manifest = ImageManifestManager(manifest_path=upload_dir) manifest.create(content) manifest.init_index() - for path in glob.glob(f'{upload_dir}/dummy_*.txt'): - os.remove(path) except Exception as ex: print(str(ex)) From d31510f9a89a987e918ffbc291570deef5c15562 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 9 Mar 2021 16:38:51 +0300 Subject: [PATCH 21/29] Fixed migration --- cvat/apps/engine/migrations/0038_manifest.py | 38 ++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/cvat/apps/engine/migrations/0038_manifest.py b/cvat/apps/engine/migrations/0038_manifest.py index 3ca0853771b8..8b7ef1113d5d 100644 --- a/cvat/apps/engine/migrations/0038_manifest.py +++ b/cvat/apps/engine/migrations/0038_manifest.py @@ -1,11 +1,17 @@ # Generated by Django 3.1.1 on 2021-02-20 08:36 -from django.db import migrations -from cvat.apps.engine.models import StorageMethodChoice, StorageChoice, DimensionType -from django.conf import settings -from utils.dataset_manifest import prepare_meta, VideoManifestManager, ImageManifestManager import glob import os +from re import search + +from django.conf import settings +from django.db import migrations + +from cvat.apps.engine.models import (DimensionType, StorageChoice, + StorageMethodChoice) +from utils.dataset_manifest import (ImageManifestManager, VideoManifestManager, + prepare_meta) + def migrate_data(apps, shema_editor): Data = apps.get_model("engine", "Data") @@ -32,7 +38,14 @@ def migrate_data(apps, shema_editor): manifest.create(meta_info) manifest.init_index() else: - sources = [os.path.join(data_dir, db_image.path) for db_image in db_data.images.all().order_by('frame')] + sources = [] + if db_data.storage == StorageChoice.LOCAL: + for (root, _, files) in os.walk(data_dir): + sources.extend([os.path.join(root, f) for f in files]) + sources.sort() + # using share, this means that we can not explicitly restore the entire data structure + else: + sources = [os.path.join(data_dir, db_image.path) for db_image in db_data.images.all().order_by('frame')] if any(list(filter(lambda x: x.dimension==DimensionType.DIM_3D, db_data.tasks.all()))): content = [] for source in sources: @@ -45,12 +58,25 @@ def migrate_data(apps, shema_editor): meta_info = prepare_meta(data_type='images', sources=sources, data_dir=data_dir) content = meta_info.content manifest = ImageManifestManager(manifest_path=upload_dir) + + if db_data.storage == StorageChoice.SHARE: + def _get_frame_step(str_): + match = search("step\s*=\s*([1-9]\d*)", str_) + return int(match.group(1)) if match else 1 + step = _get_frame_step(db_data.frame_filter) + start = db_data.start_frame + stop = db_data.stop_frame + 1 + images_range = range(start, stop, step) + result_content = [] + for i in range(stop): + item = content.pop(0) if i in images_range else dict() + result_content.append(item) + content = result_content manifest.create(content) manifest.init_index() except Exception as ex: print(str(ex)) - class Migration(migrations.Migration): dependencies = [ From 65e4b12937cdf6506f87947e1cd59a009c47aed9 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 16 Mar 2021 12:50:17 +0300 Subject: [PATCH 22/29] Refactored script to manually prepare manifest --- utils/dataset_manifest/create.py | 82 ++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index dec29987d818..45d22042f8c0 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -2,16 +2,29 @@ # # SPDX-License-Identifier: MIT import argparse +import mimetypes import os import sys +from glob import glob + +def _define_data_type(media): + media_type, _ = mimetypes.guess_type(media) + if media_type: + return media_type.split('/')[0] + +def _is_video(media_file): + return _define_data_type(media_file) == 'video' + +def _is_image(media_file): + return _define_data_type(media_file) == 'image' def get_args(): parser = argparse.ArgumentParser() - parser.add_argument('--type', type=str, choices=('video', 'images'), help='Type of datset data', required=True) + parser.add_argument('--force', action='store_true', + help='Use this flag to prepare the manifest file for video data ' + 'if by default the video does not meet the requirements and a manifest file is not prepared') parser.add_argument('manifest_directory',type=str, help='Directory where the manifest file will be saved') - parser.add_argument('--chunk_size', type=int, - help='Chunk size that will be specified when creating the task with specified video and generated manifest file') - parser.add_argument('sources', nargs='+', help='Source paths') + parser.add_argument('source', type=str, help='Source paths') return parser.parse_args() def main(): @@ -19,29 +32,56 @@ def main(): manifest_directory = os.path.abspath(args.manifest_directory) os.makedirs(manifest_directory, exist_ok=True) - try: - for source in args.sources: - assert os.path.exists(source), 'A file {} not found'.format(source) - except AssertionError as ex: - sys.exit(str(ex)) - if args.type == 'video': + source = os.path.abspath(args.source) + + sources = [] + if not os.path.isfile(source): # directory/pattern with images + data_dir = None + if os.path.isdir(source): + data_dir = source + for root, _, files in os.walk(source): + sources.extend([os.path.join(root, f) for f in files if _is_image(f)]) + else: + items = source.lstrip('/').split('/') + position = 0 + try: + for item in items: + if set(item) & {'*', '?', '[', ']'}: + break + position += 1 + else: + raise Exception('Wrong positional argument') + assert position != 0, 'Wrong pattern: there must be a common root' + data_dir = source.split(items[position])[0] + except Exception as ex: + sys.exit(str(ex)) + sources = list(filter(_is_image, glob(source, recursive=True))) + try: + assert len(sources), 'A images was not found' + meta_info = prepare_meta(data_type='images', sources=sources, + is_sorted=False, use_image_hash=True, data_dir=data_dir) + manifest = ImageManifestManager(manifest_path=manifest_directory) + manifest.create(meta_info) + except Exception as ex: + sys.exit(str(ex)) + else: # video try: - assert len(args.sources) == 1, 'Unsupporting prepare manifest file for several video files' + assert _is_video(source), 'You can specify a video path or a directory/pattern with images' meta_info, smooth_decoding = prepare_meta( - data_type=args.type, media_file=args.sources[0], chunk_size=args.chunk_size + data_type='video', media_file=source, chunk_size=36 ) + if smooth_decoding is not None and not smooth_decoding: + if not args.force: + msg = 'NOTE: prepared manifest file contains too few key frames for smooth decoding.\n' \ + 'Use --force flag if you still want to prepare a manifest file.' + print(msg) + sys.exit(0) manifest = VideoManifestManager(manifest_path=manifest_directory) manifest.create(meta_info) - if smooth_decoding is not None and not smooth_decoding: - print('NOTE: prepared manifest file contains too few key frames for smooth decoding.') except Exception as ex: - print(ex) - else: - meta_info = prepare_meta(data_type=args.type, sources=args.sources, - is_sorted=False, use_image_hash=True) - manifest = ImageManifestManager(manifest_path=manifest_directory) - manifest.create(meta_info) - print('A manifest file has been prepared ') + sys.exit(str(ex)) + + print('The manifest file has been prepared') if __name__ == "__main__": base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) From 2d4f45673484346e271767d7264e98a02a553fa1 Mon Sep 17 00:00:00 2001 From: Maya Date: Wed, 17 Mar 2021 16:36:09 +0300 Subject: [PATCH 23/29] Update documentation --- cvat/apps/documentation/data_on_fly.md | 19 +++---- utils/dataset_manifest/README.md | 69 ++++++++++++++++++++------ 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/cvat/apps/documentation/data_on_fly.md b/cvat/apps/documentation/data_on_fly.md index e82dce1530ee..c60263d933fa 100644 --- a/cvat/apps/documentation/data_on_fly.md +++ b/cvat/apps/documentation/data_on_fly.md @@ -2,22 +2,23 @@ ## Description -Data on the fly processing is a way of working with data, the main idea of which is as follows: -Minimum necessary meta information is collected, when task is created. -This meta information allows in the future to create a necessary chunks when receiving a request from a client. +Data on the fly processing is a way of working with data, the main idea of which is as follows: when creating a task, +the minimum necessary meta information is collected. This meta information allows in the future to create necessary +chunks when receiving a request from a client. -Generated chunks are stored in a cache of limited size with a policy of evicting less popular items. +Generated chunks are stored in a cache of the limited size with a policy of evicting less popular items. -When a request received from a client, the required chunk is searched for in the cache. -If the chunk does not exist yet, it is created using a prepared meta information and then put into the cache. +When a request is received from a client, the required chunk is searched for in the cache. If the chunk does not exist +yet, it is created using prepared meta information and then put into the cache. This method of working with data allows: - reduce the task creation time. -- store data in a cache of limited size with a policy of evicting less popular items. +- store data in a cache of the limited size with a policy of evicting less popular items. -Unfortunately, this method will not work for all videos with valid manifest file. -If there are not enough keyframes in the video for smooth video decoding, the task will be created in the old way. +Unfortunately, this method will not work for all videos with a valid manifest file. If there are not enough keyframes +in the video for smooth video decoding, the task will be created in another way. Namely, all chunks will be prepared +during task creation, which may take some time. #### Uploading a manifest with data diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md index 43370dc98ae9..d3c57d8b6365 100644 --- a/utils/dataset_manifest/README.md +++ b/utils/dataset_manifest/README.md @@ -2,7 +2,26 @@ ### Steps before use -When used separately from Computer Vision Annotation Tool(CVAT), the required modules must be installed +When used separately from Computer Vision Annotation Tool(CVAT), the required dependencies must be installed + +#### Ubuntu:20.04 + +Install dependencies: + +```bash +# General +sudo apt-get update && sudo apt-get --no-install-recommends install -y \ + python3-dev python3-pip python3-venv pkg-config +``` + +```bash +# Library components +sudo apt-get install --no-install-recommends -y \ + libavformat-dev libavcodec-dev libavdevice-dev \ + libavutil-dev libswscale-dev libswresample-dev libavfilter-dev +``` + +Create an environment and install the necessary python modules: ```bash python3 -m venv .env @@ -14,36 +33,56 @@ pip install -r requirements.txt ### Using ```bash -usage: python create.py [-h] --type {video,images} [--chunk_size CHUNK_SIZE] manifest_directory sources [sources ...] +usage: python create.py [-h] [--force] manifest_directory source positional arguments: - manifest_directory Directory where the manifest file will be saved - sources Source paths + manifest_directory Directory where the manifest file will be saved + source Source paths optional arguments: - -h, --help show this help message and exit - --type {video,images} - Type of datset data - --chunk_size CHUNK_SIZE - Chunk size that will be specified when creating the task with specified video and generated manifest file + -h, --help show this help message and exit + --force Use this flag to prepare the manifest file for video data if by default the video does not meet the requirements + and a manifest file is not prepared ``` -**NOTE**: If ratio of number of frames to number of key frames is small compared to the `chunk size`, -then when creating a task with prepared manifest file, you should expect that the waiting time for some chunks -will be longer than the waiting time for other chunks. (At the first iteration, when there is no chunk in the cache) +### Alternative way to use with openvino/cvat_server + +```bash +docker run -it --entrypoint python3 -v /path/to/host/data/:/path/inside/container/:rw openvino/cvat_server +utils/dataset_manifest/create.py /path/to/manifest/directory/ /path/to/data/ +``` ### Examples of using -Create a dataset manifest with video: +Create a dataset manifest with video which contains enough keyframes: ```bash -python create.py --type video ~/Documents ~/Documents/video.mp4 +python create.py ~/Documents ~/Documents/video.mp4 +``` + +Create a dataset manifest with video which does not contain enough keyframes: + +```bash +python create.py --force ~/Documents ~/Documents/video.mp4 ``` Create a dataset manifest with images: ```bash -python create.py --type images ~/Documents ~/Documents/image1.jpg ~/Documents/image2.jpg ~/Documents/image3.jpg +python create.py ~/Documents ~/Documents/images/ +``` + +Create a dataset manifest with pattern (may be used `*`, `?`, `[]`): + +```bash +python create.py ~/Documents "/home/${USER}/Documents/**/image*.jpeg" +``` + +Create a dataset manifest with `openvino/cvat_server`: + +```bash +docker run -it --entrypoint python3 -v ~/Documents/data/:${HOME}/manifest/:rw openvino/cvat_server +utils/dataset_manifest/create.py ~/manifest/ ~/manifest/images/ ``` ### Example of generated `manifest.jsonl` for video From 38b0d0f08b89c0bddf8adbfac25e5f9b8af8950b Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 19 Mar 2021 12:33:16 +0300 Subject: [PATCH 24/29] Fix some comments --- cvat/apps/engine/migrations/0038_manifest.py | 2 +- cvat/apps/engine/task.py | 3 +- cvat/apps/engine/tests/test_rest_api.py | 5 +-- utils/dataset_manifest/README.md | 25 ++++++------- utils/dataset_manifest/core.py | 37 ++++++++++++++++---- utils/dataset_manifest/create.py | 16 +++++---- 6 files changed, 58 insertions(+), 30 deletions(-) diff --git a/cvat/apps/engine/migrations/0038_manifest.py b/cvat/apps/engine/migrations/0038_manifest.py index 8b7ef1113d5d..7b3e93415c8b 100644 --- a/cvat/apps/engine/migrations/0038_manifest.py +++ b/cvat/apps/engine/migrations/0038_manifest.py @@ -30,7 +30,7 @@ def migrate_data(apps, shema_editor): data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT if hasattr(db_data, 'video'): media_file = os.path.join(data_dir, db_data.video.path) - meta_info, _ = prepare_meta( + meta_info = prepare_meta( data_type='video', media_file=media_file, ) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 19a187cbac90..d5743f9b00e4 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -367,13 +367,12 @@ def _update_status(msg): _update_status('{} Start prepare a valid manifest file.'.format(base_msg)) if not manifest_is_prepared: - meta_info, smooth_decoding = prepare_meta( + meta_info = prepare_meta( data_type='video', media_file=media_files[0], upload_dir=upload_dir, chunk_size=db_data.chunk_size ) - assert smooth_decoding == True, 'Too few keyframes for smooth video decoding.' _update_status('Start prepare a manifest file') manifest = VideoManifestManager(db_data.get_manifest_path()) manifest.create(meta_info) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 64aa0a8a20ee..d40fb464c5d8 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -1769,7 +1769,8 @@ def generate_manifest_file(data_type, manifest_path, sources): }, 'video': { 'media_file': sources[0], - 'upload_dir': os.path.dirname(sources[0]) + 'upload_dir': os.path.dirname(sources[0]), + 'force': True } } @@ -1779,7 +1780,7 @@ def generate_manifest_file(data_type, manifest_path, sources): ) if data_type == 'video': manifest = VideoManifestManager(manifest_path) - manifest.create(prepared_meta[0]) + manifest.create(prepared_meta) else: manifest = ImageManifestManager(manifest_path) manifest.create(prepared_meta) diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md index d3c57d8b6365..8341e944aaab 100644 --- a/utils/dataset_manifest/README.md +++ b/utils/dataset_manifest/README.md @@ -36,53 +36,54 @@ pip install -r requirements.txt usage: python create.py [-h] [--force] manifest_directory source positional arguments: - manifest_directory Directory where the manifest file will be saved - source Source paths + source Source paths optional arguments: - -h, --help show this help message and exit - --force Use this flag to prepare the manifest file for video data if by default the video does not meet the requirements - and a manifest file is not prepared + -h, --help show this help message and exit + --force Use this flag to prepare the manifest file for video data if by default the video does not meet the requirements + and a manifest file is not prepared + --output-dir OUTPUT_DIR + Directory where the manifest file will be saved ``` ### Alternative way to use with openvino/cvat_server ```bash docker run -it --entrypoint python3 -v /path/to/host/data/:/path/inside/container/:rw openvino/cvat_server -utils/dataset_manifest/create.py /path/to/manifest/directory/ /path/to/data/ +utils/dataset_manifest/create.py --output-dir /path/to/manifest/directory/ /path/to/data/ ``` ### Examples of using -Create a dataset manifest with video which contains enough keyframes: +Create a dataset manifest in the current directory with video which contains enough keyframes: ```bash -python create.py ~/Documents ~/Documents/video.mp4 +python create.py ~/Documents/video.mp4 ``` Create a dataset manifest with video which does not contain enough keyframes: ```bash -python create.py --force ~/Documents ~/Documents/video.mp4 +python create.py --force --output-dir ~/Documents ~/Documents/video.mp4 ``` Create a dataset manifest with images: ```bash -python create.py ~/Documents ~/Documents/images/ +python create.py --output-dir ~/Documents ~/Documents/images/ ``` Create a dataset manifest with pattern (may be used `*`, `?`, `[]`): ```bash -python create.py ~/Documents "/home/${USER}/Documents/**/image*.jpeg" +python create.py --output-dir ~/Documents "/home/${USER}/Documents/**/image*.jpeg" ``` Create a dataset manifest with `openvino/cvat_server`: ```bash docker run -it --entrypoint python3 -v ~/Documents/data/:${HOME}/manifest/:rw openvino/cvat_server -utils/dataset_manifest/create.py ~/manifest/ ~/manifest/images/ +utils/dataset_manifest/create.py --output-dir ~/manifest/ ~/manifest/images/ ``` ### Example of generated `manifest.jsonl` for video diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 5f58c5df202d..0bd22e2bad8c 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -63,6 +63,30 @@ def check_video_timestamps_sequences(self): frame_pts, frame_dts = frame.pts, frame.dts + def rough_estimate_frames_ratio(self, upper_bound): + analyzed_frames_number, key_frames_number = 0, 0 + _processing_end = False + + with closing(av.open(self.source_path, mode='r')) as container: + video_stream = self._get_video_stream(container) + for packet in container.demux(video_stream): + for frame in packet.decode(): + if frame.key_frame: + key_frames_number += 1 + analyzed_frames_number += 1 + if upper_bound == analyzed_frames_number: + _processing_end = True + break + if _processing_end: + break + # In our case no videos with non-key first frame, so 1 key frame is guaranteed + return analyzed_frames_number // key_frames_number + + def validate_frames_ratio(self, chunk_size): + upper_bound = 30 * chunk_size + ratio = self.rough_estimate_frames_ratio(upper_bound + 1) + assert ratio < upper_bound, 'Too few keyframes' + class PrepareImageInfo: def __init__(self, sources, is_sorted=True, use_image_hash=False, *args, **kwargs): self._sources = sources if is_sorted else sorted(sources) @@ -127,9 +151,6 @@ def validate_seek_key_frames(self): container.seek(offset=key_frame[1]['pts'], stream=video_stream) self.validate_key_frame(container, video_stream, key_frame) - def validate_frames_ratio(self, chunk_size): - return (len(self._key_frames) and (self.frames // len(self._key_frames)) <= 2 * chunk_size) - def save_key_frames(self): with closing(av.open(self.source_path, mode='r')) as container: video_stream = self._get_video_stream(container) @@ -156,17 +177,21 @@ def __iter__(self): for idx, key_frame in self._key_frames.items(): yield (idx, key_frame['pts'], key_frame['md5']) -def _prepare_video_meta(media_file, upload_dir=None, chunk_size=None): +def _prepare_video_meta(media_file, upload_dir=None, chunk_size=36, force=False): source_path = os.path.join(upload_dir, media_file) if upload_dir else media_file analyzer = AnalyzeVideo(source_path=source_path) analyzer.check_type_first_frame() + try: + analyzer.validate_frames_ratio(chunk_size) + except AssertionError: + if not force: + raise analyzer.check_video_timestamps_sequences() meta_info = PrepareVideoInfo(source_path=source_path) meta_info.save_key_frames() meta_info.validate_seek_key_frames() - smooth_decoding = meta_info.validate_frames_ratio(chunk_size) if chunk_size else None - return (meta_info, smooth_decoding) + return meta_info def _prepare_images_meta(sources, **kwargs): meta_info = PrepareImageInfo(sources=sources, **kwargs) diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index 45d22042f8c0..a258db96badd 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -23,14 +23,15 @@ def get_args(): parser.add_argument('--force', action='store_true', help='Use this flag to prepare the manifest file for video data ' 'if by default the video does not meet the requirements and a manifest file is not prepared') - parser.add_argument('manifest_directory',type=str, help='Directory where the manifest file will be saved') + parser.add_argument('--output-dir',type=str, help='Directory where the manifest file will be saved', + default=os.getcwd()) parser.add_argument('source', type=str, help='Source paths') return parser.parse_args() def main(): args = get_args() - manifest_directory = os.path.abspath(args.manifest_directory) + manifest_directory = os.path.abspath(args.output_dir) os.makedirs(manifest_directory, exist_ok=True) source = os.path.abspath(args.source) @@ -67,15 +68,16 @@ def main(): else: # video try: assert _is_video(source), 'You can specify a video path or a directory/pattern with images' - meta_info, smooth_decoding = prepare_meta( - data_type='video', media_file=source, chunk_size=36 - ) - if smooth_decoding is not None and not smooth_decoding: - if not args.force: + try: + meta_info = prepare_meta(data_type='video', media_file=source, force=args.force) + except AssertionError as ex: + if str(ex) == 'Too few keyframes': msg = 'NOTE: prepared manifest file contains too few key frames for smooth decoding.\n' \ 'Use --force flag if you still want to prepare a manifest file.' print(msg) sys.exit(0) + else: + raise manifest = VideoManifestManager(manifest_path=manifest_directory) manifest.create(meta_info) except Exception as ex: From c7bbd47ef5a3b5407a0d3016ad5eead2a1999a9f Mon Sep 17 00:00:00 2001 From: Maya Date: Mon, 22 Mar 2021 12:46:41 +0300 Subject: [PATCH 25/29] Fix --- utils/dataset_manifest/core.py | 2 +- utils/dataset_manifest/create.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 0bd22e2bad8c..95a5722f7cae 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -83,7 +83,7 @@ def rough_estimate_frames_ratio(self, upper_bound): return analyzed_frames_number // key_frames_number def validate_frames_ratio(self, chunk_size): - upper_bound = 30 * chunk_size + upper_bound = 3 * chunk_size ratio = self.rough_estimate_frames_ratio(upper_bound + 1) assert ratio < upper_bound, 'Too few keyframes' diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index a258db96badd..c36cf6f69b5b 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -75,7 +75,7 @@ def main(): msg = 'NOTE: prepared manifest file contains too few key frames for smooth decoding.\n' \ 'Use --force flag if you still want to prepare a manifest file.' print(msg) - sys.exit(0) + sys.exit(2) else: raise manifest = VideoManifestManager(manifest_path=manifest_directory) From 9c3a81da98218c37a3682361efafaed4dc7536a0 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 23 Mar 2021 13:48:41 +0300 Subject: [PATCH 26/29] One more fix --- cvat/apps/engine/media_extractors.py | 11 +- cvat/apps/engine/migrations/0038_manifest.py | 13 +- cvat/apps/engine/task.py | 14 +- cvat/apps/engine/tests/test_rest_api.py | 10 +- utils/dataset_manifest/__init__.py | 2 +- utils/dataset_manifest/core.py | 148 +++++++++---------- utils/dataset_manifest/create.py | 10 +- 7 files changed, 94 insertions(+), 114 deletions(-) diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 67779f6ba156..22503f38be16 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -27,7 +27,6 @@ from cvat.apps.engine.mime_types import mimetypes from utils.dataset_manifest import VideoManifestManager, ImageManifestManager -from utils.dataset_manifest.core import WorkWithVideo def get_mime(name): for type_name, type_def in MEDIA_TYPES.items(): @@ -359,10 +358,10 @@ def __iter__(self): for idx in self._frame_range: yield self._manifest[idx] -class VideoDatasetManifestReader(WorkWithVideo, FragmentMediaReader): +class VideoDatasetManifestReader(FragmentMediaReader): def __init__(self, manifest_path, **kwargs): - WorkWithVideo.__init__(self, kwargs.pop('source_path')) - FragmentMediaReader.__init__(self, **kwargs) + self.source_path = kwargs.pop('source_path') + super().__init__(**kwargs) self._manifest = VideoManifestManager(manifest_path) self._manifest.init_index() @@ -391,7 +390,9 @@ def _get_nearest_left_key_frame(self): def __iter__(self): start_decode_frame_number, start_decode_timestamp = self._get_nearest_left_key_frame() with closing(av.open(self.source_path, mode='r')) as container: - video_stream = self._get_video_stream(container) + video_stream = next(stream for stream in container.streams if stream.type == 'video') + video_stream.thread_type = 'AUTO' + container.seek(offset=start_decode_timestamp, stream=video_stream) frame_number = start_decode_frame_number - 1 diff --git a/cvat/apps/engine/migrations/0038_manifest.py b/cvat/apps/engine/migrations/0038_manifest.py index 7b3e93415c8b..7447aa6f5740 100644 --- a/cvat/apps/engine/migrations/0038_manifest.py +++ b/cvat/apps/engine/migrations/0038_manifest.py @@ -9,9 +9,7 @@ from cvat.apps.engine.models import (DimensionType, StorageChoice, StorageMethodChoice) -from utils.dataset_manifest import (ImageManifestManager, VideoManifestManager, - prepare_meta) - +from utils.dataset_manifest import ImageManifestManager, VideoManifestManager def migrate_data(apps, shema_editor): Data = apps.get_model("engine", "Data") @@ -30,14 +28,12 @@ def migrate_data(apps, shema_editor): data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT if hasattr(db_data, 'video'): media_file = os.path.join(data_dir, db_data.video.path) - meta_info = prepare_meta( - data_type='video', - media_file=media_file, - ) manifest = VideoManifestManager(manifest_path=upload_dir) + meta_info = manifest.prepare_meta(media_file=media_file) manifest.create(meta_info) manifest.init_index() else: + manifest = ImageManifestManager(manifest_path=upload_dir) sources = [] if db_data.storage == StorageChoice.LOCAL: for (root, _, files) in os.walk(data_dir): @@ -55,9 +51,8 @@ def migrate_data(apps, shema_editor): 'extension': ext }) else: - meta_info = prepare_meta(data_type='images', sources=sources, data_dir=data_dir) + meta_info = manifest.prepare_meta(sources=sources, data_dir=data_dir) content = meta_info.content - manifest = ImageManifestManager(manifest_path=upload_dir) if db_data.storage == StorageChoice.SHARE: def _get_frame_step(str_): diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index d5743f9b00e4..e24865c73690 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -17,7 +17,7 @@ from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile from cvat.apps.engine.utils import av_scan_paths from cvat.apps.engine.models import DimensionType -from utils.dataset_manifest import prepare_meta, ImageManifestManager, VideoManifestManager +from utils.dataset_manifest import ImageManifestManager, VideoManifestManager from utils.dataset_manifest.core import VideoManifestValidator import django_rq @@ -367,19 +367,18 @@ def _update_status(msg): _update_status('{} Start prepare a valid manifest file.'.format(base_msg)) if not manifest_is_prepared: - meta_info = prepare_meta( - data_type='video', + _update_status('Start prepare a manifest file') + manifest = VideoManifestManager(db_data.get_manifest_path()) + meta_info = manifest.prepare_meta( media_file=media_files[0], upload_dir=upload_dir, chunk_size=db_data.chunk_size ) - _update_status('Start prepare a manifest file') - manifest = VideoManifestManager(db_data.get_manifest_path()) manifest.create(meta_info) manifest.init_index() _update_status('A manifest had been created') - all_frames = meta_info.get_task_size() + all_frames = meta_info.get_size() video_size = meta_info.frame_sizes manifest_is_prepared = True @@ -400,8 +399,7 @@ def _update_status(msg): manifest = ImageManifestManager(db_data.get_manifest_path()) if not manifest_file: if db_task.dimension == DimensionType.DIM_2D: - meta_info = prepare_meta( - data_type='images', + meta_info = manifest.prepare_meta( sources=extractor.absolute_source_paths, data_dir=upload_dir ) diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index 7d09953669fc..67fb41b5fa90 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -32,7 +32,7 @@ Segment, StatusChoice, Task, Label, StorageMethodChoice, StorageChoice) from cvat.apps.engine.media_extractors import ValidateDimension from cvat.apps.engine.models import DimensionType -from utils.dataset_manifest import prepare_meta, ImageManifestManager, VideoManifestManager +from utils.dataset_manifest import ImageManifestManager, VideoManifestManager def create_db_users(cls): (group_admin, _) = Group.objects.get_or_create(name="admin") @@ -1984,16 +1984,12 @@ def generate_manifest_file(data_type, manifest_path, sources): } } - prepared_meta = prepare_meta( - data_type=data_type, - **kwargs[data_type] - ) if data_type == 'video': manifest = VideoManifestManager(manifest_path) - manifest.create(prepared_meta) else: manifest = ImageManifestManager(manifest_path) - manifest.create(prepared_meta) + prepared_meta = manifest.prepare_meta(**kwargs[data_type]) + manifest.create(prepared_meta) class TaskDataAPITestCase(APITestCase): _image_sizes = {} diff --git a/utils/dataset_manifest/__init__.py b/utils/dataset_manifest/__init__.py index 17485bd87517..f6547acf3583 100644 --- a/utils/dataset_manifest/__init__.py +++ b/utils/dataset_manifest/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2021 Intel Corporation # # SPDX-License-Identifier: MIT -from .core import prepare_meta, VideoManifestManager, ImageManifestManager \ No newline at end of file +from .core import VideoManifestManager, ImageManifestManager \ No newline at end of file diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 95a5722f7cae..78a00b0b98bf 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -11,9 +11,14 @@ from PIL import Image from .utils import md5_hash, rotate_image -class WorkWithVideo: +class VideoStreamReader: def __init__(self, source_path): self.source_path = source_path + self._key_frames = OrderedDict() + self.frames = 0 + + with closing(av.open(self.source_path, mode='r')) as container: + self.width, self.height = self._get_frame_size(container) @staticmethod def _get_video_stream(container): @@ -23,7 +28,7 @@ def _get_video_stream(container): @staticmethod def _get_frame_size(container): - video_stream = WorkWithVideo._get_video_stream(container) + video_stream = VideoStreamReader._get_video_stream(container) for packet in container.demux(video_stream): for frame in packet.decode(): if video_stream.metadata.get('rotate'): @@ -36,14 +41,14 @@ def _get_frame_size(container): ) return frame.width, frame.height -class AnalyzeVideo(WorkWithVideo): def check_type_first_frame(self): with closing(av.open(self.source_path, mode='r')) as container: video_stream = self._get_video_stream(container) for packet in container.demux(video_stream): for frame in packet.decode(): - assert frame.pict_type.name == 'I', 'First frame is not key frame' + if not frame.pict_type.name == 'I': + raise Exception('First frame is not key frame') return def check_video_timestamps_sequences(self): @@ -87,47 +92,7 @@ def validate_frames_ratio(self, chunk_size): ratio = self.rough_estimate_frames_ratio(upper_bound + 1) assert ratio < upper_bound, 'Too few keyframes' -class PrepareImageInfo: - def __init__(self, sources, is_sorted=True, use_image_hash=False, *args, **kwargs): - self._sources = sources if is_sorted else sorted(sources) - self._content = [] - self._data_dir = kwargs.get('data_dir', None) - self._use_image_hash = use_image_hash - - def __iter__(self): - for image in self._sources: - img = Image.open(image, mode='r') - img_name = os.path.relpath(image, self._data_dir) if self._data_dir \ - else os.path.basename(image) - name, extension = os.path.splitext(img_name) - image_properties = { - 'name': name, - 'extension': extension, - 'width': img.width, - 'height': img.height, - } - if self._use_image_hash: - image_properties['checksum'] = md5_hash(img) - yield image_properties - - def create(self): - for item in self: - self._content.append(item) - - @property - def content(self): - return self._content - -class PrepareVideoInfo(WorkWithVideo): - def __init__(self, **kwargs): - super().__init__(**kwargs) - self._key_frames = OrderedDict() - self.frames = 0 - - with closing(av.open(self.source_path, mode='r')) as container: - self.width, self.height = self._get_frame_size(container) - - def get_task_size(self): + def get_size(self): return self.frames @property @@ -173,38 +138,42 @@ def key_frames(self): def __len__(self): return len(self._key_frames) + #TODO: need to change it in future def __iter__(self): for idx, key_frame in self._key_frames.items(): yield (idx, key_frame['pts'], key_frame['md5']) -def _prepare_video_meta(media_file, upload_dir=None, chunk_size=36, force=False): - source_path = os.path.join(upload_dir, media_file) if upload_dir else media_file - analyzer = AnalyzeVideo(source_path=source_path) - analyzer.check_type_first_frame() - try: - analyzer.validate_frames_ratio(chunk_size) - except AssertionError: - if not force: - raise - analyzer.check_video_timestamps_sequences() - - meta_info = PrepareVideoInfo(source_path=source_path) - meta_info.save_key_frames() - meta_info.validate_seek_key_frames() - return meta_info - -def _prepare_images_meta(sources, **kwargs): - meta_info = PrepareImageInfo(sources=sources, **kwargs) - meta_info.create() - return meta_info - -def prepare_meta(data_type, **kwargs): - assert data_type in ('video', 'images'), 'prepare_meta: Unknown data type' - actions = { - 'video': _prepare_video_meta, - 'images': _prepare_images_meta, - } - return actions[data_type](**kwargs) + +class DatasetImagesReader: + def __init__(self, sources, is_sorted=True, use_image_hash=False, *args, **kwargs): + self._sources = sources if is_sorted else sorted(sources) + self._content = [] + self._data_dir = kwargs.get('data_dir', None) + self._use_image_hash = use_image_hash + + def __iter__(self): + for image in self._sources: + img = Image.open(image, mode='r') + img_name = os.path.relpath(image, self._data_dir) if self._data_dir \ + else os.path.basename(image) + name, extension = os.path.splitext(img_name) + image_properties = { + 'name': name, + 'extension': extension, + 'width': img.width, + 'height': img.height, + } + if self._use_image_hash: + image_properties['checksum'] = md5_hash(img) + yield image_properties + + def create(self): + for item in self: + self._content.append(item) + + @property + def content(self): + return self._content class _Manifest: FILE_NAME = 'manifest.jsonl' @@ -370,7 +339,7 @@ def create(self, content, **kwargs): 'properties': { 'name': os.path.basename(content.source_path), 'resolution': content.frame_sizes, - 'length': content.get_task_size(), + 'length': content.get_size(), }, } for key, value in base_info.items(): @@ -389,17 +358,32 @@ def create(self, content, **kwargs): def partial_update(self, number, properties): pass -#TODO: + @staticmethod + def prepare_meta(media_file, upload_dir=None, chunk_size=36, force=False): + source_path = os.path.join(upload_dir, media_file) if upload_dir else media_file + meta_info = VideoStreamReader(source_path=source_path) + meta_info.check_type_first_frame() + try: + meta_info.validate_frames_ratio(chunk_size) + except AssertionError: + if not force: + raise + meta_info.check_video_timestamps_sequences() + meta_info.save_key_frames() + meta_info.validate_seek_key_frames() + return meta_info + +#TODO: add generic manifest structure file validation class ManifestValidator: def validate_base_info(self): with open(self._manifest.path, 'r') as manifest_file: assert self._manifest.VERSION != json.loads(manifest_file.readline())['version'] assert self._manifest.TYPE != json.loads(manifest_file.readline())['type'] -class VideoManifestValidator(VideoManifestManager, WorkWithVideo): +class VideoManifestValidator(VideoManifestManager): def __init__(self, **kwargs): - WorkWithVideo.__init__(self, kwargs.pop('source_path')) - VideoManifestManager.__init__(self, **kwargs) + self.source_path = kwargs.pop('source_path') + super().__init__(self, **kwargs) def validate_key_frame(self, container, video_stream, key_frame): for packet in container.demux(video_stream): @@ -453,4 +437,10 @@ def create(self, content, **kwargs): self._manifest.is_created = True def partial_update(self, number, properties): - pass \ No newline at end of file + pass + + @staticmethod + def prepare_meta(sources, **kwargs): + meta_info = DatasetImagesReader(sources=sources, **kwargs) + meta_info.create() + return meta_info \ No newline at end of file diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index c36cf6f69b5b..680052f0cf8a 100644 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -59,17 +59,18 @@ def main(): sources = list(filter(_is_image, glob(source, recursive=True))) try: assert len(sources), 'A images was not found' - meta_info = prepare_meta(data_type='images', sources=sources, - is_sorted=False, use_image_hash=True, data_dir=data_dir) manifest = ImageManifestManager(manifest_path=manifest_directory) + meta_info = manifest.prepare_meta(sources=sources, is_sorted=False, + use_image_hash=True, data_dir=data_dir) manifest.create(meta_info) except Exception as ex: sys.exit(str(ex)) else: # video try: assert _is_video(source), 'You can specify a video path or a directory/pattern with images' + manifest = VideoManifestManager(manifest_path=manifest_directory) try: - meta_info = prepare_meta(data_type='video', media_file=source, force=args.force) + meta_info = manifest.prepare_meta(media_file=source, force=args.force) except AssertionError as ex: if str(ex) == 'Too few keyframes': msg = 'NOTE: prepared manifest file contains too few key frames for smooth decoding.\n' \ @@ -78,7 +79,6 @@ def main(): sys.exit(2) else: raise - manifest = VideoManifestManager(manifest_path=manifest_directory) manifest.create(meta_info) except Exception as ex: sys.exit(str(ex)) @@ -87,5 +87,5 @@ def main(): if __name__ == "__main__": base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) - from dataset_manifest.core import prepare_meta, VideoManifestManager, ImageManifestManager + from dataset_manifest.core import VideoManifestManager, ImageManifestManager main() \ No newline at end of file From d16022a03188ee5096a1d0da94a0d72f504e731f Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 23 Mar 2021 18:05:43 +0300 Subject: [PATCH 27/29] Update README --- utils/dataset_manifest/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md index 8341e944aaab..387fa9adcfe2 100644 --- a/utils/dataset_manifest/README.md +++ b/utils/dataset_manifest/README.md @@ -86,7 +86,14 @@ docker run -it --entrypoint python3 -v ~/Documents/data/:${HOME}/manifest/:rw op utils/dataset_manifest/create.py --output-dir ~/manifest/ ~/manifest/images/ ``` -### Example of generated `manifest.jsonl` for video +### Examples of generated `manifest.jsonl` files + +A maifest file contains some intuitive information and some specific like: + +`pts` - time at which the frame should be shown to the user +`checksum` - `md5` hash sum for the specific image/frame + +#### For a video ```json {"version":"1.0"} @@ -100,7 +107,7 @@ utils/dataset_manifest/create.py --output-dir ~/manifest/ ~/manifest/images/ {"number":675,"pts":2430000,"checksum":"0e72faf67e5218c70b506445ac91cdd7"} ``` -### Example of generated `manifest.jsonl` for dataset with images +#### For a dataset with images ```json {"version":"1.0"} From 3e58fa286a14746b6da8bc739d83f9d9793ea8b1 Mon Sep 17 00:00:00 2001 From: Maya Date: Tue, 23 Mar 2021 18:32:21 +0300 Subject: [PATCH 28/29] Revert prettier changes --- .../03_database_deployment.yml | 34 +++--- kubernetes-templates/03_redis_deployment.yml | 16 +-- .../04_cvat_backend_deployment.yml | 106 +++++++++--------- .../04_cvat_frontend_deployment.yml | 2 +- kubernetes-templates/04_database_service.yml | 4 +- kubernetes-templates/04_redis_service.yml | 4 +- .../05_cvat_backend_service.yml | 4 +- .../05_cvat_frontend_service.yml | 4 +- .../05_cvat_proxy_deployment.yml | 44 ++++---- .../05_cvat_proxy_service.yml | 4 +- kubernetes-templates/README.md | 15 +-- 11 files changed, 114 insertions(+), 123 deletions(-) diff --git a/kubernetes-templates/03_database_deployment.yml b/kubernetes-templates/03_database_deployment.yml index 3c1841fbda91..84bdf3fd62e3 100644 --- a/kubernetes-templates/03_database_deployment.yml +++ b/kubernetes-templates/03_database_deployment.yml @@ -23,25 +23,25 @@ spec: containers: - name: cvat-postgres image: postgres:10.3-alpine - imagePullPolicy: 'IfNotPresent' + imagePullPolicy: "IfNotPresent" env: - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: cvat-postgres-secret - key: POSTGRES_DB - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: cvat-postgres-secret - key: POSTGRES_USER - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: cvat-postgres-secret - key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_DB + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_PASSWORD ports: - - containerPort: 5432 + - containerPort: 5432 readinessProbe: exec: command: diff --git a/kubernetes-templates/03_redis_deployment.yml b/kubernetes-templates/03_redis_deployment.yml index 9f2f5b301c63..eafb52836d0b 100644 --- a/kubernetes-templates/03_redis_deployment.yml +++ b/kubernetes-templates/03_redis_deployment.yml @@ -19,11 +19,11 @@ spec: tier: redis-app spec: containers: - - image: redis:4.0.5-alpine - name: cvat-redis - imagePullPolicy: Always - ports: - - containerPort: 6379 - resources: - limits: - cpu: '0.1' + - image: redis:4.0.5-alpine + name: cvat-redis + imagePullPolicy: Always + ports: + - containerPort: 6379 + resources: + limits: + cpu: "0.1" diff --git a/kubernetes-templates/04_cvat_backend_deployment.yml b/kubernetes-templates/04_cvat_backend_deployment.yml index b24408eca161..7b1ea7a0f058 100644 --- a/kubernetes-templates/04_cvat_backend_deployment.yml +++ b/kubernetes-templates/04_cvat_backend_deployment.yml @@ -29,65 +29,65 @@ spec: cpu: 10m memory: 100Mi env: - - name: DJANGO_MODWSGI_EXTRA_ARGS - value: '' - - name: UI_PORT - value: '80' - - name: UI_HOST - value: 'cvat-frontend-service' - - name: ALLOWED_HOSTS - value: '*' - - name: CVAT_REDIS_HOST - value: 'cvat-redis-service' - - name: CVAT_POSTGRES_HOST - value: 'cvat-postgres-service' - - name: CVAT_POSTGRES_USER - valueFrom: - secretKeyRef: - name: cvat-postgres-secret - key: POSTGRES_USER - - name: CVAT_POSTGRES_DBNAME - valueFrom: - secretKeyRef: - name: cvat-postgres-secret - key: POSTGRES_DB - - name: CVAT_POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: cvat-postgres-secret - key: POSTGRES_PASSWORD + - name: DJANGO_MODWSGI_EXTRA_ARGS + value: "" + - name: UI_PORT + value: "80" + - name: UI_HOST + value: "cvat-frontend-service" + - name: ALLOWED_HOSTS + value: "*" + - name: CVAT_REDIS_HOST + value: "cvat-redis-service" + - name: CVAT_POSTGRES_HOST + value: "cvat-postgres-service" + - name: CVAT_POSTGRES_USER + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_USER + - name: CVAT_POSTGRES_DBNAME + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_DB + - name: CVAT_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_PASSWORD ports: - - containerPort: 8080 + - containerPort: 8080 volumeMounts: - - mountPath: /home/django/data - name: cvat-backend-data - subPath: data - - mountPath: /home/django/keys - name: cvat-backend-data - subPath: keys - - mountPath: /home/django/logs - name: cvat-backend-data - subPath: logs - - mountPath: /home/django/models - name: cvat-backend-data - subPath: models + - mountPath: /home/django/data + name: cvat-backend-data + subPath: data + - mountPath: /home/django/keys + name: cvat-backend-data + subPath: keys + - mountPath: /home/django/logs + name: cvat-backend-data + subPath: logs + - mountPath: /home/django/models + name: cvat-backend-data + subPath: models initContainers: - name: user-data-permission-fix image: busybox - command: ['/bin/chmod', '-R', '777', '/home/django'] + command: ["/bin/chmod", "-R", "777", "/home/django"] volumeMounts: - - mountPath: /home/django/data - name: cvat-backend-data - subPath: data - - mountPath: /home/django/keys - name: cvat-backend-data - subPath: keys - - mountPath: /home/django/logs - name: cvat-backend-data - subPath: logs - - mountPath: /home/django/models - name: cvat-backend-data - subPath: models + - mountPath: /home/django/data + name: cvat-backend-data + subPath: data + - mountPath: /home/django/keys + name: cvat-backend-data + subPath: keys + - mountPath: /home/django/logs + name: cvat-backend-data + subPath: logs + - mountPath: /home/django/models + name: cvat-backend-data + subPath: models volumes: - name: cvat-backend-data persistentVolumeClaim: diff --git a/kubernetes-templates/04_cvat_frontend_deployment.yml b/kubernetes-templates/04_cvat_frontend_deployment.yml index 7bd76caca050..362ebdd5a29c 100644 --- a/kubernetes-templates/04_cvat_frontend_deployment.yml +++ b/kubernetes-templates/04_cvat_frontend_deployment.yml @@ -25,5 +25,5 @@ spec: image: openvino/cvat_ui:v1.2.0 imagePullPolicy: Always ports: - - containerPort: 80 + - containerPort: 80 resources: {} diff --git a/kubernetes-templates/04_database_service.yml b/kubernetes-templates/04_database_service.yml index e586ae2c7698..bb1d04f4065a 100644 --- a/kubernetes-templates/04_database_service.yml +++ b/kubernetes-templates/04_database_service.yml @@ -9,8 +9,8 @@ metadata: spec: type: ClusterIP selector: - app: cvat-app - tier: db + app: cvat-app + tier: db ports: - port: 5432 targetPort: 5432 diff --git a/kubernetes-templates/04_redis_service.yml b/kubernetes-templates/04_redis_service.yml index 380c0231e53d..436b99b27577 100644 --- a/kubernetes-templates/04_redis_service.yml +++ b/kubernetes-templates/04_redis_service.yml @@ -9,8 +9,8 @@ metadata: spec: type: ClusterIP selector: - app: cvat-app - tier: redis-app + app: cvat-app + tier: redis-app ports: - port: 6379 targetPort: 6379 diff --git a/kubernetes-templates/05_cvat_backend_service.yml b/kubernetes-templates/05_cvat_backend_service.yml index 27fceb68d5af..d6c4659cb4f9 100644 --- a/kubernetes-templates/05_cvat_backend_service.yml +++ b/kubernetes-templates/05_cvat_backend_service.yml @@ -9,8 +9,8 @@ metadata: spec: type: ClusterIP selector: - app: cvat-app - tier: backend + app: cvat-app + tier: backend ports: - port: 8080 targetPort: 8080 diff --git a/kubernetes-templates/05_cvat_frontend_service.yml b/kubernetes-templates/05_cvat_frontend_service.yml index 52072215c368..0c97278c60e3 100644 --- a/kubernetes-templates/05_cvat_frontend_service.yml +++ b/kubernetes-templates/05_cvat_frontend_service.yml @@ -9,8 +9,8 @@ metadata: spec: type: ClusterIP selector: - app: cvat-app - tier: frontend + app: cvat-app + tier: frontend ports: - port: 80 targetPort: 80 diff --git a/kubernetes-templates/05_cvat_proxy_deployment.yml b/kubernetes-templates/05_cvat_proxy_deployment.yml index fd14c599eca4..456bec2ea901 100644 --- a/kubernetes-templates/05_cvat_proxy_deployment.yml +++ b/kubernetes-templates/05_cvat_proxy_deployment.yml @@ -21,26 +21,26 @@ spec: tier: proxy spec: containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 - volumeMounts: - - mountPath: /etc/nginx - readOnly: true - name: cvat-nginx-conf - - mountPath: /var/log/nginx - name: log + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /etc/nginx + readOnly: true + name: cvat-nginx-conf + - mountPath: /var/log/nginx + name: log volumes: - - name: cvat-nginx-conf - configMap: - name: cvat-nginx-conf - items: - - key: nginx.conf - path: nginx.conf - - key: mime.types - path: mime.types - - key: cvat.conf - path: cvat.d/cvat.conf - - name: log - emptyDir: {} + - name: cvat-nginx-conf + configMap: + name: cvat-nginx-conf + items: + - key: nginx.conf + path: nginx.conf + - key: mime.types + path: mime.types + - key: cvat.conf + path: cvat.d/cvat.conf + - name: log + emptyDir: {} diff --git a/kubernetes-templates/05_cvat_proxy_service.yml b/kubernetes-templates/05_cvat_proxy_service.yml index 4f544bdc68a8..18229d3b3dd7 100644 --- a/kubernetes-templates/05_cvat_proxy_service.yml +++ b/kubernetes-templates/05_cvat_proxy_service.yml @@ -9,8 +9,8 @@ metadata: spec: type: NodePort selector: - app: cvat-app - tier: proxy + app: cvat-app + tier: proxy ports: - port: 80 targetPort: 80 diff --git a/kubernetes-templates/README.md b/kubernetes-templates/README.md index 40de44ca7fe1..ce0c6a06d4dc 100644 --- a/kubernetes-templates/README.md +++ b/kubernetes-templates/README.md @@ -4,15 +4,13 @@ This guide will focus on how to deploy cvat in an kubernetes environment. It was tested on Kubernetes v1.19.3 but should work for >=v1.9, eventhough it is untested. ## Building the container - optional - Since prebuild container images are now available [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) this steps becomes optional. If you would like to build your one image the following steps need to be followd. - 1. Build the cvat backend and frontend images and push them to a registry that you can pull from within the cluster. 1. Replace the `openvino/...` image source in - `04_cvat_backend_deployment.yml` and `04_cvat_frontend_deployment.yml` with your newly build image. + `04_cvat_backend_deployment.yml` and `04_cvat_frontend_deployment.yml` with your newly build image. ```bash export CI_REGISTRY_IMAGE="your.private.registry" @@ -34,16 +32,14 @@ docker push $CI_REGISTRY_IMAGE/frontend:release-1.1.0 ## Adjusting the kubernetes templates 1. Replacing the domain dummy with your real domain name `cvat.my.cool.domain.com`. - Replace `{MY_SERVER_URL_COM}` in `kubernetes-templates/04_cvat_frontend_deployment.yml` - and `kubernetes-templates/05_cvat_proxy_configmap.yml`. + Replace `{MY_SERVER_URL_COM}` in `kubernetes-templates/04_cvat_frontend_deployment.yml` + and `kubernetes-templates/05_cvat_proxy_configmap.yml`. 1. Insert your choosen database password the `kubernetes-templates/02_database_secrets.yml` ## Deploying to the cluster - Deploy everything to your cluster with `kubectl apply -f kubernetes-templates/` ### Expose the deployment - The service `cvat-proxy-service` is the accesspoint to the deployment. In order to expose this resource an ingress might be handy [kubernetes ingress documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/). @@ -67,11 +63,9 @@ python3 ~/manage.py createsuperuser ``` ## Debugging hints - Due to different kubernetes versions or other deployment environments ### Incorect storage class - Depending on the selected kubernetes environment certain storage classes might not be available. The selected "standard" class is available with in all maijor kubernetes platforms (GKE, EKS, ...), but not in some local development environemnts such as miniKube. @@ -79,16 +73,13 @@ This is the case, if `kubectl describe pod -n cvat cvat-backend` shows that the To fix this, `class: standard` needs to be adjusted in `02_cvat_backend_storage.yaml` and `02_database_storage.yml`. ### Creating the django super user fails - Depending on your kuberenets version you creating the super user might not be possible with in one line. Therefore you need to get bash access within the consol and call the manage script manually. - ```bash kubectl --namespace cvat exec -it cvat-backend-7c954d5cf6-xfdcm bash python3 ~/manage.py createsuperuser ``` ### Running out of storage - By default the backend is reserving 20GB of storage if this is not enough, you will need to ajust the `02_cvat_backend_storage.yml` persistant volume claim to increase it. From 71d1b9aaf3d36135dc2e8d7579ce369e771d42fc Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Wed, 24 Mar 2021 07:45:22 +0300 Subject: [PATCH 29/29] Update utils/dataset_manifest/README.md --- utils/dataset_manifest/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dataset_manifest/README.md b/utils/dataset_manifest/README.md index 387fa9adcfe2..4a16f6151712 100644 --- a/utils/dataset_manifest/README.md +++ b/utils/dataset_manifest/README.md @@ -33,7 +33,7 @@ pip install -r requirements.txt ### Using ```bash -usage: python create.py [-h] [--force] manifest_directory source +usage: python create.py [-h] [--force] [--output-dir .] source positional arguments: source Source paths