From 5e0160b8b88a335c1cf23bf80ef08af873b0010a Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 8 Feb 2023 16:45:42 +0200 Subject: [PATCH] Supported interpolation for 3D cuboids (#5629) --- CHANGELOG.md | 2 + cvat-canvas3d/src/typescript/canvas3d.ts | 14 +- .../src/typescript/canvas3dController.ts | 15 +- cvat-canvas3d/src/typescript/canvas3dModel.ts | 53 ++++- cvat-canvas3d/src/typescript/canvas3dView.ts | 217 ++++++++++++++---- cvat-canvas3d/src/typescript/consts.ts | 6 +- cvat-canvas3d/src/typescript/index.ts | 8 +- cvat-core/src/annotation-formats.ts | 30 +-- cvat-core/src/annotations-collection.ts | 3 +- cvat-core/src/annotations-filter.ts | 28 ++- cvat-core/src/annotations-objects.ts | 45 +++- cvat-core/src/annotations.ts | 4 +- cvat-core/src/server-proxy.ts | 3 +- cvat-core/src/server-response-types.ts | 16 ++ .../views/canvas3d/canvas-wrapper3D.tsx | 63 +++-- .../controls-side-bar/controls-side-bar.tsx | 76 +++--- .../controls-side-bar/draw-shape-popover.tsx | 4 +- .../controls-side-bar/group-control.tsx | 64 +++++- .../controls-side-bar/merge-control.tsx | 40 +++- .../controls-side-bar/split-control.tsx | 38 ++- .../controls-side-bar/controls-side-bar.tsx | 84 ++++--- .../top-bar/statistics-modal.tsx | 3 +- .../controls-side-bar/draw-shape-popover.tsx | 3 +- .../controls-side-bar/controls-side-bar.tsx | 16 +- cvat-ui/src/cvat-core-wrapper.ts | 5 +- cvat/apps/dataset_manager/annotation.py | 141 +++++++++--- cvat/apps/dataset_manager/bindings.py | 28 +-- .../dataset_manager/formats/pointcloud.py | 2 +- .../dataset_manager/formats/velodynepoint.py | 10 +- cvat/apps/dataset_manager/task.py | 21 +- .../dataset_manager/tests/test_annotation.py | 6 +- .../dataset_manager/tests/test_formats.py | 12 +- ...s3d_functionality_cuboid_cancel_drawing.js | 7 +- ...56_canvas3d_functionality_basic_actions.js | 2 +- tests/cypress/support/commands_canvas3d.js | 2 +- 35 files changed, 771 insertions(+), 300 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0596b48ff8a1..0ec4d8605ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - \[SDK\] Class to represent a project as a PyTorch dataset () - Grid view and multiple context images supported () +- Interpolation is now supported for 3D cuboids. +Tracks can be exported/imported to/from Datumaro and Sly Pointcloud formats () - Support for custom file to job splits in tasks (server API & SDK only) () - \[SDK\] A PyTorch adapter setting to disable cache updates diff --git a/cvat-canvas3d/src/typescript/canvas3d.ts b/cvat-canvas3d/src/typescript/canvas3d.ts index 6333e23db754..216e52ee0fc5 100644 --- a/cvat-canvas3d/src/typescript/canvas3d.ts +++ b/cvat-canvas3d/src/typescript/canvas3d.ts @@ -1,5 +1,5 @@ // Copyright (C) 2021-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -15,6 +15,8 @@ import { ShapeProperties, GroupData, Configuration, + SplitData, + MergeData, } from './canvas3dModel'; import { Canvas3dView, Canvas3dViewImpl, ViewsDOM, CameraAction, @@ -38,6 +40,8 @@ interface Canvas3d { fitCanvas(): void; fit(): void; group(groupData: GroupData): void; + merge(mergeData: MergeData): void; + split(splitData: SplitData): void; destroy(): void; } @@ -80,6 +84,14 @@ class Canvas3dImpl implements Canvas3d { this.model.group(groupData); } + public split(splitData: SplitData): void { + this.model.split(splitData); + } + + public merge(mergeData: MergeData): void { + this.model.merge(mergeData); + } + public isAbleToChangeFrame(): boolean { return this.model.isAbleToChangeFrame(); } diff --git a/cvat-canvas3d/src/typescript/canvas3dController.ts b/cvat-canvas3d/src/typescript/canvas3dController.ts index 404b8baaa518..715db993bcfc 100644 --- a/cvat-canvas3d/src/typescript/canvas3dController.ts +++ b/cvat-canvas3d/src/typescript/canvas3dController.ts @@ -1,11 +1,12 @@ // Copyright (C) 2021-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT import { ObjectState } from '.'; import { - Canvas3dModel, Mode, DrawData, ActiveElement, GroupData, Configuration, + Canvas3dModel, Mode, DrawData, ActiveElement, + GroupData, Configuration, MergeData, SplitData, } from './canvas3dModel'; export interface Canvas3dController { @@ -17,6 +18,8 @@ export interface Canvas3dController { readonly objects: ObjectState[]; mode: Mode; group(groupData: GroupData): void; + merge(mergeData: MergeData): void; + split(splitData: SplitData): void; } export class Canvas3dControllerImpl implements Canvas3dController { @@ -61,4 +64,12 @@ export class Canvas3dControllerImpl implements Canvas3dController { public group(groupData: GroupData): void { this.model.group(groupData); } + + public merge(mergeData: MergeData): void { + this.model.merge(mergeData); + } + + public split(splitData: SplitData): void { + this.model.split(splitData); + } } diff --git a/cvat-canvas3d/src/typescript/canvas3dModel.ts b/cvat-canvas3d/src/typescript/canvas3dModel.ts index 48acec10ae4b..da706e23582b 100644 --- a/cvat-canvas3d/src/typescript/canvas3dModel.ts +++ b/cvat-canvas3d/src/typescript/canvas3dModel.ts @@ -1,5 +1,5 @@ // Copyright (C) 2021-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -18,7 +18,14 @@ export interface ActiveElement { export interface GroupData { enabled: boolean; - grouped: ObjectState[]; +} + +export interface MergeData { + enabled: boolean; +} + +export interface SplitData { + enabled: boolean; } export interface Configuration { @@ -80,6 +87,8 @@ export enum UpdateReasons { DRAG_CANVAS = 'drag_canvas', SHAPE_ACTIVATED = 'shape_activated', GROUP = 'group', + MERGE = 'merge', + SPLIT = 'split', FITTED_CANVAS = 'fitted_canvas', CONFIG_UPDATED = 'config_updated', SHAPES_CONFIG_UPDATED = 'shapes_config_updated', @@ -91,6 +100,8 @@ export enum Mode { EDIT = 'edit', DRAG_CANVAS = 'drag_canvas', GROUP = 'group', + MERGE = 'merge', + SPLIT = 'split', } export interface Canvas3dDataModel { @@ -106,6 +117,8 @@ export interface Canvas3dDataModel { objects: ObjectState[]; shapeProperties: ShapeProperties; groupData: GroupData; + mergeData: MergeData; + splitData: SplitData; configuration: Configuration; isFrameUpdating: boolean; nextSetupRequest: { @@ -119,6 +132,7 @@ export interface Canvas3dModel { data: Canvas3dDataModel; readonly imageIsDeleted: boolean; readonly groupData: GroupData; + readonly mergeData: MergeData; readonly configuration: Configuration; readonly objects: ObjectState[]; setup(frameData: any, objectStates: ObjectState[]): void; @@ -131,6 +145,8 @@ export interface Canvas3dModel { configure(configuration: Configuration): void; fit(): void; group(groupData: GroupData): void; + split(splitData: SplitData): void; + merge(mergeData: MergeData): void; destroy(): void; updateCanvasObjects(): void; unlockFrameUpdating(): void; @@ -166,7 +182,12 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { mode: Mode.IDLE, groupData: { enabled: false, - grouped: [], + }, + mergeData: { + enabled: false, + }, + splitData: { + enabled: false, }, shapeProperties: { opacity: 40, @@ -343,10 +364,30 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { return; } this.data.mode = groupData.enabled ? Mode.GROUP : Mode.IDLE; - this.data.groupData = { ...this.data.groupData, ...groupData }; + this.data.groupData = { ...groupData }; this.notify(UpdateReasons.GROUP); } + public split(splitData: SplitData): void { + if (![Mode.IDLE, Mode.SPLIT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + this.data.mode = splitData.enabled ? Mode.SPLIT : Mode.IDLE; + this.data.splitData = { ...splitData }; + this.notify(UpdateReasons.SPLIT); + } + + public merge(mergeData: MergeData): void { + if (![Mode.IDLE, Mode.MERGE].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + this.data.mode = mergeData.enabled ? Mode.MERGE : Mode.IDLE; + this.data.mergeData = { ...mergeData }; + this.notify(UpdateReasons.MERGE); + } + public configure(configuration: Configuration): void { if (typeof configuration.resetZoom === 'boolean') { this.data.configuration.resetZoom = configuration.resetZoom; @@ -391,6 +432,10 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { return { ...this.data.groupData }; } + public get mergeData(): MergeData { + return { ...this.data.mergeData }; + } + public get imageIsDeleted(): boolean { return this.data.imageIsDeleted; } diff --git a/cvat-canvas3d/src/typescript/canvas3dView.ts b/cvat-canvas3d/src/typescript/canvas3dView.ts index c247c0523712..678430a0ab2e 100644 --- a/cvat-canvas3d/src/typescript/canvas3dView.ts +++ b/cvat-canvas3d/src/typescript/canvas3dView.ts @@ -17,7 +17,7 @@ import { createResizeHelper, removeResizeHelper, createCuboidEdges, removeCuboidEdges, CuboidModel, makeCornerPointsMatrix, } from './cuboid'; -import { ObjectState } from '.'; +import { ObjectState, ObjectType } from '.'; export interface Canvas3dView { html(): ViewsDOM; @@ -108,6 +108,9 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { private isPerspectiveBeingDragged: boolean; private activatedElementID: number | null; private isCtrlDown: boolean; + private stateToBeSplitted: ObjectState | null; + private statesToBeGrouped: ObjectState[]; + private statesToBeMerged: ObjectState[]; private sceneBBox: THREE.Box3; private drawnObjects: Record { e.preventDefault(); - const selectionIsBlocked = ![Mode.GROUP, Mode.IDLE].includes(this.mode) || + const selectionIsBlocked = ![Mode.GROUP, Mode.MERGE, Mode.SPLIT, Mode.IDLE].includes(this.mode) || !this.views.perspective.rayCaster || this.isPerspectiveBeingDragged; @@ -369,20 +376,34 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { const intersectionClientID = +(intersects[0]?.object?.name) || null; const objectState = Number.isInteger(intersectionClientID) ? this.model.objects .find((state: ObjectState) => state.clientID === intersectionClientID) : null; - if ( - objectState && - this.mode === Mode.GROUP && - this.model.data.groupData.grouped - ) { - const objectStateIdx = this.model.data.groupData.grouped + + const handleClick = (targetList: ObjectState[]): void => { + const objectStateIdx = targetList .findIndex((state: ObjectState) => state.clientID === intersectionClientID); if (objectStateIdx !== -1) { - this.model.data.groupData.grouped.splice(objectStateIdx, 1); + targetList.splice(objectStateIdx, 1); } else { - this.model.data.groupData.grouped.push(objectState); + targetList.push(objectState); } this.drawnObjects[intersectionClientID].cuboid.setColor(this.receiveShapeColor(objectState)); + }; + + if (objectState && this.mode === Mode.GROUP) { + handleClick(this.statesToBeGrouped); + } else if (objectState && this.mode === Mode.MERGE) { + const [latest] = this.statesToBeMerged; + const drawnStates = Object.keys(this.drawnObjects).map((key: string): number => +key); + if (!latest || + (latest && + objectState.label.id === latest.label.id && + objectState.shapeType === latest.shapeType && + !this.statesToBeMerged.some((state) => drawnStates.includes(state.clientID))) + ) { + handleClick(this.statesToBeMerged); + } + } else if (objectState?.objectType === ObjectType.TRACK && this.mode === Mode.SPLIT) { + this.onSplitDone(objectState); } else if (this.mode === Mode.IDLE) { const intersectedClientID = intersects[0]?.object?.name || null; if (this.model.data.activeElement.clientID !== intersectedClientID) { @@ -452,6 +473,7 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { group: initState.group?.id || null, label: initState.label, shapeType: initState.shapeType, + objectType: initState.objectType, } : {}), }, duration: 0, @@ -826,22 +848,73 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { ); } - this.controller.group({ - enabled: false, - grouped: [], - }); + this.mode = Mode.IDLE; + } + + private onMergeDone(objects: any[] | null, duration?: number): void { + if (objects) { + const event: CustomEvent = new CustomEvent('canvas.merged', { + bubbles: false, + cancelable: true, + detail: { + duration, + states: objects, + }, + }); + + this.dispatchEvent(event); + } else { + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.dispatchEvent(event); + } this.mode = Mode.IDLE; } + private onSplitDone(object: ObjectState): void { + if (object) { + const event: CustomEvent = new CustomEvent('canvas.splitted', { + bubbles: false, + cancelable: true, + detail: { + state: object, + frame: object.frame, + }, + }); + + this.dispatchEvent(event); + } else { + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.dispatchEvent(event); + } + + this.controller.split({ enabled: false }); + this.mode = Mode.IDLE; + } + private receiveShapeColor(state: ObjectState | DrawnObjectData): string { + const includedInto = (states: ObjectState[]): boolean => states + .some((_state: ObjectState): boolean => _state.clientID === state.clientID); const { colorBy } = this.model.data.shapeProperties; - if (this.mode === Mode.GROUP) { - const { grouped } = this.model.data.groupData; - if (grouped.some((_state: ObjectState): boolean => _state.clientID === state.clientID)) { - return CONST.GROUPING_COLOR; - } + if (this.mode === Mode.GROUP && includedInto(this.statesToBeGrouped)) { + return CONST.GROUPING_COLOR; + } + + if (this.mode === Mode.MERGE && includedInto(this.statesToBeMerged)) { + return CONST.MERGING_COLOR; + } + + if (this.mode === Mode.SPLIT && this.stateToBeSplitted?.clientID === state.clientID) { + return CONST.SPLITTING_COLOR; } if (state instanceof ObjectState) { @@ -1027,8 +1100,18 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { } public notify(model: Canvas3dModel & Master, reason: UpdateReasons): void { + const resetColor = (list: ObjectState[]): void => { + list.forEach((state: ObjectState) => { + const { clientID } = state; + const { cuboid } = this.drawnObjects[clientID] || {}; + if (cuboid) { + cuboid.setColor(this.receiveShapeColor(state)); + } + }); + }; + if (reason === UpdateReasons.IMAGE_CHANGED) { - model.data.groupData.grouped = []; + this.statesToBeGrouped = []; this.clearScene(); const onPCDLoadFailed = (): void => { @@ -1182,30 +1265,53 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { } } + if (this.mode === Mode.MERGE) { + const { statesToBeMerged } = this; + this.statesToBeMerged = []; + resetColor(statesToBeMerged); + this.model.merge({ enabled: false }); + } + if (this.mode === Mode.GROUP) { - const { grouped } = this.model.groupData; - this.model.group({ enabled: false, grouped: [] }); - grouped.forEach((state: ObjectState) => { - const { clientID } = state; - const { cuboid } = this.drawnObjects[clientID] || {}; - if (cuboid) { - cuboid.setColor(this.receiveShapeColor(state)); - } - }); + const { statesToBeGrouped } = this; + this.statesToBeGrouped = []; + resetColor(statesToBeGrouped); + this.model.group({ enabled: false }); } - this.mode = Mode.IDLE; - model.mode = Mode.IDLE; + if (this.mode === Mode.SPLIT) { + if (this.stateToBeSplitted) { + const state = this.stateToBeSplitted; + this.stateToBeSplitted = null; + this.drawnObjects[state.clientID].cuboid.setColor(this.receiveShapeColor(state)); + } + this.model.split({ enabled: false }); + } + this.mode = Mode.IDLE; this.dispatchEvent(new CustomEvent('canvas.canceled')); } else if (reason === UpdateReasons.FITTED_CANVAS) { this.dispatchEvent(new CustomEvent('canvas.fit')); } else if (reason === UpdateReasons.GROUP) { - if (!model.groupData.enabled) { - this.onGroupDone(model.data.groupData.grouped); - } else { + if (!model.groupData.enabled && this.statesToBeGrouped.length) { + this.onGroupDone(this.statesToBeGrouped); + resetColor(this.statesToBeGrouped); + } else if (model.groupData.enabled) { this.deactivateObject(); - model.data.groupData.grouped = []; + this.statesToBeGrouped = []; + model.data.activeElement.clientID = null; + } + } else if (reason === UpdateReasons.SPLIT) { + this.deactivateObject(); + this.stateToBeSplitted = null; + model.data.activeElement.clientID = null; + } else if (reason === UpdateReasons.MERGE) { + if (!model.mergeData.enabled && this.statesToBeMerged.length) { + this.onMergeDone(this.statesToBeMerged); + resetColor(this.statesToBeMerged); + } else if (model.mergeData.enabled) { + this.deactivateObject(); + this.statesToBeMerged = []; model.data.activeElement.clientID = null; } } @@ -1475,24 +1581,37 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener { const { x, y, z } = intersection.point; object.position.set(x, y, z); } - } else if (this.mode === Mode.IDLE && !this.isPerspectiveBeingDragged && !this.isCtrlDown) { + } else { const { renderer } = this.views.perspective.rayCaster; const intersects = renderer.intersectObjects(this.getAllVisibleCuboids(), false); - if (intersects.length !== 0) { + if (intersects.length !== 0 && !this.isPerspectiveBeingDragged) { const clientID = intersects[0].object.name; - if (this.model.data.activeElement.clientID !== clientID) { - const object = this.views.perspective.scene.getObjectByName(clientID); - if (object === undefined) return; - this.dispatchEvent( - new CustomEvent('canvas.selected', { - bubbles: false, - cancelable: true, - detail: { - clientID: Number(intersects[0].object.name), - }, - }), - ); + const castedClientID = +clientID; + + if (this.mode === Mode.SPLIT) { + const objectState = Number.isInteger(castedClientID) ? this.model.objects + .find((state: ObjectState) => state.clientID === castedClientID) : null; + this.stateToBeSplitted = objectState; + this.drawnObjects[castedClientID].cuboid.setColor(this.receiveShapeColor(objectState)); + } else if (this.mode === Mode.IDLE && !this.isCtrlDown) { + if (this.model.data.activeElement.clientID !== clientID) { + const object = this.views.perspective.scene.getObjectByName(clientID); + if (object === undefined) return; + this.dispatchEvent( + new CustomEvent('canvas.selected', { + bubbles: false, + cancelable: true, + detail: { + clientID: castedClientID, + }, + }), + ); + } } + } else if (this.mode === Mode.SPLIT && this.stateToBeSplitted) { + const state = this.stateToBeSplitted; + this.stateToBeSplitted = null; + this.drawnObjects[state.clientID].cuboid.setColor(this.receiveShapeColor(state)); } } }; diff --git a/cvat-canvas3d/src/typescript/consts.ts b/cvat-canvas3d/src/typescript/consts.ts index 20df990dd212..abb78079cb2e 100644 --- a/cvat-canvas3d/src/typescript/consts.ts +++ b/cvat-canvas3d/src/typescript/consts.ts @@ -1,5 +1,5 @@ // Copyright (C) 2021-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -23,6 +23,8 @@ const FOV_INC = 0.08; const DEFAULT_GROUP_COLOR = '#e0e0e0'; const DEFAULT_OUTLINE_COLOR = '#000000'; const GROUPING_COLOR = '#8b008b'; +const MERGING_COLOR = '#0000ff'; +const SPLITTING_COLOR = '#1e90ff'; export default { BASE_GRID_WIDTH, @@ -45,4 +47,6 @@ export default { DEFAULT_GROUP_COLOR, DEFAULT_OUTLINE_COLOR, GROUPING_COLOR, + MERGING_COLOR, + SPLITTING_COLOR, }; diff --git a/cvat-canvas3d/src/typescript/index.ts b/cvat-canvas3d/src/typescript/index.ts index f9b3c572c00b..446ec788a1eb 100644 --- a/cvat-canvas3d/src/typescript/index.ts +++ b/cvat-canvas3d/src/typescript/index.ts @@ -1,9 +1,11 @@ -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT import ObjectState from 'cvat-core/src/object-state'; import { Label } from 'cvat-core/src/labels'; -import { ShapeType } from 'cvat-core/src/enums'; +import { ShapeType, ObjectType } from 'cvat-core/src/enums'; -export { ObjectState, Label, ShapeType }; +export { + ObjectState, Label, ShapeType, ObjectType, +}; diff --git a/cvat-core/src/annotation-formats.ts b/cvat-core/src/annotation-formats.ts index 1fb2a6dee71d..d976d63922bb 100644 --- a/cvat-core/src/annotation-formats.ts +++ b/cvat-core/src/annotation-formats.ts @@ -3,22 +3,21 @@ // // SPDX-License-Identifier: MIT -interface RawLoaderData { - name: string; - ext: string; - version: string; - enabled: boolean; - dimension: '2d' | '3d'; -} +import { DimensionType } from 'enums'; +import { + AnnotationExporterResponseBody, + AnnotationFormatsResponseBody, + AnnotationImporterResponseBody, +} from 'server-response-types'; export class Loader { public name: string; public format: string; public version: string; public enabled: boolean; - public dimension: '2d' | '3d'; + public dimension: DimensionType; - constructor(initialData: RawLoaderData) { + constructor(initialData: AnnotationImporterResponseBody) { const data = { name: initialData.name, format: initialData.ext, @@ -47,16 +46,14 @@ export class Loader { } } -type RawDumperData = RawLoaderData; - export class Dumper { public name: string; public format: string; public version: string; public enabled: boolean; - public dimension: '2d' | '3d'; + public dimension: DimensionType; - constructor(initialData: RawDumperData) { + constructor(initialData: AnnotationExporterResponseBody) { const data = { name: initialData.name, format: initialData.ext, @@ -85,16 +82,11 @@ export class Dumper { } } -interface AnnotationFormatRawData { - importers: RawLoaderData[]; - exporters: RawDumperData[]; -} - export class AnnotationFormats { public loaders: Loader[]; public dumpers: Dumper[]; - constructor(initialData: AnnotationFormatRawData) { + constructor(initialData: AnnotationFormatsResponseBody) { const data = { exporters: initialData.exporters.map((el) => new Dumper(el)), importers: initialData.importers.map((el) => new Loader(el)), diff --git a/cvat-core/src/annotations-collection.ts b/cvat-core/src/annotations-collection.ts index 17743099c47f..74c00836d441 100644 --- a/cvat-core/src/annotations-collection.ts +++ b/cvat-core/src/annotations-collection.ts @@ -34,7 +34,6 @@ interface ImportedCollection { export default class Collection { public flush: boolean; - private startFrame: number; private stopFrame: number; private frameMeta: any; private labels: Record @@ -49,7 +48,6 @@ export default class Collection { private injection: BasicInjection; constructor(data) { - this.startFrame = data.startFrame; this.stopFrame = data.stopFrame; this.frameMeta = data.frameMeta; @@ -78,6 +76,7 @@ export default class Collection { groups: this.groups, frameMeta: this.frameMeta, history: this.history, + dimension: data.dimension, nextClientID: () => ++this.count, groupColors: {}, getMasksOnFrame: (frame: number) => (this.shapes[frame] as MaskShape[]) diff --git a/cvat-core/src/annotations-filter.ts b/cvat-core/src/annotations-filter.ts index 63c748cdd0eb..cff1467a3f4a 100644 --- a/cvat-core/src/annotations-filter.ts +++ b/cvat-core/src/annotations-filter.ts @@ -1,24 +1,37 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT import jsonLogic from 'json-logic-js'; +import { SerializedData } from 'object-state'; import { AttributeType, ObjectType, ShapeType } from './enums'; function adjustName(name): string { return name.replace(/\./g, '\u2219'); } +interface ConvertedObjectData { + width: number | null; + height: number | null; + attr: Record>; + label: string; + serverID: number; + objectID: number; + type: ObjectType; + shape: ShapeType; + occluded: boolean; +} + export default class AnnotationsFilter { - _convertObjects(statesData) { + _convertObjects(statesData: SerializedData[]): ConvertedObjectData[] { const objects = statesData.map((state) => { const labelAttributes = state.label.attributes.reduce((acc, attr) => { acc[attr.id] = attr; return acc; }, {}); - let [width, height] = [null, null]; - + let [width, height]: (number | null)[] = [null, null]; if (state.objectType !== ObjectType.TAG) { if (state.shapeType === ShapeType.MASK) { const [xtl, ytl, xbr, ybr] = state.points.slice(-4); @@ -48,8 +61,7 @@ export default class AnnotationsFilter { } } - const attributes = {}; - Object.keys(state.attributes).reduce((acc, key) => { + const attributes = Object.keys(state.attributes).reduce>((acc, key) => { const attr = labelAttributes[key]; let value = state.attributes[key]; if (attr.inputType === AttributeType.NUMBER) { @@ -59,7 +71,7 @@ export default class AnnotationsFilter { } acc[adjustName(attr.name)] = value; return acc; - }, attributes); + }, {}); return { width, @@ -77,8 +89,8 @@ export default class AnnotationsFilter { return objects; } - filter(statesData, filters) { - if (!filters.length) return statesData; + filter(statesData: SerializedData[], filters: string[]): number[] { + if (!filters.length) return statesData.map((stateData): number => stateData.clientID); const converted = this._convertObjects(statesData); return converted .map((state) => state.objectID) diff --git a/cvat-core/src/annotations-objects.ts b/cvat-core/src/annotations-objects.ts index c390f3c5356c..e41073cec197 100644 --- a/cvat-core/src/annotations-objects.ts +++ b/cvat-core/src/annotations-objects.ts @@ -9,7 +9,7 @@ import { checkObjectType, clamp } from './common'; import { DataError, ArgumentError, ScriptingError } from './exceptions'; import { Label } from './labels'; import { - colors, Source, ShapeType, ObjectType, HistoryActions, + colors, Source, ShapeType, ObjectType, HistoryActions, DimensionType, } from './enums'; import AnnotationHistory from './annotations-history'; import { @@ -41,6 +41,7 @@ export interface BasicInjection { groupColors: Record; parentID?: number; readOnlyFields?: string[]; + dimension: DimensionType; nextClientID: () => number; getMasksOnFrame: (frame: number) => MaskShape[]; } @@ -52,11 +53,12 @@ type AnnotationInjection = BasicInjection & { class Annotation { public clientID: number; - protected taskLabels: Label[]; + protected taskLabels: Record; protected history: any; protected groupColors: Record; public serverID: number | null; protected parentID: number | null; + protected dimension: DimensionType; public group: number; public label: Label; public frame: number; @@ -79,6 +81,7 @@ class Annotation { this.clientID = clientID; this.serverID = data.id || null; this.parentID = injection.parentID || null; + this.dimension = injection.dimension; this.group = data.group; this.label = this.taskLabels[data.label_id]; this.frame = data.frame; @@ -2719,14 +2722,48 @@ export class CuboidTrack extends Track { protected interpolatePosition(leftPosition, rightPosition, offset): InterpolatedPosition { const positionOffset = leftPosition.points.map((point, index) => rightPosition.points[index] - point); - - return { + const result = { points: leftPosition.points.map((point, index) => point + positionOffset[index] * offset), rotation: leftPosition.rotation, occluded: leftPosition.occluded, outside: leftPosition.outside, zOrder: leftPosition.zOrder, }; + + if (this.dimension === DimensionType.DIMENSION_3D) { + // for 3D cuboids angle for different axies stored as a part of points array + // we need to apply interpolation using the shortest arc for each angle + + const [ + angleX, angleY, angleZ, + ] = leftPosition.points.slice(3, 6).concat(rightPosition.points.slice(3, 6)) + .map((_angle: number) => { + if (_angle < 0) { + return _angle + Math.PI * 2; + } + + return _angle; + }) + .map((_angle) => _angle * (180 / Math.PI)) + .reduce((acc: number[], angleBefore: number, index: number, arr: number[]) => { + if (index < 3) { + const angleAfter = arr[index + 3]; + let angle = (angleBefore + findAngleDiff(angleAfter, angleBefore) * offset) * (Math.PI / 180); + if (angle > Math.PI) { + angle -= Math.PI * 2; + } + acc.push(angle); + } + + return acc; + }, []); + + result.points[3] = angleX; + result.points[4] = angleY; + result.points[5] = angleZ; + } + + return result; } } diff --git a/cvat-core/src/annotations.ts b/cvat-core/src/annotations.ts index 4bcd05d067a1..af43dbb07b4c 100644 --- a/cvat-core/src/annotations.ts +++ b/cvat-core/src/annotations.ts @@ -1,5 +1,5 @@ // Copyright (C) 2019-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -49,9 +49,9 @@ async function getAnnotationsFromServer(session) { const collection = new Collection({ labels: session.labels || session.task.labels, history, - startFrame, stopFrame, frameMeta, + dimension: session.dimension, }); // eslint-disable-next-line no-unsanitized/method diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index df18a0f9b9cb..c3cae83fd0c4 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -7,6 +7,7 @@ import FormData from 'form-data'; import store from 'store'; import Axios, { AxiosResponse } from 'axios'; import * as tus from 'tus-js-client'; +import { AnnotationFormatsResponseBody } from 'server-response-types'; import { Storage } from './storage'; import { StorageLocation, WebhookSourceType } from './enums'; import { isEmail } from './common'; @@ -281,7 +282,7 @@ async function exception(exceptionObject) { } } -async function formats() { +async function formats(): Promise { const { backendAPI } = config; let response = null; diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 42e158a7263e..88835113d4b9 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -2,8 +2,24 @@ // // SPDX-License-Identifier: MIT +import { DimensionType } from 'enums'; import { SerializedModel } from 'core-types'; +export interface AnnotationImporterResponseBody { + name: string; + ext: string; + version: string; + enabled: boolean; + dimension: DimensionType; +} + +export type AnnotationExporterResponseBody = AnnotationImporterResponseBody; + +export interface AnnotationFormatsResponseBody { + importers: AnnotationImporterResponseBody[]; + exporters: AnnotationExporterResponseBody[]; +} + export interface FunctionsResponseBody { results: SerializedModel[]; count: number; diff --git a/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx b/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx index d00a75932c2d..f9f7f95a35c8 100644 --- a/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx +++ b/cvat-ui/src/components/annotation-page/canvas/views/canvas3d/canvas-wrapper3D.tsx @@ -21,8 +21,10 @@ import { editShape, groupAnnotationsAsync, groupObjects, + mergeAnnotationsAsync, resetCanvas, shapeDrawn, + splitAnnotationsAsync, updateAnnotationsAsync, updateCanvasContextMenu, } from 'actions/annotation-actions'; @@ -33,7 +35,7 @@ import { CameraAction, Canvas3d, ViewsDOM } from 'cvat-canvas3d-wrapper'; import CVATTooltip from 'components/common/cvat-tooltip'; import { LogType } from 'cvat-logger'; -import { getCore } from 'cvat-core-wrapper'; +import { getCore, ObjectState, Job } from 'cvat-core-wrapper'; const cvat = getCore(); @@ -45,7 +47,7 @@ interface StateToProps { colorBy: ColorBy; frameFetching: boolean; canvasInstance: Canvas3d; - jobInstance: any; + jobInstance: Job; frameData: any; annotations: any[]; contextMenuVisibility: boolean; @@ -62,9 +64,11 @@ interface DispatchToProps { onSetupCanvas(): void; onGroupObjects: (enabled: boolean) => void; onResetCanvas(): void; - onCreateAnnotations(sessionInstance: any, frame: number, states: any[]): void; - onUpdateAnnotations(states: any[]): void; - onGroupAnnotations(sessionInstance: any, frame: number, states: any[]): void; + onCreateAnnotations(sessionInstance: Job, frame: number, states: ObjectState[]): void; + onGroupAnnotations(sessionInstance: Job, frame: number, states: ObjectState[]): void; + onMergeAnnotations(sessionInstance: Job, frame: number, states: ObjectState[]): void; + onSplitAnnotations(sessionInstance: Job, frame: number, state: ObjectState): void; + onUpdateAnnotations(states: ObjectState[]): void; onActivateObject: (activatedStateID: number | null) => void; onShapeDrawn: () => void; onEditShape: (enabled: boolean) => void; @@ -134,15 +138,21 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { onGroupObjects(enabled: boolean): void { dispatch(groupObjects(enabled)); }, - onCreateAnnotations(sessionInstance: any, frame: number, states: any[]): void { - dispatch(createAnnotationsAsync(sessionInstance, frame, states)); - }, onShapeDrawn(): void { dispatch(shapeDrawn()); }, - onGroupAnnotations(sessionInstance: any, frame: number, states: any[]): void { + onCreateAnnotations(sessionInstance: Job, frame: number, states: ObjectState[]): void { + dispatch(createAnnotationsAsync(sessionInstance, frame, states)); + }, + onGroupAnnotations(sessionInstance: Job, frame: number, states: ObjectState[]): void { dispatch(groupAnnotationsAsync(sessionInstance, frame, states)); }, + onMergeAnnotations(sessionInstance: Job, frame: number, states: ObjectState[]): void { + dispatch(mergeAnnotationsAsync(sessionInstance, frame, states)); + }, + onSplitAnnotations(sessionInstance: Job, frame: number, state: ObjectState): void { + dispatch(splitAnnotationsAsync(sessionInstance, frame, state)); + }, onActivateObject(activatedStateID: number | null): void { if (activatedStateID === null) { dispatch(updateCanvasContextMenu(false, 0, 0)); @@ -153,7 +163,7 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { onEditShape(enabled: boolean): void { dispatch(editShape(enabled)); }, - onUpdateAnnotations(states: any[]): void { + onUpdateAnnotations(states: ObjectState[]): void { dispatch(updateAnnotationsAsync(states)); }, onUpdateContextMenu( @@ -395,8 +405,6 @@ const Canvas3DWrapperComponent = React.memo((props: Props): ReactElement => { colorBy, contextMenuVisibility, frameData, - onResetCanvas, - onSetupCanvas, annotations, frame, jobInstance, @@ -404,8 +412,14 @@ const Canvas3DWrapperComponent = React.memo((props: Props): ReactElement => { activatedStateID, resetZoom, activeObjectType, + onResetCanvas, + onSetupCanvas, onShapeDrawn, + onGroupObjects, onCreateAnnotations, + onMergeAnnotations, + onSplitAnnotations, + onGroupAnnotations, } = props; const { canvasInstance } = props as { canvasInstance: Canvas3d }; @@ -554,13 +568,20 @@ const Canvas3DWrapperComponent = React.memo((props: Props): ReactElement => { ); }; - const onCanvasObjectsGroupped = (event: any): void => { - const { onGroupAnnotations, onGroupObjects } = props; - + const onCanvasObjectsGroupped = (event: CustomEvent<{ states: ObjectState[] }>): void => { + const { states } = event.detail; onGroupObjects(false); + onGroupAnnotations(jobInstance, frame, states); + }; + const onCanvasObjectsMerged = (event: CustomEvent<{ states: ObjectState[] }>): void => { const { states } = event.detail; - onGroupAnnotations(jobInstance, frame, states); + onMergeAnnotations(jobInstance, frame, states); + }; + + const onCanvasTrackSplitted = (event: CustomEvent<{ state: ObjectState }>): void => { + const { state } = event.detail; + onSplitAnnotations(jobInstance, frame, state); }; useEffect(() => { @@ -575,7 +596,9 @@ const Canvas3DWrapperComponent = React.memo((props: Props): ReactElement => { canvasInstanceDOM.perspective.addEventListener('canvas.edited', onCanvasEditDone); canvasInstanceDOM.perspective.addEventListener('canvas.contextmenu', onContextMenu); canvasInstanceDOM.perspective.addEventListener('click', onCanvasClick); - canvasInstanceDOM.perspective.addEventListener('canvas.groupped', onCanvasObjectsGroupped); + canvasInstanceDOM.perspective.addEventListener('canvas.groupped', onCanvasObjectsGroupped as EventListener); + canvasInstanceDOM.perspective.addEventListener('canvas.merged', onCanvasObjectsMerged as EventListener); + canvasInstanceDOM.perspective.addEventListener('canvas.splitted', onCanvasTrackSplitted as EventListener); return () => { canvasInstanceDOM.perspective.removeEventListener('canvas.drawn', onCanvasShapeDrawn); @@ -583,9 +606,11 @@ const Canvas3DWrapperComponent = React.memo((props: Props): ReactElement => { canvasInstanceDOM.perspective.removeEventListener('canvas.edited', onCanvasEditDone); canvasInstanceDOM.perspective.removeEventListener('canvas.contextmenu', onContextMenu); canvasInstanceDOM.perspective.removeEventListener('click', onCanvasClick); - canvasInstanceDOM.perspective.removeEventListener('canvas.groupped', onCanvasObjectsGroupped); + canvasInstanceDOM.perspective.removeEventListener('canvas.groupped', onCanvasObjectsGroupped as EventListener); + canvasInstanceDOM.perspective.removeEventListener('canvas.merged', onCanvasObjectsMerged as EventListener); + canvasInstanceDOM.perspective.removeEventListener('canvas.splitted', onCanvasTrackSplitted as EventListener); }; - }, [frameData, annotations, activeLabelID, contextMenuVisibility]); + }, [frameData, annotations, activeLabelID, contextMenuVisibility, activeObjectType]); return <>; }); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx index 6fa50213a3e7..56b1f792ee02 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx @@ -1,5 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -194,52 +194,11 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { canvasInstance.draw({ enabled: false }); } }, - SWITCH_MERGE_MODE: (event: KeyboardEvent | undefined) => { - preventDefault(event); - const merging = activeControl === ActiveControl.MERGE; - if (!merging) { - canvasInstance.cancel(); - } - canvasInstance.merge({ enabled: !merging }); - mergeObjects(!merging); - }, - SWITCH_SPLIT_MODE: (event: KeyboardEvent | undefined) => { - preventDefault(event); - const splitting = activeControl === ActiveControl.SPLIT; - if (!splitting) { - canvasInstance.cancel(); - } - canvasInstance.split({ enabled: !splitting }); - splitTrack(!splitting); - }, - SWITCH_GROUP_MODE: (event: KeyboardEvent | undefined) => { - preventDefault(event); - const grouping = activeControl === ActiveControl.GROUP; - if (!grouping) { - canvasInstance.cancel(); - } - canvasInstance.group({ enabled: !grouping }); - groupObjects(!grouping); - }, - RESET_GROUP: (event: KeyboardEvent | undefined) => { - preventDefault(event); - const grouping = activeControl === ActiveControl.GROUP; - if (!grouping) { - return; - } - resetGroup(); - canvasInstance.group({ enabled: false }); - groupObjects(false); - }, }; subKeyMap = { ...subKeyMap, PASTE_SHAPE: keyMap.PASTE_SHAPE, SWITCH_DRAW_MODE: keyMap.SWITCH_DRAW_MODE, - SWITCH_MERGE_MODE: keyMap.SWITCH_MERGE_MODE, - SWITCH_SPLIT_MODE: keyMap.SWITCH_SPLIT_MODE, - SWITCH_GROUP_MODE: keyMap.SWITCH_GROUP_MODE, - RESET_GROUP: keyMap.RESET_GROUP, }; } @@ -349,26 +308,45 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx index 5677cc3d2af0..b4540eb031a3 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx @@ -1,5 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -155,7 +155,7 @@ function DrawShapePopoverComponent(props: Props): JSX.Element { - {is2D && shapeType !== ShapeType.MASK && ( + {shapeType !== ShapeType.MASK && ( diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/group-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/group-control.tsx index 2d2af45668c6..c105a7acc3f4 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/group-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/group-control.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -10,26 +11,36 @@ import { Canvas } from 'cvat-canvas-wrapper'; import { Canvas3d } from 'cvat-canvas3d-wrapper'; import { ActiveControl, DimensionType } from 'reducers'; import CVATTooltip from 'components/common/cvat-tooltip'; +import GlobalHotKeys, { KeyMapItem } from 'utils/mousetrap-react'; export interface Props { + groupObjects(enabled: boolean): void; + resetGroup(): void; canvasInstance: Canvas | Canvas3d; activeControl: ActiveControl; - switchGroupShortcut: string; - resetGroupShortcut: string; disabled?: boolean; jobInstance?: any; - groupObjects(enabled: boolean): void; + shortcuts: { + SWITCH_GROUP_MODE: { + details: KeyMapItem; + displayValue: string; + }; + RESET_GROUP: { + details: KeyMapItem; + displayValue: string; + }; + } } function GroupControl(props: Props): JSX.Element { const { - switchGroupShortcut, - resetGroupShortcut, + groupObjects, + resetGroup, activeControl, canvasInstance, - groupObjects, disabled, jobInstance, + shortcuts, } = props; const dynamicIconProps = @@ -53,16 +64,47 @@ function GroupControl(props: Props): JSX.Element { const title = [ `Group shapes${ jobInstance && jobInstance.dimension === DimensionType.DIM_3D ? '' : '/tracks' - } ${switchGroupShortcut}. `, - `Select and press ${resetGroupShortcut} to reset a group.`, + } ${shortcuts.SWITCH_GROUP_MODE.displayValue}. `, + `Select and press ${shortcuts.RESET_GROUP.displayValue} to reset a group.`, ].join(' '); + const shortcutHandlers = { + SWITCH_GROUP_MODE: (event: KeyboardEvent | undefined) => { + if (event) event.preventDefault(); + const grouping = activeControl === ActiveControl.GROUP; + if (!grouping) { + canvasInstance.cancel(); + } + canvasInstance.group({ enabled: !grouping }); + groupObjects(!grouping); + }, + RESET_GROUP: (event: KeyboardEvent | undefined) => { + if (event) event.preventDefault(); + const grouping = activeControl === ActiveControl.GROUP; + if (!grouping) { + return; + } + resetGroup(); + canvasInstance.group({ enabled: false }); + groupObjects(false); + }, + }; + return disabled ? ( ) : ( - - - + <> + + + + + ); } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/merge-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/merge-control.tsx index eaa1e5f25017..0868644800eb 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/merge-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/merge-control.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -7,22 +8,41 @@ import Icon from '@ant-design/icons'; import { MergeIcon } from 'icons'; import { Canvas } from 'cvat-canvas-wrapper'; +import { Canvas3d } from 'cvat-canvas3d-wrapper'; import { ActiveControl } from 'reducers'; import CVATTooltip from 'components/common/cvat-tooltip'; +import GlobalHotKeys, { KeyMapItem } from 'utils/mousetrap-react'; export interface Props { - canvasInstance: Canvas; + mergeObjects(enabled: boolean): void; + canvasInstance: Canvas | Canvas3d; activeControl: ActiveControl; - switchMergeShortcut: string; disabled?: boolean; - mergeObjects(enabled: boolean): void; + shortcuts: { + SWITCH_MERGE_MODE: { + details: KeyMapItem; + displayValue: string; + } + }; } function MergeControl(props: Props): JSX.Element { const { - switchMergeShortcut, activeControl, canvasInstance, mergeObjects, disabled, + shortcuts, activeControl, canvasInstance, mergeObjects, disabled, } = props; + const shortcutHandlers = { + SWITCH_MERGE_MODE: (event: KeyboardEvent | undefined) => { + if (event) event.preventDefault(); + const merging = activeControl === ActiveControl.MERGE; + if (!merging) { + canvasInstance.cancel(); + } + canvasInstance.merge({ enabled: !merging }); + mergeObjects(!merging); + }, + }; + const dynamicIconProps = activeControl === ActiveControl.MERGE ? { @@ -44,9 +64,15 @@ function MergeControl(props: Props): JSX.Element { return disabled ? ( ) : ( - - - + <> + + + + + ); } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/split-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/split-control.tsx index ca75a5b71685..308626b61478 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/split-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/split-control.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -7,22 +8,41 @@ import Icon from '@ant-design/icons'; import { SplitIcon } from 'icons'; import { Canvas } from 'cvat-canvas-wrapper'; +import { Canvas3d } from 'cvat-canvas3d-wrapper'; import { ActiveControl } from 'reducers'; import CVATTooltip from 'components/common/cvat-tooltip'; +import GlobalHotKeys, { KeyMapItem } from 'utils/mousetrap-react'; export interface Props { - canvasInstance: Canvas; + canvasInstance: Canvas | Canvas3d; activeControl: ActiveControl; - switchSplitShortcut: string; disabled?: boolean; splitTrack(enabled: boolean): void; + shortcuts: { + SWITCH_SPLIT_MODE: { + details: KeyMapItem; + displayValue: string; + }; + }; } function SplitControl(props: Props): JSX.Element { const { - switchSplitShortcut, activeControl, canvasInstance, splitTrack, disabled, + shortcuts, activeControl, canvasInstance, splitTrack, disabled, } = props; + const shortcutHandlers = { + SWITCH_SPLIT_MODE: (event: KeyboardEvent | undefined) => { + if (event) event.preventDefault(); + const splitting = activeControl === ActiveControl.SPLIT; + if (!splitting) { + canvasInstance.cancel(); + } + canvasInstance.split({ enabled: !splitting }); + splitTrack(!splitting); + }, + }; + const dynamicIconProps = activeControl === ActiveControl.SPLIT ? { @@ -44,9 +64,15 @@ function SplitControl(props: Props): JSX.Element { return disabled ? ( ) : ( - - - + <> + + + + + ); } diff --git a/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx index f93d33016e96..74a25b9474e7 100644 --- a/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx @@ -1,5 +1,5 @@ // Copyright (C) 2021-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -20,6 +20,12 @@ import DrawCuboidControl, { import GroupControl, { Props as GroupControlProps, } from 'components/annotation-page/standard-workspace/controls-side-bar/group-control'; +import MergeControl, { + Props as MergeControlProps, +} from 'components/annotation-page/standard-workspace/controls-side-bar/merge-control'; +import SplitControl, { + Props as SplitControlProps, +} from 'components/annotation-page/standard-workspace/controls-side-bar/split-control'; import GlobalHotKeys, { KeyMap } from 'utils/mousetrap-react'; import ControlVisibilityObserver from 'components/annotation-page/standard-workspace/controls-side-bar/control-visibility-observer'; import { filterApplicableForType } from 'utils/filter-applicable-labels'; @@ -35,6 +41,8 @@ interface Props { redrawShape(): void; pasteShape(): void; groupObjects(enabled: boolean): void; + mergeObjects(enabled: boolean): void; + splitTrack(enabled: boolean): void; resetGroup(): void; } @@ -42,6 +50,8 @@ const ObservedCursorControl = ControlVisibilityObserver(Curs const ObservedMoveControl = ControlVisibilityObserver(MoveControl); const ObservedDrawCuboidControl = ControlVisibilityObserver(DrawCuboidControl); const ObservedGroupControl = ControlVisibilityObserver(GroupControl); +const ObservedMergeControl = ControlVisibilityObserver(MergeControl); +const ObservedSplitControl = ControlVisibilityObserver(SplitControl); export default function ControlsSideBarComponent(props: Props): JSX.Element { const { @@ -54,8 +64,9 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { redrawShape, repeatDrawShape, groupObjects, + mergeObjects, + splitTrack, resetGroup, - jobInstance, } = props; const applicableLabels = filterApplicableForType(LabelType.CUBOID, labels); @@ -101,35 +112,15 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { canvasInstance.draw({ enabled: false }); } }, - SWITCH_GROUP_MODE: (event: KeyboardEvent | undefined) => { - preventDefault(event); - const grouping = activeControl === ActiveControl.GROUP; - if (!grouping) { - canvasInstance.cancel(); - } - canvasInstance.group({ enabled: !grouping }); - groupObjects(!grouping); - }, - RESET_GROUP: (event: KeyboardEvent | undefined) => { - preventDefault(event); - const grouping = activeControl === ActiveControl.GROUP; - if (!grouping) { - return; - } - resetGroup(); - canvasInstance.group({ enabled: false }); - groupObjects(false); - }, }; subKeyMap = { ...subKeyMap, PASTE_SHAPE: keyMap.PASTE_SHAPE, SWITCH_DRAW_MODE: keyMap.SWITCH_DRAW_MODE, - SWITCH_GROUP_MODE: keyMap.SWITCH_GROUP_MODE, - RESET_GROUP: keyMap.RESET_GROUP, }; } + const controlsDisabled = !applicableLabels.length; return ( @@ -142,16 +133,51 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { - + + + + ); diff --git a/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx b/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx index 4b7a803dc468..c4b735c6f630 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/statistics-modal.tsx @@ -1,4 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -139,7 +140,7 @@ function StatisticsModalComponent(props: StateToProps & DispatchToProps): JSX.El }); const makeShapesTracksTitle = (title: string): JSX.Element => ( - + {title} diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx index ddd34f0befb1..33bf06932332 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx @@ -9,6 +9,7 @@ import { RadioChangeEvent } from 'antd/lib/radio'; import { CombinedState, ShapeType, ObjectType } from 'reducers'; import { rememberObject } from 'actions/annotation-actions'; import { Canvas, RectDrawingMethod, CuboidDrawingMethod } from 'cvat-canvas-wrapper'; +import { Canvas3d } from 'cvat-canvas3d-wrapper'; import DrawShapePopoverComponent from 'components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover'; import { Label } from 'cvat-core-wrapper'; @@ -29,7 +30,7 @@ interface DispatchToProps { interface StateToProps { normalizedKeyMap: Record; - canvasInstance: Canvas; + canvasInstance: Canvas | Canvas3d; shapeType: ShapeType; labels: any[]; jobInstance: any; diff --git a/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx index 809c6de6f6ae..f142c86c7c88 100644 --- a/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx @@ -1,13 +1,15 @@ // Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT import { connect } from 'react-redux'; import { KeyMap } from 'utils/mousetrap-react'; -import { Canvas } from 'cvat-canvas-wrapper'; import { Canvas3d } from 'cvat-canvas3d-wrapper'; import { groupObjects, + splitTrack, + mergeObjects, pasteShapeAsync, redrawShapeAsync, repeatDrawShapeAsync, @@ -17,7 +19,7 @@ import ControlsSideBarComponent from 'components/annotation-page/standard3D-work import { ActiveControl, CombinedState } from 'reducers'; interface StateToProps { - canvasInstance: Canvas | Canvas3d; + canvasInstance: Canvas3d; activeControl: ActiveControl; keyMap: KeyMap; normalizedKeyMap: Record; @@ -31,6 +33,8 @@ interface DispatchToProps { pasteShape(): void; resetGroup(): void; groupObjects(enabled: boolean): void; + mergeObjects(enabled: boolean): void; + splitTrack(enabled: boolean): void; } function mapStateToProps(state: CombinedState): StateToProps { @@ -43,7 +47,7 @@ function mapStateToProps(state: CombinedState): StateToProps { } = state; return { - canvasInstance, + canvasInstance: canvasInstance as Canvas3d, activeControl, normalizedKeyMap, keyMap, @@ -69,6 +73,12 @@ function dispatchToProps(dispatch: any): DispatchToProps { resetGroup(): void { dispatch(resetAnnotationsGroup()); }, + mergeObjects(enabled: boolean): void { + dispatch(mergeObjects(enabled)); + }, + splitTrack(enabled: boolean): void { + dispatch(splitTrack(enabled)); + }, }; } diff --git a/cvat-ui/src/cvat-core-wrapper.ts b/cvat-ui/src/cvat-core-wrapper.ts index 5bf26107d25a..1e3aa6447793 100644 --- a/cvat-ui/src/cvat-core-wrapper.ts +++ b/cvat-ui/src/cvat-core-wrapper.ts @@ -1,5 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022-2023 CVAT.ai Corporation +// Copyright (C) 2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -11,6 +11,7 @@ import { ModelProvider } from 'cvat-core/src/lambda-manager'; import { Label, Attribute, RawAttribute, RawLabel, } from 'cvat-core/src/labels'; +import { Job, Task } from 'cvat-core/src/session'; import { ShapeType, LabelType, ModelKind, ModelProviders, ModelReturnType, } from 'cvat-core/src/enums'; @@ -34,6 +35,8 @@ export { getCore, ObjectState, Label, + Job, + Task, Attribute, ShapeType, LabelType, diff --git a/cvat/apps/dataset_manager/annotation.py b/cvat/apps/dataset_manager/annotation.py index 16e67691a5f0..6d99a2711467 100644 --- a/cvat/apps/dataset_manager/annotation.py +++ b/cvat/apps/dataset_manager/annotation.py @@ -1,21 +1,24 @@ # Copyright (C) 2019-2022 Intel Corporation +# Copyright (C) 2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT from copy import copy, deepcopy +import math import numpy as np from itertools import chain from scipy.optimize import linear_sum_assignment from shapely import geometry -from cvat.apps.engine.models import ShapeType +from cvat.apps.engine.models import ShapeType, DimensionType from cvat.apps.engine.serializers import LabeledDataSerializer class AnnotationIR: - def __init__(self, data=None): + def __init__(self, dimension, data=None): self.reset() + self.dimension = dimension if data: self.tags = getattr(data, 'tags', []) or data['tags'] self.shapes = getattr(data, 'shapes', []) or data['shapes'] @@ -80,7 +83,7 @@ def has_overlap(a, b): return False @classmethod - def _slice_track(cls, track_, start, stop): + def _slice_track(cls, track_, start, stop, dimension): def filter_track_shapes(shapes): shapes = [s for s in shapes if cls._is_shape_inside(s, start, stop)] drop_count = 0 @@ -97,9 +100,9 @@ def filter_track_shapes(shapes): if len(segment_shapes) < len(track['shapes']): for element in track.get('elements', []): - element = cls._slice_track(element, start, stop) + element = cls._slice_track(element, start, stop, dimension) interpolated_shapes = TrackManager.get_interpolated_shapes( - track, start, stop) + track, start, stop, dimension) scoped_shapes = filter_track_shapes(interpolated_shapes) if scoped_shapes: @@ -121,8 +124,8 @@ def filter_track_shapes(shapes): return track def slice(self, start, stop): - #makes a data copy from specified frame interval - splitted_data = AnnotationIR() + # makes a data copy from specified frame interval + splitted_data = AnnotationIR(self.dimension) splitted_data.tags = [deepcopy(t) for t in self.tags if self._is_shape_inside(t, start, stop)] splitted_data.shapes = [deepcopy(s) @@ -130,7 +133,7 @@ def slice(self, start, stop): splitted_tracks = [] for t in self.tracks: if self._is_track_inside(t, start, stop): - track = self._slice_track(t, start, stop) + track = self._slice_track(t, start, stop, self.dimension) if 0 < len(track['shapes']): splitted_tracks.append(track) splitted_data.tracks = splitted_tracks @@ -147,19 +150,19 @@ class AnnotationManager: def __init__(self, data): self.data = data - def merge(self, data, start_frame, overlap): + def merge(self, data, start_frame, overlap, dimension): tags = TagManager(self.data.tags) - tags.merge(data.tags, start_frame, overlap) + tags.merge(data.tags, start_frame, overlap, dimension) shapes = ShapeManager(self.data.shapes) - shapes.merge(data.shapes, start_frame, overlap) + shapes.merge(data.shapes, start_frame, overlap, dimension) - tracks = TrackManager(self.data.tracks) - tracks.merge(data.tracks, start_frame, overlap) + tracks = TrackManager(self.data.tracks, dimension) + tracks.merge(data.tracks, start_frame, overlap, dimension) - def to_shapes(self, end_frame): + def to_shapes(self, end_frame, dimension): shapes = self.data.shapes - tracks = TrackManager(self.data.tracks) + tracks = TrackManager(self.data.tracks, dimension) return shapes + tracks.to_shapes(end_frame) @@ -190,7 +193,7 @@ def _get_cost_threshold(): raise NotImplementedError() @staticmethod - def _calc_objects_similarity(obj0, obj1, start_frame, overlap): + def _calc_objects_similarity(obj0, obj1, start_frame, overlap, dimension): raise NotImplementedError() @staticmethod @@ -200,7 +203,7 @@ def _unite_objects(obj0, obj1): def _modify_unmached_object(self, obj, end_frame): raise NotImplementedError() - def merge(self, objects, start_frame, overlap): + def merge(self, objects, start_frame, overlap, dimension): # 1. Split objects on two parts: new and which can be intersected # with existing objects. new_objects = [obj for obj in objects @@ -239,7 +242,7 @@ def merge(self, objects, start_frame, overlap): for i, int_obj in enumerate(int_objects): for j, old_obj in enumerate(old_objects): cost_matrix[i][j] = 1 - self._calc_objects_similarity( - int_obj, old_obj, start_frame, overlap) + int_obj, old_obj, start_frame, overlap, dimension) # 6. Find optimal solution using Hungarian algorithm. row_ind, col_ind = linear_sum_assignment(cost_matrix) @@ -274,7 +277,7 @@ def _get_cost_threshold(): return 0.25 @staticmethod - def _calc_objects_similarity(obj0, obj1, start_frame, overlap): + def _calc_objects_similarity(obj0, obj1, start_frame, overlap, dimension): # TODO: improve the trivial implementation, compare attributes return 1 if obj0["label_id"] == obj1["label_id"] else 0 @@ -320,7 +323,7 @@ def _get_cost_threshold(): return 0.25 @staticmethod - def _calc_objects_similarity(obj0, obj1, start_frame, overlap): + def _calc_objects_similarity(obj0, obj1, start_frame, overlap, dimension): def _calc_polygons_similarity(p0, p1): if p0.is_valid and p1.is_valid: # check validity of polygons overlap_area = p0.intersection(p1).area @@ -335,17 +338,61 @@ def _calc_polygons_similarity(p0, p1): has_same_label = obj0.get("label_id") == obj1.get("label_id") if has_same_type and has_same_label: if obj0["type"] == ShapeType.RECTANGLE: + # FIXME: need to consider rotated boxes p0 = geometry.box(*obj0["points"]) p1 = geometry.box(*obj1["points"]) return _calc_polygons_similarity(p0, p1) + elif obj0["type"] == ShapeType.CUBOID and dimension == DimensionType.DIM_3D: + [x_c0, y_c0, z_c0] = obj0["points"][0:3] + [x_c1, y_c1, z_c1] = obj1["points"][0:3] + + [x_len0, y_len0, z_len0] = obj0["points"][6:9] + [x_len1, y_len1, z_len1] = obj1["points"][6:9] + + top_view_0 = [ + x_c0 - x_len0 / 2, + y_c0 - y_len0 / 2, + x_c0 + x_len0 / 2, + y_c0 + y_len0 / 2 + ] + + top_view_1 = [ + x_c1 - x_len1 / 2, + y_c1 - y_len1 / 2, + x_c1 + x_len1 / 2, + y_c1 + y_len1 / 2 + ] + + p_top0 = geometry.box(*top_view_0) + p_top1 = geometry.box(*top_view_1) + top_similarity =_calc_polygons_similarity(p_top0, p_top1) + + side_view_0 = [ + x_c0 - x_len0 / 2, + z_c0 - z_len0 / 2, + x_c0 + x_len0 / 2, + z_c0 + z_len0 / 2 + ] + + side_view_1 = [ + x_c1 - x_len1 / 2, + z_c1 - z_len1 / 2, + x_c1 + x_len1 / 2, + z_c1 + z_len1 / 2 + ] + p_side0 = geometry.box(*side_view_0) + p_side1 = geometry.box(*side_view_1) + side_similarity =_calc_polygons_similarity(p_side0, p_side1) + + return top_similarity * side_similarity elif obj0["type"] == ShapeType.POLYGON: p0 = geometry.Polygon(pairwise(obj0["points"])) p1 = geometry.Polygon(pairwise(obj1["points"])) return _calc_polygons_similarity(p0, p1) else: - return 0 # FIXME: need some similarity for points and polylines + return 0 # FIXME: need some similarity for points, polylines, ellipses and 2D cuboids return 0 @staticmethod @@ -357,11 +404,15 @@ def _modify_unmached_object(self, obj, end_frame): pass class TrackManager(ObjectManager): + def __init__(self, objects, dimension): + self._dimension = dimension + super().__init__(objects) + def to_shapes(self, end_frame, end_skeleton_frame=None): shapes = [] for idx, track in enumerate(self.objects): track_shapes = [] - for shape in TrackManager.get_interpolated_shapes(track, 0, end_frame): + for shape in TrackManager.get_interpolated_shapes(track, 0, end_frame, self._dimension): shape["label_id"] = track["label_id"] shape["group"] = track["group"] shape["track_id"] = idx @@ -375,7 +426,7 @@ def to_shapes(self, end_frame, end_skeleton_frame=None): track_shapes.append(shape) if len(track.get("elements", [])): - element_tracks = TrackManager(track["elements"]) + element_tracks = TrackManager(track["elements"], self._dimension) element_shapes = element_tracks.to_shapes(end_frame, end_skeleton_frame=track_shapes[-1]["frame"]) for i in range(len(element_shapes) // len(track_shapes)): @@ -404,14 +455,14 @@ def _get_cost_threshold(): return 0.5 @staticmethod - def _calc_objects_similarity(obj0, obj1, start_frame, overlap): + def _calc_objects_similarity(obj0, obj1, start_frame, overlap, dimension): if obj0["label_id"] == obj1["label_id"]: # Here start_frame is the start frame of next segment # and stop_frame is the stop frame of current segment # end_frame == stop_frame + 1 end_frame = start_frame + overlap - obj0_shapes = TrackManager.get_interpolated_shapes(obj0, start_frame, end_frame) - obj1_shapes = TrackManager.get_interpolated_shapes(obj1, start_frame, end_frame) + obj0_shapes = TrackManager.get_interpolated_shapes(obj0, start_frame, end_frame, dimension) + obj1_shapes = TrackManager.get_interpolated_shapes(obj1, start_frame, end_frame, dimension) obj0_shapes_by_frame = {shape["frame"]:shape for shape in obj0_shapes} obj1_shapes_by_frame = {shape["frame"]:shape for shape in obj1_shapes} assert obj0_shapes_by_frame and obj1_shapes_by_frame @@ -424,7 +475,7 @@ def _calc_objects_similarity(obj0, obj1, start_frame, overlap): if shape0["outside"] != shape1["outside"]: error += 1 else: - error += 1 - ShapeManager._calc_objects_similarity(shape0, shape1, start_frame, overlap) + error += 1 - ShapeManager._calc_objects_similarity(shape0, shape1, start_frame, overlap, dimension) count += 1 elif shape0 or shape1: error += 1 @@ -446,7 +497,7 @@ def _modify_unmached_object(self, obj, end_frame): self._modify_unmached_object(element, end_frame) @staticmethod - def get_interpolated_shapes(track, start_frame, end_frame): + def get_interpolated_shapes(track, start_frame, end_frame, dimension): def copy_shape(source, frame, points=None, rotation=None): copied = deepcopy(source) copied["keyframe"] = False @@ -483,6 +534,23 @@ def simple_interpolation(shape0, shape1): return shapes + def simple_3d_interpolation(shape0, shape1): + result = simple_interpolation(shape0, shape1) + angles = (shape0["points"][3:6] + shape1["points"][3:6]) + distance = shape1["frame"] - shape0["frame"] + + for shape in result: + offset = (shape["frame"] - shape0["frame"]) / distance + for i, angle0 in enumerate(angles): + if i < 3: + angle1 = angles[i + 3] + angle0 = (angle0 if angle0 >= 0 else angle0 + math.pi * 2) * 180 / math.pi + angle1 = (angle1 if angle1 >= 0 else angle1 + math.pi * 2) * 180 / math.pi + angle = angle0 + find_angle_diff(angle1, angle0) * offset * math.pi / 180 + shape["points"][i + 3] = angle if angle <= math.pi else angle - math.pi * 2 + + return result + def points_interpolation(shape0, shape1): if len(shape0["points"]) == 2 and len(shape1["points"]) == 2: return simple_interpolation(shape0, shape1) @@ -725,6 +793,8 @@ def interpolate(shape0, shape1): raise NotImplementedError() shapes = [] + if dimension == DimensionType.DIM_3D: + shapes = simple_3d_interpolation(shape0, shape1) if is_rectangle or is_cuboid or is_ellipse or is_skeleton: shapes = simple_interpolation(shape0, shape1) elif is_points: @@ -741,11 +811,15 @@ def interpolate(shape0, shape1): for shape in sorted(track["shapes"], key=lambda shape: shape["frame"]): curr_frame = shape["frame"] if end_frame <= curr_frame: - if not prev_shape: - shape["keyframe"] = True - shapes.append(shape) - prev_shape = shape - break + # if we exceed endframe, we still need to interpolate using the next keyframe + # but we keep the results only up to end_frame + interpolated = interpolate(prev_shape, deepcopy(shape)) + for shape in sorted(interpolated, key=lambda shape: shape["frame"]): + if shape["frame"] < end_frame: + shapes.append(shape) + else: + break + return shapes if prev_shape: assert shape["frame"] > prev_shape["frame"] @@ -760,6 +834,7 @@ def interpolate(shape0, shape1): prev_shape = shape if not prev_shape["outside"]: + # valid when the latest keyframe of a track less than end_frame and it is not outside, so, need to propagate shape = deepcopy(prev_shape) shape["frame"] = end_frame shapes.extend(interpolate(prev_shape, shape)) diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index c417a7c09100..090e18bee53a 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -1,6 +1,6 @@ # Copyright (C) 2019-2022 Intel Corporation -# Copyright (C) 2022 CVAT.ai Corporation +# Copyright (C) 2022-2023 CVAT.ai Corporation # # SPDX-License-Identifier: MIT @@ -180,6 +180,7 @@ class CommonData(InstanceLabelData): Labels = namedtuple('Label', 'id, name, color, type') def __init__(self, annotation_ir, db_task, host='', create_callback=None) -> None: + self._dimension = annotation_ir.dimension self._annotation_ir = annotation_ir self._host = host self._create_callback = create_callback @@ -332,7 +333,7 @@ def _export_tag(self, tag): def _export_track(self, track, idx): track['shapes'] = list(filter(lambda x: not self._is_frame_deleted(x['frame']), track['shapes'])) tracked_shapes = TrackManager.get_interpolated_shapes( - track, 0, len(self)) + track, 0, len(self), self._annotation_ir.dimension) for tracked_shape in tracked_shapes: tracked_shape["attributes"] += track["attributes"] tracked_shape["track_id"] = idx @@ -384,9 +385,9 @@ def get_frame(idx): get_frame(idx) anno_manager = AnnotationManager(self._annotation_ir) - shape_data = '' - for shape in sorted(anno_manager.to_shapes(len(self)), + for shape in sorted(anno_manager.to_shapes(len(self), self._annotation_ir.dimension), key=lambda shape: shape.get("z_order", 0)): + shape_data = '' if shape['frame'] not in self._frame_info or self._is_frame_deleted(shape['frame']): # After interpolation there can be a finishing frame # outside of the task boundaries. Filter it out to avoid errors. @@ -1013,7 +1014,7 @@ def _export_tag(self, tag: dict, task_id: int): def _export_track(self, track: dict, task_id: int, task_size: int, idx: int): track['shapes'] = list(filter(lambda x: (task_id, x['frame']) not in self._deleted_frames, track['shapes'])) tracked_shapes = TrackManager.get_interpolated_shapes( - track, 0, task_size + track, 0, task_size, self._annotation_irs[task_id].dimension ) for tracked_shape in tracked_shapes: tracked_shape["attributes"] += track["attributes"] @@ -1060,7 +1061,7 @@ def get_frame(task_id: int, idx: int) -> ProjectData.Frame: for task in self._db_tasks.values(): anno_manager = AnnotationManager(self._annotation_irs[task.id]) - for shape in sorted(anno_manager.to_shapes(task.data.size), + for shape in sorted(anno_manager.to_shapes(task.data.size, self._annotation_irs[task.id].dimension), key=lambda shape: shape.get("z_order", 0)): if (task.id, shape['frame']) not in self._frame_info or (task.id, shape['frame']) in self._deleted_frames: continue @@ -1534,11 +1535,7 @@ def convert_attrs(label, cvat_attrs): attributes=anno_attr, group=anno_group) item_anno.append(anno) - shapes = [] - if hasattr(cvat_frame_anno, 'shapes'): - for shape in cvat_frame_anno.shapes: - shapes.append({"id": shape.id, "label_id": shape.label_id}) - + num_of_tracks = reduce(lambda a, x: a + (1 if getattr(x, 'track_id', None) is not None else 0), cvat_frame_anno.labeled_shapes, 0) for index, shape_obj in enumerate(cvat_frame_anno.labeled_shapes): anno_group = shape_obj.group or 0 anno_label = map_label(shape_obj.label) @@ -1592,10 +1589,9 @@ def convert_attrs(label, cvat_attrs): z_order=shape_obj.z_order) elif shape_obj.type == ShapeType.CUBOID: if dimension == DimensionType.DIM_3D: - if format_name == "sly_pointcloud": - anno_id = shapes[index]["id"] - else: - anno_id = index + anno_id = getattr(shape_obj, 'track_id', None) + if anno_id is None: + anno_id = num_of_tracks + index position, rotation, scale = anno_points[0:3], anno_points[3:6], anno_points[6:9] anno = dm.Cuboid3d( id=anno_id, position=position, rotation=rotation, scale=scale, @@ -1759,7 +1755,7 @@ def reduce_fn(acc, v): if ann.attributes.get('source', '').lower() in {'auto', 'manual'} else 'manual' shape_type = shapes[ann.type] - if track_id is None or dm_dataset.format != 'cvat' : + if track_id is None or 'keyframe' not in ann.attributes or dm_dataset.format not in ['cvat', 'datumaro', 'sly_pointcloud']: elements = [] if ann.type == dm.AnnotationType.skeleton: for element in ann.elements: diff --git a/cvat/apps/dataset_manager/formats/pointcloud.py b/cvat/apps/dataset_manager/formats/pointcloud.py index 5be57e596b15..6eb9cfa4cfb9 100644 --- a/cvat/apps/dataset_manager/formats/pointcloud.py +++ b/cvat/apps/dataset_manager/formats/pointcloud.py @@ -21,7 +21,7 @@ def _export_images(dst_file, temp_dir, task_data, save_images=False): task_data, include_images=save_images, format_type='sly_pointcloud', dimension=DimensionType.DIM_3D), env=dm_env) - dataset.export(temp_dir, 'sly_pointcloud', save_images=save_images) + dataset.export(temp_dir, 'sly_pointcloud', save_images=save_images, allow_undeclared_attrs=True) make_zip_archive(temp_dir, dst_file) diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index 1c917b3e75b9..77156e3894f1 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -6,6 +6,7 @@ import zipfile from datumaro.components.dataset import Dataset +from datumaro.components.extractor import ItemTransform from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ import_dm_annotations @@ -16,13 +17,20 @@ from .registry import exporter, importer +class RemoveTrackingInformation(ItemTransform): + def transform_item(self, item): + annotations = list(item.annotations) + for anno in annotations: + if hasattr(anno, 'attributes') and 'track_id' in anno.attributes: + del anno.attributes['track_id'] + return item.wrap(annotations=annotations) @exporter(name='Kitti Raw Format', ext='ZIP', version='1.0', dimension=DimensionType.DIM_3D) def _export_images(dst_file, temp_dir, task_data, save_images=False): dataset = Dataset.from_extractors(GetCVATDataExtractor( task_data, include_images=save_images, format_type="kitti_raw", dimension=DimensionType.DIM_3D), env=dm_env) - + dataset.transform(RemoveTrackingInformation) dataset.export(temp_dir, 'kitti_raw', save_images=save_images, reindex=True) make_zip_archive(temp_dir, dst_file) diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py index 66acbf17c30c..f85005ff4cd6 100644 --- a/cvat/apps/dataset_manager/task.py +++ b/cvat/apps/dataset_manager/task.py @@ -87,7 +87,7 @@ def __init__(self, pk, is_prefetched=False): db_segment = self.db_job.segment self.start_frame = db_segment.start_frame self.stop_frame = db_segment.stop_frame - self.ir_data = AnnotationIR() + self.ir_data = AnnotationIR(db_segment.task.dimension) self.db_labels = {db_label.id:db_label for db_label in (db_segment.task.project.label_set.all() @@ -576,7 +576,7 @@ def export(self, dst_file, exporter, host='', **options): def import_annotations(self, src_file, importer, **options): job_data = JobData( - annotation_ir=AnnotationIR(), + annotation_ir=AnnotationIR(self.db_job.segment.task.dimension), db_job=self.db_job, create_callback=self.create, ) @@ -597,13 +597,13 @@ def __init__(self, pk): # Postgres doesn't guarantee an order by default without explicit order_by self.db_jobs = models.Job.objects.select_related("segment").filter(segment__task_id=pk).order_by('id') - self.ir_data = AnnotationIR() + self.ir_data = AnnotationIR(self.db_task.dimension) def reset(self): self.ir_data.reset() def _patch_data(self, data, action): - _data = data if isinstance(data, AnnotationIR) else AnnotationIR(data) + _data = data if isinstance(data, AnnotationIR) else AnnotationIR(self.db_task.dimension, data) splitted_data = {} jobs = {} for db_job in self.db_jobs: @@ -614,18 +614,18 @@ def _patch_data(self, data, action): splitted_data[jid] = _data.slice(start, stop) for jid, job_data in splitted_data.items(): - _data = AnnotationIR() + _data = AnnotationIR(self.db_task.dimension) if action is None: _data.data = put_job_data(jid, job_data) else: _data.data = patch_job_data(jid, job_data, action) if _data.version > self.ir_data.version: self.ir_data.version = _data.version - self._merge_data(_data, jobs[jid]["start"], self.db_task.overlap) + self._merge_data(_data, jobs[jid]["start"], self.db_task.overlap, self.db_task.dimension) - def _merge_data(self, data, start_frame, overlap): + def _merge_data(self, data, start_frame, overlap, dimension): annotation_manager = AnnotationManager(self.ir_data) - annotation_manager.merge(data, start_frame, overlap) + annotation_manager.merge(data, start_frame, overlap, dimension) def put(self, data): self._patch_data(data, None) @@ -654,7 +654,8 @@ def init_from_db(self): db_segment = db_job.segment start_frame = db_segment.start_frame overlap = self.db_task.overlap - self._merge_data(annotation.ir_data, start_frame, overlap) + dimension = self.db_task.dimension + self._merge_data(annotation.ir_data, start_frame, overlap, dimension) def export(self, dst_file, exporter, host='', **options): task_data = TaskData( @@ -670,7 +671,7 @@ def export(self, dst_file, exporter, host='', **options): def import_annotations(self, src_file, importer, **options): task_data = TaskData( - annotation_ir=AnnotationIR(), + annotation_ir=AnnotationIR(self.db_task.dimension), db_task=self.db_task, create_callback=self.create, ) diff --git a/cvat/apps/dataset_manager/tests/test_annotation.py b/cvat/apps/dataset_manager/tests/test_annotation.py index 0a26a89e7d00..644ea9d4cc32 100644 --- a/cvat/apps/dataset_manager/tests/test_annotation.py +++ b/cvat/apps/dataset_manager/tests/test_annotation.py @@ -9,7 +9,7 @@ class TrackManagerTest(TestCase): def _check_interpolation(self, track): - interpolated = TrackManager.get_interpolated_shapes(track, 0, 7) + interpolated = TrackManager.get_interpolated_shapes(track, 0, 7, '2d') self.assertEqual(len(interpolated), 6) self.assertTrue(interpolated[0]["keyframe"]) @@ -254,7 +254,7 @@ def test_outside_bbox_interpolation(self): } ] - interpolated_shapes = TrackManager.get_interpolated_shapes(track, 0, 5) + interpolated_shapes = TrackManager.get_interpolated_shapes(track, 0, 5, '2d') self.assertEqual(expected_shapes, interpolated_shapes) def test_outside_polygon_interpolation(self): @@ -314,5 +314,5 @@ def test_outside_polygon_interpolation(self): } ] - interpolated_shapes = TrackManager.get_interpolated_shapes(track, 0, 3) + interpolated_shapes = TrackManager.get_interpolated_shapes(track, 0, 3, '2d') self.assertEqual(expected_shapes, interpolated_shapes) diff --git a/cvat/apps/dataset_manager/tests/test_formats.py b/cvat/apps/dataset_manager/tests/test_formats.py index 302991ef0a9c..5ee54d6cd2cc 100644 --- a/cvat/apps/dataset_manager/tests/test_formats.py +++ b/cvat/apps/dataset_manager/tests/test_formats.py @@ -440,7 +440,7 @@ def test_cant_make_rel_frame_id_from_unknown(self): images = self._generate_task_images(3) images['frame_filter'] = 'step=2' task = self._generate_task(images) - task_data = TaskData(AnnotationIR(), Task.objects.get(pk=task['id'])) + task_data = TaskData(AnnotationIR('2d'), Task.objects.get(pk=task['id']),) with self.assertRaisesRegex(ValueError, r'Unknown'): task_data.rel_frame_id(1) # the task has only 0 and 2 frames @@ -450,7 +450,7 @@ def test_can_make_rel_frame_id_from_known(self): images['frame_filter'] = 'step=2' images['start_frame'] = 1 task = self._generate_task(images) - task_data = TaskData(AnnotationIR(), Task.objects.get(pk=task['id'])) + task_data = TaskData(AnnotationIR('2d'), Task.objects.get(pk=task['id'])) self.assertEqual(2, task_data.rel_frame_id(5)) @@ -458,7 +458,7 @@ def test_cant_make_abs_frame_id_from_unknown(self): images = self._generate_task_images(3) images['frame_filter'] = 'step=2' task = self._generate_task(images) - task_data = TaskData(AnnotationIR(), Task.objects.get(pk=task['id'])) + task_data = TaskData(AnnotationIR('2d'), Task.objects.get(pk=task['id'])) with self.assertRaisesRegex(ValueError, r'Unknown'): task_data.abs_frame_id(2) # the task has only 0 and 1 indices @@ -468,7 +468,7 @@ def test_can_make_abs_frame_id_from_known(self): images['frame_filter'] = 'step=2' images['start_frame'] = 1 task = self._generate_task(images) - task_data = TaskData(AnnotationIR(), Task.objects.get(pk=task['id'])) + task_data = TaskData(AnnotationIR('2d'), Task.objects.get(pk=task['id'])) self.assertEqual(5, task_data.abs_frame_id(2)) @@ -569,7 +569,7 @@ def test_frame_matching(self): images = self._generate_task_images(task_paths) task = self._generate_task(images) - task_data = TaskData(AnnotationIR(), Task.objects.get(pk=task["id"])) + task_data = TaskData(AnnotationIR('2d'), Task.objects.get(pk=task["id"])) for input_path, expected, root in [ ('z.jpg', None, ''), # unknown item @@ -594,7 +594,7 @@ def test_dataset_root(self): with self.subTest(expected=expected): images = self._generate_task_images(task_paths) task = self._generate_task(images) - task_data = TaskData(AnnotationIR(), + task_data = TaskData(AnnotationIR('2d'), Task.objects.get(pk=task["id"])) dataset = [ datumaro.components.extractor.DatasetItem( diff --git a/tests/cypress/integration/canvas3d_functionality/case_85_canvas3d_functionality_cuboid_cancel_drawing.js b/tests/cypress/integration/canvas3d_functionality/case_85_canvas3d_functionality_cuboid_cancel_drawing.js index ee255fdc66ee..f9a6f45f7c9d 100644 --- a/tests/cypress/integration/canvas3d_functionality/case_85_canvas3d_functionality_cuboid_cancel_drawing.js +++ b/tests/cypress/integration/canvas3d_functionality/case_85_canvas3d_functionality_cuboid_cancel_drawing.js @@ -14,6 +14,8 @@ context('Canvas 3D functionality. Cancel drawing.', () => { before(() => { cy.openTask(taskName); cy.openJob(); + + // eslint-disable-next-line cypress/no-unnecessary-waiting cy.wait(1000); // Waiting for the point cloud to display }); @@ -26,7 +28,7 @@ context('Canvas 3D functionality. Cancel drawing.', () => { .within(() => { cy.contains(new RegExp(`^${labelName}$`)).click(); }); - cy.get('.cvat-draw-cuboid-popover').find('button').click(); + cy.get('.cvat-draw-cuboid-popover').contains('Shape').click(); cy.get('.cvat-canvas3d-perspective').trigger('mousemove'); cy.customScreenshot('.cvat-canvas3d-perspective', 'canvas3d_perspective_drawning'); cy.get('body').type('{Esc}'); @@ -38,8 +40,7 @@ context('Canvas 3D functionality. Cancel drawing.', () => { ); }); - // Temporarily disabling the test until it is fixed https://github.com/openvinotoolkit/cvat/issues/3438#issuecomment-892432089 - it.skip('Repeat draw.', () => { + it('Repeat draw.', () => { cy.get('body').type('n'); cy.get('.cvat-canvas3d-perspective').trigger('mousemove'); cy.get('.cvat-canvas3d-perspective').trigger('mousemove', 450, 250).dblclick(450, 250); diff --git a/tests/cypress/integration/canvas3d_functionality_2/case_56_canvas3d_functionality_basic_actions.js b/tests/cypress/integration/canvas3d_functionality_2/case_56_canvas3d_functionality_basic_actions.js index 349c0575ed47..49e3a08c8655 100644 --- a/tests/cypress/integration/canvas3d_functionality_2/case_56_canvas3d_functionality_basic_actions.js +++ b/tests/cypress/integration/canvas3d_functionality_2/case_56_canvas3d_functionality_basic_actions.js @@ -137,7 +137,7 @@ context('Canvas 3D functionality. Basic actions.', () => { cy.get('.cvat-canvas-controls-sidebar') .find('[role="img"]') .then(($controlButtons) => { - expect($controlButtons.length).to.be.equal(4); + expect($controlButtons.length).to.be.equal(6); }); cy.get('.cvat-canvas-controls-sidebar') .should('exist') diff --git a/tests/cypress/support/commands_canvas3d.js b/tests/cypress/support/commands_canvas3d.js index e94734696783..7df6e5802c06 100644 --- a/tests/cypress/support/commands_canvas3d.js +++ b/tests/cypress/support/commands_canvas3d.js @@ -20,7 +20,7 @@ Cypress.Commands.add('compareImagesAndCheckResult', (baseImage, afterImage, noCh Cypress.Commands.add('create3DCuboid', (cuboidCreationParams) => { cy.interactControlButton('draw-cuboid'); cy.switchLabel(cuboidCreationParams.labelName, 'draw-cuboid'); - cy.get('.cvat-draw-cuboid-popover').find('button').click(); + cy.get('.cvat-draw-cuboid-popover').contains('Shape').click(); cy.get('.cvat-canvas3d-perspective') .trigger('mousemove', cuboidCreationParams.x, cuboidCreationParams.y) .dblclick(cuboidCreationParams.x, cuboidCreationParams.y);