From f1aee89589f32d80cd29e445e9cd7a92cba78715 Mon Sep 17 00:00:00 2001 From: Boris Sekachev <40690378+bsekachev@users.noreply.github.com> Date: Fri, 17 Apr 2020 18:52:11 +0300 Subject: [PATCH] React UI: ReID algorithm (#1406) * Initial commit * Connected storage * Added core API method * Done implementation * Removed rule * Removed double cancel * Updated changelog * Fixed: Cannot read property toFixed of undefined * Update CHANGELOG.md --- CHANGELOG.md | 5 +- cvat-core/src/annotations.js | 28 +++ cvat-core/src/session.js | 60 +++++ cvat-ui/src/actions/plugins-actions.ts | 6 +- .../top-bar/annotation-menu.tsx | 4 + .../annotation-page/top-bar/reid-plugin.tsx | 227 ++++++++++++++++++ .../top-bar/annotation-menu.tsx | 7 + cvat-ui/src/reducers/interfaces.ts | 1 + cvat-ui/src/reducers/plugins-reducer.ts | 1 + cvat-ui/src/utils/dextr-utils.ts | 2 - cvat-ui/src/utils/plugin-checker.ts | 3 + cvat-ui/src/utils/reid-utils.ts | 96 ++++++++ cvat/apps/dextr_segmentation/dextr.py | 2 +- cvat/apps/dextr_segmentation/urls.py | 2 +- cvat/apps/reid/urls.py | 3 +- cvat/apps/reid/views.py | 3 + 16 files changed, 441 insertions(+), 9 deletions(-) create mode 100644 cvat-ui/src/components/annotation-page/top-bar/reid-plugin.tsx create mode 100644 cvat-ui/src/utils/reid-utils.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6a09a6aef7..56206927ceb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.0.0-beta.2] - Unreleased ### Added -- +- Re-Identification algorithm to merging bounding boxes automatically to the new UI (https://github.com/opencv/cvat/pull/1406) +- Methods ``import`` and ``export`` to import/export raw annotations for Job and Task in ``cvat-core`` (https://github.com/opencv/cvat/pull/1406) ### Changed - @@ -34,7 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to create one tracked point (https://github.com/opencv/cvat/pull/1383) - Ability to draw/edit polygons and polylines with automatic bordering feature (https://github.com/opencv/cvat/pull/1394) - Tutorial: instructions for CVAT over HTTPS -- Added deep extreme cut (semi-automatic segmentation) to the new UI (https://github.com/opencv/cvat/pull/1398) +- Deep extreme cut (semi-automatic segmentation) to the new UI (https://github.com/opencv/cvat/pull/1398) ### Changed - Increase preview size of a task till 256, 256 on the server diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 3ee70d3cb483..608faf1eee96 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -247,6 +247,32 @@ return result; } + function importAnnotations(session, data) { + const sessionType = session instanceof Task ? 'task' : 'job'; + const cache = getCache(sessionType); + + if (cache.has(session)) { + return cache.get(session).collection.import(data); + } + + throw new DataError( + 'Collection has not been initialized yet. Call annotations.get() or annotations.clear(true) before', + ); + } + + function exportAnnotations(session) { + const sessionType = session instanceof Task ? 'task' : 'job'; + const cache = getCache(sessionType); + + if (cache.has(session)) { + return cache.get(session).collection.export(); + } + + throw new DataError( + 'Collection has not been initialized yet. Call annotations.get() or annotations.clear(true) before', + ); + } + async function exportDataset(session, format) { if (!(format instanceof String || typeof format === 'string')) { throw new ArgumentError( @@ -332,6 +358,8 @@ selectObject, uploadAnnotations, dumpAnnotations, + importAnnotations, + exportAnnotations, exportDataset, undoActions, redoActions, diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index 0ae7fb1ab461..9643cb1bf173 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -97,6 +97,18 @@ return result; }, + async import(data) { + const result = await PluginRegistry + .apiWrapper.call(this, prototype.annotations.import, data); + return result; + }, + + async export() { + const result = await PluginRegistry + .apiWrapper.call(this, prototype.annotations.export); + return result; + }, + async exportDataset(format) { const result = await PluginRegistry .apiWrapper.call(this, prototype.annotations.exportDataset, format); @@ -391,6 +403,28 @@ * @throws {module:API.cvat.exceptions.PluginError} * @instance */ + /** + * + * Import raw data in a collection + * @method import + * @memberof Session.annotations + * @param {Object} data + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ArgumentError} + * @instance + * @async + */ + /** + * + * Export a collection as a row data + * @method export + * @memberof Session.annotations + * @returns {Object} data + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ArgumentError} + * @instance + * @async + */ /** * Export as a dataset. * Method builds a dataset in the specified format. @@ -695,6 +729,8 @@ search: Object.getPrototypeOf(this).annotations.search.bind(this), upload: Object.getPrototypeOf(this).annotations.upload.bind(this), select: Object.getPrototypeOf(this).annotations.select.bind(this), + import: Object.getPrototypeOf(this).annotations.import.bind(this), + export: Object.getPrototypeOf(this).annotations.export.bind(this), statistics: Object.getPrototypeOf(this).annotations.statistics.bind(this), hasUnsavedChanges: Object.getPrototypeOf(this) .annotations.hasUnsavedChanges.bind(this), @@ -1245,6 +1281,8 @@ search: Object.getPrototypeOf(this).annotations.search.bind(this), upload: Object.getPrototypeOf(this).annotations.upload.bind(this), select: Object.getPrototypeOf(this).annotations.select.bind(this), + import: Object.getPrototypeOf(this).annotations.import.bind(this), + export: Object.getPrototypeOf(this).annotations.export.bind(this), statistics: Object.getPrototypeOf(this).annotations.statistics.bind(this), hasUnsavedChanges: Object.getPrototypeOf(this) .annotations.hasUnsavedChanges.bind(this), @@ -1326,6 +1364,8 @@ annotationsStatistics, uploadAnnotations, dumpAnnotations, + importAnnotations, + exportAnnotations, exportDataset, undoActions, redoActions, @@ -1490,6 +1530,16 @@ return result; }; + Job.prototype.annotations.import.implementation = function (data) { + const result = importAnnotations(this, data); + return result; + }; + + Job.prototype.annotations.export.implementation = function () { + const result = exportAnnotations(this); + return result; + }; + Job.prototype.annotations.dump.implementation = async function (name, dumper) { const result = await dumpAnnotations(this, name, dumper); return result; @@ -1739,6 +1789,16 @@ return result; }; + Task.prototype.annotations.import.implementation = function (data) { + const result = importAnnotations(this, data); + return result; + }; + + Task.prototype.annotations.export.implementation = function () { + const result = exportAnnotations(this); + return result; + }; + Task.prototype.annotations.exportDataset.implementation = async function (format) { const result = await exportDataset(this, format); return result; diff --git a/cvat-ui/src/actions/plugins-actions.ts b/cvat-ui/src/actions/plugins-actions.ts index 361756837711..510f6590e827 100644 --- a/cvat-ui/src/actions/plugins-actions.ts +++ b/cvat-ui/src/actions/plugins-actions.ts @@ -33,6 +33,7 @@ export function checkPluginsAsync(): ThunkAction { GIT_INTEGRATION: false, TF_ANNOTATION: false, TF_SEGMENTATION: false, + REID: false, DEXTR_SEGMENTATION: false, }; @@ -43,11 +44,12 @@ export function checkPluginsAsync(): ThunkAction { PluginChecker.check(SupportedPlugins.TF_ANNOTATION), PluginChecker.check(SupportedPlugins.TF_SEGMENTATION), PluginChecker.check(SupportedPlugins.DEXTR_SEGMENTATION), + PluginChecker.check(SupportedPlugins.REID), ]; const values = await Promise.all(promises); - [plugins.ANALYTICS, plugins.AUTO_ANNOTATION, plugins.GIT_INTEGRATION, - plugins.TF_ANNOTATION, plugins.TF_SEGMENTATION, plugins.DEXTR_SEGMENTATION] = values; + [plugins.ANALYTICS, plugins.AUTO_ANNOTATION, plugins.GIT_INTEGRATION, plugins.TF_ANNOTATION, + plugins.TF_SEGMENTATION, plugins.DEXTR_SEGMENTATION, plugins.REID] = values; dispatch(pluginActions.checkedAllPlugins(plugins)); }; } diff --git a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx index 85d057a6f126..76a0cf98e88f 100644 --- a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx @@ -9,6 +9,7 @@ import Modal from 'antd/lib/modal'; import DumpSubmenu from 'components/actions-menu/dump-submenu'; import LoadSubmenu from 'components/actions-menu/load-submenu'; import ExportSubmenu from 'components/actions-menu/export-submenu'; +import ReIDPlugin from './reid-plugin'; interface Props { taskMode: string; @@ -18,6 +19,7 @@ interface Props { loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; + installedReID: boolean; onClickMenu(params: ClickParam, file?: File): void; } @@ -39,6 +41,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { loadActivity, dumpActivities, exportActivities, + installedReID, } = props; let latestParams: ClickParam | null = null; @@ -120,6 +123,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element { Open the task + { installedReID && } ); } diff --git a/cvat-ui/src/components/annotation-page/top-bar/reid-plugin.tsx b/cvat-ui/src/components/annotation-page/top-bar/reid-plugin.tsx new file mode 100644 index 000000000000..d413df158f26 --- /dev/null +++ b/cvat-ui/src/components/annotation-page/top-bar/reid-plugin.tsx @@ -0,0 +1,227 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import ReactDOM from 'react-dom'; +import React, { useState, useEffect } from 'react'; +import { Row, Col } from 'antd/lib/grid'; +import Modal from 'antd/lib/modal'; +import Menu from 'antd/lib/menu'; +import Text from 'antd/lib/typography/Text'; +import InputNumber from 'antd/lib/input-number'; +import Tooltip from 'antd/lib/tooltip'; + +import { clamp } from 'utils/math'; +import { run, cancel } from 'utils/reid-utils'; +import { connect } from 'react-redux'; +import { CombinedState } from 'reducers/interfaces'; +import { fetchAnnotationsAsync } from 'actions/annotation-actions'; + +interface InputModalProps { + visible: boolean; + onCancel(): void; + onSubmit(threshold: number, distance: number): void; +} + +function InputModal(props: InputModalProps): JSX.Element { + const { visible, onCancel, onSubmit } = props; + const [threshold, setThreshold] = useState(0.5); + const [distance, setDistance] = useState(50); + + const [thresholdMin, thresholdMax] = [0.05, 0.95]; + const [distanceMin, distanceMax] = [1, 1000]; + return ( + onSubmit(threshold, distance)} + okText='Merge' + > + + + + Similarity threshold: + + + + { + if (typeof (value) === 'number') { + setThreshold(clamp(value, thresholdMin, thresholdMax)); + } + }} + /> + + + + + + Max pixel distance: + + + + { + if (typeof (value) === 'number') { + setDistance(clamp(value, distanceMin, distanceMax)); + } + }} + /> + + + + ); +} + +interface InProgressDialogProps { + visible: boolean; + progress: number; + onCancel(): void; +} + +function InProgressDialog(props: InProgressDialogProps): JSX.Element { + const { visible, onCancel, progress } = props; + return ( + + {`Merging is in progress ${progress}%`} + + ); +} + +const reidContainer = window.document.createElement('div'); +reidContainer.setAttribute('id', 'cvat-reid-wrapper'); +window.document.body.appendChild(reidContainer); + + +interface StateToProps { + jobInstance: any | null; +} + +interface DispatchToProps { + updateAnnotations(): void; +} + +function mapStateToProps(state: CombinedState): StateToProps { + const { + annotation: { + job: { + instance: jobInstance, + }, + }, + } = state; + + return { + jobInstance, + }; +} + +function mapDispatchToProps(dispatch: any): DispatchToProps { + return { + updateAnnotations(): void { + dispatch(fetchAnnotationsAsync()); + }, + }; +} + + +function ReIDPlugin(props: StateToProps & DispatchToProps): JSX.Element { + const { jobInstance, updateAnnotations, ...rest } = props; + const [showInputDialog, setShowInputDialog] = useState(false); + const [showInProgressDialog, setShowInProgressDialog] = useState(false); + const [progress, setProgress] = useState(0); + + useEffect(() => { + ReactDOM.render(( + <> + { + cancel(jobInstance.id); + }} + /> + setShowInputDialog(false)} + onSubmit={async (threshold: number, distance: number) => { + setProgress(0); + setShowInputDialog(false); + setShowInProgressDialog(true); + + const onUpdatePercentage = (percent: number): void => { + setProgress(percent); + }; + + try { + const annotations = await jobInstance.annotations.export(); + const merged = await run({ + threshold, + distance, + onUpdatePercentage, + jobID: jobInstance.id, + annotations, + }); + await jobInstance.annotations.clear(); + updateAnnotations(); // one more call to do not confuse canvas + await jobInstance.annotations.import(merged); + updateAnnotations(); + } catch (error) { + Modal.error({ + title: 'Could not merge annotations', + content: error.toString(), + }); + } finally { + setShowInProgressDialog(false); + } + }} + /> + + ), reidContainer); + }); + + return ( + { + if (jobInstance) { + setShowInputDialog(true); + } + }} + > + Run ReID merge + + ); +} + +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ReIDPlugin); diff --git a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx index c4d0a35bc8fb..8c73137d46f0 100644 --- a/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx +++ b/cvat-ui/src/containers/annotation-page/top-bar/annotation-menu.tsx @@ -27,6 +27,7 @@ interface StateToProps { loadActivity: string | null; dumpActivities: string[] | null; exportActivities: string[] | null; + installedReID: boolean; } interface DispatchToProps { @@ -57,6 +58,9 @@ function mapStateToProps(state: CombinedState): StateToProps { exports: activeExports, }, }, + plugins: { + list, + }, } = state; const taskID = jobInstance.task.id; @@ -70,6 +74,7 @@ function mapStateToProps(state: CombinedState): StateToProps { jobInstance, annotationFormats, exporters, + installedReID: list.REID, }; } @@ -105,6 +110,7 @@ function AnnotationMenuContainer(props: Props): JSX.Element { loadActivity, dumpActivities, exportActivities, + installedReID, } = props; const loaders = annotationFormats @@ -157,6 +163,7 @@ function AnnotationMenuContainer(props: Props): JSX.Element { loadActivity={loadActivity} dumpActivities={dumpActivities} exportActivities={exportActivities} + installedReID={installedReID} onClickMenu={onClickMenu} /> ); diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 27f7f93d9f72..237426bca0d3 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -77,6 +77,7 @@ export enum SupportedPlugins { TF_SEGMENTATION = 'TF_SEGMENTATION', DEXTR_SEGMENTATION = 'DEXTR_SEGMENTATION', ANALYTICS = 'ANALYTICS', + REID = 'REID', } export interface PluginsState { diff --git a/cvat-ui/src/reducers/plugins-reducer.ts b/cvat-ui/src/reducers/plugins-reducer.ts index 334c51527dc0..0bdc6fd2fe0a 100644 --- a/cvat-ui/src/reducers/plugins-reducer.ts +++ b/cvat-ui/src/reducers/plugins-reducer.ts @@ -19,6 +19,7 @@ const defaultState: PluginsState = { TF_SEGMENTATION: false, DEXTR_SEGMENTATION: false, ANALYTICS: false, + REID: false, }, }; diff --git a/cvat-ui/src/utils/dextr-utils.ts b/cvat-ui/src/utils/dextr-utils.ts index 2a0b1e03ec80..54189519158f 100644 --- a/cvat-ui/src/utils/dextr-utils.ts +++ b/cvat-ui/src/utils/dextr-utils.ts @@ -143,8 +143,6 @@ function serverRequest( reject(error); }); }); - - // start checking } const plugin: DEXTRPlugin = { diff --git a/cvat-ui/src/utils/plugin-checker.ts b/cvat-ui/src/utils/plugin-checker.ts index 93b4768f55a0..530a97367b32 100644 --- a/cvat-ui/src/utils/plugin-checker.ts +++ b/cvat-ui/src/utils/plugin-checker.ts @@ -41,6 +41,9 @@ class PluginChecker { case SupportedPlugins.ANALYTICS: { return isReachable(`${serverHost}/analytics/app/kibana`, 'GET'); } + case SupportedPlugins.REID: { + return isReachable(`${serverHost}/reid/enabled`, 'GET'); + } default: return false; } diff --git a/cvat-ui/src/utils/reid-utils.ts b/cvat-ui/src/utils/reid-utils.ts new file mode 100644 index 000000000000..181d566c92d1 --- /dev/null +++ b/cvat-ui/src/utils/reid-utils.ts @@ -0,0 +1,96 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import getCore from 'cvat-core'; +import { ShapeType, RQStatus } from 'reducers/interfaces'; + + +const core = getCore(); +const baseURL = core.config.backendAPI.slice(0, -7); + +type Params = { + threshold: number; + distance: number; + onUpdatePercentage(percentage: number): void; + jobID: number; + annotations: any; +}; + +export function run(params: Params): Promise { + return new Promise((resolve, reject) => { + const { + threshold, + distance, + onUpdatePercentage, + jobID, + annotations, + } = params; + const { shapes, ...rest } = annotations; + + const boxes = shapes.filter((shape: any): boolean => shape.type === ShapeType.RECTANGLE); + const others = shapes.filter((shape: any): boolean => shape.type !== ShapeType.RECTANGLE); + + core.server.request( + `${baseURL}/reid/start/job/${params.jobID}`, { + method: 'POST', + data: JSON.stringify({ + threshold, + maxDistance: distance, + boxes, + }), + headers: { + 'Content-Type': 'application/json', + }, + }, + ).then(() => { + const timeoutCallback = (): void => { + core.server.request( + `${baseURL}/reid/check/${jobID}`, { + method: 'GET', + }, + ).then((response: any) => { + const { status } = response; + if (status === RQStatus.finished) { + if (!response.result) { + // cancelled + resolve(annotations); + } + + const result = JSON.parse(response.result); + const collection = rest; + Array.prototype.push.apply(collection.tracks, result); + collection.shapes = others; + resolve(collection); + } else if (status === RQStatus.started) { + const { progress } = response; + if (typeof (progress) === 'number') { + onUpdatePercentage(+progress.toFixed(2)); + } + setTimeout(timeoutCallback, 1000); + } else if (status === RQStatus.failed) { + reject(new Error(response.stderr)); + } else if (status === RQStatus.unknown) { + reject(new Error('Unknown REID status has been received')); + } else { + setTimeout(timeoutCallback, 1000); + } + }).catch((error: Error) => { + reject(error); + }); + }; + + setTimeout(timeoutCallback, 1000); + }).catch((error: Error) => { + reject(error); + }); + }); +} + +export function cancel(jobID: number): void { + core.server.request( + `${baseURL}/reid/cancel/${jobID}`, { + method: 'GET', + }, + ); +} diff --git a/cvat/apps/dextr_segmentation/dextr.py b/cvat/apps/dextr_segmentation/dextr.py index 9bb3f3ba3112..d6eb20022248 100644 --- a/cvat/apps/dextr_segmentation/dextr.py +++ b/cvat/apps/dextr_segmentation/dextr.py @@ -1,5 +1,5 @@ -# Copyright (C) 2018 Intel Corporation +# Copyright (C) 2018-2020 Intel Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/dextr_segmentation/urls.py b/cvat/apps/dextr_segmentation/urls.py index 11a92983ef38..6b3120b67939 100644 --- a/cvat/apps/dextr_segmentation/urls.py +++ b/cvat/apps/dextr_segmentation/urls.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Intel Corporation +# Copyright (C) 2018-2020 Intel Corporation # # SPDX-License-Identifier: MIT diff --git a/cvat/apps/reid/urls.py b/cvat/apps/reid/urls.py index 4decc5cf7467..431b11192ec7 100644 --- a/cvat/apps/reid/urls.py +++ b/cvat/apps/reid/urls.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Intel Corporation +# Copyright (C) 2018-2020 Intel Corporation # # SPDX-License-Identifier: MIT @@ -9,4 +9,5 @@ path('start/job/', views.start), path('cancel/', views.cancel), path('check/', views.check), + path('enabled', views.enabled), ] diff --git a/cvat/apps/reid/views.py b/cvat/apps/reid/views.py index d0753e46d664..100151ee9d1b 100644 --- a/cvat/apps/reid/views.py +++ b/cvat/apps/reid/views.py @@ -94,3 +94,6 @@ def cancel(request, jid): return HttpResponseBadRequest(str(e)) return HttpResponse() + +def enabled(request): + return HttpResponse()