Fixed two issues with GT annotations (#7299)

changelog.d/20231228_131558_boris_fixed_couple_of_gt_bugs.md

@@ -0,0 +1,7 @@
+### Fixed
+
+- UI crashes if user highligts conflict related to annotations hidden by a filter
+  (<https://github.com/opencv/cvat/pull/7299>)
+- Annotations conflicts are not highligted properly on the first frame of a job
+  (<https://github.com/opencv/cvat/pull/7299>)
+

cvat-canvas/src/typescript/canvas.ts

@@ -109,7 +109,7 @@ class CanvasImpl implements Canvas {
         this.model.activate(clientID, attributeID);
     }
 
-    public highlight(clientIDs: number[] | null, severity: HighlightSeverity | null = null): void {
+    public highlight(clientIDs: number[], severity: HighlightSeverity | null = null): void {
         this.model.highlight(clientIDs, severity);
     }
 

cvat-canvas/src/typescript/canvasModel.ts

@@ -56,8 +56,8 @@ export enum HighlightSeverity {
 }
 
 export interface HighlightedElements {
-    elementsIDs: number [];
-    severity: HighlightSeverity;
+    elementsIDs: number[];
+    severity: HighlightSeverity | null;
 }
 
 export enum RectDrawingMethod {
@@ -267,7 +267,7 @@ export interface CanvasModel {
     setup(frameData: any, objectStates: any[], zLayer: number): void;
     setupIssueRegions(issueRegions: Record<number, { hidden: boolean; points: number[] }>): void;
     activate(clientID: number | null, attributeID: number | null): void;
-    highlight(clientIDs: number[] | null, severity: HighlightSeverity): void;
+    highlight(clientIDs: number[], severity: HighlightSeverity): void;
     rotate(rotationAngle: number): void;
     focus(clientID: number, padding: number): void;
     fit(): void;
@@ -641,18 +641,11 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
         this.notify(UpdateReasons.SHAPE_ACTIVATED);
     }
 
-    public highlight(clientIDs: number[] | null, severity: HighlightSeverity | null): void {
-        if (Array.isArray(clientIDs)) {
-            this.data.highlightedElements = {
-                elementsIDs: clientIDs,
-                severity,
-            };
-        } else {
-            this.data.highlightedElements = {
-                elementsIDs: [],
-                severity: null,
-            };
-        }
+    public highlight(clientIDs: number[], severity: HighlightSeverity | null): void {
+        this.data.highlightedElements = {
+            elementsIDs: clientIDs,
+            severity,
+        };
 
         this.notify(UpdateReasons.SHAPE_HIGHLIGHTED);
     }

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "14.0.1",
+  "version": "14.0.2",
   "type": "module",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",

cvat-core/src/api-implementation.ts

@@ -29,7 +29,7 @@ import Webhook from './webhook';
 import { ArgumentError } from './exceptions';
 import { SerializedAsset } from './server-response-types';
 import QualityReport from './quality-report';
-import QualityConflict from './quality-conflict';
+import QualityConflict, { ConflictSeverity } from './quality-conflict';
 import QualitySettings from './quality-settings';
 import { FramesMetaData } from './frames';
 import AnalyticsReport from './analytics-report';
@@ -427,9 +427,74 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
             };
         }
 
-        const reportsData = await serverProxy.analytics.quality.conflicts(updatedParams);
+        const conflictsData = await serverProxy.analytics.quality.conflicts(updatedParams);
+        const conflicts = conflictsData.map((conflict) => new QualityConflict({ ...conflict }));
+        const frames = Array.from(new Set(conflicts.map((conflict) => conflict.frame)))
+            .sort((a, b) => a - b);
+
+        // each QualityConflict may have several AnnotationConflicts bound
+        // at the same time, many quality conflicts may refer
+        // to the same labeled object (e.g. mismatch label, low overlap)
+        // the code below unites quality conflicts bound to the same object into one QualityConflict object
+        const mergedConflicts: QualityConflict[] = [];
+
+        for (const frame of frames) {
+            const frameConflicts = conflicts.filter((conflict) => conflict.frame === frame);
+            const conflictsByObject: Record<string, QualityConflict[]> = {};
+
+            frameConflicts.forEach((qualityConflict: QualityConflict) => {
+                const { type, serverID } = qualityConflict.annotationConflicts[0];
+                const firstObjID = `${type}_${serverID}`;
+                conflictsByObject[firstObjID] = conflictsByObject[firstObjID] || [];
+                conflictsByObject[firstObjID].push(qualityConflict);
+            });
+
+            for (const objectConflicts of Object.values(conflictsByObject)) {
+                if (objectConflicts.length === 1) {
+                    // only one quality conflict refers to the object on current frame
+                    mergedConflicts.push(objectConflicts[0]);
+                } else {
+                    const mainObjectConflict = objectConflicts
+                        .find((conflict) => conflict.severity === ConflictSeverity.ERROR) || objectConflicts[0];
+                    const descriptionList: string[] = [mainObjectConflict.description];
+
+                    for (const objectConflict of objectConflicts) {
+                        if (objectConflict !== mainObjectConflict) {
+                            descriptionList.push(objectConflict.description);
+
+                            for (const annotationConflict of objectConflict.annotationConflicts) {
+                                if (!mainObjectConflict.annotationConflicts.find((_annotationConflict) => (
+                                    _annotationConflict.serverID === annotationConflict.serverID &&
+                                    _annotationConflict.type === annotationConflict.type))
+                                ) {
+                                    mainObjectConflict.annotationConflicts.push(annotationConflict);
+                                }
+                            }
+                        }
+                    }
+
+                    // decorate the original conflict to avoid changing it
+                    const description = descriptionList.join(', ');
+                    const visibleConflict = new Proxy(mainObjectConflict, {
+                        get(target, prop) {
+                            if (prop === 'description') {
+                                return description;
+                            }
+
+                            // By default, it looks like Reflect.get(target, prop, receiver)
+                            // which has a different value of `this`. It doesn't allow to
+                            // work with methods / properties that use private members.
+                            const val = Reflect.get(target, prop);
+                            return typeof val === 'function' ? (...args: any[]) => val.apply(target, args) : val;
+                        },
+                    });
+
+                    mergedConflicts.push(visibleConflict);
+                }
+            }
+        }
 
-        return reportsData.map((conflict) => new QualityConflict({ ...conflict }));
+        return mergedConflicts;
     });
     implementationMixin(cvat.analytics.quality.settings.get, async (taskID: number) => {
         const settings = await serverProxy.analytics.quality.settings.get(taskID);

cvat-core/src/quality-conflict.ts

@@ -2,6 +2,8 @@
 //
 // SPDX-License-Identifier: MIT
 
+import { ObjectType } from './enums';
+
 export enum QualityConflictType {
     EXTRA = 'extra_annotation',
     MISMATCHING = 'mismatching_label',
@@ -26,8 +28,7 @@ export interface SerializedQualityConflictData {
 export interface SerializedAnnotationConflictData {
     job_id?: number;
     obj_id?: number;
-    client_id?: number;
-    type?: string;
+    type?: ObjectType;
     shape_type?: string | null;
     conflict_type?: string;
     severity?: string;
@@ -36,8 +37,7 @@ export interface SerializedAnnotationConflictData {
 export class AnnotationConflict {
     #jobID: number;
     #serverID: number;
-    #clientID: number;
-    #type: string;
+    #type: ObjectType;
     #shapeType: string | null;
     #conflictType: QualityConflictType;
     #severity: ConflictSeverity;
@@ -46,7 +46,6 @@ export class AnnotationConflict {
     constructor(initialData: SerializedAnnotationConflictData) {
         this.#jobID = initialData.job_id;
         this.#serverID = initialData.obj_id;
-        this.#clientID = initialData.client_id;
         this.#type = initialData.type;
         this.#shapeType = initialData.shape_type;
         this.#conflictType = initialData.conflict_type as QualityConflictType;
@@ -64,15 +63,7 @@ export class AnnotationConflict {
         return this.#serverID;
     }
 
-    get clientID(): number {
-        return this.#clientID;
-    }
-
-    set clientID(newID: number) {
-        this.#clientID = newID;
-    }
-
-    get type(): string {
+    get type(): ObjectType {
         return this.#type;
     }
 

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.61.0",
+  "version": "1.61.1",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-context-menu.tsx

@@ -125,7 +125,10 @@ export default function CanvasContextMenu(props: Props): JSX.Element | null {
 
     const state = objectStates.find((_state: any): boolean => _state.clientID === contextMenuClientID);
     const conflict = frameConflicts.find((qualityConflict: QualityConflict) => qualityConflict.annotationConflicts.some(
-        (annotationConflict: AnnotationConflict) => annotationConflict.clientID === state.clientID,
+        (annotationConflict: AnnotationConflict) => (
+            annotationConflict.serverID === state.serverID &&
+            annotationConflict.type === state.objectType
+        ),
     ));
 
     const copyObject = state?.isGroundTruth ? state : null;

cvat-ui/src/components/annotation-page/canvas/views/canvas2d/canvas-wrapper.tsx

@@ -72,7 +72,7 @@ interface StateToProps {
     activatedStateID: number | null;
     activatedElementID: number | null;
     activatedAttributeID: number | null;
-    annotations: any[];
+    annotations: ObjectState[];
     frameData: any;
     frameAngle: number;
     canvasIsReady: boolean;
@@ -481,12 +481,15 @@ class CanvasWrapperComponent extends React.PureComponent<Props> {
         }
 
         if (prevProps.highlightedConflict !== highlightedConflict) {
-            const severity: HighlightSeverity | null =
-                highlightedConflict?.severity ? (highlightedConflict?.severity as any) : null;
-            const highlightedElementsIDs = highlightedConflict?.annotationConflicts.map(
-                (conflict: AnnotationConflict) => conflict.clientID,
-            );
-            canvasInstance.highlight(highlightedElementsIDs || null, severity);
+            const severity: HighlightSeverity | undefined = highlightedConflict
+                ?.severity as unknown as HighlightSeverity;
+            const highlightedClientIDs = (highlightedConflict?.annotationConflicts || [])
+                .map((conflict: AnnotationConflict) => annotations
+                    .find((state) => state.serverID === conflict.serverID && state.objectType === conflict.type),
+                ).filter((state: ObjectState | undefined) => !!state)
+                .map((state) => state?.clientID) as number[];
+
+            canvasInstance.highlight(highlightedClientIDs, severity || null);
         }
 
         if (gridSize !== prevProps.gridSize) {

cvat-ui/src/components/annotation-page/review/issues-aggregator.tsx

@@ -26,7 +26,7 @@ interface ConflictMappingElement {
     severity: ConflictSeverity;
     x: number;
     y: number;
-    clientID: number;
+    serverID: number;
     conflict: QualityConflict;
 }
 
@@ -47,7 +47,8 @@ export default function IssueAggregatorComponent(): JSX.Element | null {
     const showConflicts = useSelector((state: CombinedState) => state.settings.shapes.showGroundTruth);
     const highlightedConflict = useSelector((state: CombinedState) => state.annotation.annotations.highlightedConflict);
 
-    const highlightedObjectsIDs = highlightedConflict?.annotationConflicts?.map((c: AnnotationConflict) => c.clientID);
+    const highlightedObjectsIDs = highlightedConflict?.annotationConflicts
+        ?.map((annotationConflict: AnnotationConflict) => annotationConflict.serverID);
 
     const activeControl = useSelector((state: CombinedState) => state.annotation.canvas.activeControl);
 
@@ -127,30 +128,37 @@ export default function IssueAggregatorComponent(): JSX.Element | null {
     }, [newIssuePosition, frameIssues, issuesResolvedHidden, issuesHidden, canvasReady, showConflicts]);
 
     useEffect(() => {
-        if (canvasReady && showConflicts) {
-            const newconflictMapping = qualityConflicts.map((conflict: QualityConflict) => {
-                const c = conflict.annotationConflicts[0];
-                const state = objectStates.find((s: ObjectState) => s.serverID === c.serverID);
-                if (state) {
-                    const points = canvasInstance.setupConflictRegions(state);
-                    if (points) {
-                        return {
-                            description: conflict.description,
-                            severity: conflict.severity,
-                            x: points[0],
-                            y: points[1],
-                            clientID: c.clientID,
-                            conflict,
-                        };
+        if (canvasReady && showConflicts && qualityConflicts.length) {
+            const updatedConflictMapping = qualityConflicts
+                .map((conflict: QualityConflict) => {
+                    const mainAnnotationsConflict = conflict.annotationConflicts[0];
+                    const state = objectStates.find((_state: ObjectState) => (
+                        _state.serverID === mainAnnotationsConflict.serverID &&
+                        _state.objectType === mainAnnotationsConflict.type
+                    ));
+
+                    if (state) {
+                        const points = canvasInstance.setupConflictRegions(state);
+                        if (points) {
+                            return {
+                                description: conflict.description,
+                                severity: conflict.severity,
+                                x: points[0],
+                                y: points[1],
+                                serverID: state.serverID,
+                                conflict,
+                            };
+                        }
                     }
-                }
-                return null;
-            }).filter((element) => element);
-            setConflictMapping(newconflictMapping as ConflictMappingElement[]);
+
+                    return null;
+                }).filter((element) => element) as ConflictMappingElement[];
+
+            setConflictMapping(updatedConflictMapping);
         } else {
             setConflictMapping([]);
         }
-    }, [geometry, objectStates, showConflicts, canvasReady]);
+    }, [geometry, objectStates, showConflicts, canvasReady, qualityConflicts]);
 
     if (!canvasReady || !geometry) {
         return null;
@@ -238,6 +246,7 @@ export default function IssueAggregatorComponent(): JSX.Element | null {
         null;
 
     for (const conflict of conflictMapping) {
+        const isConflictHighligted = highlightedObjectsIDs?.includes(conflict.serverID) || false;
         conflictLabels.push(
             <ConflictLabel
                 key={(Math.random() + 1).toString(36).substring(7)}
@@ -247,13 +256,11 @@ export default function IssueAggregatorComponent(): JSX.Element | null {
                 angle={-geometry.angle}
                 scale={1 / geometry.scale}
                 severity={conflict.severity}
-                darken={!!highlightedConflict && !!highlightedObjectsIDs &&
-                        (!highlightedObjectsIDs.includes(conflict.clientID))}
+                darken={!isConflictHighligted}
                 conflict={conflict.conflict}
                 onEnter={onEnter}
                 onLeave={onLeave}
-                tooltipVisible={!!highlightedConflict && !!highlightedObjectsIDs &&
-                        highlightedObjectsIDs.includes(conflict.clientID)}
+                tooltipVisible={isConflictHighligted}
             />,
         );
     }

cvat-ui/src/reducers/annotation-reducer.ts

@@ -1017,15 +1017,19 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
         case AnnotationActionTypes.HIGHLIGHT_CONFLICT: {
             const { conflict } = action.payload;
             if (conflict) {
-                const { annotationConflicts } = conflict;
-                const [mainConflict] = annotationConflicts;
-                const { clientID } = mainConflict;
+                const { annotationConflicts: [mainConflict] } = conflict;
+
+                // object may be hidden using annotations filter
+                // it is not guaranteed to be visible
+                const conflictObject = state.annotations.states
+                    .find((_state) => _state.serverID === mainConflict.serverID);
+
                 return {
                     ...state,
                     annotations: {
                         ...state.annotations,
                         highlightedConflict: conflict,
-                        activatedStateID: clientID,
+                        activatedStateID: conflictObject?.clientID || null,
                         activatedElementID: null,
                         activatedAttributeID: null,
                     },