Fixed issue: object interpolated incorrectly if a frame with the object keyframe is deleted

changelog.d/20250116_191626_sekachev.bs_fixed_deleted_frames.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- A track will be interpolated incorrectly if to delete an image containing the object keyframe
+  (<https://github.com/cvat-ai/cvat/pull/8951>)

cvat-core/src/annotations-collection.ts

@@ -1,12 +1,12 @@
 // Copyright (C) 2019-2022 Intel Corporation
-// Copyright (C) 2022-2024 CVAT.ai Corporation
+// Copyright (C) 2022-2025 CVAT.ai Corporation
 //
 // SPDX-License-Identifier: MIT
 
 import {
     shapeFactory, trackFactory, Track, Shape, Tag,
-    MaskShape, BasicInjection,
-    SkeletonShape, SkeletonTrack, PolygonShape, CuboidShape,
+    MaskShape, BasicInjection, SkeletonShape,
+    SkeletonTrack, PolygonShape, CuboidShape,
     RectangleShape, PolylineShape, PointsShape, EllipseShape,
 } from './annotations-objects';
 import { SerializedCollection, SerializedShape, SerializedTrack } from './server-response-types';
@@ -22,7 +22,6 @@ import {
     HistoryActions, ShapeType, ObjectType, colors, Source, DimensionType, JobType,
 } from './enums';
 import AnnotationHistory from './annotations-history';
-import { Job } from './session';
 
 const validateAttributesList = (
     attributes: { spec_id: number, value: string }[],
@@ -48,14 +47,9 @@ const labelAttributesAsDict = (label: Label): Record<number, Attribute> => (
     }, {})
 );
 
-export type FrameMeta = Record<number, Awaited<ReturnType<Job['frames']['get']>>> & {
-    deleted_frames: Record<number, boolean>
-};
-
 export default class Collection {
     public flush: boolean;
     private stopFrame: number;
-    private frameMeta: FrameMeta;
     private labels: Record<number, Label>;
     private annotationsFilter: AnnotationsFilter;
     private history: AnnotationHistory;
@@ -71,11 +65,11 @@ export default class Collection {
         history: AnnotationHistory;
         stopFrame: number;
         dimension: DimensionType;
-        frameMeta: Collection['frameMeta'];
+        framesInfo: BasicInjection['framesInfo'];
         jobType: JobType;
+        isFrameDeleted: (frame: number) => boolean;
     }) {
         this.stopFrame = data.stopFrame;
-        this.frameMeta = data.frameMeta;
 
         this.labels = data.labels.reduce((labelAccumulator, label) => {
             labelAccumulator[label.id] = label;
@@ -96,15 +90,17 @@ export default class Collection {
         this.groups = {
             max: 0,
         }; // it is an object to we can pass it as an argument by a reference
+
         this.injection = {
             labels: this.labels,
             groups: this.groups,
-            frameMeta: this.frameMeta,
+            framesInfo: data.framesInfo,
             history: this.history,
             dimension: data.dimension,
             jobType: data.jobType,
-            nextClientID: () => ++config.globalObjectsCounter,
             groupColors: {},
+            nextClientID: () => ++config.globalObjectsCounter,
+            isFrameDeleted: data.isFrameDeleted,
             getMasksOnFrame: (frame: number) => (this.shapes[frame] as MaskShape[])
                 .filter((object) => object instanceof MaskShape),
         };
@@ -239,9 +235,13 @@ export default class Collection {
     }
 
     public get(frame: number, allTracks: boolean, filters: object[]): ObjectState[] {
+        if (this.injection.isFrameDeleted(frame)) {
+            return [];
+        }
+
         const { tracks } = this;
-        const shapes = this.shapes[frame] || [];
-        const tags = this.tags[frame] || [];
+        const shapes = this.shapes[frame] ?? [];
+        const tags = this.tags[frame] ?? [];
 
         const objects = [].concat(tracks, shapes, tags);
         const visible = [];
@@ -771,7 +771,7 @@ export default class Collection {
             );
         }
 
-        const { width, height } = this.frameMeta[slicedObject.frame];
+        const { width, height } = this.injection.framesInfo[slicedObject.frame];
         if (slicedObject instanceof MaskShape) {
             points1.push(slicedObject.left, slicedObject.top, slicedObject.right, slicedObject.bottom);
             points2.push(slicedObject.left, slicedObject.top, slicedObject.right, slicedObject.bottom);
@@ -923,7 +923,7 @@ export default class Collection {
                     count -= 1;
                 }
                 for (let i = start + 1; lastIsKeyframe ? i < stop : i <= stop; i++) {
-                    if (this.frameMeta.deleted_frames[i]) {
+                    if (this.injection.isFrameDeleted(i)) {
                         count--;
                     }
                 }
@@ -936,7 +936,7 @@ export default class Collection {
             const keyframes = Object.keys(track.shapes)
                 .sort((a, b) => +a - +b)
                 .map((el) => +el)
-                .filter((frame) => !this.frameMeta.deleted_frames[frame]);
+                .filter((frame) => !this.injection.isFrameDeleted(frame));
 
             let prevKeyframe = keyframes[0];
             let visible = false;
@@ -987,19 +987,19 @@ export default class Collection {
             }
 
             const { name: label } = object.label;
-            if (objectType === 'tag' && !this.frameMeta.deleted_frames[object.frame]) {
+            if (objectType === 'tag' && !this.injection.isFrameDeleted(object.frame)) {
                 labels[label].tag++;
                 labels[label].manually++;
                 labels[label].total++;
             } else if (objectType === 'track') {
                 scanTrack(object);
-            } else if (!this.frameMeta.deleted_frames[object.frame]) {
+            } else if (!this.injection.isFrameDeleted(object.frame)) {
                 const { shapeType } = object as Shape;
                 labels[label][shapeType].shape++;
                 labels[label].manually++;
                 labels[label].total++;
                 if (shapeType === ShapeType.SKELETON) {
-                    (object as SkeletonShape).elements.forEach((element) => {
+                    (object as unknown as SkeletonShape).elements.forEach((element) => {
                         const combinedName = [label, element.label.name].join(sep);
                         labels[combinedName][element.shapeType].shape++;
                         labels[combinedName].manually++;
@@ -1086,7 +1086,7 @@ export default class Collection {
                         outside: state.outside || false,
                         occluded: state.occluded || false,
                         points: state.shapeType === 'mask' ? (() => {
-                            const { width, height } = this.frameMeta[state.frame];
+                            const { width, height } = this.injection.framesInfo[state.frame];
                             return cropMask(state.points, width, height);
                         })() : state.points,
                         rotation: state.rotation || 0,
@@ -1292,7 +1292,7 @@ export default class Collection {
         const predicate = sign > 0 ? (frame) => frame <= frameTo : (frame) => frame >= frameTo;
         const update = sign > 0 ? (frame) => frame + 1 : (frame) => frame - 1;
         for (let frame = frameFrom; predicate(frame); frame = update(frame)) {
-            if (!allowDeletedFrames && this.frameMeta[frame].deleted) {
+            if (!allowDeletedFrames && this.injection.isFrameDeleted(frame)) {
                 continue;
             }
 
@@ -1359,7 +1359,7 @@ export default class Collection {
         if (!annotationsFilters) {
             let frame = frameFrom;
             while (predicate(frame)) {
-                if (!allowDeletedFrames && this.frameMeta[frame].deleted) {
+                if (!allowDeletedFrames && this.injection.isFrameDeleted(frame)) {
                     frame = update(frame);
                     continue;
                 }
@@ -1374,7 +1374,7 @@ export default class Collection {
         const linearSearch = filtersStr.match(/"var":"width"/) || filtersStr.match(/"var":"height"/);
 
         for (let frame = frameFrom; predicate(frame); frame = update(frame)) {
-            if (!allowDeletedFrames && this.frameMeta[frame].deleted) {
+            if (!allowDeletedFrames && this.injection.isFrameDeleted(frame)) {
                 continue;
             }
 

cvat-core/src/annotations-objects.ts

@@ -1,5 +1,5 @@
 // Copyright (C) 2019-2022 Intel Corporation
-// Copyright (C) 2022-2024 CVAT.ai Corporation
+// Copyright (C) 2022-2025 CVAT.ai Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -61,9 +61,7 @@ function computeNewSource(currentSource: Source): Source {
 export interface BasicInjection {
     labels: Record<number, Label>;
     groups: { max: number };
-    frameMeta: {
-        deleted_frames: Record<number, boolean>;
-    };
+    framesInfo: Readonly<Record<number, Readonly<{ width: number; height: number; }>>>
     history: AnnotationHistory;
     groupColors: Record<number, string>;
     parentID?: number;
@@ -72,6 +70,7 @@ export interface BasicInjection {
     jobType: JobType;
     nextClientID: () => number;
     getMasksOnFrame: (frame: number) => MaskShape[];
+    isFrameDeleted: (frame: number) => boolean;
 }
 
 type AnnotationInjection = BasicInjection & {
@@ -394,15 +393,17 @@ class Annotation {
 }
 
 class Drawn extends Annotation {
-    protected frameMeta: AnnotationInjection['frameMeta'];
+    protected isFrameDeleted: (frame: number) => boolean;
+    protected framesInfo: AnnotationInjection['framesInfo'];
     protected descriptions: string[];
     public hidden: boolean;
     protected pinned: boolean;
     public shapeType: ShapeType;
 
     constructor(data, clientID: number, color: string, injection: AnnotationInjection) {
         super(data, clientID, color, injection);
-        this.frameMeta = injection.frameMeta;
+        this.isFrameDeleted = injection.isFrameDeleted;
+        this.framesInfo = injection.framesInfo;
         this.descriptions = data.descriptions || [];
         this.hidden = false;
         this.pinned = true;
@@ -487,16 +488,10 @@ class Drawn extends Annotation {
             checkObjectType('points', data.points, null, Array);
             checkNumberOfPoints(this.shapeType, data.points);
             // cut points
-            const { width, height, filename } = this.frameMeta[frame];
+            const { width, height } = this.framesInfo[frame];
             fittedPoints = this.fitPoints(data.points, data.rotation, width, height);
-            let check = true;
-            if (filename && filename.slice(filename.length - 3) === 'pcd') {
-                check = false;
-            }
-            if (check) {
-                if (!checkShapeArea(this.shapeType, fittedPoints)) {
-                    fittedPoints = [];
-                }
+            if (this.dimension === DimensionType.DIMENSION_2D && !checkShapeArea(this.shapeType, fittedPoints)) {
+                fittedPoints = [];
             }
         }
 
@@ -960,7 +955,7 @@ export class Track extends Drawn {
         let last = Number.MIN_SAFE_INTEGER;
 
         for (const frame of frames) {
-            if (frame in this.frameMeta.deleted_frames) {
+            if (this.isFrameDeleted(frame)) {
                 continue;
             }
 
@@ -2216,7 +2211,7 @@ export class MaskShape extends Shape {
     constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) {
         super(data, clientID, color, injection);
         const [left, top, right, bottom] = this.points.slice(-4);
-        const { width, height } = this.frameMeta[this.frame];
+        const { width, height } = this.framesInfo[this.frame];
         if (left >= width || top >= height || right >= width || bottom >= height) {
             this.points = cropMask(this.points, width, height);
         }
@@ -2229,7 +2224,7 @@ export class MaskShape extends Shape {
     protected validateStateBeforeSave(data: ObjectState, updated: ObjectState['updateFlags'], frame?: number): number[] {
         super.validateStateBeforeSave(data, updated, frame);
         if (updated.points) {
-            const { width, height } = this.frameMeta[frame];
+            const { width, height } = this.framesInfo[frame];
             return cropMask(data.points, width, height);
         }
 
@@ -2610,7 +2605,7 @@ class PolyTrack extends Track {
                 return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);
             }
 
-            function minimizeSegment(baseLength: number, N: number, startInterpolated, stopInterpolated): void {
+            function minimizeSegment(baseLength: number, N: number, startInterpolated, stopInterpolated): Point2D[] {
                 const threshold = baseLength / (2 * N);
                 const minimized = [interpolatedPoints[startInterpolated]];
                 let latestPushed = startInterpolated;

cvat-core/src/annotations.ts

@@ -1,18 +1,18 @@
 // Copyright (C) 2019-2022 Intel Corporation
-// Copyright (C) 2022-2024 CVAT.ai Corporation
+// Copyright (C) 2022-2025 CVAT.ai Corporation
 //
 // SPDX-License-Identifier: MIT
 
 import { Storage } from './storage';
 import serverProxy from './server-proxy';
-import AnnotationsCollection, { FrameMeta } from './annotations-collection';
+import AnnotationsCollection from './annotations-collection';
 import AnnotationsSaver from './annotations-saver';
 import AnnotationsHistory from './annotations-history';
 import { checkObjectType } from './common';
 import Project from './project';
 import { Task, Job } from './session';
 import { ArgumentError } from './exceptions';
-import { getDeletedFrames } from './frames';
+import { getFramesMeta, getJobFramesMetaSync } from './frames';
 import { JobType } from './enums';
 
 const jobCollectionCache = new WeakMap<Task | Job, { collection: AnnotationsCollection; saver: AnnotationsSaver; }>();
@@ -88,22 +88,27 @@ async function getAnnotationsFromServer(session: Job | Task): Promise<void> {
         const serializedAnnotations = await serverProxy.annotations.getAnnotations(sessionType, session.id);
 
         // Get meta information about frames
-        const startFrame = session instanceof Job ? session.startFrame : 0;
-        const stopFrame = session instanceof Job ? session.stopFrame : session.size - 1;
-        const frameMeta: Partial<FrameMeta> = {};
-        for (let i = startFrame; i <= stopFrame; i++) {
-            frameMeta[i] = await session.frames.get(i);
-        }
-        frameMeta.deleted_frames = await getDeletedFrames(sessionType, session.id);
+        const frameMeta = await getFramesMeta(sessionType, session.id);
+        const frameNumbers = frameMeta.getSegmentFrameNumbers(session instanceof Job ? session.startFrame : 0);
 
         const history = cache.history.has(session) ? cache.history.get(session) : new AnnotationsHistory();
         const collection = new AnnotationsCollection({
-            labels: session.labels,
-            history,
-            stopFrame,
-            frameMeta: frameMeta as FrameMeta,
             jobType: session instanceof Job ? session.type : JobType.ANNOTATION,
+            stopFrame: session instanceof Job ? session.stopFrame : session.size - 1,
+            labels: session.labels,
             dimension: session.dimension,
+            isFrameDeleted: session instanceof Job ?
+                (frame: number) => !!getJobFramesMetaSync(session.id).deletedFrames[frame] :
+                (frame: number) => !!frameMeta.deletedFrames[frame],
+            framesInfo: frameMeta.frames.reduce((acc, frameInfo, idx) => {
+                // keep only static information
+                acc[frameNumbers[idx]] = {
+                    width: frameInfo.width,
+                    height: frameInfo.height,
+                };
+                return acc;
+            }, {}),
+            history,
         });
 
         // eslint-disable-next-line no-unsanitized/method
@@ -127,15 +132,19 @@ export function clearCache(session): void {
     }
 }
 
-export async function getAnnotations(session, frame, allTracks, filters): Promise<ReturnType<AnnotationsCollection['get']>> {
+export async function getAnnotations(
+    session: Job | Task,
+    frame: number,
+    allTracks: boolean,
+    filters: object[],
+): Promise<ReturnType<AnnotationsCollection['get']>> {
     try {
         return getCollection(session).get(frame, allTracks, filters);
     } catch (error) {
         if (error instanceof InstanceNotInitializedError) {
             await getAnnotationsFromServer(session);
             return getCollection(session).get(frame, allTracks, filters);
         }
-
         throw error;
     }
 }