Fixed nan values in .pcd files, fixed broken navigation in 3D workspace, updated three js

CHANGELOG.md

@@ -44,6 +44,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- PCD files with nan values could not be opened on 3D workspace
+ (<https://github.com/opencv/cvat/pull/6862>)
+- Fixed direct navigation to neightbour chunk on 3D workspace
+ (<https://github.com/opencv/cvat/pull/6862>)
+- Intencity level from .bin lidar data ignored when converting .bin -> .pcd
+ (<https://github.com/opencv/cvat/pull/6862>)
 - Incorrectly determined video frame count when the video contains an MP4 edit list
   (<https://github.com/opencv/cvat/pull/6929>)
 - Internal server error when retrieving data from CS and cache=True

cvat-canvas3d/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas3d",
-  "version": "0.0.8",
+  "version": "0.0.9",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas3D library",
   "main": "src/canvas3d.ts",
   "scripts": {
@@ -17,8 +17,8 @@
   "devDependencies": {},
   "dependencies": {
     "cvat-core": "link:./../cvat-core",
-    "@types/three": "^0.125.3",
+    "@types/three": "^0.156.0",
     "camera-controls": "^1.25.3",
-    "three": "^0.126.1"
+    "three": "^0.156.1"
   }
 }

cvat-canvas3d/src/typescript/canvas3dModel.ts

@@ -31,7 +31,7 @@ export interface SplitData {
 export interface Image {
     renderWidth: number;
     renderHeight: number;
-    imageData: ImageData | CanvasImageSource;
+    imageData: Blob;
 }
 
 export interface DrawData {
@@ -102,7 +102,7 @@ export enum Mode {
 export interface Canvas3dDataModel {
     activeElement: ActiveElement;
     canvasSize: Size;
-    image: Image | null;
+    image: { imageData: Blob } | null;
     imageID: number | null;
     imageOffset: number;
     imageSize: Size;
@@ -236,7 +236,7 @@ export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel {
                 this.data.image = null;
                 this.notify(UpdateReasons.IMAGE_CHANGED);
             })
-            .then((data: Image): void => {
+            .then((data: { imageData: Blob }): void => {
                 this.data.imageSize = {
                     height: frameData.height as number,
                     width: frameData.width as number,

cvat-canvas3d/src/typescript/canvas3dView.ts

@@ -1149,10 +1149,6 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener {
             this.statesToBeGrouped = [];
             this.clearScene();
 
-            const onPCDLoadFailed = (): void => {
-                model.unlockFrameUpdating();
-            };
-
             const onPCDLoadSuccess = (points: any): void => {
                 try {
                     this.onSceneImageLoaded(points);
@@ -1164,14 +1160,8 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener {
             };
 
             try {
-                if (!model.data.image) {
-                    throw new Error('No image data found');
-                }
-
-                const loader = new PCDLoader();
-                const objectURL = URL.createObjectURL(model.data.image.imageData);
-
-                try {
+                if (model.data.image) {
+                    const loader = new PCDLoader();
                     this.views.perspective.renderer.dispose();
                     if (this.controller.imageIsDeleted) {
                         try {
@@ -1202,14 +1192,97 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener {
                             model.unlockFrameUpdating();
                         }
                     } else {
-                        loader.load(objectURL, onPCDLoadSuccess, () => {}, onPCDLoadFailed);
+                        model.data.image.imageData.arrayBuffer().then((data) => {
+                            const defaultImpl = console.error;
+                            // loader.parse will throw some errors to console when nan values are in PCD file
+                            // there is not way to prevent it, because three.js opinion is that nan values
+                            // in input data is incorrect
+                            let cloud = null;
+                            try {
+                                console.error = () => {};
+                                cloud = loader.parse(data) as THREE.Points;
+                            } finally {
+                                console.error = defaultImpl;
+                            }
+
+                            let {
+                                position, color, normal, intensity, label,
+                            } = cloud.geometry.attributes;
+                            cloud.material.vertexColors = true;
+
+                            ({
+                                color, position, normal, intensity, label,
+                            } = position.array.reduce((acc, _, i, array) => {
+                                if (
+                                    i % 3 === 0 &&
+                                    Number.isFinite(array[i]) &&
+                                    Number.isFinite(array[i + 1]) &&
+                                    Number.isFinite(array[i + 2])
+                                ) {
+                                    acc.position.push(array[i], array[i + 1], array[i + 2]);
+
+                                    if (
+                                        color &&
+                                        Number.isFinite(color.array[i]) &&
+                                        Number.isFinite(color.array[i + 1]) &&
+                                        Number.isFinite(color.array[i + 2])
+                                    ) {
+                                        acc.color.push(color.array[i], color.array[i + 1], color.array[i + 2]);
+                                    } else {
+                                        acc.color.push(255, 255, 255);
+                                    }
+
+                                    if (
+                                        normal &&
+                                        Number.isFinite(normal.array[i]) &&
+                                        Number.isFinite(normal.array[i + 1]) &&
+                                        Number.isFinite(normal.array[i + 2])
+                                    ) {
+                                        acc.normal.push(normal.array[i], normal.array[i + 1], normal.array[i + 2]);
+                                    }
+
+                                    if (intensity) {
+                                        acc.intensity.push(intensity.array[i / 3]);
+                                    }
+
+                                    if (label) {
+                                        acc.label.push(label.array[i / 3]);
+                                    }
+                                }
+
+                                return acc;
+                            }, {
+                                position: [], color: [], normal: [], intensity: [], label: [],
+                            }));
+
+                            if (position.length) {
+                                cloud.geometry.setAttribute('position', new THREE.Float32BufferAttribute(position, 3));
+                            }
+
+                            if (color.length) {
+                                cloud.geometry.setAttribute('color', new THREE.Float32BufferAttribute(color, 3));
+                            }
+
+                            if (normal.length) {
+                                cloud.geometry.setAttribute('normal', new THREE.Float32BufferAttribute(normal, 3));
+                            }
+
+                            if (intensity.length) {
+                                cloud.geometry.setAttribute('intensity', new THREE.Float32BufferAttribute(intensity, 1));
+                            }
+
+                            if (label.length) {
+                                cloud.geometry.setAttribute('label', new THREE.Float32BufferAttribute(label, 1));
+                            }
+
+                            cloud.geometry.computeBoundingSphere();
+                            onPCDLoadSuccess(cloud);
+                        });
                         const [overlay] = window.document.getElementsByClassName('cvat_3d_canvas_deleted_overlay');
                         if (overlay) {
                             overlay.remove();
                         }
                     }
-                } finally {
-                    URL.revokeObjectURL(objectURL);
                 }
             } catch (error: any) {
                 model.unlockFrameUpdating();
@@ -1476,7 +1549,7 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener {
         // Setup TopView
         const canvasTopView = this.views.top.renderer.domElement;
         const topScenePlane = new THREE.Mesh(
-            new THREE.PlaneBufferGeometry(
+            new THREE.PlaneGeometry(
                 canvasTopView.offsetHeight,
                 canvasTopView.offsetWidth,
                 canvasTopView.offsetHeight,
@@ -1499,7 +1572,7 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener {
         // Setup Side View
         const canvasSideView = this.views.side.renderer.domElement;
         const sideScenePlane = new THREE.Mesh(
-            new THREE.PlaneBufferGeometry(
+            new THREE.PlaneGeometry(
                 canvasSideView.offsetHeight,
                 canvasSideView.offsetWidth,
                 canvasSideView.offsetHeight,
@@ -1521,7 +1594,7 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener {
         // Setup front View
         const canvasFrontView = this.views.front.renderer.domElement;
         const frontScenePlane = new THREE.Mesh(
-            new THREE.PlaneBufferGeometry(
+            new THREE.PlaneGeometry(
                 canvasFrontView.offsetHeight,
                 canvasFrontView.offsetWidth,
                 canvasFrontView.offsetHeight,
@@ -1604,7 +1677,8 @@ export class Canvas3dViewImpl implements Canvas3dView, Listener {
     private renderRayCaster = (viewType: RenderView): void => {
         viewType.rayCaster.renderer.setFromCamera(viewType.rayCaster.mouseVector, viewType.camera);
         if (this.mode === Mode.DRAW) {
-            const [intersection] = viewType.rayCaster.renderer.intersectObjects(this.views.perspective.scene.children);
+            const [intersection] = viewType.rayCaster.renderer
+                .intersectObjects(this.views.perspective.scene.children, false);
             if (intersection) {
                 const object = this.views.perspective.scene.getObjectByName('drawTemplate');
                 const { x, y, z } = intersection.point;

cvat-canvas3d/src/typescript/cuboid.ts

@@ -59,23 +59,23 @@ export class CuboidModel {
         this.front = new THREE.Mesh(geometry, material);
 
         const planeTop = new THREE.Mesh(
-            new THREE.PlaneBufferGeometry(1, 1, 1, 1),
+            new THREE.PlaneGeometry(1, 1, 1, 1),
             new THREE.MeshBasicMaterial({
                 color: 0xff0000,
                 visible: false,
             }),
         );
 
         const planeSide = new THREE.Mesh(
-            new THREE.PlaneBufferGeometry(1, 1, 1, 1),
+            new THREE.PlaneGeometry(1, 1, 1, 1),
             new THREE.MeshBasicMaterial({
                 color: 0xff0000,
                 visible: false,
             }),
         );
 
         const planeFront = new THREE.Mesh(
-            new THREE.PlaneBufferGeometry(1, 1, 1, 1),
+            new THREE.PlaneGeometry(1, 1, 1, 1),
             new THREE.MeshBasicMaterial({
                 color: 0xff0000,
                 visible: false,

cvat/apps/engine/media_extractors.py

@@ -935,10 +935,10 @@ def convert_bin_to_pcd(path, delete_source=True):
         def write_header(fileObj, width, height):
             fileObj.writelines(f'{line}\n' for line in [
                 'VERSION 0.7',
-                'FIELDS x y z',
-                'SIZE 4 4 4',
-                'TYPE F F F',
-                'COUNT 1 1 1',
+                'FIELDS x y z intensity',
+                'SIZE 4 4 4 4',
+                'TYPE F F F F',
+                'COUNT 1 1 1 1',
                 f'WIDTH {width}',
                 f'HEIGHT {height}',
                 'VIEWPOINT 0 0 0 1 0 0 0',
@@ -952,8 +952,8 @@ def write_header(fileObj, width, height):
             size_float = 4
             byte = f.read(size_float * 4)
             while byte:
-                x, y, z, _ = struct.unpack("ffff", byte)
-                list_pcd.append([x, y, z])
+                x, y, z, intensity = struct.unpack("ffff", byte)
+                list_pcd.append([x, y, z, intensity])
                 byte = f.read(size_float * 4)
         np_pcd = np.asarray(list_pcd)
         pcd_filename = path.replace(".bin", ".pcd")

tests/cypress/e2e/canvas3d_functionality_2/case_82_canvas3d_functionality_cuboid_opacity_outlined_borders.js

@@ -77,7 +77,9 @@ context('Canvas 3D functionality. Opacity. Outlined borders.', () => {
                 cy.contains('Ok').click();
             });
             cy.get('.cvat-canvas3d-perspective canvas').then(([el]) => {
-                expect({ ...getWireframe(el).material.color }).to.deep.equal({ r: 1, g: 0, b: 0.48627450980392156 });
+                expect({ ...getWireframe(el).material.color }).to.deep.equal({
+                    isColor: true, r: 1, g: 0, b: 0.20155625378383743,
+                });
             });
 
             cy.get('.cvat-appearance-outlinded-borders-checkbox').find('[type="checkbox"]').uncheck();

yarn.lock

@@ -2285,10 +2285,20 @@
   resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c"
   integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==
 
-"@types/three@^0.125.3":
-  version "0.125.3"
-  resolved "https://registry.yarnpkg.com/@types/three/-/three-0.125.3.tgz#5d5c29acea66af42a3a32a13a9cc8e88e367d8a3"
-  integrity sha512-tUPMzKooKDvMOhqcNVUPwkt+JNnF8ASgWSsrLgleVd0SjLj4boJhteSsF9f6YDjye0mmUjO+BDMWW83F97ehXA==
+"@types/stats.js@*":
+  version "0.17.0"
+  resolved "https://registry.yarnpkg.com/@types/stats.js/-/stats.js-0.17.0.tgz#0ed81d48e03b590c24da85540c1d952077a9fe20"
+  integrity sha512-9w+a7bR8PeB0dCT/HBULU2fMqf6BAzvKbxFboYhmDtDkKPiyXYbjoe2auwsXlEFI7CFNMF1dCv3dFH5Poy9R1w==
+
+"@types/three@^0.156.0":
+  version "0.156.0"
+  resolved "https://registry.yarnpkg.com/@types/three/-/three-0.156.0.tgz#cd49f2a12e858400962ea818d1e1c45e638141a8"
+  integrity sha512-733bXDSRdlrxqOmQuOmfC1UBRuJ2pREPk8sWnx9MtIJEVDQMx8U0NQO5MVVaOrjzDPyLI+cFPim2X/ss9v0+LQ==
+  dependencies:
+    "@types/stats.js" "*"
+    "@types/webxr" "*"
+    fflate "~0.6.10"
+    meshoptimizer "~0.18.1"
 
 "@types/tough-cookie@*":
   version "4.0.2"
@@ -2305,6 +2315,11 @@
   resolved "https://registry.yarnpkg.com/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz#b6725d5f4af24ace33b36fafd295136e75509f43"
   integrity sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==
 
+"@types/webxr@*":
+  version "0.5.4"
+  resolved "https://registry.yarnpkg.com/@types/webxr/-/webxr-0.5.4.tgz#3d55a6427f9281d456843d754c99bf7804657fe3"
+  integrity sha512-41gfGLTtqXZhcmoDlLDHqMJDuwAMwhHwXf9Q2job3TUBsvkNfPNI/3IWVEtLH4tyY1ElWtfwIaoNeqeEX238/Q==
+
 "@types/ws@^8.5.1":
   version "8.5.4"
   resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.4.tgz#bb10e36116d6e570dd943735f86c933c1587b8a5"
@@ -4049,10 +4064,10 @@ custom-error-instance@2.1.1:
 "cvat-canvas3d@link:./cvat-canvas3d":
   version "0.0.8"
   dependencies:
-    "@types/three" "^0.125.3"
+    "@types/three" "^0.156.0"
     camera-controls "^1.25.3"
     cvat-core "link:./cvat-core"
-    three "^0.126.1"
+    three "^0.156.1"
 
 "cvat-canvas@link:./cvat-canvas":
   version "2.17.5"
@@ -5133,6 +5148,11 @@ fb-watchman@^2.0.0:
   dependencies:
     bser "2.1.1"
 
+fflate@~0.6.10:
+  version "0.6.10"
+  resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.6.10.tgz#5f40f9659205936a2d18abf88b2e7781662b6d43"
+  integrity sha512-IQrh3lEPM93wVCEczc9SaAOvkmcoQn/G8Bo1e8ZPlY3X3bnAxWaBdvTdvM1hP62iZp0BXWDy4vTAy4fF0+Dlpg==
+
 figgy-pudding@^3.5.1:
   version "3.5.2"
   resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e"
@@ -7824,6 +7844,11 @@ merge2@^1.3.0, merge2@^1.4.1:
   resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
   integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
 
+meshoptimizer@~0.18.1:
+  version "0.18.1"
+  resolved "https://registry.yarnpkg.com/meshoptimizer/-/meshoptimizer-0.18.1.tgz#cdb90907f30a7b5b1190facd3b7ee6b7087797d8"
+  integrity sha512-ZhoIoL7TNV4s5B6+rx5mC//fw8/POGyNxS/DZyCJeiZ12ScLfVwRE/GfsxwiTkMYYD5DmK2/JXnEVXqL4rF+Sw==
+
 methods@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
@@ -12243,10 +12268,10 @@ text-table@^0.2.0:
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==
 
-three@^0.126.1:
-  version "0.126.1"
-  resolved "https://registry.yarnpkg.com/three/-/three-0.126.1.tgz#cf4e4e52060fd952f6f0d5440cbd422c66bc4be7"
-  integrity sha512-eOEXnZeE1FDV0XgL1u08auIP13jxdN9LQBAEmlErYzMxtIIfuGIAZbijOyookALUhqVzVOx0Tywj6n192VM+nQ==
+three@^0.156.1:
+  version "0.156.1"
+  resolved "https://registry.yarnpkg.com/three/-/three-0.156.1.tgz#bab4fec121a5b3975eb4f4d227d9c912171eb399"
+  integrity sha512-kP7H0FK9d/k6t/XvQ9FO6i+QrePoDcNhwl0I02+wmUJRNSLCUIDMcfObnzQvxb37/0Uc9TDT0T1HgsRRrO6SYQ==
 
 through@^2.3.8:
   version "2.3.8"