pam-diff: new pipeline support

koush · Mar 21, 2023 · 01d0f4c · 01d0f4c
1 parent 9fe3f1a
commit 01d0f4c
Show file tree

Hide file tree

Showing 3 changed files with 104 additions and 8 deletions.
diff --git a/plugins/pam-diff/package-lock.json b/plugins/pam-diff/package-lock.json
diff --git a/plugins/pam-diff/package.json b/plugins/pam-diff/package.json
@@ -43,5 +43,5 @@
    "devDependencies": {
       "@scrypted/sdk": "file:../../sdk"
    },
-   "version": "0.0.16"
+   "version": "0.0.17"
 }
diff --git a/plugins/pam-diff/src/main.ts b/plugins/pam-diff/src/main.ts
@@ -1,11 +1,10 @@
-import { ObjectDetectionResult, FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionSession, ObjectsDetected, ScryptedDeviceBase, ScryptedInterface, ScryptedMimeTypes } from '@scrypted/sdk';
-import sdk from '@scrypted/sdk';
-import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "../../../common/src/media-helpers";
-
+import sdk, { FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionGeneratorResult, ObjectDetectionGeneratorSession, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionSession, ObjectsDetected, ScryptedDeviceBase, ScryptedInterface, ScryptedMimeTypes, VideoFrame } from '@scrypted/sdk';
 import child_process, { ChildProcess } from 'child_process';
+import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "../../../common/src/media-helpers";
 
 import PD from 'pam-diff';
 import P2P from 'pipe2pam';
+import { PassThrough, Writable } from 'stream';
 
 const { mediaManager } = sdk;
 
@@ -51,6 +50,101 @@ class PamDiff extends ScryptedDeviceBase implements ObjectDetection {
         pds.timeout = setTimeout(() => this.endSession(id), duration);
     }
 
+    async * generateObjectDetectionsInternal(videoFrames: AsyncGenerator<VideoFrame, any, unknown>, session: ObjectDetectionGeneratorSession): AsyncGenerator<ObjectDetectionGeneratorResult, any, unknown> {
+        videoFrames = await sdk.connectRPCObject(videoFrames);
+
+        const width = 640;
+        const height = 360;
+        const p2p: Writable = new P2P();
+        const pt = new PassThrough();
+        const pamDiff = new PD({
+            difference: parseInt(session.settings?.difference) || defaultDifference,
+            percent: parseInt(session.settings?.percent) || defaultPercentage,
+            response: session?.settings?.motionAsObjects ? 'blobs' : 'percent',
+        });
+        pt.pipe(p2p).pipe(pamDiff);
+
+        const queued: ObjectsDetected[] = [];
+        pamDiff.on('diff', async (data: any) => {
+            const trigger = data.trigger[0];
+            // console.log(trigger.blobs.length);
+            const { blobs } = trigger;
+
+            const detections: ObjectDetectionResult[] = [];
+            if (blobs?.length) {
+                for (const blob of blobs) {
+                    detections.push(
+                        {
+                            className: 'motion',
+                            score: 1,
+                            boundingBox: [blob.minX, blob.minY, blob.maxX - blob.minX, blob.maxY - blob.minY],
+                        }
+                    )
+                }
+            }
+            else {
+                detections.push(
+                    {
+                        className: 'motion',
+                        score: trigger.percent / 100,
+                    }
+                )
+            }
+            const event: ObjectsDetected = {
+                timestamp: Date.now(),
+                running: true,
+                inputDimensions: [width, height],
+                detections,
+            }
+            queued.push(event);
+        });
+
+
+        for await (const videoFrame of videoFrames) {
+            const header = `P7
+WIDTH ${width}
+HEIGHT ${height}
+DEPTH 3
+MAXVAL 255
+TUPLTYPE RGB
+ENDHDR
+`;
+
+            const buffer = await videoFrame.toBuffer({
+                resize: {
+                    width,
+                    height,
+                },
+                format: 'rgb',
+            });
+            pt.write(Buffer.from(header));
+            pt.write(buffer);
+
+            if (!queued.length) {
+                yield {
+                    __json_copy_serialize_children: true,
+                    videoFrame,
+                    detected: {
+                        timestamp: Date.now(),
+                        detections: [],
+                    }
+                }
+            }
+            while (queued.length) {
+                yield {
+                    __json_copy_serialize_children: true,
+                    detected: queued.pop(),
+                    videoFrame,
+                };
+            }
+        }
+    }
+
+
+    async generateObjectDetections(videoFrames: AsyncGenerator<VideoFrame, any, unknown>, session: ObjectDetectionGeneratorSession): Promise<AsyncGenerator<ObjectDetectionGeneratorResult, any, unknown>> {
+        return this.generateObjectDetectionsInternal(videoFrames, session);
+    }
+
     async detectObjects(mediaObject: MediaObject, session?: ObjectDetectionSession, callbacks?: ObjectDetectionCallbacks): Promise<ObjectsDetected> {
         if (mediaObject && mediaObject.mimeType?.startsWith('image/'))
             throw new Error('can not run motion detection on image')
@@ -209,6 +303,8 @@ class PamDiff extends ScryptedDeviceBase implements ObjectDetection {
         return {
             name: '@scrypted/pam-diff',
             classes: ['motion'],
+            inputFormat: 'rgb',
+            inputSize: [640, 360],
             settings: [
                 {
                     title: 'Motion Difference',
@@ -229,4 +325,4 @@ class PamDiff extends ScryptedDeviceBase implements ObjectDetection {
     }
 }
 
-export default new PamDiff();
+export default PamDiff;