diff --git a/plugins/pam-diff/package-lock.json b/plugins/pam-diff/package-lock.json index 33ac32683f..ad2f22fa7d 100644 --- a/plugins/pam-diff/package-lock.json +++ b/plugins/pam-diff/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/pam-diff", - "version": "0.0.16", + "version": "0.0.17", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/pam-diff", - "version": "0.0.16", + "version": "0.0.17", "hasInstallScript": true, "dependencies": { "@types/node": "^16.6.1", diff --git a/plugins/pam-diff/package.json b/plugins/pam-diff/package.json index f4b343d5f3..3a4155c93b 100644 --- a/plugins/pam-diff/package.json +++ b/plugins/pam-diff/package.json @@ -43,5 +43,5 @@ "devDependencies": { "@scrypted/sdk": "file:../../sdk" }, - "version": "0.0.16" + "version": "0.0.17" } diff --git a/plugins/pam-diff/src/main.ts b/plugins/pam-diff/src/main.ts index 6810019ec9..1d7940b88c 100644 --- a/plugins/pam-diff/src/main.ts +++ b/plugins/pam-diff/src/main.ts @@ -1,11 +1,10 @@ -import { ObjectDetectionResult, FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionModel, ObjectDetectionSession, ObjectsDetected, ScryptedDeviceBase, ScryptedInterface, ScryptedMimeTypes } from '@scrypted/sdk'; -import sdk from '@scrypted/sdk'; -import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "../../../common/src/media-helpers"; - +import sdk, { FFmpegInput, MediaObject, ObjectDetection, ObjectDetectionCallbacks, ObjectDetectionGeneratorResult, ObjectDetectionGeneratorSession, ObjectDetectionModel, ObjectDetectionResult, ObjectDetectionSession, ObjectsDetected, ScryptedDeviceBase, ScryptedInterface, ScryptedMimeTypes, VideoFrame } from '@scrypted/sdk'; import child_process, { ChildProcess } from 'child_process'; +import { ffmpegLogInitialOutput, safeKillFFmpeg, safePrintFFmpegArguments } from "../../../common/src/media-helpers"; import PD from 'pam-diff'; import P2P from 'pipe2pam'; +import { PassThrough, Writable } from 'stream'; const { mediaManager } = sdk; @@ -51,6 +50,101 @@ class PamDiff extends ScryptedDeviceBase implements ObjectDetection { pds.timeout = setTimeout(() => this.endSession(id), duration); } + async * generateObjectDetectionsInternal(videoFrames: AsyncGenerator, session: ObjectDetectionGeneratorSession): AsyncGenerator { + videoFrames = await sdk.connectRPCObject(videoFrames); + + const width = 640; + const height = 360; + const p2p: Writable = new P2P(); + const pt = new PassThrough(); + const pamDiff = new PD({ + difference: parseInt(session.settings?.difference) || defaultDifference, + percent: parseInt(session.settings?.percent) || defaultPercentage, + response: session?.settings?.motionAsObjects ? 'blobs' : 'percent', + }); + pt.pipe(p2p).pipe(pamDiff); + + const queued: ObjectsDetected[] = []; + pamDiff.on('diff', async (data: any) => { + const trigger = data.trigger[0]; + // console.log(trigger.blobs.length); + const { blobs } = trigger; + + const detections: ObjectDetectionResult[] = []; + if (blobs?.length) { + for (const blob of blobs) { + detections.push( + { + className: 'motion', + score: 1, + boundingBox: [blob.minX, blob.minY, blob.maxX - blob.minX, blob.maxY - blob.minY], + } + ) + } + } + else { + detections.push( + { + className: 'motion', + score: trigger.percent / 100, + } + ) + } + const event: ObjectsDetected = { + timestamp: Date.now(), + running: true, + inputDimensions: [width, height], + detections, + } + queued.push(event); + }); + + + for await (const videoFrame of videoFrames) { + const header = `P7 +WIDTH ${width} +HEIGHT ${height} +DEPTH 3 +MAXVAL 255 +TUPLTYPE RGB +ENDHDR +`; + + const buffer = await videoFrame.toBuffer({ + resize: { + width, + height, + }, + format: 'rgb', + }); + pt.write(Buffer.from(header)); + pt.write(buffer); + + if (!queued.length) { + yield { + __json_copy_serialize_children: true, + videoFrame, + detected: { + timestamp: Date.now(), + detections: [], + } + } + } + while (queued.length) { + yield { + __json_copy_serialize_children: true, + detected: queued.pop(), + videoFrame, + }; + } + } + } + + + async generateObjectDetections(videoFrames: AsyncGenerator, session: ObjectDetectionGeneratorSession): Promise> { + return this.generateObjectDetectionsInternal(videoFrames, session); + } + async detectObjects(mediaObject: MediaObject, session?: ObjectDetectionSession, callbacks?: ObjectDetectionCallbacks): Promise { if (mediaObject && mediaObject.mimeType?.startsWith('image/')) throw new Error('can not run motion detection on image') @@ -209,6 +303,8 @@ class PamDiff extends ScryptedDeviceBase implements ObjectDetection { return { name: '@scrypted/pam-diff', classes: ['motion'], + inputFormat: 'rgb', + inputSize: [640, 360], settings: [ { title: 'Motion Difference', @@ -229,4 +325,4 @@ class PamDiff extends ScryptedDeviceBase implements ObjectDetection { } } -export default new PamDiff(); +export default PamDiff;