diff --git a/.storybook/stories/FaceControls.stories.tsx b/.storybook/stories/FaceControls.stories.tsx index 0037aa5f1..4dbb4e926 100644 --- a/.storybook/stories/FaceControls.stories.tsx +++ b/.storybook/stories/FaceControls.stories.tsx @@ -4,8 +4,9 @@ import { Meta, StoryObj } from '@storybook/react' import { Setup } from '../Setup' -import { FaceLandmarker, FaceControls, Box } from '../../src' -import { ComponentProps } from 'react' +import { FaceLandmarker, FaceControls, Box, WebcamVideoTexture } from '../../src' +import { ComponentProps, ElementRef, useRef, useState } from 'react' +import { FaceLandmarkerResult } from '@mediapipe/tasks-vision' export default { title: 'Controls/FaceControls', @@ -17,10 +18,13 @@ export default { ), ], + tags: ['!autodocs'], // FaceLandmarker cannot have multiple instances accross different SB iframes } satisfies Meta type Story = StoryObj +// + function FaceControlsScene(props: ComponentProps) { return ( <> @@ -44,3 +48,47 @@ export const FaceControlsSt = { render: (args) => , name: 'Default', } satisfies Story + +// + +function FaceControlsScene2(props: ComponentProps) { + const faceLandmarkerRef = useRef>(null) + const videoTextureRef = useRef>(null) + + const [faceLandmarkerResult, setFaceLandmarkerResult] = useState() + + return ( + <> + + + + + + { + const faceLandmarker = faceLandmarkerRef.current + const videoTexture = videoTextureRef.current + if (!faceLandmarker || !videoTexture) return + + const videoFrame = videoTexture.source.data + const result = faceLandmarker.detectForVideo(videoFrame, now) + setFaceLandmarkerResult(result) + }} + /> + + + + + + + + + + ) +} + +export const FaceControlsSt2 = { + render: (args) => , + name: 'manualDetect', +} satisfies Story diff --git a/src/web/FaceControls.tsx b/src/web/FaceControls.tsx index 994779df5..723a4d3d7 100644 --- a/src/web/FaceControls.tsx +++ b/src/web/FaceControls.tsx @@ -13,17 +13,17 @@ import { RefObject, createContext, useContext, + ElementRef, } from 'react' import { useFrame, useThree } from '@react-three/fiber' -import type { FaceLandmarkerResult } from '@mediapipe/tasks-vision' +import type { FaceLandmarker, FaceLandmarkerResult } from '@mediapipe/tasks-vision' import { easing } from 'maath' -import { suspend, clear } from 'suspend-react' -import { useVideoTexture } from '../core/VideoTexture' +import { VideoTexture, VideoTextureProps, WebcamVideoTexture } from '..' import { Facemesh, FacemeshApi, FacemeshProps } from './Facemesh' import { useFaceLandmarker } from './FaceLandmarker' -type VideoTextureSrc = Parameters[0] // useVideoTexture 1st arg `src` type +type VideoFrame = Parameters[0] function mean(v1: THREE.Vector3, v2: THREE.Vector3) { return v1.clone().add(v2).multiplyScalar(0.5) @@ -40,55 +40,43 @@ function localToLocal(objSrc: THREE.Object3D, v: THREE.Vector3, objDst: THREE.Ob // export type FaceControlsProps = { - /** The camera to be controlled, default: global state camera */ + /** The camera to be controlled */ camera?: THREE.Camera - /** Whether to autostart the webcam, default: true */ - autostart?: boolean - /** Enable/disable the webcam, default: true */ - webcam?: boolean - /** A custom video URL or mediaStream, default: undefined */ - webcamVideoTextureSrc?: VideoTextureSrc - /** Disable the rAF camera position/rotation update, default: false */ - manualUpdate?: boolean - /** Disable the rVFC face-detection, default: false */ + /** VideoTexture or WebcamVideoTexture options */ + videoTexture: VideoTextureProps + /** Disable the automatic face-detection => you should provide `faceLandmarkerResult` yourself in this case */ manualDetect?: boolean - /** Callback function to call on "videoFrame" event, default: undefined */ - onVideoFrame?: (e: THREE.Event) => void + /** FaceLandmarker result */ + faceLandmarkerResult?: FaceLandmarkerResult + /** Disable the rAF camera position/rotation update */ + manualUpdate?: boolean /** Reference this FaceControls instance as state's `controls` */ makeDefault?: boolean /** Approximate time to reach the target. A smaller value will reach the target faster. */ smoothTime?: number /** Apply position offset extracted from `facialTransformationMatrix` */ offset?: boolean - /** Offset sensitivity factor, less is more sensible, default: 80 */ + /** Offset sensitivity factor, less is more sensible */ offsetScalar?: number /** Enable eye-tracking */ eyes?: boolean - /** Force Facemesh's `origin` to be the middle of the 2 eyes, default: true */ + /** Force Facemesh's `origin` to be the middle of the 2 eyes */ eyesAsOrigin?: boolean - /** Constant depth of the Facemesh, default: .15 */ + /** Constant depth of the Facemesh */ depth?: number - /** Enable debug mode, default: false */ + /** Enable debug mode */ debug?: boolean /** Facemesh options, default: undefined */ facemesh?: FacemeshProps } export type FaceControlsApi = THREE.EventDispatcher & { - /** Detect faces from the video */ - detect: (video: HTMLVideoElement, time: number) => FaceLandmarkerResult | undefined /** Compute the target for the camera */ computeTarget: () => THREE.Object3D /** Update camera's position/rotation to the `target` */ update: (delta: number, target?: THREE.Object3D) => void /** ref api */ facemeshApiRef: RefObject - /** ref api */ - webcamApiRef: RefObject - /** Play the video */ - play: () => void - /** Pause the video */ - pause: () => void } const FaceControlsContext = /* @__PURE__ */ createContext({} as FaceControlsApi) @@ -107,12 +95,11 @@ export const FaceControls = /* @__PURE__ */ forwardRef { @@ -131,8 +117,6 @@ export const FaceControls = /* @__PURE__ */ forwardRef state.get) const explCamera = camera || defaultCamera - const webcamApiRef = useRef(null) - const facemeshApiRef = useRef(null) // @@ -223,67 +207,49 @@ export const FaceControls = /* @__PURE__ */ forwardRef { + if (manualUpdate) return + update(delta) + }) + // - // detect() + // onVideoFrame (only used if !manualDetect) // - const [faces, setFaces] = useState() - const faceLandmarker = useFaceLandmarker() - const detect = useCallback( - (video, time) => { - const result = faceLandmarker?.detectForVideo(video, time) - setFaces(result) + const videoTextureRef = useRef>(null) - return result + const [_faceLandmarkerResult, setFaceLandmarkerResult] = useState() + const faceLandmarker = useFaceLandmarker() + const onVideoFrame = useCallback>( + (now, metadata) => { + const texture = videoTextureRef.current + if (!texture) return + const videoFrame = texture.source.data as VideoFrame + const result = faceLandmarker?.detectForVideo(videoFrame, now) + setFaceLandmarkerResult(result) }, [faceLandmarker] ) - useFrame((_, delta) => { - if (!manualUpdate) { - update(delta) - } - }) - + // // Ref API + // + const api = useMemo( () => Object.assign(Object.create(THREE.EventDispatcher.prototype), { - detect, computeTarget, update, facemeshApiRef, - webcamApiRef, - // shorthands - play: () => { - webcamApiRef.current?.videoTextureApiRef.current?.texture.source.data.play() - }, - pause: () => { - webcamApiRef.current?.videoTextureApiRef.current?.texture.source.data.pause() - }, }), - [detect, computeTarget, update] + [computeTarget, update] ) useImperativeHandle(fref, () => api, [api]) // - // events callbacks + // makeDefault (`controls` global state) // - useEffect(() => { - const onVideoFrameCb = (e: THREE.Event) => { - if (!manualDetect) detect(e.texture.source.data, e.time) - if (onVideoFrame) onVideoFrame(e) - } - - api.addEventListener('videoFrame', onVideoFrameCb) - - return () => { - api.removeEventListener('videoFrame', onVideoFrameCb) - } - }, [api, detect, faceLandmarker, manualDetect, onVideoFrame]) - - // `controls` global state useEffect(() => { if (makeDefault) { const old = get().controls @@ -292,14 +258,27 @@ export const FaceControls = /* @__PURE__ */ forwardRef - {webcam && ( + {!manualDetect && ( - + {'src' in videoTextureProps ? ( + + ) : ( + + )} )} @@ -326,103 +305,3 @@ export const FaceControls = /* @__PURE__ */ forwardRef useContext(FaceControlsContext) - -// -// Webcam -// - -type WebcamApi = { - videoTextureApiRef: RefObject -} - -type WebcamProps = { - videoTextureSrc?: VideoTextureSrc - autostart?: boolean -} - -const Webcam = /* @__PURE__ */ forwardRef(({ videoTextureSrc, autostart = true }, fref) => { - const videoTextureApiRef = useRef(null) - - const faceControls = useFaceControls() - - const stream: MediaStream | null = suspend(async () => { - return !videoTextureSrc - ? await navigator.mediaDevices.getUserMedia({ - audio: false, - video: { facingMode: 'user' }, - }) - : Promise.resolve(null) - }, [videoTextureSrc]) - - useEffect(() => { - faceControls.dispatchEvent({ type: 'stream', stream }) - - return () => { - stream?.getTracks().forEach((track) => track.stop()) - clear([videoTextureSrc]) - } - }, [stream, faceControls, videoTextureSrc]) - - // ref-api - const api = useMemo( - () => ({ - videoTextureApiRef, - }), - [] - ) - useImperativeHandle(fref, () => api, [api]) - - return ( - - - - ) -}) - -// -// VideoTexture -// - -type VideoTextureApi = { texture: THREE.VideoTexture } -type VideoTextureProps = { src: VideoTextureSrc; start: boolean } - -const VideoTexture = /* @__PURE__ */ forwardRef(({ src, start }, fref) => { - const texture = useVideoTexture(src, { start }) - const video = texture.source.data - - const faceControls = useFaceControls() - const onVideoFrame = useCallback( - (time: number) => { - faceControls.dispatchEvent({ type: 'videoFrame', texture, time }) - }, - [texture, faceControls] - ) - useVideoFrame(video, onVideoFrame) - - // ref-api - const api = useMemo( - () => ({ - texture, - }), - [texture] - ) - useImperativeHandle(fref, () => api, [api]) - - return <> -}) - -const useVideoFrame = (video: HTMLVideoElement, f: (...args: any) => any) => { - // https://web.dev/requestvideoframecallback-rvfc/ - // https://www.remotion.dev/docs/video-manipulation - useEffect(() => { - if (!video || !video.requestVideoFrameCallback) return - let handle: number - function callback(...args: any) { - f(...args) - handle = video.requestVideoFrameCallback(callback) - } - video.requestVideoFrameCallback(callback) - - return () => video.cancelVideoFrameCallback(handle) - }, [video, f]) -} diff --git a/src/web/FaceLandmarker.tsx b/src/web/FaceLandmarker.tsx index 7cb1b8906..68c29f5bd 100644 --- a/src/web/FaceLandmarker.tsx +++ b/src/web/FaceLandmarker.tsx @@ -1,6 +1,6 @@ /* eslint react-hooks/exhaustive-deps: 1 */ import * as React from 'react' -import { createContext, ReactNode, useContext, useEffect } from 'react' +import { createContext, forwardRef, ReactNode, useContext, useEffect, useImperativeHandle } from 'react' import type { FaceLandmarker as FaceLandmarkerImpl, FaceLandmarkerOptions } from '@mediapipe/tasks-vision' import { clear, suspend } from 'suspend-react' @@ -26,28 +26,28 @@ export const FaceLandmarkerDefaults = { } as FaceLandmarkerOptions, } -export function FaceLandmarker({ - basePath = FaceLandmarkerDefaults.basePath, - options = FaceLandmarkerDefaults.options, - children, -}: FaceLandmarkerProps) { - const opts = JSON.stringify(options) - - const faceLandmarker = suspend(async () => { - const { FilesetResolver, FaceLandmarker } = await import('@mediapipe/tasks-vision') - const vision = await FilesetResolver.forVisionTasks(basePath) - return FaceLandmarker.createFromOptions(vision, options) - }, [basePath, opts]) - - useEffect(() => { - return () => { - faceLandmarker?.close() - clear([basePath, opts]) - } - }, [faceLandmarker, basePath, opts]) - - return {children} -} +export const FaceLandmarker = forwardRef( + ({ basePath = FaceLandmarkerDefaults.basePath, options = FaceLandmarkerDefaults.options, children }, fref) => { + const opts = JSON.stringify(options) + + const faceLandmarker = suspend(async () => { + const { FilesetResolver, FaceLandmarker } = await import('@mediapipe/tasks-vision') + const vision = await FilesetResolver.forVisionTasks(basePath) + return FaceLandmarker.createFromOptions(vision, options) + }, [basePath, opts]) + + useEffect(() => { + return () => { + faceLandmarker?.close() + clear([basePath, opts]) + } + }, [faceLandmarker, basePath, opts]) + + useImperativeHandle(fref, () => faceLandmarker, [faceLandmarker]) // expose faceLandmarker through ref + + return {children} + } +) export function useFaceLandmarker() { return useContext(FaceLandmarkerContext)