Add a way to pass a transform matrix along with Stream::setAcquiredIm…

…age (#8496) BUGS=[399959254] --------- Co-authored-by: Syed Idris Shah <idrisshah@google.com> Co-authored-by: Haneul Kim <haneulk730@gmail.com> Co-authored-by: Powei Feng <powei@google.com> Co-authored-by: Doris Wu <doriswu@google.com> Co-authored-by: Mathias Agopian <mathias@google.com>
google · Mar 10, 2025 · b9a39bc · b9a39bc
1 parent e825f43
commit b9a39bc
Show file tree

Hide file tree

Showing 14 changed files with 39 additions and 23 deletions.
diff --git a/filament/backend/include/backend/AcquiredImage.h b/filament/backend/include/backend/AcquiredImage.h
@@ -18,6 +18,7 @@
 #define TNT_FILAMENT_BACKEND_PRIVATE_ACQUIREDIMAGE_H
 
 #include <backend/DriverEnums.h>
+#include <math/mat3.h>
 
 namespace filament::backend {
 

diff --git a/filament/backend/include/private/backend/Driver.h b/filament/backend/include/private/backend/Driver.h
@@ -24,6 +24,7 @@
 #include <backend/Handle.h>
 #include <backend/PipelineState.h>
 #include <backend/TargetBufferInfo.h>
+#include <backend/AcquiredImage.h>
 
 #include <utils/CString.h>
 #include <utils/compiler.h>

diff --git a/filament/backend/include/private/backend/DriverAPI.inc b/filament/backend/include/private/backend/DriverAPI.inc
@@ -339,7 +339,7 @@ DECL_DRIVER_API_N(destroyDescriptorSet,         backend::DescriptorSetHandle, ds
 DECL_DRIVER_API_SYNCHRONOUS_0(void, terminate)
 DECL_DRIVER_API_SYNCHRONOUS_N(backend::StreamHandle, createStreamNative, void*, stream)
 DECL_DRIVER_API_SYNCHRONOUS_0(backend::StreamHandle, createStreamAcquired)
-DECL_DRIVER_API_SYNCHRONOUS_N(void, setAcquiredImage, backend::StreamHandle, stream, void*, image, backend::CallbackHandler*, handler, backend::StreamCallback, cb, void*, userData)
+DECL_DRIVER_API_SYNCHRONOUS_N(void, setAcquiredImage, backend::StreamHandle, stream, void*, image, const math::mat3f&, transform, backend::CallbackHandler*, handler, backend::StreamCallback, cb, void*, userData)
 DECL_DRIVER_API_SYNCHRONOUS_N(void, setStreamDimensions, backend::StreamHandle, stream, uint32_t, width, uint32_t, height)
 DECL_DRIVER_API_SYNCHRONOUS_N(int64_t, getStreamTimestamp, backend::StreamHandle, stream)
 DECL_DRIVER_API_SYNCHRONOUS_N(void, updateStreams, backend::DriverApi*, driver)

diff --git a/filament/backend/include/private/backend/HandleAllocator.h b/filament/backend/include/private/backend/HandleAllocator.h
@@ -40,7 +40,7 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#define HandleAllocatorGL  HandleAllocator<32,  96, 136>    // ~4520 / pool / MiB
+#define HandleAllocatorGL  HandleAllocator<32,  96, 184>    // ~4520 / pool / MiB
 #define HandleAllocatorVK  HandleAllocator<64, 160, 312>    // ~1820 / pool / MiB
 #define HandleAllocatorMTL HandleAllocator<32,  64, 552>    // ~1660 / pool / MiB
 

diff --git a/filament/backend/src/Driver.cpp b/filament/backend/src/Driver.cpp
@@ -140,8 +140,8 @@ void DriverBase::scheduleDestroySlow(BufferDescriptor&& buffer) noexcept {
 // This is called from an async driver method so it's in the GL thread, but purge is called
 // on the user thread. This is typically called 0 or 1 times per frame.
 void DriverBase::scheduleRelease(AcquiredImage const& image) noexcept {
-    scheduleCallback(image.handler, [image]() {
-        image.callback(image.image, image.userData);
+    scheduleCallback(image.handler, [callback = image.callback, image = image.image, userData = image.userData]() {
+        callback(image, userData);
     });
 }
 

diff --git a/filament/backend/src/metal/MetalDriver.mm b/filament/backend/src/metal/MetalDriver.mm
@@ -984,7 +984,7 @@
     return {};
 }
 
-void MetalDriver::setAcquiredImage(Handle<HwStream> sh, void* image,
+void MetalDriver::setAcquiredImage(Handle<HwStream> sh, void* image, const math::mat3f& transform,
         CallbackHandler* handler, StreamCallback cb, void* userData) {
 }
 

diff --git a/filament/backend/src/noop/NoopDriver.cpp b/filament/backend/src/noop/NoopDriver.cpp
@@ -133,7 +133,7 @@ Handle<HwStream> NoopDriver::createStreamAcquired() {
     return {};
 }
 
-void NoopDriver::setAcquiredImage(Handle<HwStream> sh, void* image,
+void NoopDriver::setAcquiredImage(Handle<HwStream> sh, void* image, const math::mat3f& transform,
         CallbackHandler* handler, StreamCallback cb, void* userData) {
 }
 

diff --git a/filament/backend/src/opengl/OpenGLDriver.cpp b/filament/backend/src/opengl/OpenGLDriver.cpp
@@ -2086,7 +2086,7 @@ Handle<HwStream> OpenGLDriver::createStreamAcquired() {
 // setAcquiredImage should be called by the user outside of beginFrame / endFrame, and should be
 // called only once per frame. If the user pushes images to the same stream multiple times in a
 // single frame, we emit a warning and honor only the final image, but still invoke all callbacks.
-void OpenGLDriver::setAcquiredImage(Handle<HwStream> sh, void* hwbuffer,
+void OpenGLDriver::setAcquiredImage(Handle<HwStream> sh, void* hwbuffer, const math::mat3f& transform,
         CallbackHandler* handler, StreamCallback cb, void* userData) {
     GLStream* glstream = handle_cast<GLStream*>(sh);
     assert_invariant(glstream->streamType == StreamType::ACQUIRED);
@@ -2122,7 +2122,7 @@ void OpenGLDriver::updateStreams(DriverApi* driver) {
 
             // Bind the stashed EGLImage to its corresponding GL texture as soon as we start
             // making the GL calls for the upcoming frame.
-            driver->queueCommand([this, s, image = s->user_thread.acquired.image, previousImage]() {
+            driver->queueCommand([this, s, image = s->user_thread.acquired.image, previousImage, transform = s->user_thread.transform]() {
 
                 auto& streams = mTexturesWithStreamsAttached;
                 auto pos = std::find_if(streams.begin(), streams.end(),
@@ -2137,6 +2137,7 @@ void OpenGLDriver::updateStreams(DriverApi* driver) {
                         t->gl.target = t->externalTexture->target;
                         t->gl.id = t->externalTexture->id;
                         bindTexture(OpenGLContext::DUMMY_TEXTURE_BINDING, t);
+                        s->transform = transform;
                     }
                 }
 
@@ -2167,8 +2168,12 @@ int64_t OpenGLDriver::getStreamTimestamp(Handle<HwStream> sh) {
 
 math::mat3f OpenGLDriver::getStreamTransformMatrix(Handle<HwStream> sh) {
     if (sh) {
-        GLStream* s = handle_cast<GLStream*>(sh);        
-        return mPlatform.getTransformMatrix(s->stream);
+        GLStream* s = handle_cast<GLStream*>(sh);
+        if (s->streamType == StreamType::NATIVE) {
+            return mPlatform.getTransformMatrix(s->stream);
+        } else {
+            return s->transform;
+        }
     }
     return math::mat3f();
 }

diff --git a/filament/backend/src/opengl/OpenGLDriver.h b/filament/backend/src/opengl/OpenGLDriver.h
@@ -161,7 +161,10 @@ class OpenGLDriver final : public OpenGLDriverBase {
             uint8_t cur = 0;
             AcquiredImage acquired;
             AcquiredImage pending;
+            math::mat3f transform;
         } user_thread;
+
+         math::mat3f transform;
     };
 
     struct GLRenderTarget : public HwRenderTarget {

diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp
@@ -907,7 +907,7 @@ Handle<HwStream> VulkanDriver::createStreamAcquired() {
     return {};
 }
 
-void VulkanDriver::setAcquiredImage(Handle<HwStream> sh, void* image,
+void VulkanDriver::setAcquiredImage(Handle<HwStream> sh, void* image, const math::mat3f& transform,
         CallbackHandler* handler, StreamCallback cb, void* userData) {
 }
 

diff --git a/filament/include/filament/Stream.h b/filament/include/filament/Stream.h
@@ -24,6 +24,8 @@
 
 #include <utils/compiler.h>
 
+#include <math/mat3.h>
+
 #include <stdint.h>
 
 namespace filament {
@@ -189,9 +191,10 @@ class UTILS_PUBLIC Stream : public FilamentAPI {
      *                   The callback tales two arguments: the AHardwareBuffer and the userdata.
      * @param userdata   Optional closure data. Filament will pass this into the callback when it
      *                   releases the image.
+     * @param transform  Optional transform matrix to apply to the image.
      */
     void setAcquiredImage(void* UTILS_NONNULL image,
-            Callback UTILS_NONNULL callback, void* UTILS_NULLABLE userdata) noexcept;
+            Callback UTILS_NONNULL callback, void* UTILS_NULLABLE userdata, math::mat3f const& transform = math::mat3f()) noexcept;
 
     /**
      * @see setAcquiredImage(void*, Callback, void*)
@@ -202,10 +205,11 @@ class UTILS_PUBLIC Stream : public FilamentAPI {
      *                   It callback tales two arguments: the AHardwareBuffer and the userdata.
      * @param userdata   Optional closure data. Filament will pass this into the callback when it
      *                   releases the image.
+     * @param transform  Optional transform matrix to apply to the image.
      */
     void setAcquiredImage(void* UTILS_NONNULL image,
             backend::CallbackHandler* UTILS_NULLABLE handler,
-            Callback UTILS_NONNULL callback, void* UTILS_NULLABLE userdata) noexcept;
+            Callback UTILS_NONNULL callback, void* UTILS_NULLABLE userdata, math::mat3f const& transform = math::mat3f()) noexcept;
 
     /**
      * Updates the size of the incoming stream. Whether this value is used is

diff --git a/filament/src/Stream.cpp b/filament/src/Stream.cpp
@@ -24,13 +24,13 @@ StreamType Stream::getStreamType() const noexcept {
     return downcast(this)->getStreamType();
 }
 
-void Stream::setAcquiredImage(void* image, Callback const callback, void* userdata) noexcept {
-    downcast(this)->setAcquiredImage(image, callback, userdata);
+void Stream::setAcquiredImage(void* image, Callback const callback, void* userdata, math::mat3f const& transform) noexcept {
+    downcast(this)->setAcquiredImage(image, callback, userdata, transform);
 }
 
 void Stream::setAcquiredImage(void* image,
-        CallbackHandler* handler, Callback const callback, void* userdata) noexcept {
-    downcast(this)->setAcquiredImage(image, handler, callback, userdata);
+        CallbackHandler* handler, Callback const callback, void* userdata, math::mat3f const& transform) noexcept {
+    downcast(this)->setAcquiredImage(image, handler, callback, userdata, transform);
 }
 
 void Stream::setDimensions(uint32_t const width, uint32_t const height) noexcept {

diff --git a/filament/src/details/Stream.cpp b/filament/src/details/Stream.cpp
@@ -95,13 +95,13 @@ void FStream::terminate(FEngine& engine) noexcept {
 }
 
 void FStream::setAcquiredImage(void* image,
-        Callback const callback, void* userdata) noexcept {
-    mEngine.getDriverApi().setAcquiredImage(mStreamHandle, image, nullptr, callback, userdata);
+    Callback const callback, void* userdata, math::mat3f const& transform) noexcept {
+    mEngine.getDriverApi().setAcquiredImage(mStreamHandle, image, transform, nullptr, callback, userdata);
 }
 
 void FStream::setAcquiredImage(void* image,
-        CallbackHandler* handler, Callback const callback, void* userdata) noexcept {
-    mEngine.getDriverApi().setAcquiredImage(mStreamHandle, image, handler, callback, userdata);
+    CallbackHandler* handler, Callback const callback, void* userdata, math::mat3f const& transform) noexcept {
+    mEngine.getDriverApi().setAcquiredImage(mStreamHandle, image, transform, handler, callback, userdata);
 }
 
 void FStream::setDimensions(uint32_t const width, uint32_t const height) noexcept {

diff --git a/filament/src/details/Stream.h b/filament/src/details/Stream.h
@@ -25,6 +25,8 @@
 
 #include <utils/compiler.h>
 
+#include <math/mat3.h>
+
 namespace filament {
 
 class FEngine;
@@ -36,8 +38,8 @@ class FStream : public Stream {
 
     backend::Handle<backend::HwStream> getHandle() const noexcept { return mStreamHandle; }
 
-    void setAcquiredImage(void* image, Callback callback, void* userdata) noexcept;
-    void setAcquiredImage(void* image, backend::CallbackHandler* handler, Callback callback, void* userdata) noexcept;
+    void setAcquiredImage(void* image, Callback callback, void* userdata, math::mat3f const& transform) noexcept;
+    void setAcquiredImage(void* image, backend::CallbackHandler* handler, Callback callback, void* userdata, math::mat3f const& transform) noexcept;
 
     void setDimensions(uint32_t width, uint32_t height) noexcept;