Implement MultipleInputVideoGraph

Also adds DebugTraceUtil category to track MultipleInputVideoGraph events. PiperOrigin-RevId: 564749202
androidx · Sep 12, 2023 · e2882c0 · e2882c0
1 parent b042943
commit e2882c0
Show file tree

Hide file tree

Showing 8 changed files with 560 additions and 38 deletions.
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DebugTraceUtil.java b/libraries/effect/src/main/java/androidx/media3/effect/DebugTraceUtil.java
@@ -55,7 +55,9 @@ public final class DebugTraceUtil {
     EVENT_VFP_QUEUE_BITMAP,
     EVENT_VFP_QUEUE_TEXTURE,
     EVENT_VFP_RENDERED_TO_OUTPUT_SURFACE,
+    EVENT_VFP_OUTPUT_TEXTURE_RENDERED,
     EVENT_VFP_FINISH_PROCESSING_INPUT_STREAM,
+    EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED,
     EVENT_ENCODER_ENCODED_FRAME,
     EVENT_MUXER_CAN_WRITE_SAMPLE_VIDEO,
     EVENT_MUXER_WRITE_SAMPLE_VIDEO,
@@ -84,7 +86,10 @@ public final class DebugTraceUtil {
   public static final String EVENT_VFP_QUEUE_BITMAP = "VFP-QueueBitmap";
   public static final String EVENT_VFP_QUEUE_TEXTURE = "VFP-QueueTexture";
   public static final String EVENT_VFP_RENDERED_TO_OUTPUT_SURFACE = "VFP-RenderedToOutputSurface";
+  public static final String EVENT_VFP_OUTPUT_TEXTURE_RENDERED = "VFP-OutputTextureRendered";
   public static final String EVENT_VFP_FINISH_PROCESSING_INPUT_STREAM = "VFP-FinishOneInputStream";
+  public static final String EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED =
+      "COMP-OutputTextureRendered";
   public static final String EVENT_ENCODER_ENCODED_FRAME = "Encoder-EncodedFrame";
   public static final String EVENT_MUXER_CAN_WRITE_SAMPLE_VIDEO = "Muxer-CanWriteSample_Video";
   public static final String EVENT_MUXER_WRITE_SAMPLE_VIDEO = "Muxer-WriteSample_Video";
@@ -115,7 +120,9 @@ public final class DebugTraceUtil {
           EVENT_VFP_QUEUE_BITMAP,
           EVENT_VFP_QUEUE_TEXTURE,
           EVENT_VFP_RENDERED_TO_OUTPUT_SURFACE,
+          EVENT_VFP_OUTPUT_TEXTURE_RENDERED,
           EVENT_VFP_FINISH_PROCESSING_INPUT_STREAM,
+          EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED,
           EVENT_ENCODER_ENCODED_FRAME,
           EVENT_MUXER_CAN_WRITE_SAMPLE_VIDEO,
           EVENT_MUXER_WRITE_SAMPLE_VIDEO,

diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
@@ -15,7 +15,6 @@
  */
 package androidx.media3.effect;
 
-import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
 import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
@@ -87,8 +86,8 @@ public static final class Factory implements VideoFrameProcessor.Factory {
     /** A builder for {@link DefaultVideoFrameProcessor.Factory} instances. */
     public static final class Builder {
       private boolean enableColorTransfers;
+      @Nullable private ExecutorService executorService;
       private @MonotonicNonNull GlObjectsProvider glObjectsProvider;
-      private @MonotonicNonNull ExecutorService executorService;
       private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener;
       private int textureOutputCapacity;
 
@@ -97,6 +96,14 @@ public Builder() {
         enableColorTransfers = true;
       }
 
+      private Builder(Factory factory) {
+        enableColorTransfers = factory.enableColorTransfers;
+        executorService = factory.executorService;
+        glObjectsProvider = factory.glObjectsProvider;
+        textureOutputListener = factory.textureOutputListener;
+        textureOutputCapacity = factory.textureOutputCapacity;
+      }
+
       /**
        * Sets whether to transfer colors to an intermediate color space when applying effects.
        *
@@ -122,18 +129,18 @@ public Builder setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
       /**
        * Sets the {@link Util#newSingleThreadScheduledExecutor} to execute GL commands from.
        *
-       * <p>If set, the {@link ExecutorService} must be {@linkplain ExecutorService#shutdown shut
-       * down} by the caller after all {@linkplain VideoFrameProcessor VideoFrameProcessors} using
-       * it have been {@linkplain #release released}.
+       * <p>If set to a non-null value, the {@link ExecutorService} must be {@linkplain
+       * ExecutorService#shutdown shut down} by the caller after all {@linkplain VideoFrameProcessor
+       * VideoFrameProcessors} using it have been {@linkplain #release released}.
        *
        * <p>The default value is a new {@link Util#newSingleThreadScheduledExecutor}, owned and
-       * {@link ExecutorService#shutdown} by the created {@link DefaultVideoFrameProcessor}.
+       * {@link ExecutorService#shutdown} by the created {@link DefaultVideoFrameProcessor}. Setting
+       * a {@code null} {@link ExecutorService} is equivalent to using the default value.
        *
        * @param executorService The {@link ExecutorService}.
        */
       @CanIgnoreReturnValue
-      @VisibleForTesting(otherwise = PACKAGE_PRIVATE)
-      public Builder setExecutorService(ExecutorService executorService) {
+      public Builder setExecutorService(@Nullable ExecutorService executorService) {
         this.executorService = executorService;
         return this;
       }
@@ -156,7 +163,6 @@ public Builder setExecutorService(ExecutorService executorService) {
        * @param textureOutputCapacity The amount of output textures that may be allocated at a time
        *     before texture output blocks. Must be greater than or equal to 1.
        */
-      @VisibleForTesting
       @CanIgnoreReturnValue
       public Builder setTextureOutput(
           GlTextureProducer.Listener textureOutputListener,
@@ -197,6 +203,10 @@ private Factory(
       this.textureOutputCapacity = textureOutputCapacity;
     }
 
+    public Builder buildUpon() {
+      return new Builder(this);
+    }
+
     /**
      * {@inheritDoc}
      *
@@ -266,6 +276,7 @@ public DefaultVideoFrameProcessor create(
       boolean shouldShutdownExecutorService = executorService == null;
       ExecutorService instanceExecutorService =
           executorService == null ? Util.newSingleThreadExecutor(THREAD_NAME) : executorService;
+
       VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
           new VideoFrameProcessingTaskExecutor(
               instanceExecutorService, shouldShutdownExecutorService, listener::onError);
@@ -427,6 +438,7 @@ public boolean queueInputTexture(int textureId, long presentationTimeUs) {
     if (!inputStreamRegisteredCondition.isOpen()) {
       return false;
     }
+
     inputSwitcher.activeTextureManager().queueInputTexture(textureId, presentationTimeUs);
     return true;
   }