diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DebugTraceUtil.java b/libraries/effect/src/main/java/androidx/media3/effect/DebugTraceUtil.java index d237972fb30..092c532fb5a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DebugTraceUtil.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DebugTraceUtil.java @@ -55,9 +55,7 @@ public final class DebugTraceUtil { EVENT_VFP_QUEUE_BITMAP, EVENT_VFP_QUEUE_TEXTURE, EVENT_VFP_RENDERED_TO_OUTPUT_SURFACE, - EVENT_VFP_OUTPUT_TEXTURE_RENDERED, EVENT_VFP_FINISH_PROCESSING_INPUT_STREAM, - EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED, EVENT_ENCODER_ENCODED_FRAME, EVENT_MUXER_CAN_WRITE_SAMPLE_VIDEO, EVENT_MUXER_WRITE_SAMPLE_VIDEO, @@ -86,10 +84,7 @@ public final class DebugTraceUtil { public static final String EVENT_VFP_QUEUE_BITMAP = "VFP-QueueBitmap"; public static final String EVENT_VFP_QUEUE_TEXTURE = "VFP-QueueTexture"; public static final String EVENT_VFP_RENDERED_TO_OUTPUT_SURFACE = "VFP-RenderedToOutputSurface"; - public static final String EVENT_VFP_OUTPUT_TEXTURE_RENDERED = "VFP-OutputTextureRendered"; public static final String EVENT_VFP_FINISH_PROCESSING_INPUT_STREAM = "VFP-FinishOneInputStream"; - public static final String EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED = - "COMP-OutputTextureRendered"; public static final String EVENT_ENCODER_ENCODED_FRAME = "Encoder-EncodedFrame"; public static final String EVENT_MUXER_CAN_WRITE_SAMPLE_VIDEO = "Muxer-CanWriteSample_Video"; public static final String EVENT_MUXER_WRITE_SAMPLE_VIDEO = "Muxer-WriteSample_Video"; @@ -120,9 +115,7 @@ public final class DebugTraceUtil { EVENT_VFP_QUEUE_BITMAP, EVENT_VFP_QUEUE_TEXTURE, EVENT_VFP_RENDERED_TO_OUTPUT_SURFACE, - EVENT_VFP_OUTPUT_TEXTURE_RENDERED, EVENT_VFP_FINISH_PROCESSING_INPUT_STREAM, - EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED, EVENT_ENCODER_ENCODED_FRAME, EVENT_MUXER_CAN_WRITE_SAMPLE_VIDEO, EVENT_MUXER_WRITE_SAMPLE_VIDEO, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 89f6a69e5e5..0e5762a9564 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; @@ -86,8 +87,8 @@ public static final class Factory implements VideoFrameProcessor.Factory { /** A builder for {@link DefaultVideoFrameProcessor.Factory} instances. */ public static final class Builder { private boolean enableColorTransfers; - @Nullable private ExecutorService executorService; private @MonotonicNonNull GlObjectsProvider glObjectsProvider; + private @MonotonicNonNull ExecutorService executorService; private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener; private int textureOutputCapacity; @@ -96,14 +97,6 @@ public Builder() { enableColorTransfers = true; } - private Builder(Factory factory) { - enableColorTransfers = factory.enableColorTransfers; - executorService = factory.executorService; - glObjectsProvider = factory.glObjectsProvider; - textureOutputListener = factory.textureOutputListener; - textureOutputCapacity = factory.textureOutputCapacity; - } - /** * Sets whether to transfer colors to an intermediate color space when applying effects. * @@ -129,18 +122,18 @@ public Builder setGlObjectsProvider(GlObjectsProvider glObjectsProvider) { /** * Sets the {@link Util#newSingleThreadScheduledExecutor} to execute GL commands from. * - *

If set to a non-null value, the {@link ExecutorService} must be {@linkplain - * ExecutorService#shutdown shut down} by the caller after all {@linkplain VideoFrameProcessor - * VideoFrameProcessors} using it have been {@linkplain #release released}. + *

If set, the {@link ExecutorService} must be {@linkplain ExecutorService#shutdown shut + * down} by the caller after all {@linkplain VideoFrameProcessor VideoFrameProcessors} using + * it have been {@linkplain #release released}. * *

The default value is a new {@link Util#newSingleThreadScheduledExecutor}, owned and - * {@link ExecutorService#shutdown} by the created {@link DefaultVideoFrameProcessor}. Setting - * a {@code null} {@link ExecutorService} is equivalent to using the default value. + * {@link ExecutorService#shutdown} by the created {@link DefaultVideoFrameProcessor}. * * @param executorService The {@link ExecutorService}. */ @CanIgnoreReturnValue - public Builder setExecutorService(@Nullable ExecutorService executorService) { + @VisibleForTesting(otherwise = PACKAGE_PRIVATE) + public Builder setExecutorService(ExecutorService executorService) { this.executorService = executorService; return this; } @@ -163,6 +156,7 @@ public Builder setExecutorService(@Nullable ExecutorService executorService) { * @param textureOutputCapacity The amount of output textures that may be allocated at a time * before texture output blocks. Must be greater than or equal to 1. */ + @VisibleForTesting @CanIgnoreReturnValue public Builder setTextureOutput( GlTextureProducer.Listener textureOutputListener, @@ -203,10 +197,6 @@ private Factory( this.textureOutputCapacity = textureOutputCapacity; } - public Builder buildUpon() { - return new Builder(this); - } - /** * {@inheritDoc} * @@ -276,7 +266,6 @@ public DefaultVideoFrameProcessor create( boolean shouldShutdownExecutorService = executorService == null; ExecutorService instanceExecutorService = executorService == null ? Util.newSingleThreadExecutor(THREAD_NAME) : executorService; - VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor = new VideoFrameProcessingTaskExecutor( instanceExecutorService, shouldShutdownExecutorService, listener::onError); @@ -438,7 +427,6 @@ public boolean queueInputTexture(int textureId, long presentationTimeUs) { if (!inputStreamRegisteredCondition.isOpen()) { return false; } - inputSwitcher.activeTextureManager().queueInputTexture(textureId, presentationTimeUs); return true; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java deleted file mode 100644 index a12920da782..00000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java +++ /dev/null @@ -1,482 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.transformer; - -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; -import static androidx.media3.common.util.Util.contains; -import static androidx.media3.common.util.Util.newSingleThreadScheduledExecutor; -import static androidx.media3.effect.DebugTraceUtil.EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED; -import static androidx.media3.effect.DebugTraceUtil.EVENT_VFP_OUTPUT_TEXTURE_RENDERED; -import static androidx.media3.effect.DebugTraceUtil.logEvent; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -import android.content.Context; -import android.opengl.EGLContext; -import android.opengl.EGLDisplay; -import android.opengl.EGLSurface; -import android.util.SparseArray; -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.FrameInfo; -import androidx.media3.common.GlObjectsProvider; -import androidx.media3.common.GlTextureInfo; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.util.Consumer; -import androidx.media3.common.util.GlUtil; -import androidx.media3.effect.DefaultGlObjectsProvider; -import androidx.media3.effect.DefaultVideoCompositor; -import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.effect.GlTextureProducer; -import androidx.media3.effect.VideoCompositor; -import com.google.common.util.concurrent.MoreExecutors; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -/** A {@link VideoGraph} that handles multiple input streams. */ -/* package */ final class MultipleInputVideoGraph implements VideoGraph { - - public static final class Factory implements VideoGraph.Factory { - - @Override - public MultipleInputVideoGraph create( - Context context, - ColorInfo inputColorInfo, - ColorInfo outputColorInfo, - Consumer errorConsumer, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - List compositionEffects, - long initialTimestampOffsetUs) { - return new MultipleInputVideoGraph( - context, - inputColorInfo, - outputColorInfo, - errorConsumer, - debugViewProvider, - listener, - listenerExecutor, - compositionEffects, - initialTimestampOffsetUs); - } - } - - private static final String SHARED_EXECUTOR_NAME = "Transformer:MultipleInputVideoGraph:Thread"; - - private static final long RELEASE_WAIT_TIME_MS = 1_000; - private static final int PRE_COMPOSITOR_TEXTURE_OUTPUT_CAPACITY = 2; - private static final int COMPOSITOR_TEXTURE_OUTPUT_CAPACITY = 1; - - private final Context context; - private final ColorInfo inputColorInfo; - private final ColorInfo outputColorInfo; - private final Consumer errorConsumer; - private final GlObjectsProvider glObjectsProvider; - private final DebugViewProvider debugViewProvider; - private final Listener listener; - private final Executor listenerExecutor; - private final List compositionEffects; - private final List preProcessingWrappers; - - private final ExecutorService sharedExecutorService; - - private final DefaultVideoFrameProcessor.Factory videoFrameProcessorFactory; - private final Queue compositorOutputTextures; - private final SparseArray compositorOutputTextureReleases; - - private final long initialTimestampOffsetUs; - - @Nullable private VideoFrameProcessor compositionVideoFrameProcessor; - @Nullable private VideoCompositor videoCompositor; - - private boolean compositionVideoFrameProcessorInputStreamRegistered; - private boolean compositionVideoFrameProcessorInputStreamRegistrationCompleted; - private boolean compositorEnded; - private boolean released; - private long lastRenderedPresentationTimeUs; - - private volatile boolean hasProducedFrameWithTimestampZero; - - // TODO - b/289986435: Remove errorConsumer and use Listener.onError(). - private MultipleInputVideoGraph( - Context context, - ColorInfo inputColorInfo, - ColorInfo outputColorInfo, - Consumer errorConsumer, - DebugViewProvider debugViewProvider, - Listener listener, - Executor listenerExecutor, - List compositionEffects, - long initialTimestampOffsetUs) { - this.context = context; - this.inputColorInfo = inputColorInfo; - this.outputColorInfo = outputColorInfo; - this.errorConsumer = errorConsumer; - this.debugViewProvider = debugViewProvider; - this.listener = listener; - this.listenerExecutor = listenerExecutor; - this.compositionEffects = new ArrayList<>(compositionEffects); - this.initialTimestampOffsetUs = initialTimestampOffsetUs; - lastRenderedPresentationTimeUs = C.TIME_UNSET; - preProcessingWrappers = new ArrayList<>(); - sharedExecutorService = newSingleThreadScheduledExecutor(SHARED_EXECUTOR_NAME); - glObjectsProvider = new SingleContextGlObjectsProvider(); - // TODO - b/289986435: Support injecting VideoFrameProcessor.Factory. - videoFrameProcessorFactory = - new DefaultVideoFrameProcessor.Factory.Builder() - .setGlObjectsProvider(glObjectsProvider) - .setExecutorService(sharedExecutorService) - .build(); - compositorOutputTextures = new ArrayDeque<>(); - compositorOutputTextureReleases = new SparseArray<>(); - } - - /** - * {@inheritDoc} - * - *

This method must be called at most once. - */ - @Override - public void initialize() throws VideoFrameProcessingException { - checkState( - preProcessingWrappers.isEmpty() - && videoCompositor == null - && compositionVideoFrameProcessor == null - && !released); - - // Setting up the compositionVideoFrameProcessor - compositionVideoFrameProcessor = - videoFrameProcessorFactory.create( - context, - debugViewProvider, - // Pre-processing VideoFrameProcessors have converted the inputColor to outputColor - // already. - /* inputColorInfo= */ outputColorInfo, - outputColorInfo, - /* renderFramesAutomatically= */ true, - /* listenerExecutor= */ MoreExecutors.directExecutor(), - new VideoFrameProcessor.Listener() { - @Override - public void onInputStreamRegistered( - @VideoFrameProcessor.InputType int inputType, - List effects, - FrameInfo frameInfo) { - compositionVideoFrameProcessorInputStreamRegistrationCompleted = true; - queueCompositionOutputInternal(); - } - - @Override - public void onOutputSizeChanged(int width, int height) { - checkNotNull(compositionVideoFrameProcessor) - .setOutputSurfaceInfo(listener.onOutputSizeChanged(width, height)); - } - - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - if (presentationTimeUs == 0) { - hasProducedFrameWithTimestampZero = true; - } - lastRenderedPresentationTimeUs = presentationTimeUs; - } - - @Override - public void onError(VideoFrameProcessingException exception) { - handleException(exception); - } - - @Override - public void onEnded() { - listenerExecutor.execute(() -> listener.onEnded(lastRenderedPresentationTimeUs)); - } - }); - // Release the compositor's output texture. - compositionVideoFrameProcessor.setOnInputFrameProcessedListener( - (textureId, syncObject) -> { - checkState(contains(compositorOutputTextureReleases, textureId)); - compositorOutputTextureReleases.get(textureId).release(); - compositorOutputTextureReleases.remove(textureId); - queueCompositionOutputInternal(); - }); - - // Setting up the compositor. - videoCompositor = - new DefaultVideoCompositor( - context, - glObjectsProvider, - new DefaultVideoCompositor.Settings(), - sharedExecutorService, - new VideoCompositor.Listener() { - @Override - public void onError(VideoFrameProcessingException exception) { - handleException(exception); - } - - @Override - public void onEnded() { - compositorEnded = true; - if (compositorOutputTextures.isEmpty()) { - compositionVideoFrameProcessor.signalEndOfInput(); - } else { - queueCompositionOutputInternal(); - } - } - }, - /* textureOutputListener= */ this::processCompositorOutputTexture, - COMPOSITOR_TEXTURE_OUTPUT_CAPACITY); - } - - @Override - public GraphInput createInput() throws VideoFrameProcessingException { - checkStateNotNull(videoCompositor); - - int videoCompositorInputId = videoCompositor.registerInputSource(); - // Creating a new VideoFrameProcessor for the input. - VideoFrameProcessingWrapper preProcessingVideoFrameProcessorWrapper = - new VideoFrameProcessingWrapper( - context, - videoFrameProcessorFactory - .buildUpon() - .setTextureOutput( - // Texture output to compositor. - (textureProducer, texture, presentationTimeUs, syncObject) -> { - logEvent(EVENT_VFP_OUTPUT_TEXTURE_RENDERED, presentationTimeUs); - checkNotNull(videoCompositor) - .queueInputTexture( - videoCompositorInputId, - textureProducer, - texture, - // Color is converted to outputColor in pre processing. - /* colorInfo= */ outputColorInfo, - presentationTimeUs); - }, - PRE_COMPOSITOR_TEXTURE_OUTPUT_CAPACITY) - .build(), - inputColorInfo, - outputColorInfo, - DebugViewProvider.NONE, - listenerExecutor, - new VideoFrameProcessor.Listener() { - @Override - public void onInputStreamRegistered( - @VideoFrameProcessor.InputType int inputType, - List effects, - FrameInfo frameInfo) { - // Do nothing. - } - - @Override - public void onOutputSizeChanged(int width, int height) {} - - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) {} - - @Override - public void onError(VideoFrameProcessingException ex) { - errorConsumer.accept(ExportException.createForVideoFrameProcessingException(ex)); - } - - @Override - public void onEnded() { - checkNotNull(videoCompositor).signalEndOfInputSource(videoCompositorInputId); - } - }, - /* renderFramesAutomatically= */ true, - /* presentation= */ null, - initialTimestampOffsetUs); - preProcessingWrappers.add(preProcessingVideoFrameProcessorWrapper); - return preProcessingVideoFrameProcessorWrapper; - } - - @Override - public boolean hasProducedFrameWithTimestampZero() { - return hasProducedFrameWithTimestampZero; - } - - @Override - public void release() { - if (released) { - return; - } - - // Needs to release the frame processors before their internal executor services are released. - for (int i = 0; i < preProcessingWrappers.size(); i++) { - preProcessingWrappers.get(i).release(); - } - preProcessingWrappers.clear(); - - if (videoCompositor != null) { - videoCompositor.release(); - videoCompositor = null; - } - - if (compositionVideoFrameProcessor != null) { - compositionVideoFrameProcessor.release(); - compositionVideoFrameProcessor = null; - } - - sharedExecutorService.shutdown(); - try { - sharedExecutorService.awaitTermination(RELEASE_WAIT_TIME_MS, MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - listenerExecutor.execute(() -> listener.onError(VideoFrameProcessingException.from(e))); - } - - released = true; - } - - private void handleException(Exception e) { - errorConsumer.accept( - ExportException.createForVideoFrameProcessingException( - e instanceof VideoFrameProcessingException - ? (VideoFrameProcessingException) e - : VideoFrameProcessingException.from(e))); - } - - private void processCompositorOutputTexture( - GlTextureProducer textureProducer, - GlTextureInfo outputTexture, - long presentationTimeUs, - long syncObject) { - checkStateNotNull(compositionVideoFrameProcessor); - checkState(!compositorEnded); - logEvent(EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED, presentationTimeUs); - - compositorOutputTextures.add( - new CompositorOutputTextureInfo(outputTexture, presentationTimeUs)); - compositorOutputTextureReleases.put( - outputTexture.texId, - new CompositorOutputTextureRelease(textureProducer, presentationTimeUs)); - - if (!compositionVideoFrameProcessorInputStreamRegistered) { - checkNotNull(compositionVideoFrameProcessor) - .registerInputStream( - INPUT_TYPE_TEXTURE_ID, - compositionEffects, - new FrameInfo.Builder(outputTexture.width, outputTexture.height).build()); - compositionVideoFrameProcessorInputStreamRegistered = true; - // Return as the VideoFrameProcessor rejects input textures until the input is registered. - return; - } - queueCompositionOutputInternal(); - } - - private void queueCompositionOutputInternal() { - checkStateNotNull(compositionVideoFrameProcessor); - if (!compositionVideoFrameProcessorInputStreamRegistrationCompleted) { - return; - } - - @Nullable CompositorOutputTextureInfo outputTexture = compositorOutputTextures.peek(); - if (outputTexture == null) { - return; - } - - checkState( - checkNotNull(compositionVideoFrameProcessor) - .queueInputTexture( - outputTexture.glTextureInfo.texId, outputTexture.presentationTimeUs)); - compositorOutputTextures.remove(); - if (compositorEnded && compositorOutputTextures.isEmpty()) { - checkNotNull(compositionVideoFrameProcessor).signalEndOfInput(); - } - } - - private static final class CompositorOutputTextureInfo { - public final GlTextureInfo glTextureInfo; - public final long presentationTimeUs; - - private CompositorOutputTextureInfo(GlTextureInfo glTextureInfo, long presentationTimeUs) { - this.glTextureInfo = glTextureInfo; - this.presentationTimeUs = presentationTimeUs; - } - } - - private static final class CompositorOutputTextureRelease { - private final GlTextureProducer textureProducer; - private final long presentationTimeUs; - - public CompositorOutputTextureRelease( - GlTextureProducer textureProducer, long presentationTimeUs) { - this.textureProducer = textureProducer; - this.presentationTimeUs = presentationTimeUs; - } - - public void release() { - textureProducer.releaseOutputTexture(presentationTimeUs); - } - } - - /** - * A {@link GlObjectsProvider} that creates a new {@link EGLContext} in {@link #createEglContext} - * with the same shared EGLContext. - */ - private static final class SingleContextGlObjectsProvider implements GlObjectsProvider { - private final GlObjectsProvider glObjectsProvider; - private @MonotonicNonNull EGLContext singleEglContext; - - public SingleContextGlObjectsProvider() { - this.glObjectsProvider = new DefaultGlObjectsProvider(); - } - - @Override - public EGLContext createEglContext( - EGLDisplay eglDisplay, int openGlVersion, int[] configAttributes) - throws GlUtil.GlException { - if (singleEglContext == null) { - singleEglContext = - glObjectsProvider.createEglContext(eglDisplay, openGlVersion, configAttributes); - } - return singleEglContext; - } - - @Override - public EGLSurface createEglSurface( - EGLDisplay eglDisplay, - Object surface, - @C.ColorTransfer int colorTransfer, - boolean isEncoderInputSurface) - throws GlUtil.GlException { - return glObjectsProvider.createEglSurface( - eglDisplay, surface, colorTransfer, isEncoderInputSurface); - } - - @Override - public EGLSurface createFocusedPlaceholderEglSurface( - EGLContext eglContext, EGLDisplay eglDisplay) throws GlUtil.GlException { - return glObjectsProvider.createFocusedPlaceholderEglSurface(eglContext, eglDisplay); - } - - @Override - public GlTextureInfo createBuffersForTexture(int texId, int width, int height) - throws GlUtil.GlException { - return glObjectsProvider.createBuffersForTexture(texId, width, height); - } - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java index 0c117aff3f9..c51b5ae5f3f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java @@ -125,17 +125,7 @@ private SingleInputVideoGraph( *

This method must be called at most once. */ @Override - public void initialize() { - // Initialization is deferred to createInput(). - } - - /** - * {@inheritDoc} - * - *

This method must only be called once. - */ - @Override - public GraphInput createInput() throws VideoFrameProcessingException { + public void initialize() throws VideoFrameProcessingException { checkStateNotNull(videoFrameProcessingWrapper == null && !released); videoFrameProcessingWrapper = @@ -185,7 +175,12 @@ public void onEnded() { renderFramesAutomatically, presentation, initialTimestampOffsetUs); - return videoFrameProcessingWrapper; + } + + /** Returns the {@link GraphInput}. */ + @Override + public GraphInput getInput() { + return checkNotNull(videoFrameProcessingWrapper); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 3a8a1c6f15e..0bee9c80582 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -782,10 +782,12 @@ public void removeAllListeners() { *

This method is under development. A {@link Composition} must meet the following conditions: * *

* *

{@linkplain EditedMediaItemSequence Sequences} within the {@link Composition} must meet the @@ -970,6 +972,11 @@ private void verifyApplicationThread() { private void startInternal( Composition composition, MuxerWrapper muxerWrapper, ComponentListener componentListener) { checkArgument(composition.effects.audioProcessors.isEmpty()); + // Only supports Presentation in video effects. + ImmutableList videoEffects = composition.effects.videoEffects; + checkArgument( + videoEffects.isEmpty() + || (videoEffects.size() == 1 && videoEffects.get(0) instanceof Presentation)); verifyApplicationThread(); checkState(transformerInternal == null, "There is already an export in progress."); HandlerWrapper applicationHandler = clock.createHandler(looper, /* callback= */ null); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index f7391c27b80..76f3a5fc99e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -570,6 +570,7 @@ private void createDecodedSampleExporter(Format assetLoaderOutputFormat) fallbackListener)); } else { + ImmutableList compositionVideoEffects = composition.effects.videoEffects; // TODO(b/267301878): Pass firstAssetLoaderOutputFormat once surface creation not in VSP. assetLoaderInputTracker.registerSampleExporter( C.TRACK_TYPE_VIDEO, @@ -577,15 +578,14 @@ private void createDecodedSampleExporter(Format assetLoaderOutputFormat) context, firstAssetLoaderInputFormat, transformationRequest, - composition.effects.videoEffects, + compositionVideoEffects, videoFrameProcessorFactory, encoderFactory, muxerWrapper, /* errorConsumer= */ this::onError, fallbackListener, debugViewProvider, - videoSampleTimestampOffsetUs, - /* hasMultipleInputs= */ composition.sequences.size() > 1)); + videoSampleTimestampOffsetUs)); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java index 1043eebe1ab..4a94f12a285 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoGraph.java @@ -68,23 +68,14 @@ interface Listener { * * @param width The new output width in pixels. * @param height The new output width in pixels. - * @return A {@link SurfaceInfo} to which the {@link VideoGraph} renders to, or {@code null} if - * the output is not needed. + * @return A {@link SurfaceInfo} to which {@link SingleInputVideoGraph} renders to, or {@code + * null} if the output is not needed. */ - // TODO - b/289985577: Consider returning void from this method. @Nullable SurfaceInfo onOutputSizeChanged(int width, int height); - /** Called after the {@link VideoGraph} has rendered its final output frame. */ + /** Called after the {@link SingleInputVideoGraph} has rendered its final output frame. */ void onEnded(long finalFramePresentationTimeUs); - - /** - * Called when an exception occurs during video frame processing. - * - *

If this is called, the calling {@link VideoGraph} must immediately be {@linkplain - * #release() released}. - */ - void onError(VideoFrameProcessingException exception); } /** @@ -102,21 +93,15 @@ interface Listener { *

This method must be called after successfully {@linkplain #initialize() initializing} the * {@code VideoGraph}. * - *

This method must called exactly once for every input stream. - * *

If the method throws any {@link Exception}, the caller must call {@link #release}. */ - GraphInput createInput() throws VideoFrameProcessingException; + GraphInput getInput() throws VideoFrameProcessingException; /** * Returns whether the {@code VideoGraph} has produced a frame with zero presentation timestamp. */ boolean hasProducedFrameWithTimestampZero(); - /** - * Releases the associated resources. - * - *

This {@code VideoGraph} instance must not be used after this method is called. - */ + /** Releases the associated resources. */ void release(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index ab1a58702c7..0c098fd3726 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -84,8 +84,7 @@ public VideoSampleExporter( Consumer errorConsumer, FallbackListener fallbackListener, DebugViewProvider debugViewProvider, - long initialTimestampOffsetUs, - boolean hasMultipleInputs) + long initialTimestampOffsetUs) throws ExportException { // TODO(b/278259383) Consider delaying configuration of VideoSampleExporter to use the decoder // output format instead of the extractor output format, to match AudioSampleExporter behavior. @@ -144,9 +143,7 @@ public VideoSampleExporter( videoGraph = new VideoGraphWrapper( context, - hasMultipleInputs - ? new MultipleInputVideoGraph.Factory() - : new SingleInputVideoGraph.Factory(videoFrameProcessorFactory), + new SingleInputVideoGraph.Factory(videoFrameProcessorFactory), videoGraphInputColor, videoGraphOutputColor, errorConsumer, @@ -162,7 +159,7 @@ public VideoSampleExporter( public GraphInput getInput(EditedMediaItem editedMediaItem, Format format) throws ExportException { try { - return videoGraph.createInput(); + return videoGraph.getInput(); } catch (VideoFrameProcessingException e) { throw ExportException.createForVideoFrameProcessingException(e); } @@ -520,19 +517,14 @@ public void onEnded(long finalFramePresentationTimeUs) { } } - @Override - public void onError(VideoFrameProcessingException e) { - errorConsumer.accept(ExportException.createForVideoFrameProcessingException(e)); - } - @Override public void initialize() throws VideoFrameProcessingException { videoGraph.initialize(); } @Override - public GraphInput createInput() throws VideoFrameProcessingException { - return videoGraph.createInput(); + public GraphInput getInput() throws VideoFrameProcessingException { + return videoGraph.getInput(); } @Override