From d3885375b8daaf97e784d73f4b7f14da72b6d7cb Mon Sep 17 00:00:00 2001 From: tofunmi Date: Tue, 12 Dec 2023 11:51:30 -0800 Subject: [PATCH] Trim optimization: fallback on format mismatches Manual testing: tested manually with pixel 4a PiperOrigin-RevId: 590284361 --- .../transformer/TransformerEndToEndTest.java | 40 ++++++++++++++- .../transformer/DefaultEncoderFactory.java | 5 ++ .../exoplayer2/transformer/ExportResult.java | 7 +++ .../exoplayer2/transformer/MuxerWrapper.java | 12 +++++ .../exoplayer2/transformer/Transformer.java | 49 ++++++++++++++----- 5 files changed, 101 insertions(+), 12 deletions(-) diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java index f1c32f5ff48..48e9b65c5fc 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java @@ -27,6 +27,7 @@ import static com.google.android.exoplayer2.transformer.AndroidTestUtil.generateTextureFromBitmap; import static com.google.android.exoplayer2.transformer.AndroidTestUtil.recordTestSkipped; import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_ABANDONED; +import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH; import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_SUCCEEDED; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Util.isRunningOnEmulator; @@ -423,7 +424,7 @@ public void clippedMedia_completesWithClippedDuration() throws Exception { } Transformer transformer = new Transformer.Builder(context).build(); long clippingStartMs = 10_000; - long clippingEndMs = 11_000; + long clippingEndMs = 13_000; MediaItem mediaItem = new MediaItem.Builder() .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)) @@ -442,6 +443,43 @@ public void clippedMedia_completesWithClippedDuration() throws Exception { assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); } + @Test + public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch() + throws Exception { + String testId = "clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch"; + if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT, + /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) { + return; + } + Transformer transformer = + new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build(); + long clippingStartMs = 10_000; + long clippingEndMs = 13_000; + // The file is made artificially on computer software so phones will not have the encoder + // available to match the csd. + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(clippingStartMs) + .setEndPositionMs(clippingEndMs) + .build()) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, mediaItem); + + assertThat(result.exportResult.optimizationResult) + .isEqualTo(OPTIMIZATION_FAILED_FORMAT_MISMATCH); + assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); + } + @Test public void clippedMedia_trimOptimizationEnabled_completesWithOptimizationApplied() throws Exception { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java index 91d1efbde6c..f2ad5c2bd5d 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java @@ -207,6 +207,11 @@ public DefaultCodec createForAudioEncoding(Format format) throws ExportException */ @Override public DefaultCodec createForVideoEncoding(Format format) throws ExportException { + // TODO b/304476154 - Investigate how to match initialization data without running into + // mediaCodec errors under API 29. + if (SDK_INT < 29) { + format = format.buildUpon().setInitializationData(null).build(); + } if (format.frameRate == Format.NO_VALUE || deviceNeedsDefaultFrameRateWorkaround()) { format = format.buildUpon().setFrameRate(DEFAULT_FRAME_RATE).build(); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExportResult.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExportResult.java index 6b6a9382b01..f0056d571b4 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExportResult.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExportResult.java @@ -307,6 +307,7 @@ public ProcessedInput( OPTIMIZATION_SUCCEEDED, OPTIMIZATION_ABANDONED, OPTIMIZATION_FAILED_EXTRACTION_FAILED, + OPTIMIZATION_FAILED_FORMAT_MISMATCH }) @interface OptimizationResult {} @@ -328,6 +329,12 @@ public ProcessedInput( */ public static final int OPTIMIZATION_FAILED_EXTRACTION_FAILED = 3; + /** + * The optimization failed because the format between the two parts of the media to be put + * together did not match. Normal export proceeded. + */ + public static final int OPTIMIZATION_FAILED_FORMAT_MISMATCH = 4; + /** The list of {@linkplain ProcessedInput processed inputs}. */ public final ImmutableList processedInputs; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java index fdcdc206914..4cd43141c55 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java @@ -298,6 +298,18 @@ public void addTrackFormat(Format format) throws Muxer.MuxerException { } } + /** + * Returns the {@link Format} of given {@code trackType} that was {@linkplain #addTrackFormat + * added}. + * + * @throws IllegalArgumentException If the {@code trackType} has not been {@linkplain + * #addTrackFormat added}. + */ + public Format getTrackFormat(@C.TrackType int trackType) { + checkArgument(contains(trackTypeToInfo, trackType)); + return trackTypeToInfo.get(trackType).format; + } + /** * Attempts to write a sample to the muxer. * diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 987df092a8c..cd713ead53b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -19,6 +19,7 @@ import static com.google.android.exoplayer2.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR; import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_ABANDONED; import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED; +import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH; import static com.google.android.exoplayer2.transformer.TransformerUtil.shouldTranscodeAudio; import static com.google.android.exoplayer2.transformer.TransformerUtil.shouldTranscodeVideo; import static com.google.android.exoplayer2.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes; @@ -929,7 +930,8 @@ public void start(Composition composition, String path) { composition, new MuxerWrapper(path, muxerFactory, componentListener, MuxerWrapper.MUXER_MODE_DEFAULT), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } else { processMediaBeforeFirstSyncSampleAfterTrimStartTime(); } @@ -1117,7 +1119,8 @@ private void processFullInput() { componentListener, MuxerWrapper.MUXER_MODE_DEFAULT), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } private void remuxProcessedVideo() { @@ -1153,7 +1156,8 @@ public void onSuccess(TransmuxTranscodeHelper.ResumeMetadata resumeMetadata) { /* clippingEndPositionUs= */ resumeMetadata.lastSyncSampleTimestampUs), checkNotNull(remuxingMuxerWrapper), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } @Override @@ -1181,7 +1185,8 @@ private void processRemainingVideo() { videoOnlyComposition, remuxingMuxerWrapper, componentListener, - /* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs); + /* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs, + /* matchInitializationData= */ false); } private void processAudio() { @@ -1196,7 +1201,8 @@ private void processAudio() { componentListener, MuxerWrapper.MUXER_MODE_DEFAULT), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } // TODO: b/308253384 - Move copy output logic into MuxerWrapper. @@ -1301,7 +1307,8 @@ && shouldTranscodeAudio( trancodeComposition, checkNotNull(remuxingMuxerWrapper), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ true); } @Override @@ -1315,8 +1322,13 @@ public void onFailure(Throwable t) { private void remuxRemainingMedia() { transformerState = TRANSFORMER_STATE_REMUX_REMAINING_MEDIA; - // TODO: b/304476154 - check original file format against transcode file format here to fail - // fast if necessary. + if (!doesFormatsMatch()) { + remuxingMuxerWrapper = null; + transformerInternal = null; + exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH); + processFullInput(); + return; + } MediaItem firstMediaItem = checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem; long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs; @@ -1336,7 +1348,21 @@ private void remuxRemainingMedia() { remuxingMuxerWrapper, componentListener, /* initialTimestampOffsetUs= */ mp4MetadataInfo.firstSyncSampleTimestampUsAfterTimeUs - - trimStartTimeUs); + - trimStartTimeUs, + /* matchInitializationData= */ false); + } + + private boolean doesFormatsMatch() { + checkNotNull(mp4MetadataInfo); + boolean videoFormatMatches = + checkNotNull(remuxingMuxerWrapper) + .getTrackFormat(C.TRACK_TYPE_VIDEO) + .initializationDataEquals(checkNotNull(mp4MetadataInfo.videoFormat)); + boolean audioFormatMatches = + mp4MetadataInfo.audioFormat == null + || mp4MetadataInfo.audioFormat.initializationDataEquals( + checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO)); + return videoFormatMatches && audioFormatMatches; } private boolean isMultiAsset() { @@ -1354,7 +1380,8 @@ private void startInternal( Composition composition, MuxerWrapper muxerWrapper, ComponentListener componentListener, - long initialTimestampOffsetUs) { + long initialTimestampOffsetUs, + boolean matchInitializationData) { checkArgument(composition.effects.audioProcessors.isEmpty()); checkState(transformerInternal == null, "There is already an export in progress."); TransformationRequest transformationRequest = this.transformationRequest; @@ -1391,7 +1418,7 @@ private void startInternal( debugViewProvider, clock, initialTimestampOffsetUs, - /* matchInitializationData= */ trimOptimizationEnabled); + matchInitializationData); transformerInternal.start(); }