Skip to content

Commit

Permalink
Trim optimization: fallback on format mismatches
Browse files Browse the repository at this point in the history 
Manual testing: tested manually with pixel 4a

PiperOrigin-RevId: 590284361
  • Loading branch information
@tof-tof @copybara-github
tof-tof authored and copybara-github committed Dec 12, 2023
1 parent 9b4d599 commit d388537
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 12 deletions.
40 changes: 39 additions & 1 deletion ...c/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import static com.google.android.exoplayer2.transformer.AndroidTestUtil.generateTextureFromBitmap;
import static com.google.android.exoplayer2.transformer.AndroidTestUtil.recordTestSkipped;
import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_ABANDONED;
import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH;
import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_SUCCEEDED;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Util.isRunningOnEmulator;
Expand Down Expand Up @@ -423,7 +424,7 @@ public void clippedMedia_completesWithClippedDuration() throws Exception {
}
Transformer transformer = new Transformer.Builder(context).build();
long clippingStartMs = 10_000;
long clippingEndMs = 11_000;
long clippingEndMs = 13_000;
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
Expand All @@ -442,6 +443,43 @@ public void clippedMedia_completesWithClippedDuration() throws Exception {
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
}

@Test
public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch()
throws Exception {
String testId = "clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch";
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
/* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) {
return;
}
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
long clippingStartMs = 10_000;
long clippingEndMs = 13_000;
// The file is made artificially on computer software so phones will not have the encoder
// available to match the csd.
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(clippingStartMs)
.setEndPositionMs(clippingEndMs)
.build())
.build();

ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, mediaItem);

assertThat(result.exportResult.optimizationResult)
.isEqualTo(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
}

@Test
public void clippedMedia_trimOptimizationEnabled_completesWithOptimizationApplied()
throws Exception {
Expand Down
5 changes: 5 additions & 0 deletions ...former/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,11 @@ public DefaultCodec createForAudioEncoding(Format format) throws ExportException
*/
@Override
public DefaultCodec createForVideoEncoding(Format format) throws ExportException {
// TODO b/304476154 - Investigate how to match initialization data without running into
// mediaCodec errors under API 29.
if (SDK_INT < 29) {
format = format.buildUpon().setInitializationData(null).build();
}
if (format.frameRate == Format.NO_VALUE || deviceNeedsDefaultFrameRateWorkaround()) {
format = format.buildUpon().setFrameRate(DEFAULT_FRAME_RATE).build();
}
Expand Down
7 changes: 7 additions & 0 deletions ...ary/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExportResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ public ProcessedInput(
OPTIMIZATION_SUCCEEDED,
OPTIMIZATION_ABANDONED,
OPTIMIZATION_FAILED_EXTRACTION_FAILED,
OPTIMIZATION_FAILED_FORMAT_MISMATCH
})
@interface OptimizationResult {}

Expand All @@ -328,6 +329,12 @@ public ProcessedInput(
*/
public static final int OPTIMIZATION_FAILED_EXTRACTION_FAILED = 3;

/**
* The optimization failed because the format between the two parts of the media to be put
* together did not match. Normal export proceeded.
*/
public static final int OPTIMIZATION_FAILED_FORMAT_MISMATCH = 4;

/** The list of {@linkplain ProcessedInput processed inputs}. */
public final ImmutableList<ProcessedInput> processedInputs;

Expand Down
12 changes: 12 additions & 0 deletions ...ary/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,18 @@ public void addTrackFormat(Format format) throws Muxer.MuxerException {
}
}

/**
* Returns the {@link Format} of given {@code trackType} that was {@linkplain #addTrackFormat
* added}.
*
* @throws IllegalArgumentException If the {@code trackType} has not been {@linkplain
* #addTrackFormat added}.
*/
public Format getTrackFormat(@C.TrackType int trackType) {
checkArgument(contains(trackTypeToInfo, trackType));
return trackTypeToInfo.get(trackType).format;
}

/**
* Attempts to write a sample to the muxer.
*
Expand Down
49 changes: 38 additions & 11 deletions library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static com.google.android.exoplayer2.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR;
import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_ABANDONED;
import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED;
import static com.google.android.exoplayer2.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH;
import static com.google.android.exoplayer2.transformer.TransformerUtil.shouldTranscodeAudio;
import static com.google.android.exoplayer2.transformer.TransformerUtil.shouldTranscodeVideo;
import static com.google.android.exoplayer2.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes;
Expand Down Expand Up @@ -929,7 +930,8 @@ public void start(Composition composition, String path) {
composition,
new MuxerWrapper(path, muxerFactory, componentListener, MuxerWrapper.MUXER_MODE_DEFAULT),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
} else {
processMediaBeforeFirstSyncSampleAfterTrimStartTime();
}
Expand Down Expand Up @@ -1117,7 +1119,8 @@ private void processFullInput() {
componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
}

private void remuxProcessedVideo() {
Expand Down Expand Up @@ -1153,7 +1156,8 @@ public void onSuccess(TransmuxTranscodeHelper.ResumeMetadata resumeMetadata) {
/* clippingEndPositionUs= */ resumeMetadata.lastSyncSampleTimestampUs),
checkNotNull(remuxingMuxerWrapper),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
}

@Override
Expand Down Expand Up @@ -1181,7 +1185,8 @@ private void processRemainingVideo() {
videoOnlyComposition,
remuxingMuxerWrapper,
componentListener,
/* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs);
/* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs,
/* matchInitializationData= */ false);
}

private void processAudio() {
Expand All @@ -1196,7 +1201,8 @@ private void processAudio() {
componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
}

// TODO: b/308253384 - Move copy output logic into MuxerWrapper.
Expand Down Expand Up @@ -1301,7 +1307,8 @@ && shouldTranscodeAudio(
trancodeComposition,
checkNotNull(remuxingMuxerWrapper),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ true);
}

@Override
Expand All @@ -1315,8 +1322,13 @@ public void onFailure(Throwable t) {

private void remuxRemainingMedia() {
transformerState = TRANSFORMER_STATE_REMUX_REMAINING_MEDIA;
// TODO: b/304476154 - check original file format against transcode file format here to fail
// fast if necessary.
if (!doesFormatsMatch()) {
remuxingMuxerWrapper = null;
transformerInternal = null;
exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
processFullInput();
return;
}
MediaItem firstMediaItem =
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem;
long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs;
Expand All @@ -1336,7 +1348,21 @@ private void remuxRemainingMedia() {
remuxingMuxerWrapper,
componentListener,
/* initialTimestampOffsetUs= */ mp4MetadataInfo.firstSyncSampleTimestampUsAfterTimeUs
- trimStartTimeUs);
- trimStartTimeUs,
/* matchInitializationData= */ false);
}

private boolean doesFormatsMatch() {
checkNotNull(mp4MetadataInfo);
boolean videoFormatMatches =
checkNotNull(remuxingMuxerWrapper)
.getTrackFormat(C.TRACK_TYPE_VIDEO)
.initializationDataEquals(checkNotNull(mp4MetadataInfo.videoFormat));
boolean audioFormatMatches =
mp4MetadataInfo.audioFormat == null
|| mp4MetadataInfo.audioFormat.initializationDataEquals(
checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO));
return videoFormatMatches && audioFormatMatches;
}

private boolean isMultiAsset() {
Expand All @@ -1354,7 +1380,8 @@ private void startInternal(
Composition composition,
MuxerWrapper muxerWrapper,
ComponentListener componentListener,
long initialTimestampOffsetUs) {
long initialTimestampOffsetUs,
boolean matchInitializationData) {
checkArgument(composition.effects.audioProcessors.isEmpty());
checkState(transformerInternal == null, "There is already an export in progress.");
TransformationRequest transformationRequest = this.transformationRequest;
Expand Down Expand Up @@ -1391,7 +1418,7 @@ private void startInternal(
debugViewProvider,
clock,
initialTimestampOffsetUs,
/* matchInitializationData= */ trimOptimizationEnabled);
matchInitializationData);
transformerInternal.start();
}

Expand Down

0 comments on commit d388537

Please sign in to comment.