From bb20eb49758ee002f8f8ee61f721e94fd461e352 Mon Sep 17 00:00:00 2001 From: dancho Date: Mon, 25 Nov 2024 08:38:19 -0800 Subject: [PATCH] Frame Extractor HDR: tone map to SDR Support extracting frames from HDR input by tone mapping to SDR (BT.709). ExperimentalFrameExtractor must be public because HDR tests live in a different package. PiperOrigin-RevId: 699994112 --- .../media3/transformer/AndroidTestUtil.java | 18 ++++ .../transformer/mh/FrameExtractorHdrTest.java | 91 +++++++++++++++++++ .../ExperimentalFrameExtractor.java | 17 +++- 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index ec25ee44e7a..fb5b9eace9a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -530,6 +530,24 @@ public String toString() { .build()) .build(); + public static final AssetInfo MP4_ASSET_COLOR_TEST_1080P_HLG10 = + new AssetInfo.Builder("asset:///media/mp4/hlg10-color-test.mp4") + .setVideoFormat( + new Format.Builder() + .setSampleMimeType(VIDEO_H265) + .setWidth(1920) + .setHeight(1080) + .setFrameRate(30.000f) + .setColorInfo( + new ColorInfo.Builder() + .setColorSpace(C.COLOR_SPACE_BT2020) + .setColorRange(C.COLOR_RANGE_LIMITED) + .setColorTransfer(C.COLOR_TRANSFER_HLG) + .build()) + .setCodecs("hvc1.2.4.L153") + .build()) + .build(); + public static final AssetInfo MP4_ASSET_720P_4_SECOND_HDR10 = new AssetInfo.Builder("asset:///media/mp4/hdr10-720p.mp4") .setVideoFormat( diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java new file mode 100644 index 00000000000..5b258a933ec --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameExtractorHdrTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer.mh; + +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.media3.test.utils.TestUtil.assertBitmapsAreSimilar; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_COLOR_TEST_1080P_HLG10; +import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceSupportsOpenGlToneMapping; +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; + +import android.content.Context; +import android.graphics.Bitmap; +import androidx.media3.common.MediaItem; +import androidx.media3.transformer.ExperimentalFrameExtractor; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListenableFuture; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** End-to-end HDR instrumentation test for {@link ExperimentalFrameExtractor}. */ +@RunWith(AndroidJUnit4.class) +public class FrameExtractorHdrTest { + // This file is generated on a Pixel 7, because the emulator isn't able to decode HLG to generate + // this file. + private static final String TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH = + "test-generated-goldens/sample_mp4_first_frame/electrical_colors/tone_map_hlg_to_sdr.png"; + private static final long TIMEOUT_SECONDS = 10; + private static final float PSNR_THRESHOLD = 25f; + + @Rule public final TestName testName = new TestName(); + + private final Context context = ApplicationProvider.getApplicationContext(); + + private String testId; + private @MonotonicNonNull ExperimentalFrameExtractor frameExtractor; + + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + + @After + public void tearDown() { + if (frameExtractor != null) { + frameExtractor.release(); + } + } + + @Test + public void extractFrame_oneFrameHlg_returnsToneMappedFrame() throws Exception { + assumeDeviceSupportsOpenGlToneMapping(testId, MP4_ASSET_COLOR_TEST_1080P_HLG10.videoFormat); + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(MP4_ASSET_COLOR_TEST_1080P_HLG10.uri), + /* effects= */ ImmutableList.of()); + + ListenableFuture frameFuture = + frameExtractor.getFrame(/* positionMs= */ 0); + ExperimentalFrameExtractor.Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); + Bitmap actualBitmap = frame.bitmap; + Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH); + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + + assertThat(frame.presentationTimeMs).isEqualTo(0); + assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java index 231cd7745dc..738a6240c87 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -16,6 +16,8 @@ package androidx.media3.transformer; +import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED; +import static androidx.media3.common.ColorInfo.isTransferHdr; import static androidx.media3.common.PlaybackException.ERROR_CODE_FAILED_RUNTIME_CHECK; import static androidx.media3.common.Player.DISCONTINUITY_REASON_SEEK; import static androidx.media3.common.util.Assertions.checkNotNull; @@ -30,6 +32,7 @@ import android.opengl.GLES20; import android.os.Handler; import android.os.Looper; +import androidx.annotation.CallSuper; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.Effect; @@ -42,6 +45,7 @@ import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.NullableType; +import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.effect.GlEffect; import androidx.media3.effect.GlShaderProgram; @@ -79,7 +83,8 @@ * *

Frame extractor instances must be accessed from a single application thread. */ -/* package */ final class ExperimentalFrameExtractor implements AnalyticsListener { +@UnstableApi +public final class ExperimentalFrameExtractor implements AnalyticsListener { /** Configuration for the frame extractor. */ public static final class Configuration { @@ -428,6 +433,16 @@ public void setVideoEffects(List effects) { setEffectsWithRotation(); } + @CallSuper + @Override + protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException { + if (isTransferHdr(format.colorInfo)) { + // Setting the VideoSink format to SDR_BT709_LIMITED tone maps to SDR. + format = format.buildUpon().setColorInfo(SDR_BT709_LIMITED).build(); + } + super.onReadyToInitializeCodec(format); + } + @Override @Nullable protected DecoderReuseEvaluation onInputFormatChanged(FormatHolder formatHolder)