From fab66d972ef84599cdaa2b498b91f21d104fbf26 Mon Sep 17 00:00:00 2001
From: michaelkatz <michaelkatz@google.com>
Date: Fri, 11 Nov 2022 11:23:41 +0000
Subject: [PATCH] Changed decoder list sort to order by functional support of
format
Added new method to check if codec just functionally supports a format. Changed getDecoderInfosSortedByFormatSupport to use new function to order by functional support. This allows decoders that only support functionally and are more preferred by the MediaCodecSelector to keep their preferred position in the sorted list.
UnitTests included
-Two MediaCodecVideoRenderer tests that verify hw vs sw does not have an effect on sort of the decoder list, it is only based on functional support
Issue: google/ExoPlayer#10604
PiperOrigin-RevId: 487779284
---
RELEASENOTES.md | 6 +
.../exoplayer/RendererCapabilities.java | 6 +-
.../exoplayer/mediacodec/MediaCodecInfo.java | 24 +++-
.../mediacodec/MediaCodecRenderer.java | 8 ++
.../exoplayer/mediacodec/MediaCodecUtil.java | 13 +-
.../video/MediaCodecVideoRendererTest.java | 121 ++++++++++++++++++
6 files changed, 158 insertions(+), 20 deletions(-)
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index ecb022a96fe..6980d669675 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -30,6 +30,12 @@ Release notes
([#10684](https://github.com/google/ExoPlayer/issues/10684)).
* Add `Player.getVideoSurfaceSize` that returns the size of the surface on
which the video is rendered.
+ * Tweak the renderer's decoder ordering logic to uphold the
+ `MediaCodecSelector`'s preferences, even if a decoder reports it may not
+ be able to play the media performantly. For example with default
+ selector, hardware decoder with only functional support will be
+ preferred over software decoder that fully supports the format
+ ([#10604](https://github.com/google/ExoPlayer/issues/10604)).
* Build:
* Avoid publishing block when included in another gradle build.
* Downloads:
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/RendererCapabilities.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/RendererCapabilities.java
index 604b607842e..dbc2fa059e0 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/RendererCapabilities.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/RendererCapabilities.java
@@ -144,13 +144,13 @@ public interface RendererCapabilities {
/** A mask to apply to {@link Capabilities} to obtain {@link DecoderSupport} only. */
int MODE_SUPPORT_MASK = 0b11 << 7;
/**
- * The renderer will use a decoder for fallback mimetype if possible as format's MIME type is
- * unsupported
+ * The format's MIME type is unsupported and the renderer may use a decoder for a fallback MIME
+ * type.
*/
int DECODER_SUPPORT_FALLBACK_MIMETYPE = 0b10 << 7;
/** The renderer is able to use the primary decoder for the format's MIME type. */
int DECODER_SUPPORT_PRIMARY = 0b1 << 7;
- /** The renderer will use a fallback decoder. */
+ /** The format exceeds the primary decoder's capabilities but is supported by fallback decoder */
int DECODER_SUPPORT_FALLBACK = 0;
/**
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java
index 72733a90e9a..e49a9a5a3cc 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java
@@ -245,7 +245,8 @@ public int getMaxSupportedInstances() {
}
/**
- * Returns whether the decoder may support decoding the given {@code format}.
+ * Returns whether the decoder may support decoding the given {@code format} both functionally and
+ * performantly.
*
* @param format The input media format.
* @return Whether the decoder may support decoding the given {@code format}.
@@ -256,7 +257,7 @@ public boolean isFormatSupported(Format format) throws MediaCodecUtil.DecoderQue
return false;
}
- if (!isCodecProfileAndLevelSupported(format)) {
+ if (!isCodecProfileAndLevelSupported(format, /* checkPerformanceCapabilities= */ true)) {
return false;
}
@@ -283,15 +284,24 @@ public boolean isFormatSupported(Format format) throws MediaCodecUtil.DecoderQue
}
}
+ /**
+ * Returns whether the decoder may functionally support decoding the given {@code format}.
+ *
+ * @param format The input media format.
+ * @return Whether the decoder may functionally support decoding the given {@code format}.
+ */
+ public boolean isFormatFunctionallySupported(Format format) {
+ return isSampleMimeTypeSupported(format)
+ && isCodecProfileAndLevelSupported(format, /* checkPerformanceCapabilities= */ false);
+ }
+
private boolean isSampleMimeTypeSupported(Format format) {
return mimeType.equals(format.sampleMimeType)
|| mimeType.equals(MediaCodecUtil.getAlternativeCodecMimeType(format));
}
- private boolean isCodecProfileAndLevelSupported(Format format) {
- if (format.codecs == null) {
- return true;
- }
+ private boolean isCodecProfileAndLevelSupported(
+ Format format, boolean checkPerformanceCapabilities) {
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
if (codecProfileAndLevel == null) {
// If we don't know any better, we assume that the profile and level are supported.
@@ -327,7 +337,7 @@ private boolean isCodecProfileAndLevelSupported(Format format) {
for (CodecProfileLevel profileLevel : profileLevels) {
if (profileLevel.profile == profile
- && profileLevel.level >= level
+ && (profileLevel.level >= level || !checkPerformanceCapabilities)
&& !needsProfileExcludedWorkaround(mimeType, profile)) {
return true;
}
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
index 79c3b9ca7a4..4a4c43e6d4a 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
@@ -1113,6 +1113,14 @@ private void initCodec(MediaCodecInfo codecInfo, MediaCrypto crypto) throws Exce
}
codecInitializedTimestamp = SystemClock.elapsedRealtime();
+ if (!codecInfo.isFormatSupported(inputFormat)) {
+ Log.w(
+ TAG,
+ Util.formatInvariant(
+ "Format exceeds selected codec's capabilities [%s, %s]",
+ Format.toLogString(inputFormat), codecName));
+ }
+
this.codecInfo = codecInfo;
this.codecOperatingRate = codecOperatingRate;
codecInputFormat = inputFormat;
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java
index c3200150d0d..e97e053084d 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java
@@ -190,22 +190,15 @@ public static synchronized List<MediaCodecInfo> getDecoderInfos(
}
/**
- * Returns a copy of the provided decoder list sorted such that decoders with format support are
- * listed first. The returned list is modifiable for convenience.
+ * Returns a copy of the provided decoder list sorted such that decoders with functional format
+ * support are listed first. The returned list is modifiable for convenience.
*/
@CheckResult
public static List<MediaCodecInfo> getDecoderInfosSortedByFormatSupport(
List<MediaCodecInfo> decoderInfos, Format format) {
decoderInfos = new ArrayList<>(decoderInfos);
sortByScore(
- decoderInfos,
- decoderInfo -> {
- try {
- return decoderInfo.isFormatSupported(format) ? 1 : 0;
- } catch (DecoderQueryException e) {
- return -1;
- }
- });
+ decoderInfos, decoderInfo -> decoderInfo.isFormatFunctionallySupported(format) ? 1 : 0);
return decoderInfos;
}
diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
index 4e3f48ea062..d42ab38fe02 100644
--- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
@@ -58,6 +58,7 @@
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.Collections;
+import java.util.List;
import java.util.stream.Collectors;
import org.junit.After;
import org.junit.Before;
@@ -84,6 +85,32 @@ public class MediaCodecVideoRendererTest {
.setHeight(1080)
.build();
+ private static final MediaCodecInfo H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO =
+ MediaCodecInfo.newInstance(
+ /* name= */ "h264-codec-hw",
+ /* mimeType= */ MimeTypes.VIDEO_H264,
+ /* codecMimeType= */ MimeTypes.VIDEO_H264,
+ /* capabilities= */ createCodecCapabilities(
+ CodecProfileLevel.AVCProfileHigh, CodecProfileLevel.AVCLevel4),
+ /* hardwareAccelerated= */ true,
+ /* softwareOnly= */ false,
+ /* vendor= */ false,
+ /* forceDisableAdaptive= */ false,
+ /* forceSecure= */ false);
+
+ private static final MediaCodecInfo H264_PROFILE8_LEVEL5_SW_MEDIA_CODEC_INFO =
+ MediaCodecInfo.newInstance(
+ /* name= */ "h264-codec-sw",
+ /* mimeType= */ MimeTypes.VIDEO_H264,
+ /* codecMimeType= */ MimeTypes.VIDEO_H264,
+ /* capabilities= */ createCodecCapabilities(
+ CodecProfileLevel.AVCProfileHigh, CodecProfileLevel.AVCLevel5),
+ /* hardwareAccelerated= */ false,
+ /* softwareOnly= */ true,
+ /* vendor= */ false,
+ /* forceDisableAdaptive= */ false,
+ /* forceSecure= */ false);
+
private Looper testMainLooper;
private Surface surface;
private MediaCodecVideoRenderer mediaCodecVideoRenderer;
@@ -711,6 +738,100 @@ public void supportsFormat_withDolbyVision_setsDecoderSupportFlagsByDisplayDolby
.isEqualTo(RendererCapabilities.DECODER_SUPPORT_PRIMARY);
}
+ @Test
+ public void getDecoderInfo_withNonPerformantHardwareDecoder_returnsHardwareDecoderFirst()
+ throws Exception {
+ // AVC Format, Profile: 8, Level: 8192
+ Format avcFormat =
+ new Format.Builder()
+ .setSampleMimeType(MimeTypes.VIDEO_H264)
+ .setCodecs("avc1.64002a")
+ .build();
+ // Provide hardware and software AVC decoders
+ MediaCodecSelector mediaCodecSelector =
+ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> {
+ if (!mimeType.equals(MimeTypes.VIDEO_H264)) {
+ return ImmutableList.of();
+ }
+ // Hardware decoder supports above format functionally but not performantly as
+ // it supports MIME type & Profile but not Level
+ // Software decoder supports format functionally and peformantly as it supports
+ // MIME type, Profile, and Level(assuming resolution/frame rate support too)
+ return ImmutableList.of(
+ H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO, H264_PROFILE8_LEVEL5_SW_MEDIA_CODEC_INFO);
+ };
+ MediaCodecVideoRenderer renderer =
+ new MediaCodecVideoRenderer(
+ ApplicationProvider.getApplicationContext(),
+ mediaCodecSelector,
+ /* allowedJoiningTimeMs= */ 0,
+ /* eventHandler= */ new Handler(testMainLooper),
+ /* eventListener= */ eventListener,
+ /* maxDroppedFramesToNotify= */ 1);
+ renderer.init(/* index= */ 0, PlayerId.UNSET);
+
+ List<MediaCodecInfo> mediaCodecInfoList =
+ renderer.getDecoderInfos(mediaCodecSelector, avcFormat, false);
+ @Capabilities int capabilities = renderer.supportsFormat(avcFormat);
+
+ assertThat(mediaCodecInfoList).hasSize(2);
+ assertThat(mediaCodecInfoList.get(0).hardwareAccelerated).isTrue();
+ assertThat(RendererCapabilities.getFormatSupport(capabilities)).isEqualTo(C.FORMAT_HANDLED);
+ assertThat(RendererCapabilities.getDecoderSupport(capabilities))
+ .isEqualTo(RendererCapabilities.DECODER_SUPPORT_FALLBACK);
+ }
+
+ @Test
+ public void getDecoderInfo_softwareDecoderPreferred_returnsSoftwareDecoderFirst()
+ throws Exception {
+ // AVC Format, Profile: 8, Level: 8192
+ Format avcFormat =
+ new Format.Builder()
+ .setSampleMimeType(MimeTypes.VIDEO_H264)
+ .setCodecs("avc1.64002a")
+ .build();
+ // Provide software and hardware AVC decoders
+ MediaCodecSelector mediaCodecSelector =
+ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> {
+ if (!mimeType.equals(MimeTypes.VIDEO_H264)) {
+ return ImmutableList.of();
+ }
+ // Hardware decoder supports above format functionally but not performantly as
+ // it supports MIME type & Profile but not Level
+ // Software decoder supports format functionally and peformantly as it supports
+ // MIME type, Profile, and Level(assuming resolution/frame rate support too)
+ return ImmutableList.of(
+ H264_PROFILE8_LEVEL5_SW_MEDIA_CODEC_INFO, H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO);
+ };
+ MediaCodecVideoRenderer renderer =
+ new MediaCodecVideoRenderer(
+ ApplicationProvider.getApplicationContext(),
+ mediaCodecSelector,
+ /* allowedJoiningTimeMs= */ 0,
+ /* eventHandler= */ new Handler(testMainLooper),
+ /* eventListener= */ eventListener,
+ /* maxDroppedFramesToNotify= */ 1);
+ renderer.init(/* index= */ 0, PlayerId.UNSET);
+
+ List<MediaCodecInfo> mediaCodecInfoList =
+ renderer.getDecoderInfos(mediaCodecSelector, avcFormat, false);
+ @Capabilities int capabilities = renderer.supportsFormat(avcFormat);
+
+ assertThat(mediaCodecInfoList).hasSize(2);
+ assertThat(mediaCodecInfoList.get(0).hardwareAccelerated).isFalse();
+ assertThat(RendererCapabilities.getFormatSupport(capabilities)).isEqualTo(C.FORMAT_HANDLED);
+ assertThat(RendererCapabilities.getDecoderSupport(capabilities))
+ .isEqualTo(RendererCapabilities.DECODER_SUPPORT_PRIMARY);
+ }
+
+ private static CodecCapabilities createCodecCapabilities(int profile, int level) {
+ CodecCapabilities capabilities = new CodecCapabilities();
+ capabilities.profileLevels = new CodecProfileLevel[] {new CodecProfileLevel()};
+ capabilities.profileLevels[0].profile = profile;
+ capabilities.profileLevels[0].level = level;
+ return capabilities;
+ }
+
@Test
public void getCodecMaxInputSize_videoH263() {
MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_H263);