From 0a8dc41632fa4c95ff78bd33ccb06b586b09bbdb Mon Sep 17 00:00:00 2001 From: olly Date: Wed, 25 Jan 2017 06:56:26 -0800 Subject: [PATCH] Set max resolution from codec capabilities for ABR where resolutions are unknown Issue: #2096 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=145542983 --- .../audio/MediaCodecAudioRenderer.java | 3 +- .../exoplayer2/mediacodec/MediaCodecInfo.java | 64 +++++----- .../mediacodec/MediaCodecRenderer.java | 7 +- .../video/MediaCodecVideoRenderer.java | 109 +++++++++++++++--- 4 files changed, 136 insertions(+), 47 deletions(-) diff --git a/library/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index b4813d90a20..f8501c38581 100644 --- a/library/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -183,7 +183,8 @@ protected boolean allowPassthrough(String mimeType) { } @Override - protected void configureCodec(MediaCodec codec, Format format, MediaCrypto crypto) { + protected void configureCodec(MediaCodecInfo codecInfo, MediaCodec codec, Format format, + MediaCrypto crypto) { if (passthroughEnabled) { // Override the MIME type used to configure the codec if we are using a passthrough decoder. passthroughMediaFormat = format.getFrameworkMediaFormatV16(); diff --git a/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java b/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java index 166de37c50b..6914b2f52c3 100644 --- a/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java +++ b/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java @@ -16,6 +16,7 @@ package com.google.android.exoplayer2.mediacodec; import android.annotation.TargetApi; +import android.graphics.Point; import android.media.MediaCodec; import android.media.MediaCodecInfo.AudioCapabilities; import android.media.MediaCodecInfo.CodecCapabilities; @@ -23,6 +24,7 @@ import android.media.MediaCodecInfo.VideoCapabilities; import android.util.Log; import android.util.Pair; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; @@ -142,70 +144,68 @@ public boolean isCodecSupported(String codec) { } /** - * Whether the decoder supports video with a specified width and height. + * Whether the decoder supports video with a given width, height and frame rate. *

* Must not be called if the device SDK version is less than 21. * * @param width Width in pixels. * @param height Height in pixels. - * @return Whether the decoder supports video with the given width and height. + * @param frameRate Optional frame rate in frames per second. Ignored if set to + * {@link Format#NO_VALUE} or any value less than or equal to 0. + * @return Whether the decoder supports video with the given width, height and frame rate. */ @TargetApi(21) - public boolean isVideoSizeSupportedV21(int width, int height) { + public boolean isVideoSizeAndRateSupportedV21(int width, int height, double frameRate) { if (capabilities == null) { - logNoSupport("size.caps"); + logNoSupport("sizeAndRate.caps"); return false; } VideoCapabilities videoCapabilities = capabilities.getVideoCapabilities(); if (videoCapabilities == null) { - logNoSupport("size.vCaps"); + logNoSupport("sizeAndRate.vCaps"); return false; } - if (!videoCapabilities.isSizeSupported(width, height)) { + if (!areSizeAndRateSupported(videoCapabilities, width, height, frameRate)) { // Capabilities are known to be inaccurately reported for vertical resolutions on some devices // (b/31387661). If the video is vertical and the capabilities indicate support if the width // and height are swapped, we assume that the vertical resolution is also supported. - if (width >= height || !videoCapabilities.isSizeSupported(height, width)) { - logNoSupport("size.support, " + width + "x" + height); + if (width >= height + || !areSizeAndRateSupported(videoCapabilities, height, width, frameRate)) { + logNoSupport("sizeAndRate.support, " + width + "x" + height + "x" + frameRate); return false; } - logAssumedSupport("size.rotated, " + width + "x" + height); + logAssumedSupport("sizeAndRate.rotated, " + width + "x" + height + "x" + frameRate); } return true; } /** - * Whether the decoder supports video with a given width, height and frame rate. + * Returns the smallest video size greater than or equal to a specified size that also satisfies + * the {@link MediaCodec}'s width and height alignment requirements. *

* Must not be called if the device SDK version is less than 21. * * @param width Width in pixels. * @param height Height in pixels. - * @param frameRate Frame rate in frames per second. - * @return Whether the decoder supports video with the given width, height and frame rate. + * @return The smallest video size greater than or equal to the specified size that also satisfies + * the {@link MediaCodec}'s width and height alignment requirements, or null if not a video + * codec. */ @TargetApi(21) - public boolean isVideoSizeAndRateSupportedV21(int width, int height, double frameRate) { + public Point alignVideoSizeV21(int width, int height) { if (capabilities == null) { - logNoSupport("sizeAndRate.caps"); - return false; + logNoSupport("align.caps"); + return null; } VideoCapabilities videoCapabilities = capabilities.getVideoCapabilities(); if (videoCapabilities == null) { - logNoSupport("sizeAndRate.vCaps"); - return false; + logNoSupport("align.vCaps"); + return null; } - if (!videoCapabilities.areSizeAndRateSupported(width, height, frameRate)) { - // Capabilities are known to be inaccurately reported for vertical resolutions on some devices - // (b/31387661). If the video is vertical and the capabilities indicate support if the width - // and height are swapped, we assume that the vertical resolution is also supported. - if (width >= height || !videoCapabilities.areSizeAndRateSupported(height, width, frameRate)) { - logNoSupport("sizeAndRate.support, " + width + "x" + height + "x" + frameRate); - return false; - } - logAssumedSupport("sizeAndRate.rotated, " + width + "x" + height + "x" + frameRate); - } - return true; + int widthAlignment = videoCapabilities.getWidthAlignment(); + int heightAlignment = videoCapabilities.getHeightAlignment(); + return new Point(Util.ceilDivide(width, widthAlignment) * widthAlignment, + Util.ceilDivide(height, heightAlignment) * heightAlignment); } /** @@ -279,6 +279,14 @@ private static boolean isAdaptiveV19(CodecCapabilities capabilities) { return capabilities.isFeatureSupported(CodecCapabilities.FEATURE_AdaptivePlayback); } + @TargetApi(21) + private static boolean areSizeAndRateSupported(VideoCapabilities capabilities, int width, + int height, double frameRate) { + return frameRate == Format.NO_VALUE || frameRate <= 0 + ? capabilities.isSizeSupported(width, height) + : capabilities.areSizeAndRateSupported(width, height, frameRate); + } + private static boolean isTunneling(CodecCapabilities capabilities) { return Util.SDK_INT >= 21 && isTunnelingV21(capabilities); } diff --git a/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index 7e8b83b84c9..70445466a6d 100644 --- a/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -276,11 +276,14 @@ protected MediaCodecInfo getDecoderInfo(MediaCodecSelector mediaCodecSelector, /** * Configures a newly created {@link MediaCodec}. * + * @param codecInfo Information about the {@link MediaCodec} being configured. * @param codec The {@link MediaCodec} to configure. * @param format The format for which the codec is being configured. * @param crypto For drm protected playbacks, a {@link MediaCrypto} to use for decryption. + * @throws DecoderQueryException If an error occurs querying {@code codecInfo}. */ - protected abstract void configureCodec(MediaCodec codec, Format format, MediaCrypto crypto); + protected abstract void configureCodec(MediaCodecInfo codecInfo, MediaCodec codec, Format format, + MediaCrypto crypto) throws DecoderQueryException; @SuppressWarnings("deprecation") protected final void maybeInitCodec() throws ExoPlaybackException { @@ -345,7 +348,7 @@ protected final void maybeInitCodec() throws ExoPlaybackException { codec = MediaCodec.createByCodecName(codecName); TraceUtil.endSection(); TraceUtil.beginSection("configureCodec"); - configureCodec(codec, format, mediaCrypto); + configureCodec(decoderInfo, codec, format, mediaCrypto); TraceUtil.endSection(); TraceUtil.beginSection("startCodec"); codec.start(); diff --git a/library/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 62224a64d68..280f0042114 100644 --- a/library/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -18,6 +18,7 @@ import android.annotation.SuppressLint; import android.annotation.TargetApi; import android.content.Context; +import android.graphics.Point; import android.media.MediaCodec; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCrypto; @@ -56,6 +57,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private static final String KEY_CROP_BOTTOM = "crop-bottom"; private static final String KEY_CROP_TOP = "crop-top"; + // Long edge length in pixels for standard video formats, in decreasing in order. + private static final int[] STANDARD_LONG_EDGE_VIDEO_PX = new int[] { + 1920, 1600, 1440, 1280, 960, 854, 640, 540, 480}; + private final VideoFrameReleaseTimeHelper frameReleaseTimeHelper; private final EventDispatcher eventDispatcher; private final long allowedJoiningTimeMs; @@ -186,12 +191,8 @@ protected int supportsFormat(MediaCodecSelector mediaCodecSelector, Format forma boolean decoderCapable = decoderInfo.isCodecSupported(format.codecs); if (decoderCapable && format.width > 0 && format.height > 0) { if (Util.SDK_INT >= 21) { - if (format.frameRate > 0) { - decoderCapable = decoderInfo.isVideoSizeAndRateSupportedV21(format.width, format.height, - format.frameRate); - } else { - decoderCapable = decoderInfo.isVideoSizeSupportedV21(format.width, format.height); - } + decoderCapable = decoderInfo.isVideoSizeAndRateSupportedV21(format.width, format.height, + format.frameRate); } else { decoderCapable = format.width * format.height <= MediaCodecUtil.maxH264DecodableFrameSize(); if (!decoderCapable) { @@ -318,8 +319,9 @@ protected boolean shouldInitCodec() { } @Override - protected void configureCodec(MediaCodec codec, Format format, MediaCrypto crypto) { - codecMaxValues = getCodecMaxValues(format, streamFormats); + protected void configureCodec(MediaCodecInfo codecInfo, MediaCodec codec, Format format, + MediaCrypto crypto) throws DecoderQueryException { + codecMaxValues = getCodecMaxValues(codecInfo, format, streamFormats); MediaFormat mediaFormat = getMediaFormat(format, codecMaxValues, deviceNeedsAutoFrcWorkaround, tunnelingAudioSessionId); codec.configure(mediaFormat, surface, crypto, 0); @@ -597,29 +599,92 @@ private static void configureTunnelingV21(MediaFormat mediaFormat, int tunneling * Returns {@link CodecMaxValues} suitable for configuring a codec for {@code format} in a way * that will allow possible adaptation to other compatible formats in {@code streamFormats}. * + * @param codecInfo Information about the {@link MediaCodec} being configured. * @param format The format for which the codec is being configured. * @param streamFormats The possible stream formats. * @return Suitable {@link CodecMaxValues}. + * @throws DecoderQueryException If an error occurs querying {@code codecInfo}. */ - private static CodecMaxValues getCodecMaxValues(Format format, Format[] streamFormats) { + private static CodecMaxValues getCodecMaxValues(MediaCodecInfo codecInfo, Format format, + Format[] streamFormats) throws DecoderQueryException { int maxWidth = format.width; int maxHeight = format.height; int maxInputSize = getMaxInputSize(format); + if (streamFormats.length == 1) { + // The single entry in streamFormats must correspond to the format for which the codec is + // being configured. + return new CodecMaxValues(maxWidth, maxHeight, maxInputSize); + } + boolean haveUnknownDimensions = false; for (Format streamFormat : streamFormats) { if (areAdaptationCompatible(format, streamFormat)) { + haveUnknownDimensions |= (streamFormat.width == Format.NO_VALUE + || streamFormat.height == Format.NO_VALUE); maxWidth = Math.max(maxWidth, streamFormat.width); maxHeight = Math.max(maxHeight, streamFormat.height); maxInputSize = Math.max(maxInputSize, getMaxInputSize(streamFormat)); } } + if (haveUnknownDimensions) { + Log.w(TAG, "Resolutions unknown. Codec max resolution: " + maxWidth + "x" + maxHeight); + Point codecMaxSize = getCodecMaxSize(codecInfo, format); + if (codecMaxSize != null) { + maxWidth = Math.max(maxWidth, codecMaxSize.x); + maxHeight = Math.max(maxHeight, codecMaxSize.y); + maxInputSize = Math.max(maxInputSize, + getMaxInputSize(format.sampleMimeType, maxWidth, maxHeight)); + Log.w(TAG, "Codec max resolution adjusted to: " + maxWidth + "x" + maxHeight); + } + } return new CodecMaxValues(maxWidth, maxHeight, maxInputSize); } + /** + * Returns a maximum video size to use when configuring a codec for {@code format} in a way + * that will allow possible adaptation to other compatible formats that are expected to have the + * same aspect ratio, but whose sizes are unknown. + * + * @param codecInfo Information about the {@link MediaCodec} being configured. + * @param format The format for which the codec is being configured. + * @return The maximum video size to use, or null if the size of {@code format} should be used. + * @throws DecoderQueryException If an error occurs querying {@code codecInfo}. + */ + private static Point getCodecMaxSize(MediaCodecInfo codecInfo, Format format) + throws DecoderQueryException { + boolean isVerticalVideo = format.height > format.width; + int formatLongEdgePx = isVerticalVideo ? format.height : format.width; + int formatShortEdgePx = isVerticalVideo ? format.width : format.height; + float aspectRatio = (float) formatShortEdgePx / formatLongEdgePx; + for (int longEdgePx : STANDARD_LONG_EDGE_VIDEO_PX) { + int shortEdgePx = (int) (longEdgePx * aspectRatio); + if (longEdgePx <= formatLongEdgePx || shortEdgePx <= formatShortEdgePx) { + // Don't return a size not larger than the format for which the codec is being configured. + return null; + } else if (Util.SDK_INT >= 21) { + Point alignedSize = codecInfo.alignVideoSizeV21(isVerticalVideo ? shortEdgePx : longEdgePx, + isVerticalVideo ? longEdgePx : shortEdgePx); + float frameRate = format.frameRate; + if (codecInfo.isVideoSizeAndRateSupportedV21(alignedSize.x, alignedSize.y, frameRate)) { + return alignedSize; + } + } else { + // Conservatively assume the codec requires 16px width and height alignment. + longEdgePx = Util.ceilDivide(longEdgePx, 16) * 16; + shortEdgePx = Util.ceilDivide(shortEdgePx, 16) * 16; + if (longEdgePx * shortEdgePx <= MediaCodecUtil.maxH264DecodableFrameSize()) { + return new Point(isVerticalVideo ? shortEdgePx : longEdgePx, + isVerticalVideo ? longEdgePx : shortEdgePx); + } + } + } + return null; + } + /** * Returns a maximum input size for a given format. * * @param format The format. - * @return An maximum input size in bytes, or {@link Format#NO_VALUE} if a maximum could not be + * @return A maximum input size in bytes, or {@link Format#NO_VALUE} if a maximum could not be * determined. */ private static int getMaxInputSize(Format format) { @@ -627,8 +692,20 @@ private static int getMaxInputSize(Format format) { // The format defines an explicit maximum input size. return format.maxInputSize; } + return getMaxInputSize(format.sampleMimeType, format.width, format.height); + } - if (format.width == Format.NO_VALUE || format.height == Format.NO_VALUE) { + /** + * Returns a maximum input size for a given mime type, width and height. + * + * @param sampleMimeType The format mime type. + * @param width The width in pixels. + * @param height The height in pixels. + * @return A maximum input size in bytes, or {@link Format#NO_VALUE} if a maximum could not be + * determined. + */ + private static int getMaxInputSize(String sampleMimeType, int width, int height) { + if (width == Format.NO_VALUE || height == Format.NO_VALUE) { // We can't infer a maximum input size without video dimensions. return Format.NO_VALUE; } @@ -636,10 +713,10 @@ private static int getMaxInputSize(Format format) { // Attempt to infer a maximum input size from the format. int maxPixels; int minCompressionRatio; - switch (format.sampleMimeType) { + switch (sampleMimeType) { case MimeTypes.VIDEO_H263: case MimeTypes.VIDEO_MP4V: - maxPixels = format.width * format.height; + maxPixels = width * height; minCompressionRatio = 2; break; case MimeTypes.VIDEO_H264: @@ -649,17 +726,17 @@ private static int getMaxInputSize(Format format) { return Format.NO_VALUE; } // Round up width/height to an integer number of macroblocks. - maxPixels = ((format.width + 15) / 16) * ((format.height + 15) / 16) * 16 * 16; + maxPixels = Util.ceilDivide(width, 16) * Util.ceilDivide(height, 16) * 16 * 16; minCompressionRatio = 2; break; case MimeTypes.VIDEO_VP8: // VPX does not specify a ratio so use the values from the platform's SoftVPX.cpp. - maxPixels = format.width * format.height; + maxPixels = width * height; minCompressionRatio = 2; break; case MimeTypes.VIDEO_H265: case MimeTypes.VIDEO_VP9: - maxPixels = format.width * format.height; + maxPixels = width * height; minCompressionRatio = 4; break; default: