Add support to fetch ColorInfo from avcC box in AtomParsers

#minor-release PiperOrigin-RevId: 526082823
androidx · Apr 24, 2023 · 8a73d8d · 8a73d8d
1 parent a0fe3b2
commit 8a73d8d
Show file tree

Hide file tree

Showing 19 changed files with 177 additions and 27 deletions.
diff --git a/libraries/common/src/main/java/androidx/media3/common/util/NalUnitUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/NalUnitUtil.java
@@ -63,6 +63,9 @@ public static final class SpsData {
     public final int picOrderCountType;
     public final int picOrderCntLsbLength;
     public final boolean deltaPicOrderAlwaysZeroFlag;
+    public final @C.ColorSpace int colorSpace;
+    public final @C.ColorRange int colorRange;
+    public final @C.ColorTransfer int colorTransfer;
 
     public SpsData(
         int profileIdc,
@@ -78,7 +81,10 @@ public SpsData(
         int frameNumLength,
         int picOrderCountType,
         int picOrderCntLsbLength,
-        boolean deltaPicOrderAlwaysZeroFlag) {
+        boolean deltaPicOrderAlwaysZeroFlag,
+        @C.ColorSpace int colorSpace,
+        @C.ColorRange int colorRange,
+        @C.ColorTransfer int colorTransfer) {
       this.profileIdc = profileIdc;
       this.constraintsFlagsAndReservedZero2Bits = constraintsFlagsAndReservedZero2Bits;
       this.levelIdc = levelIdc;
@@ -93,6 +99,9 @@ public SpsData(
       this.picOrderCountType = picOrderCountType;
       this.picOrderCntLsbLength = picOrderCntLsbLength;
       this.deltaPicOrderAlwaysZeroFlag = deltaPicOrderAlwaysZeroFlag;
+      this.colorSpace = colorSpace;
+      this.colorRange = colorRange;
+      this.colorTransfer = colorTransfer;
     }
   }
 
@@ -443,6 +452,9 @@ public static SpsData parseSpsNalUnitPayload(byte[] nalData, int nalOffset, int
       frameHeight -= (frameCropTopOffset + frameCropBottomOffset) * cropUnitY;
     }
 
+    @C.ColorSpace int colorSpace = Format.NO_VALUE;
+    @C.ColorRange int colorRange = Format.NO_VALUE;
+    @C.ColorTransfer int colorTransfer = Format.NO_VALUE;
     float pixelWidthHeightRatio = 1;
     boolean vuiParametersPresentFlag = data.readBit();
     if (vuiParametersPresentFlag) {
@@ -461,6 +473,23 @@ public static SpsData parseSpsNalUnitPayload(byte[] nalData, int nalOffset, int
           Log.w(TAG, "Unexpected aspect_ratio_idc value: " + aspectRatioIdc);
         }
       }
+      if (data.readBit()) { // overscan_info_present_flag
+        data.skipBit(); // overscan_appropriate_flag
+      }
+      if (data.readBit()) { // video_signal_type_present_flag
+        data.skipBits(3); // video_format
+        colorRange =
+            data.readBit() ? C.COLOR_RANGE_FULL : C.COLOR_RANGE_LIMITED; // video_full_range_flag
+        if (data.readBit()) { // colour_description_present_flag
+          int colorPrimaries = data.readBits(8); // colour_primaries
+          int transferCharacteristics = data.readBits(8); // transfer_characteristics
+          data.skipBits(8); // matrix_coeffs
+
+          colorSpace = ColorInfo.isoColorPrimariesToColorSpace(colorPrimaries);
+          colorTransfer =
+              ColorInfo.isoTransferCharacteristicsToColorTransfer(transferCharacteristics);
+        }
+      }
     }
 
     return new SpsData(
@@ -477,7 +506,10 @@ public static SpsData parseSpsNalUnitPayload(byte[] nalData, int nalOffset, int
         frameNumLength,
         picOrderCntType,
         picOrderCntLsbLength,
-        deltaPicOrderAlwaysZeroFlag);
+        deltaPicOrderAlwaysZeroFlag,
+        colorSpace,
+        colorRange,
+        colorTransfer);
   }
 
   /**
@@ -505,10 +537,6 @@ public static H265SpsData parseH265SpsNalUnit(byte[] nalData, int nalOffset, int
   public static H265SpsData parseH265SpsNalUnitPayload(
       byte[] nalData, int nalOffset, int nalLimit) {
     ParsableNalUnitBitArray data = new ParsableNalUnitBitArray(nalData, nalOffset, nalLimit);
-    // HDR related metadata.
-    @C.ColorSpace int colorSpace = Format.NO_VALUE;
-    @C.ColorRange int colorRange = Format.NO_VALUE;
-    @C.ColorTransfer int colorTransfer = Format.NO_VALUE;
     data.skipBits(4); // sps_video_parameter_set_id
     int maxSubLayersMinus1 = data.readBits(3);
     data.skipBit(); // sps_temporal_id_nesting_flag
@@ -595,6 +623,9 @@ public static H265SpsData parseH265SpsNalUnitPayload(
       }
     }
     data.skipBits(2); // sps_temporal_mvp_enabled_flag, strong_intra_smoothing_enabled_flag
+    @C.ColorSpace int colorSpace = Format.NO_VALUE;
+    @C.ColorRange int colorRange = Format.NO_VALUE;
+    @C.ColorTransfer int colorTransfer = Format.NO_VALUE;
     float pixelWidthHeightRatio = 1;
     if (data.readBit()) { // vui_parameters_present_flag
       if (data.readBit()) { // aspect_ratio_info_present_flag
@@ -616,14 +647,14 @@ public static H265SpsData parseH265SpsNalUnitPayload(
       }
       if (data.readBit()) { // video_signal_type_present_flag
         data.skipBits(3); // video_format
-        boolean fullRangeFlag = data.readBit(); // video_full_range_flag
+        colorRange =
+            data.readBit() ? C.COLOR_RANGE_FULL : C.COLOR_RANGE_LIMITED; // video_full_range_flag
         if (data.readBit()) { // colour_description_present_flag
           int colorPrimaries = data.readBits(8); // colour_primaries
           int transferCharacteristics = data.readBits(8); // transfer_characteristics
           data.skipBits(8); // matrix_coeffs
 
           colorSpace = ColorInfo.isoColorPrimariesToColorSpace(colorPrimaries);
-          colorRange = fullRangeFlag ? C.COLOR_RANGE_FULL : C.COLOR_RANGE_LIMITED;
           colorTransfer =
               ColorInfo.isoTransferCharacteristicsToColorTransfer(transferCharacteristics);
         }

diff --git a/libraries/common/src/test/java/androidx/media3/common/util/NalUnitUtilTest.java b/libraries/common/src/test/java/androidx/media3/common/util/NalUnitUtilTest.java
@@ -31,8 +31,8 @@ public final class NalUnitUtilTest {
   private static final int TEST_NAL_POSITION = 10;
   private static final byte[] SPS_TEST_DATA =
       createByteArray(
-          0x00, 0x00, 0x01, 0x67, 0x4D, 0x40, 0x16, 0xEC, 0xA0, 0x50, 0x17, 0xFC, 0xB8, 0x08, 0x80,
-          0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x0F, 0x47, 0x8B, 0x16, 0xCB);
+          0x00, 0x00, 0x01, 0x67, 0x4D, 0x40, 0x16, 0xEC, 0xA0, 0x50, 0x17, 0xFC, 0xB8, 0x0A, 0x90,
+          0x91, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x0F, 0x47, 0x8B, 0x16, 0xCB);
   private static final int SPS_TEST_DATA_OFFSET = 3;
 
   @Test
@@ -135,6 +135,9 @@ public void parseSpsNalUnit() {
     assertThat(data.pixelWidthHeightRatio).isEqualTo(1.0f);
     assertThat(data.picOrderCountType).isEqualTo(0);
     assertThat(data.separateColorPlaneFlag).isFalse();
+    assertThat(data.colorSpace).isEqualTo(6);
+    assertThat(data.colorRange).isEqualTo(2);
+    assertThat(data.colorTransfer).isEqualTo(6);
   }
 
   @Test

diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/AvcConfig.java b/libraries/extractor/src/main/java/androidx/media3/extractor/AvcConfig.java
@@ -16,6 +16,7 @@
 package androidx.media3.extractor;
 
 import androidx.annotation.Nullable;
+import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.ParserException;
 import androidx.media3.common.util.CodecSpecificDataUtil;
@@ -57,6 +58,9 @@ public static AvcConfig parse(ParsableByteArray data) throws ParserException {
 
       int width = Format.NO_VALUE;
       int height = Format.NO_VALUE;
+      @C.ColorSpace int colorSpace = Format.NO_VALUE;
+      @C.ColorRange int colorRange = Format.NO_VALUE;
+      @C.ColorTransfer int colorTransfer = Format.NO_VALUE;
       float pixelWidthHeightRatio = 1;
       @Nullable String codecs = null;
       if (numSequenceParameterSets > 0) {
@@ -66,6 +70,9 @@ public static AvcConfig parse(ParsableByteArray data) throws ParserException {
                 initializationData.get(0), nalUnitLengthFieldLength, sps.length);
         width = spsData.width;
         height = spsData.height;
+        colorSpace = spsData.colorSpace;
+        colorRange = spsData.colorRange;
+        colorTransfer = spsData.colorTransfer;
         pixelWidthHeightRatio = spsData.pixelWidthHeightRatio;
         codecs =
             CodecSpecificDataUtil.buildAvcCodecString(
@@ -77,6 +84,9 @@ public static AvcConfig parse(ParsableByteArray data) throws ParserException {
           nalUnitLengthFieldLength,
           width,
           height,
+          colorSpace,
+          colorRange,
+          colorTransfer,
           pixelWidthHeightRatio,
           codecs);
     } catch (ArrayIndexOutOfBoundsException e) {
@@ -100,6 +110,22 @@ public static AvcConfig parse(ParsableByteArray data) throws ParserException {
   /** The height of each decoded frame, or {@link Format#NO_VALUE} if unknown. */
   public final int height;
 
+  /**
+   * The {@link C.ColorSpace} of the video, or {@link Format#NO_VALUE} if unknown or not applicable.
+   */
+  public final @C.ColorSpace int colorSpace;
+
+  /**
+   * The {@link C.ColorRange} of the video, or {@link Format#NO_VALUE} if unknown or not applicable.
+   */
+  public final @C.ColorRange int colorRange;
+
+  /**
+   * The {@link C.ColorTransfer} of the video, or {@link Format#NO_VALUE} if unknown or not
+   * applicable.
+   */
+  public final @C.ColorTransfer int colorTransfer;
+
   /** The pixel width to height ratio. */
   public final float pixelWidthHeightRatio;
 
@@ -115,12 +141,18 @@ private AvcConfig(
       int nalUnitLengthFieldLength,
       int width,
       int height,
+      @C.ColorSpace int colorSpace,
+      @C.ColorRange int colorRange,
+      @C.ColorTransfer int colorTransfer,
       float pixelWidthHeightRatio,
       @Nullable String codecs) {
     this.initializationData = initializationData;
     this.nalUnitLengthFieldLength = nalUnitLengthFieldLength;
     this.width = width;
     this.height = height;
+    this.colorSpace = colorSpace;
+    this.colorRange = colorRange;
+    this.colorTransfer = colorTransfer;
     this.pixelWidthHeightRatio = pixelWidthHeightRatio;
     this.codecs = codecs;
   }

diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/HevcConfig.java b/libraries/extractor/src/main/java/androidx/media3/extractor/HevcConfig.java
@@ -114,11 +114,11 @@ public static HevcConfig parse(ParsableByteArray data) throws ParserException {
           lengthSizeMinusOne + 1,
           width,
           height,
-          pixelWidthHeightRatio,
-          codecs,
           colorSpace,
           colorRange,
-          colorTransfer);
+          colorTransfer,
+          pixelWidthHeightRatio,
+          codecs);
     } catch (ArrayIndexOutOfBoundsException e) {
       throw ParserException.createForMalformedContainer("Error parsing HEVC config", e);
     }
@@ -142,9 +142,6 @@ public static HevcConfig parse(ParsableByteArray data) throws ParserException {
   /** The height of each decoded frame, or {@link Format#NO_VALUE} if unknown. */
   public final int height;
 
-  /** The pixel width to height ratio. */
-  public final float pixelWidthHeightRatio;
-
   /**
    * The {@link C.ColorSpace} of the video or {@link Format#NO_VALUE} if unknown or not applicable.
    */
@@ -161,6 +158,9 @@ public static HevcConfig parse(ParsableByteArray data) throws ParserException {
    */
   public final @C.ColorTransfer int colorTransfer;
 
+  /** The pixel width to height ratio. */
+  public final float pixelWidthHeightRatio;
+
   /**
    * An RFC 6381 codecs string representing the video format, or {@code null} if not known.
    *
@@ -173,19 +173,19 @@ private HevcConfig(
       int nalUnitLengthFieldLength,
       int width,
       int height,
-      float pixelWidthHeightRatio,
-      @Nullable String codecs,
       @C.ColorSpace int colorSpace,
       @C.ColorRange int colorRange,
-      @C.ColorTransfer int colorTransfer) {
+      @C.ColorTransfer int colorTransfer,
+      float pixelWidthHeightRatio,
+      @Nullable String codecs) {
     this.initializationData = initializationData;
     this.nalUnitLengthFieldLength = nalUnitLengthFieldLength;
     this.width = width;
     this.height = height;
-    this.pixelWidthHeightRatio = pixelWidthHeightRatio;
-    this.codecs = codecs;
     this.colorSpace = colorSpace;
     this.colorRange = colorRange;
     this.colorTransfer = colorTransfer;
+    this.pixelWidthHeightRatio = pixelWidthHeightRatio;
+    this.codecs = codecs;
   }
 }
diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java
@@ -1197,6 +1197,9 @@ private static void parseVideoSampleEntry(
           pixelWidthHeightRatio = avcConfig.pixelWidthHeightRatio;
         }
         codecs = avcConfig.codecs;
+        colorSpace = avcConfig.colorSpace;
+        colorRange = avcConfig.colorRange;
+        colorTransfer = avcConfig.colorTransfer;
       } else if (childAtomType == Atom.TYPE_hvcC) {
         ExtractorUtil.checkContainerInput(mimeType == null, /* message= */ null);
         mimeType = MimeTypes.VIDEO_H265;
@@ -1309,12 +1312,13 @@ private static void parseVideoSampleEntry(
           }
         }
       } else if (childAtomType == Atom.TYPE_colr) {
-        // Only modify these values if they have not been previously established by the bitstream.
-        // If 'Atom.TYPE_hvcC' atom or 'Atom.TYPE_vpcC' is available, they will take precedence and
-        // overwrite any existing values.
-        if (colorSpace == Format.NO_VALUE
-            && colorRange == Format.NO_VALUE
-            && colorTransfer == Format.NO_VALUE) {
+        // Only modify these values if 'colorSpace' and 'colorTransfer' have not been previously
+        // established by the bitstream. The absence of color descriptors ('colorSpace' and
+        // 'colorTransfer') does not necessarily mean that 'colorRange' has default values, hence it
+        // is not being verified here.
+        // If 'Atom.TYPE_avcC', 'Atom.TYPE_hvcC' or 'Atom.TYPE_vpcC' is available, they will take
+        // precedence and overwrite any existing values.
+        if (colorSpace == Format.NO_VALUE && colorTransfer == Format.NO_VALUE) {
           int colorType = parent.readInt();
           if (colorType == TYPE_nclx || colorType == TYPE_nclc) {
             // For more info on syntax, see Section 8.5.2.2 in ISO/IEC 14496-12:2012(E) and

diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.0.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.0.dump
@@ -17,6 +17,11 @@ track 0:
     width = 1280
     height = 720
     frameRate = 13.307984
+    colorInfo:
+      colorSpace = 2
+      colorRange = 1
+      colorTransfer = -1
+      hdrStaticInfo = length 0, hash 0
     metadata = entries=[mdta: key=com.android.capture.fps, value=43700000]
     initializationData:
       data = length 22, hash 4CF81805

diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.1.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.1.dump
@@ -17,6 +17,11 @@ track 0:
     width = 1280
     height = 720
     frameRate = 13.307984
+    colorInfo:
+      colorSpace = 2
+      colorRange = 1
+      colorTransfer = -1
+      hdrStaticInfo = length 0, hash 0
     metadata = entries=[mdta: key=com.android.capture.fps, value=43700000]
     initializationData:
       data = length 22, hash 4CF81805

diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.2.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.2.dump
@@ -17,6 +17,11 @@ track 0:
     width = 1280
     height = 720
     frameRate = 13.307984
+    colorInfo:
+      colorSpace = 2
+      colorRange = 1
+      colorTransfer = -1
+      hdrStaticInfo = length 0, hash 0
     metadata = entries=[mdta: key=com.android.capture.fps, value=43700000]
     initializationData:
       data = length 22, hash 4CF81805

diff --git a/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.3.dump b/libraries/test_data/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.3.dump
@@ -17,6 +17,11 @@ track 0:
     width = 1280
     height = 720
     frameRate = 13.307984
+    colorInfo:
+      colorSpace = 2
+      colorRange = 1
+      colorTransfer = -1
+      hdrStaticInfo = length 0, hash 0
     metadata = entries=[mdta: key=com.android.capture.fps, value=43700000]
     initializationData:
       data = length 22, hash 4CF81805

diff --git a/...ata/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.unknown_length.dump b/...ata/src/test/assets/extractordumps/mp4/sample_android_slow_motion.mp4.unknown_length.dump
@@ -17,6 +17,11 @@ track 0:
     width = 1280
     height = 720
     frameRate = 13.307984
+    colorInfo:
+      colorSpace = 2
+      colorRange = 1
+      colorTransfer = -1
+      hdrStaticInfo = length 0, hash 0
     metadata = entries=[mdta: key=com.android.capture.fps, value=43700000]
     initializationData:
       data = length 22, hash 4CF81805

diff --git a/libraries/test_data/src/test/assets/muxerdumps/mp4_with_0_orientation.mp4.dump b/libraries/test_data/src/test/assets/muxerdumps/mp4_with_0_orientation.mp4.dump
@@ -16,6 +16,11 @@ track 0:
     maxInputSize = 85
     width = 12
     height = 10
+    colorInfo:
+      colorSpace = -1
+      colorRange = 1
+      colorTransfer = -1
+      hdrStaticInfo = length 0, hash 0
     initializationData:
       data = length 28, hash 410B510
       data = length 9, hash FBADD682

diff --git a/libraries/test_data/src/test/assets/muxerdumps/mp4_with_180_orientation.mp4.dump b/libraries/test_data/src/test/assets/muxerdumps/mp4_with_180_orientation.mp4.dump
@@ -17,6 +17,11 @@ track 0:
     width = 12
     height = 10
     rotationDegrees = 180
+    colorInfo:
+      colorSpace = -1
+      colorRange = 1
+      colorTransfer = -1
+      hdrStaticInfo = length 0, hash 0
     initializationData:
       data = length 28, hash 410B510
       data = length 9, hash FBADD682