Mitigate the need for FLAG_ALLOW_NON_IDR_KEYFRAMES

Use random access indicator in transport streams Issue:#1967 Issue:#2020 Issue:#2182 Issue:#2469 Issue:#2581 Issue:#2748 Issue:#2939 Issue:#2979 Issue:#3316 Issue:#3574 Issue:#3709 Issue:#3747 Issue:#4103 Issue:#4184 Issue:#4355 Issue:#4538 Issue:#4719 Issue:#4861 Issue:#4925 Issue:#4951 Issue:#5108 Issue:#5186 PiperOrigin-RevId: 225798044
google · Dec 19, 2018 · f6165f7 · f6165f7
1 parent 7426c14
commit f6165f7
Show file tree

Hide file tree

Showing 22 changed files with 168 additions and 85 deletions.
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
@@ -4,16 +4,18 @@
 
 * Captions: Support PNG subtitles in SMPTE-TT
   ([#1583](https://github.com/google/ExoPlayer/issues/1583)).
-* Workaround for MiTV (dangal) issue when swapping output surface
-  ([#5169](https://github.com/google/ExoPlayer/issues/5169)).
-* Fix decode-only frame skipping on Nvidia Shield TV devices.
+* MPEG-TS: Use the random access indicator to minimize the need for
+  `FLAG_ALLOW_NON_IDR_KEYFRAMES`.
+* MP3:
+  * Use the true bitrate for constant-bitrate MP3 seeking.
+  * Fix issue where streams would play twice on some Samsung devices
+     ([#4519](https://github.com/google/ExoPlayer/issues/4519)).
 * Fix regression where some audio formats were incorrectly marked as being
   unplayable due to under-reporting of platform decoder capabilities
   ([#5145](https://github.com/google/ExoPlayer/issues/5145)).
- * MP3:
-   * Use the true bitrate for constant-bitrate MP3 seeking.
-   * Fix issue where streams would play twice on some Samsung devices
-     ([#4519](https://github.com/google/ExoPlayer/issues/4519)).
+* Fix decode-only frame skipping on Nvidia Shield TV devices.
+* Workaround for MiTV (dangal) issue when swapping output surface
+  ([#5169](https://github.com/google/ExoPlayer/issues/5169)).
 
 ### 2.9.2 ###
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Extractor.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Extractor.java
@@ -15,6 +15,8 @@
  */
 package com.google.android.exoplayer2.extractor.ts;
 
+import static com.google.android.exoplayer2.extractor.ts.TsPayloadReader.FLAG_DATA_ALIGNMENT_INDICATOR;
+
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.audio.Ac3Util;
 import com.google.android.exoplayer2.extractor.Extractor;
@@ -140,7 +142,7 @@ public int read(ExtractorInput input, PositionHolder seekPosition) throws IOExce
 
     if (!startedPacket) {
       // Pass data to the reader as though it's contained within a single infinitely long packet.
-      reader.packetStarted(firstSampleTimestampUs, true);
+      reader.packetStarted(firstSampleTimestampUs, FLAG_DATA_ALIGNMENT_INDICATOR);
       startedPacket = true;
     }
     // TODO: Make it possible for the reader to consume the dataSource directly, so that it becomes

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Reader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Ac3Reader.java
@@ -100,7 +100,7 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator gener
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
     timeUs = pesTimeUs;
   }
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/AdtsExtractor.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/AdtsExtractor.java
@@ -15,6 +15,8 @@
  */
 package com.google.android.exoplayer2.extractor.ts;
 
+import static com.google.android.exoplayer2.extractor.ts.TsPayloadReader.FLAG_DATA_ALIGNMENT_INDICATOR;
+
 import android.support.annotation.IntDef;
 import android.support.annotation.Nullable;
 import com.google.android.exoplayer2.C;
@@ -202,7 +204,7 @@ public int read(ExtractorInput input, PositionHolder seekPosition)
 
     if (!startedPacket) {
       // Pass data to the reader as though it's contained within a single infinitely long packet.
-      reader.packetStarted(firstSampleTimestampUs, true);
+      reader.packetStarted(firstSampleTimestampUs, FLAG_DATA_ALIGNMENT_INDICATOR);
       startedPacket = true;
     }
     // TODO: Make it possible for reader to consume the dataSource directly, so that it becomes

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/AdtsReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/AdtsReader.java
@@ -141,7 +141,7 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
     timeUs = pesTimeUs;
   }
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/DtsReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/DtsReader.java
@@ -80,7 +80,7 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
     timeUs = pesTimeUs;
   }
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/DvbSubtitleReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/DvbSubtitleReader.java
@@ -15,6 +15,8 @@
  */
 package com.google.android.exoplayer2.extractor.ts;
 
+import static com.google.android.exoplayer2.extractor.ts.TsPayloadReader.FLAG_DATA_ALIGNMENT_INDICATOR;
+
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.extractor.ExtractorOutput;
@@ -73,8 +75,8 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
-    if (!dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
+    if ((flags & FLAG_DATA_ALIGNMENT_INDICATOR) == 0) {
       return;
     }
     writingSample = true;

diff --git a/...core/src/main/java/com/google/android/exoplayer2/extractor/ts/ElementaryStreamReader.java b/...core/src/main/java/com/google/android/exoplayer2/extractor/ts/ElementaryStreamReader.java
@@ -43,9 +43,9 @@ public interface ElementaryStreamReader {
    * Called when a packet starts.
    *
    * @param pesTimeUs The timestamp associated with the packet.
-   * @param dataAlignmentIndicator The data alignment indicator associated with the packet.
+   * @param flags See {@link TsPayloadReader.Flags}.
    */
-  void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator);
+  void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags);
 
   /**
    * Consumes (possibly partial) data from the current packet.

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H262Reader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H262Reader.java
@@ -107,7 +107,8 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
+    // TODO (Internal b/32267012): Consider using random access indicator.
     this.pesTimeUs = pesTimeUs;
   }
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java
@@ -15,6 +15,8 @@
  */
 package com.google.android.exoplayer2.extractor.ts;
 
+import static com.google.android.exoplayer2.extractor.ts.TsPayloadReader.FLAG_RANDOM_ACCESS_INDICATOR;
+
 import android.util.SparseArray;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
@@ -56,9 +58,12 @@ public final class H264Reader implements ElementaryStreamReader {
   // State that should not be reset on seek.
   private boolean hasOutputFormat;
 
-  // Per packet state that gets reset at the start of each packet.
+  // Per PES packet state that gets reset at the start of each PES packet.
   private long pesTimeUs;
 
+  // State inherited from the TS packet header.
+  private boolean randomAccessIndicator;
+
   // Scratch variables to avoid allocations.
   private final ParsableByteArray seiWrapper;
 
@@ -88,6 +93,7 @@ public void seek() {
     sei.reset();
     sampleReader.reset();
     totalBytesWritten = 0;
+    randomAccessIndicator = false;
   }
 
   @Override
@@ -100,8 +106,9 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
     this.pesTimeUs = pesTimeUs;
+    randomAccessIndicator |= (flags & FLAG_RANDOM_ACCESS_INDICATOR) != 0;
   }
 
   @Override
@@ -220,12 +227,17 @@ private void endNalUnit(long position, int offset, int discardPadding, long pesT
       seiWrapper.setPosition(4); // NAL prefix and nal_unit() header.
       seiReader.consume(pesTimeUs, seiWrapper);
     }
-    sampleReader.endNalUnit(position, offset);
+    boolean sampleIsKeyFrame =
+        sampleReader.endNalUnit(position, offset, hasOutputFormat, randomAccessIndicator);
+    if (sampleIsKeyFrame) {
+      // This is either an IDR frame or the first I-frame since the random access indicator, so mark
+      // it as a keyframe. Clear the flag so that subsequent non-IDR I-frames are not marked as
+      // keyframes until we see another random access indicator.
+      randomAccessIndicator = false;
+    }
   }
 
-  /**
-   * Consumes a stream of NAL units and outputs samples.
-   */
+  /** Consumes a stream of NAL units and outputs samples. */
   private static final class SampleReader {
 
     private static final int DEFAULT_BUFFER_SIZE = 128;
@@ -430,11 +442,12 @@ public void appendToNalUnit(byte[] data, int offset, int limit) {
       isFilling = false;
     }
 
-    public void endNalUnit(long position, int offset) {
+    public boolean endNalUnit(
+        long position, int offset, boolean hasOutputFormat, boolean randomAccessIndicator) {
       if (nalUnitType == NAL_UNIT_TYPE_AUD
           || (detectAccessUnits && sliceHeader.isFirstVclNalUnitOfPicture(previousSliceHeader))) {
         // If the NAL unit ending is the start of a new sample, output the previous one.
-        if (readingSample) {
+        if (hasOutputFormat && readingSample) {
           int nalUnitLength = (int) (position - nalUnitStartPosition);
           outputSample(offset + nalUnitLength);
         }
@@ -443,8 +456,12 @@ public void endNalUnit(long position, int offset) {
         sampleIsKeyframe = false;
         readingSample = true;
       }
-      sampleIsKeyframe |= nalUnitType == NAL_UNIT_TYPE_IDR || (allowNonIdrKeyframes
-          && nalUnitType == NAL_UNIT_TYPE_NON_IDR && sliceHeader.isISlice());
+      boolean treatIFrameAsKeyframe =
+          allowNonIdrKeyframes ? sliceHeader.isISlice() : randomAccessIndicator;
+      sampleIsKeyframe |=
+          nalUnitType == NAL_UNIT_TYPE_IDR
+              || (treatIFrameAsKeyframe && nalUnitType == NAL_UNIT_TYPE_NON_IDR);
+      return sampleIsKeyframe;
     }
 
     private void outputSample(int offset) {
@@ -486,10 +503,21 @@ public void setSliceType(int sliceType) {
         hasSliceType = true;
       }
 
-      public void setAll(SpsData spsData, int nalRefIdc, int sliceType, int frameNum,
-          int picParameterSetId, boolean fieldPicFlag, boolean bottomFieldFlagPresent,
-          boolean bottomFieldFlag, boolean idrPicFlag, int idrPicId, int picOrderCntLsb,
-          int deltaPicOrderCntBottom, int deltaPicOrderCnt0, int deltaPicOrderCnt1) {
+      public void setAll(
+          SpsData spsData,
+          int nalRefIdc,
+          int sliceType,
+          int frameNum,
+          int picParameterSetId,
+          boolean fieldPicFlag,
+          boolean bottomFieldFlagPresent,
+          boolean bottomFieldFlag,
+          boolean idrPicFlag,
+          int idrPicId,
+          int picOrderCntLsb,
+          int deltaPicOrderCntBottom,
+          int deltaPicOrderCnt0,
+          int deltaPicOrderCnt1) {
         this.spsData = spsData;
         this.nalRefIdc = nalRefIdc;
         this.sliceType = sliceType;
@@ -514,23 +542,26 @@ public boolean isISlice() {
 
       private boolean isFirstVclNalUnitOfPicture(SliceHeaderData other) {
         // See ISO 14496-10 subsection 7.4.1.2.4.
-        return isComplete && (!other.isComplete || frameNum != other.frameNum
-            || picParameterSetId != other.picParameterSetId || fieldPicFlag != other.fieldPicFlag
-            || (bottomFieldFlagPresent && other.bottomFieldFlagPresent
-                && bottomFieldFlag != other.bottomFieldFlag)
-            || (nalRefIdc != other.nalRefIdc && (nalRefIdc == 0 || other.nalRefIdc == 0))
-            || (spsData.picOrderCountType == 0 && other.spsData.picOrderCountType == 0
-                && (picOrderCntLsb != other.picOrderCntLsb
-                    || deltaPicOrderCntBottom != other.deltaPicOrderCntBottom))
-            || (spsData.picOrderCountType == 1 && other.spsData.picOrderCountType == 1
-                && (deltaPicOrderCnt0 != other.deltaPicOrderCnt0
-                    || deltaPicOrderCnt1 != other.deltaPicOrderCnt1))
-            || idrPicFlag != other.idrPicFlag
-            || (idrPicFlag && other.idrPicFlag && idrPicId != other.idrPicId));
+        return isComplete
+            && (!other.isComplete
+                || frameNum != other.frameNum
+                || picParameterSetId != other.picParameterSetId
+                || fieldPicFlag != other.fieldPicFlag
+                || (bottomFieldFlagPresent
+                    && other.bottomFieldFlagPresent
+                    && bottomFieldFlag != other.bottomFieldFlag)
+                || (nalRefIdc != other.nalRefIdc && (nalRefIdc == 0 || other.nalRefIdc == 0))
+                || (spsData.picOrderCountType == 0
+                    && other.spsData.picOrderCountType == 0
+                    && (picOrderCntLsb != other.picOrderCntLsb
+                        || deltaPicOrderCntBottom != other.deltaPicOrderCntBottom))
+                || (spsData.picOrderCountType == 1
+                    && other.spsData.picOrderCountType == 1
+                    && (deltaPicOrderCnt0 != other.deltaPicOrderCnt0
+                        || deltaPicOrderCnt1 != other.deltaPicOrderCnt1))
+                || idrPicFlag != other.idrPicFlag
+                || (idrPicFlag && other.idrPicFlag && idrPicId != other.idrPicId));
       }
-
     }
-
   }
-
 }
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H265Reader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H265Reader.java
@@ -104,7 +104,8 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
+    // TODO (Internal b/32267012): Consider using random access indicator.
     this.pesTimeUs = pesTimeUs;
   }
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Id3Reader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/Id3Reader.java
@@ -15,6 +15,8 @@
  */
 package com.google.android.exoplayer2.extractor.ts;
 
+import static com.google.android.exoplayer2.extractor.ts.TsPayloadReader.FLAG_DATA_ALIGNMENT_INDICATOR;
+
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.extractor.ExtractorOutput;
@@ -63,8 +65,8 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
-    if (!dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
+    if ((flags & FLAG_DATA_ALIGNMENT_INDICATOR) == 0) {
       return;
     }
     writingSample = true;

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/LatmReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/LatmReader.java
@@ -93,7 +93,7 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
     timeUs = pesTimeUs;
   }
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/MpegAudioReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/MpegAudioReader.java
@@ -83,7 +83,7 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
   }
 
   @Override
-  public void packetStarted(long pesTimeUs, boolean dataAlignmentIndicator) {
+  public void packetStarted(long pesTimeUs, @TsPayloadReader.Flags int flags) {
     timeUs = pesTimeUs;
   }
 

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/PesReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/PesReader.java
@@ -78,9 +78,8 @@ public final void seek() {
   }
 
   @Override
-  public final void consume(ParsableByteArray data, boolean payloadUnitStartIndicator)
-      throws ParserException {
-    if (payloadUnitStartIndicator) {
+  public final void consume(ParsableByteArray data, @Flags int flags) throws ParserException {
+    if ((flags & FLAG_PAYLOAD_UNIT_START_INDICATOR) != 0) {
       switch (state) {
         case STATE_FINDING_HEADER:
         case STATE_READING_HEADER:
@@ -122,7 +121,8 @@ public final void consume(ParsableByteArray data, boolean payloadUnitStartIndica
           if (continueRead(data, pesScratch.data, readLength)
               && continueRead(data, null, extendedHeaderLength)) {
             parseHeaderExtension();
-            reader.packetStarted(timeUs, dataAlignmentIndicator);
+            flags |= dataAlignmentIndicator ? FLAG_DATA_ALIGNMENT_INDICATOR : 0;
+            reader.packetStarted(timeUs, flags);
             setState(STATE_READING_BODY);
           }
           break;

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/PsExtractor.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/PsExtractor.java
@@ -343,7 +343,7 @@ public void consume(ParsableByteArray data) throws ParserException {
       data.readBytes(pesScratch.data, 0, extendedHeaderLength);
       pesScratch.setPosition(0);
       parseHeaderExtension();
-      pesPayloadReader.packetStarted(timeUs, true);
+      pesPayloadReader.packetStarted(timeUs, TsPayloadReader.FLAG_DATA_ALIGNMENT_INDICATOR);
       pesPayloadReader.consume(data);
       // We always have complete PES packets with program stream.
       pesPayloadReader.packetFinished();

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/SectionReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/SectionReader.java
@@ -57,7 +57,8 @@ public void seek() {
   }
 
   @Override
-  public void consume(ParsableByteArray data, boolean payloadUnitStartIndicator) {
+  public void consume(ParsableByteArray data, @Flags int flags) {
+    boolean payloadUnitStartIndicator = (flags & FLAG_PAYLOAD_UNIT_START_INDICATOR) != 0;
     int payloadStartPosition = C.POSITION_UNSET;
     if (payloadUnitStartIndicator) {
       int payloadStartOffset = data.readUnsignedByte();