Skip to content

Commit

Permalink
Prepend Ogg ID and Comment Header Pages to offloaded Opus stream
Browse files Browse the repository at this point in the history 
Add Ogg ID Header and Comment Header Pages to the Ogg encapsulated Opus for offload playback. This further matches the RFC 7845 spec and provides initialization data to decoders.

PiperOrigin-RevId: 548080222
(cherry picked from commit 847f6f2)
  • Loading branch information
@microkatz @tianyif
microkatz authored and tianyif committed Jul 14, 2023
1 parent 57d0c19 commit 0983500
Show file tree
Hide file tree
Showing 6 changed files with 315 additions and 51 deletions.
3 changes: 3 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
created after the playlist is cleared.
* Add fields streaming format (sf), stream type (st) and version (v) to
Common Media Client Data (CMCD) logging.
* Audio Offload:
* Prepend Ogg ID Header and Comment Header Pages to bitstream for
offloaded Opus playback in accordance with RFC 7845.
* Text:
* CEA-608: Change cue truncation logic to only consider visible text.
Previously indent and tab offset were included when limiting the cue
Expand Down
197 changes: 160 additions & 37 deletions ...aries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/OggOpusAudioPacketizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,37 @@
import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER;
import static androidx.media3.common.util.Assertions.checkNotNull;

import androidx.annotation.Nullable;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.extractor.OpusUtil;
import com.google.common.primitives.UnsignedBytes;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;

/** A packetizer that encapsulates OPUS audio encodings in OGG packets. */
/** A packetizer that encapsulates Opus audio encodings in Ogg packets. */
@UnstableApi
public final class OggOpusAudioPacketizer {

private static final int CHECKSUM_INDEX = 22;

/** ID Header and Comment Header pages are 0 and 1 respectively */
private static final int FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE = 2;
private static final int FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER = 2;

private static final int OGG_PACKET_HEADER_LENGTH = 28;
private static final int SERIAL_NUMBER = 0;
private static final byte[] OGG_DEFAULT_ID_HEADER_PAGE =
new byte[] {
79, 103, 103, 83, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, -43, -59, -9, 1,
19, 79, 112, 117, 115, 72, 101, 97, 100, 1, 2, 56, 1, -128, -69, 0, 0, 0, 0, 0
};
private static final byte[] OGG_DEFAULT_COMMENT_HEADER_PAGE =
new byte[] {
79, 103, 103, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, -103, 87, 83, 1,
16, 79, 112, 117, 115, 84, 97, 103, 115, 0, 0, 0, 0, 0, 0, 0, 0
};

private ByteBuffer outputBuffer;
private int pageSequenceNumber;
Expand All @@ -40,7 +58,7 @@ public final class OggOpusAudioPacketizer {
public OggOpusAudioPacketizer() {
outputBuffer = EMPTY_BUFFER;
granulePosition = 0;
pageSequenceNumber = FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE;
pageSequenceNumber = FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER;
}

/**
Expand All @@ -49,13 +67,23 @@ public OggOpusAudioPacketizer() {
* @param inputBuffer The input buffer to packetize. It must be a direct {@link ByteBuffer} with
* LITTLE_ENDIAN order. The contents will be overwritten with the Ogg packet. The caller
* retains ownership of the provided buffer.
* @param initializationData contains set-up data for the Opus Decoder. The data will be provided
* in an Ogg ID Header Page prepended to the bitstream. The list should contain either one or
* three byte arrays. The first item is the payload for the Ogg ID Header Page. If three
* items, then it also contains the Opus pre-skip and seek pre-roll values in that order.
*/
public void packetize(DecoderInputBuffer inputBuffer) {
public void packetize(DecoderInputBuffer inputBuffer, List<byte[]> initializationData) {
checkNotNull(inputBuffer.data);
if (inputBuffer.data.limit() - inputBuffer.data.position() == 0) {
return;
}
outputBuffer = packetizeInternal(inputBuffer.data);
@Nullable
byte[] providedOggIdHeaderPayloadBytes =
pageSequenceNumber == FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER
&& (initializationData.size() == 1 || initializationData.size() == 3)
? initializationData.get(0)
: null;
outputBuffer = packetizeInternal(inputBuffer.data, providedOggIdHeaderPayloadBytes);
inputBuffer.clear();
inputBuffer.ensureSpaceForWrite(outputBuffer.remaining());
inputBuffer.data.put(outputBuffer);
Expand All @@ -66,16 +94,24 @@ public void packetize(DecoderInputBuffer inputBuffer) {
public void reset() {
outputBuffer = EMPTY_BUFFER;
granulePosition = 0;
pageSequenceNumber = FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE;
pageSequenceNumber = FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER;
}

/**
* Fill outputBuffer with an Ogg packet encapsulating the inputBuffer.
*
* @param inputBuffer contains Opus to wrap in Ogg packet
* <p>If {@code providedOggIdHeaderPayloadBytes} is {@code null} and {@link #pageSequenceNumber}
* is {@link #FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER}, then {@link #OGG_DEFAULT_ID_HEADER_PAGE}
* will be prepended to the Ogg Opus Audio packets for the Ogg ID Header Page.
*
* @param inputBuffer contains Opus to wrap in Ogg packet.
* @param providedOggIdHeaderPayloadBytes containing the Ogg ID Header Page payload. Expected to
* be {@code null} if {@link #pageSequenceNumber} is not {@link
* #FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER}.
* @return {@link ByteBuffer} containing Ogg packet
*/
private ByteBuffer packetizeInternal(ByteBuffer inputBuffer) {
private ByteBuffer packetizeInternal(
ByteBuffer inputBuffer, @Nullable byte[] providedOggIdHeaderPayloadBytes) {
int position = inputBuffer.position();
int limit = inputBuffer.limit();
int inputBufferSize = limit - position;
Expand All @@ -86,38 +122,37 @@ private ByteBuffer packetizeInternal(ByteBuffer inputBuffer) {

int outputPacketSize = headerSize + inputBufferSize;

// If first audio sample in stream, then the packetizer will add Ogg ID Header and Comment
// Header Pages. Include additional page lengths in buffer size calculation.
int oggIdHeaderPageSize = 0;
if (pageSequenceNumber == FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER) {
oggIdHeaderPageSize =
providedOggIdHeaderPayloadBytes != null
? OGG_PACKET_HEADER_LENGTH + providedOggIdHeaderPayloadBytes.length
: OGG_DEFAULT_ID_HEADER_PAGE.length;
outputPacketSize += oggIdHeaderPageSize + OGG_DEFAULT_COMMENT_HEADER_PAGE.length;
}

// Resample the little endian input and update the output buffers.
ByteBuffer buffer = replaceOutputBuffer(outputPacketSize);

// Capture Pattern for Page [OggS]
buffer.put((byte) 'O');
buffer.put((byte) 'g');
buffer.put((byte) 'g');
buffer.put((byte) 'S');

// StreamStructure Version
buffer.put((byte) 0);

// header_type_flag
buffer.put((byte) 0x00);
// If first audio sample in stream then insert Ogg ID Header and Comment Header Pages
if (pageSequenceNumber == FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER) {
if (providedOggIdHeaderPayloadBytes != null) {
writeOggIdHeaderPage(buffer, /* idHeaderPayloadBytes= */ providedOggIdHeaderPayloadBytes);
} else {
// Write default Ogg ID Header Payload
buffer.put(OGG_DEFAULT_ID_HEADER_PAGE);
}
buffer.put(OGG_DEFAULT_COMMENT_HEADER_PAGE);
}

// granule_position
int numSamples = OpusUtil.parsePacketAudioSampleCount(inputBuffer);
granulePosition += numSamples;
buffer.putLong(granulePosition);

// bitstream_serial_number
buffer.putInt(0);

// page_sequence_number
buffer.putInt(pageSequenceNumber);
pageSequenceNumber++;

// CRC_checksum
buffer.putInt(0);

// number_page_segments
buffer.put((byte) numSegments);
writeOggPacketHeader(
buffer, granulePosition, pageSequenceNumber, numSegments, /* isIdHeaderPacket= */ false);

// Segment_table
int bytesLeft = inputBufferSize;
Expand All @@ -131,23 +166,111 @@ private ByteBuffer packetizeInternal(ByteBuffer inputBuffer) {
}
}

// Write Opus audio data
for (int i = position; i < limit; i++) {
buffer.put(inputBuffer.get(i));
}

inputBuffer.position(inputBuffer.limit());
buffer.flip();

int checksum;
if (pageSequenceNumber == FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE_NUMBER) {
checksum =
Util.crc32(
buffer.array(),
/* start= */ buffer.arrayOffset()
+ oggIdHeaderPageSize
+ OGG_DEFAULT_COMMENT_HEADER_PAGE.length,
/* end= */ buffer.limit() - buffer.position(),
/* initialValue= */ 0);
buffer.putInt(
oggIdHeaderPageSize + OGG_DEFAULT_COMMENT_HEADER_PAGE.length + CHECKSUM_INDEX, checksum);
} else {
checksum =
Util.crc32(
buffer.array(),
/* start= */ buffer.arrayOffset(),
/* end= */ buffer.limit() - buffer.position(),
/* initialValue= */ 0);
buffer.putInt(CHECKSUM_INDEX, checksum);
}

// Increase pageSequenceNumber for next packet
pageSequenceNumber++;

return buffer;
}

/**
* Write Ogg ID Header Page packet to {@link ByteBuffer}.
*
* @param buffer to write into.
* @param idHeaderPayloadBytes containing the Ogg ID Header Page payload.
*/
private void writeOggIdHeaderPage(ByteBuffer buffer, byte[] idHeaderPayloadBytes) {
// TODO(b/290195621): Use starting position to calculate correct 'pre-skip' value
writeOggPacketHeader(
buffer,
/* granulePosition= */ 0,
/* pageSequenceNumber= */ 0,
/* numberPageSegments= */ 1,
/* isIdHeaderPacket= */ true);
buffer.put(UnsignedBytes.checkedCast(idHeaderPayloadBytes.length));
buffer.put(idHeaderPayloadBytes);
int checksum =
Util.crc32(
buffer.array(),
buffer.arrayOffset(),
buffer.limit() - buffer.position(),
/* start= */ buffer.arrayOffset(),
/* end= */ OGG_PACKET_HEADER_LENGTH + idHeaderPayloadBytes.length,
/* initialValue= */ 0);
buffer.putInt(22, checksum);
buffer.position(0);
buffer.putInt(/* index= */ CHECKSUM_INDEX, checksum);
buffer.position(OGG_PACKET_HEADER_LENGTH + idHeaderPayloadBytes.length);
}

return buffer;
/**
* Write header for an Ogg Page Packet to {@link ByteBuffer}.
*
* @param byteBuffer to write unto.
* @param granulePosition is the number of audio samples in the stream up to and including this
* packet.
* @param pageSequenceNumber of the page this header is for.
* @param numberPageSegments the data of this Ogg page will span.
* @param isIdHeaderPacket where if this header is start of the bitstream.
*/
private void writeOggPacketHeader(
ByteBuffer byteBuffer,
long granulePosition,
int pageSequenceNumber,
int numberPageSegments,
boolean isIdHeaderPacket) {
// Capture Pattern for Ogg Page [OggS]
byteBuffer.put((byte) 'O');
byteBuffer.put((byte) 'g');
byteBuffer.put((byte) 'g');
byteBuffer.put((byte) 'S');

// StreamStructure Version
byteBuffer.put((byte) 0);

// Header-type
byteBuffer.put(isIdHeaderPacket ? (byte) 0x02 : (byte) 0x00);

// Granule_position
byteBuffer.putLong(granulePosition);

// bitstream_serial_number
byteBuffer.putInt(SERIAL_NUMBER);

// Page_sequence_number
byteBuffer.putInt(pageSequenceNumber);

// CRC_checksum
// Will be overwritten with calculated checksum after rest of page is written to buffer.
byteBuffer.putInt(0);

// Number_page_segments
byteBuffer.put(UnsignedBytes.checkedCast(numberPageSegments));
}

/**
Expand Down
3 changes: 2 additions & 1 deletion ...ries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,7 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb
bypassBatchBuffer.clear();
bypassSampleBuffer.clear();
bypassSampleBufferPending = false;
oggOpusAudioPacketizer.reset();
} else {
flushOrReinitializeCodec();
}
Expand Down Expand Up @@ -2332,7 +2333,7 @@ private void bypassRead() throws ExoPlaybackException {
if (inputFormat != null
&& inputFormat.sampleMimeType != null
&& inputFormat.sampleMimeType.equals(MimeTypes.AUDIO_OPUS)) {
oggOpusAudioPacketizer.packetize(bypassSampleBuffer);
oggOpusAudioPacketizer.packetize(bypassSampleBuffer, inputFormat.initializationData);
}

if (!bypassBatchBuffer.append(bypassSampleBuffer)) {
Expand Down
36 changes: 33 additions & 3 deletions libraries/extractor/src/main/java/androidx/media3/extractor/OpusUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,45 @@ public static List<byte[]> buildInitializationData(byte[] header) {
*/
public static int parseOggPacketAudioSampleCount(ByteBuffer buffer) {
// RFC 3433 section 6 - The Ogg page format.
int numPageSegments = buffer.get(/* index= */ 26);
int indexFirstOpusPacket = 27 + numPageSegments; // Skip Ogg header and segment table.
int preAudioPacketByteCount = parseOggPacketForPreAudioSampleByteCount(buffer);
int numPageSegments = buffer.get(/* index= */ 26 + preAudioPacketByteCount);
// Skip Ogg header + segment table.
int indexFirstOpusPacket = 27 + numPageSegments + preAudioPacketByteCount;
long packetDurationUs =
getPacketDurationUs(
buffer.get(indexFirstOpusPacket),
buffer.limit() > 1 ? buffer.get(indexFirstOpusPacket + 1) : 0);
buffer.limit() - indexFirstOpusPacket > 1 ? buffer.get(indexFirstOpusPacket + 1) : 0);
return (int) (packetDurationUs * SAMPLE_RATE / C.MICROS_PER_SECOND);
}

/**
* Calculate the offset from the start of the buffer to audio sample Ogg packets.
*
* @param buffer containing the Ogg Encapsulated Opus audio bitstream.
* @return the offset before the Ogg packet containing audio samples.
*/
public static int parseOggPacketForPreAudioSampleByteCount(ByteBuffer buffer) {
// Parse Ogg Packet Type from Header at index 5
if ((buffer.get(/* index= */ 5) & 0x02) == 0) {
// Ogg Page packet header type is not beginning of logical stream. Must be an Audio page.
return 0;
}
// ID Header Page size is Ogg packet header size + sum(lacing values: 1..number_page_segments).
int idHeaderPageSize = 28;
int idHeaderPageNumOfSegments = buffer.get(/* index= */ 26);
for (int i = 0; i < idHeaderPageNumOfSegments; i++) {
idHeaderPageSize += buffer.get(/* index= */ 27 + i);
}
// Comment Header Page size is Ogg packet header size + sum(lacing values:
// 1..number_page_segments).
int commentHeaderPageSize = 28;
int commentHeaderPageSizeNumOfSegments = buffer.get(/* index= */ idHeaderPageSize + 26);
for (int i = 0; i < commentHeaderPageSizeNumOfSegments; i++) {
commentHeaderPageSize += buffer.get(/* index= */ idHeaderPageSize + 27 + i);
}
return idHeaderPageSize + commentHeaderPageSize;
}

/**
* Returns the number of audio samples in the given audio packet.
*
Expand Down
Loading

0 comments on commit 0983500

Please sign in to comment.