Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add IAMF support (#1415) #1416

Merged
merged 3 commits into from
Oct 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CONTRIBUTORS
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ Daniel Cantarín <canta@canta.com.ar>
David Cavar <pal3thorn@gmail.com>
Dennis E. Mungai (Brainiarc7) <dmngaie@gmail.com>
Evgeny Zajcev <zevlg@yandex.ru>
Felicia Lim <flim@google.com>
Gabe Kopley <gabe@philo.com>
Geoff Jukes <geoff@jukes.org>
Haoming Chen <hmchen@google.com>
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ Shaka Packager supports:
| FLAC | I / O | - | - | - | - |
| Opus | I / O³ | I / O | - | - | - |
| Vorbis | - | I / O | - | - | - |
| IAMF | I / O | - | - | - | - |

NOTES:
- I for input and O for output.
Expand Down
32 changes: 32 additions & 0 deletions packager/app/test/packager_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1104,6 +1104,38 @@ def testAv1WebM(self):
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('av1-webm')

def testIamfWithBaseProfileAndPcm(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-base-pcm.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-base-pcm-mp4')

def testIamfWithBaseProfileAndOpus(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-base-opus.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-base-opus-mp4')

def testIamfWithSimpleProfileAndAacLc(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-simple-aac-lc.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-simple-aac-lc-mp4')

def testIamfWithSimpleProfileAndFlac(self):
self.assertPackageSuccess(
self._GetStreams(['audio'],
output_format='mp4',
test_files=['bear-iamf-simple-flac.mp4']),
self._GetFlags(output_dash=True, output_hls=True))
self._CheckTestResults('iamf-simple-flac-mp4')

def testEncryption(self):
self.assertPackageSuccess(
self._GetStreams(['audio', 'video']),
Expand Down
Binary file not shown.
9 changes: 9 additions & 0 deletions packager/app/test/testdata/iamf-base-opus-mp4/output.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>

#EXT-X-INDEPENDENT-SEGMENTS

#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"

#EXT-X-STREAM-INF:BANDWIDTH=136796,AVERAGE-BANDWIDTH=134443,CODECS="iamf.001.001.Opus",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8
15 changes: 15 additions & 0 deletions packager/app/test/testdata/iamf-base-opus-mp4/output.mpd
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="136796" codecs="iamf.001.001.Opus" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-base-opus-audio.mp4</BaseURL>
<SegmentBase indexRange="892-959" timescale="48000">
<Initialization range="0-891"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>
16 changes: 16 additions & 0 deletions packager/app/test/testdata/iamf-base-opus-mp4/stream_0.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:2
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-base-opus-audio.mp4",BYTERANGE="892@0"
#EXTINF:1.014,
#EXT-X-BYTERANGE:16835@960
bear-iamf-base-opus-audio.mp4
#EXTINF:1.000,
#EXT-X-BYTERANGE:16789
bear-iamf-base-opus-audio.mp4
#EXTINF:0.726,
#EXT-X-BYTERANGE:12422
bear-iamf-base-opus-audio.mp4
#EXT-X-ENDLIST
Binary file not shown.
9 changes: 9 additions & 0 deletions packager/app/test/testdata/iamf-base-pcm-mp4/output.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>

#EXT-X-INDEPENDENT-SEGMENTS

#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"

#EXT-X-STREAM-INF:BANDWIDTH=2177226,AVERAGE-BANDWIDTH=1803234,CODECS="iamf.001.001.ipcm",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8
15 changes: 15 additions & 0 deletions packager/app/test/testdata/iamf-base-pcm-mp4/output.mpd
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="2177226" codecs="iamf.001.001.ipcm" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-base-pcm-audio.mp4</BaseURL>
<SegmentBase indexRange="850-917" timescale="48000">
<Initialization range="0-849"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>
16 changes: 16 additions & 0 deletions packager/app/test/testdata/iamf-base-pcm-mp4/stream_0.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:1
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-base-pcm-audio.mp4",BYTERANGE="850@0"
#EXTINF:1.000,
#EXT-X-BYTERANGE:208108@918
bear-iamf-base-pcm-audio.mp4
#EXTINF:1.000,
#EXT-X-BYTERANGE:208108
bear-iamf-base-pcm-audio.mp4
#EXTINF:0.740,
#EXT-X-BYTERANGE:201382
bear-iamf-base-pcm-audio.mp4
#EXT-X-ENDLIST
Binary file not shown.
9 changes: 9 additions & 0 deletions packager/app/test/testdata/iamf-simple-aac-lc-mp4/output.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>

#EXT-X-INDEPENDENT-SEGMENTS

#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"

#EXT-X-STREAM-INF:BANDWIDTH=604125,AVERAGE-BANDWIDTH=590976,CODECS="iamf.000.000.mp4a.40.2",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8
15 changes: 15 additions & 0 deletions packager/app/test/testdata/iamf-simple-aac-lc-mp4/output.mpd
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="604125" codecs="iamf.000.000.mp4a.40.2" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-simple-aac-lc-audio.mp4</BaseURL>
<SegmentBase indexRange="900-967" timescale="48000">
<Initialization range="0-899"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>
16 changes: 16 additions & 0 deletions packager/app/test/testdata/iamf-simple-aac-lc-mp4/stream_0.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:2
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-simple-aac-lc-audio.mp4",BYTERANGE="900@0"
#EXTINF:1.003,
#EXT-X-BYTERANGE:75717@968
bear-iamf-simple-aac-lc-audio.mp4
#EXTINF:1.003,
#EXT-X-BYTERANGE:72441
bear-iamf-simple-aac-lc-audio.mp4
#EXTINF:0.735,
#EXT-X-BYTERANGE:54248
bear-iamf-simple-aac-lc-audio.mp4
#EXT-X-ENDLIST
Binary file not shown.
9 changes: 9 additions & 0 deletions packager/app/test/testdata/iamf-simple-flac-mp4/output.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#EXTM3U
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>

#EXT-X-INDEPENDENT-SEGMENTS

#EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="0"

#EXT-X-STREAM-INF:BANDWIDTH=977240,AVERAGE-BANDWIDTH=926327,CODECS="iamf.000.000.fLaC",AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
stream_0.m3u8
15 changes: 15 additions & 0 deletions packager/app/test/testdata/iamf-simple-flac-mp4/output.mpd
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.739958S">
<Period id="0">
<AdaptationSet id="0" contentType="audio" subsegmentAlignment="true">
<Representation id="0" bandwidth="977240" codecs="iamf.000.000.fLaC" mimeType="audio/mp4" audioSamplingRate="0">
<AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="0"/>
<BaseURL>bear-iamf-simple-flac-audio.mp4</BaseURL>
<SegmentBase indexRange="883-950" timescale="48000">
<Initialization range="0-882"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>
16 changes: 16 additions & 0 deletions packager/app/test/testdata/iamf-simple-flac-mp4/stream_0.m3u8
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#EXTM3U
#EXT-X-VERSION:6
## Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>
#EXT-X-TARGETDURATION:1
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-MAP:URI="bear-iamf-simple-flac-audio.mp4",BYTERANGE="883@0"
#EXTINF:1.000,
#EXT-X-BYTERANGE:116763@951
bear-iamf-simple-flac-audio.mp4
#EXTINF:1.000,
#EXT-X-BYTERANGE:122155
bear-iamf-simple-flac-audio.mp4
#EXTINF:0.740,
#EXT-X-BYTERANGE:78344
bear-iamf-simple-flac-audio.mp4
#EXT-X-ENDLIST
1 change: 1 addition & 0 deletions packager/media/base/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ target_link_libraries(media_handler_test_base
add_executable(media_base_unittest
aes_cryptor_unittest.cc
aes_pattern_cryptor_unittest.cc
audio_stream_info_unittest.cc
audio_timestamp_helper_unittest.cc
bit_reader_unittest.cc
bit_writer_unittest.cc
Expand Down
35 changes: 35 additions & 0 deletions packager/media/base/audio_stream_info.cc
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ std::string AudioCodecToString(Codec codec) {
return "AC4";
case kCodecFlac:
return "FLAC";
case kCodecIAMF:
return "IAMF";
case kCodecOpus:
return "Opus";
case kCodecVorbis:
Expand Down Expand Up @@ -168,6 +170,39 @@ std::string AudioStreamInfo::GetCodecString(Codec codec,
(audio_object_type & 0x18) >> 3, audio_object_type & 0x7);
case kCodecFlac:
return "flac";
case kCodecIAMF: {
// https://aomediacodec.github.io/iamf/#codecsparameter
// The codecs parameter string is composed as
//
// iamf.xxx.yyy.<standalone_codec_string>
//
// - xxx is the IAMF primary profile
// - yyy is the IAMF additional profile
// - <standalone_codec_string> are the elements of the codecs parameter
// string if that stream was carried in its own track
//
// audio_object_type is composed of primary_profile (2 bits),
// additional_profile (2 bits) and (IAMF codec - kCodecAudio) (4 bits).
const int iamf_codec = (audio_object_type & 0xF) + kCodecAudio;

const std::string iamf_codec_string =
absl::StrFormat("iamf.%03d.%03d", (audio_object_type & 0xC0) >> 6,
(audio_object_type & 0x30) >> 4);

switch (iamf_codec) {
case kCodecOpus:
return absl::StrFormat("%s.%s", iamf_codec_string, "Opus");
case kCodecAAC:
return absl::StrFormat("%s.%s", iamf_codec_string, "mp4a.40.2");
case kCodecFlac:
return absl::StrFormat("%s.%s", iamf_codec_string, "fLaC");
case kCodecPcm:
return absl::StrFormat("%s.%s", iamf_codec_string, "ipcm");
default:
LOG(WARNING) << "Unknown IAMF codec: " << iamf_codec;
return "unknown";
}
}
case kCodecOpus:
return "opus";
case kCodecMP3:
Expand Down
73 changes: 73 additions & 0 deletions packager/media/base/audio_stream_info_unittest.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright 2024 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd

#include <packager/media/base/audio_stream_info.h>

#include <gtest/gtest.h>

namespace shaka {
namespace media {

const int kSimpleProfile = 0;
const int kBaseProfile = 1;

TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndPcm) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecPcm - kCodecAudio) & 0xF)); // IAMF codec

std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.ipcm", codec_string);
}

TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndOpus) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecOpus - kCodecAudio) & 0xF)); // IAMF codec

std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.Opus", codec_string);
}

TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndMp4a) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecAAC - kCodecAudio) & 0xF)); // IAMF codec

std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.mp4a.40.2", codec_string);
}

TEST(AudioStreamInfo, IamfGetCodecStringForSimpleProfilesAndFlac) {
const uint8_t audio_object_type =
((kSimpleProfile << 6) | // primary_profile
((kSimpleProfile << 4) & 0x3F) | // additional_profile
((kCodecFlac - kCodecAudio) & 0xF)); // IAMF codec

std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.000.000.fLaC", codec_string);
}

TEST(AudioStreamInfo, IamfGetCodecStringForBaseProfilesAndPcm) {
const uint8_t audio_object_type =
((kBaseProfile << 6) | // primary_profile = 1
((kBaseProfile << 4) & 0x3F) | // additional_profile = 1
((kCodecPcm - kCodecAudio) & 0xF)); // IAMF codec

std::string codec_string =
AudioStreamInfo::GetCodecString(kCodecIAMF, audio_object_type);
EXPECT_EQ("iamf.001.001.ipcm", codec_string);
}

} // namespace media
} // namespace shaka
3 changes: 3 additions & 0 deletions packager/media/base/fourccs.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,10 @@ enum FourCC : uint32_t {
FOURCC_hvc1 = 0x68766331,
FOURCC_hvcC = 0x68766343,
FOURCC_hvcE = 0x68766345,
FOURCC_iamf = 0x69616d66,
FOURCC_iacb = 0x69616362,
FOURCC_iden = 0x6964656e,
FOURCC_ipcm = 0x6970636d,
FOURCC_iso6 = 0x69736f36,
FOURCC_iso8 = 0x69736f38,
FOURCC_isom = 0x69736f6d,
Expand Down
2 changes: 2 additions & 0 deletions packager/media/base/stream_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ enum Codec {
kCodecDTSX,
kCodecEAC3,
kCodecFlac,
kCodecIAMF,
kCodecOpus,
kCodecPcm,
kCodecVorbis,
kCodecMP3,
kCodecMha1,
Expand Down
2 changes: 2 additions & 0 deletions packager/media/codecs/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ add_library(media_codecs STATIC
h26x_byte_to_unit_stream_converter.cc
hevc_decoder_configuration_record.cc
hls_audio_util.cc
iamf_audio_util.cc
nal_unit_to_byte_stream_converter.cc
nalu_reader.cc
video_slice_header_parser.cc
Expand Down Expand Up @@ -52,6 +53,7 @@ add_executable(media_codecs_unittest
h26x_bit_reader_unittest.cc
hevc_decoder_configuration_record_unittest.cc
hls_audio_util_unittest.cc
iamf_audio_util_unittest.cc
nal_unit_to_byte_stream_converter_unittest.cc
nalu_reader_unittest.cc
video_slice_header_parser_unittest.cc
Expand Down
Loading