From 2ab121abe7914d3a8f07a78d0e3807bd20fd8e21 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Mon, 14 May 2018 16:14:46 -0300 Subject: [PATCH 01/32] Replace libav with ffmpeg 3.4.2 --- erizo/src/erizo/media/ExternalInput.cpp | 4 +- erizo/src/erizo/media/ExternalOutput.cpp | 87 +++++++++++++-------- erizo/src/erizo/media/ExternalOutput.h | 8 +- erizo/src/erizo/media/codecs/AudioCodec.h | 1 + erizo/src/erizo/media/codecs/VideoCodec.cpp | 4 +- scripts/installMacDeps.sh | 16 ++-- scripts/installUbuntuDeps.sh | 36 ++++----- scripts/travisInstallDeps.sh | 34 ++++---- 8 files changed, 101 insertions(+), 89 deletions(-) diff --git a/erizo/src/erizo/media/ExternalInput.cpp b/erizo/src/erizo/media/ExternalInput.cpp index 5933a14314..994703a15e 100644 --- a/erizo/src/erizo/media/ExternalInput.cpp +++ b/erizo/src/erizo/media/ExternalInput.cpp @@ -27,7 +27,7 @@ ExternalInput::~ExternalInput() { thread_.join(); if (needTranscoding_) encodeThread_.join(); - av_free_packet(&avpacket_); + av_packet_unref(&avpacket_); if (context_ != NULL) avformat_free_context(context_); ELOG_DEBUG("ExternalInput closed"); @@ -221,7 +221,7 @@ void ExternalInput::receiveLoop() { } } } - av_free_packet(&orig_pkt); + av_packet_unref(&orig_pkt); } ELOG_DEBUG("Ended stream to play %s", url_.c_str()); running_ = false; diff --git a/erizo/src/erizo/media/ExternalOutput.cpp b/erizo/src/erizo/media/ExternalOutput.cpp index 3c4d2da7b2..121fa65f98 100644 --- a/erizo/src/erizo/media/ExternalOutput.cpp +++ b/erizo/src/erizo/media/ExternalOutput.cpp @@ -27,8 +27,8 @@ ExternalOutput::ExternalOutput(std::shared_ptr worker, const std::string audio_stream_{nullptr}, video_source_ssrc_{0}, first_video_timestamp_{-1}, first_audio_timestamp_{-1}, first_data_received_{}, video_offset_ms_{-1}, audio_offset_ms_{-1}, - need_to_send_fir_{true}, rtp_mappings_{rtp_mappings}, video_codec_{AV_CODEC_ID_NONE}, - audio_codec_{AV_CODEC_ID_NONE}, pipeline_initialized_{false}, ext_processor_{ext_mappings} { + need_to_send_fir_{true}, rtp_mappings_{rtp_mappings}, video_codec_id_{AV_CODEC_ID_NONE}, + audio_codec_id_{AV_CODEC_ID_NONE}, pipeline_initialized_{false}, ext_processor_{ext_mappings} { ELOG_DEBUG("Creating output to %s", output_url.c_str()); fb_sink_ = nullptr; @@ -114,12 +114,14 @@ void ExternalOutput::syncClose() { av_write_trailer(context_); } - if (video_stream_ && video_stream_->codec != nullptr) { - avcodec_close(video_stream_->codec); + if (video_ctx_ != nullptr) { + avcodec_close(video_ctx_); + avcodec_free_context(&video_ctx_); } - if (audio_stream_ && audio_stream_->codec != nullptr) { - avcodec_close(audio_stream_->codec); + if (audio_ctx_ != nullptr) { + avcodec_close(audio_ctx_); + avcodec_free_context(&audio_ctx_); } if (context_ != nullptr) { @@ -192,7 +194,7 @@ void ExternalOutput::writeAudioData(char* buf, int len) { } long long timestamp_to_write = (current_timestamp - first_audio_timestamp_) / // NOLINT - (audio_stream_->codec->sample_rate / audio_stream_->time_base.den); + (audio_ctx_->sample_rate / audio_stream_->time_base.den); // generally 48000 / 1000 for the denominator portion, at least for opus // Adjust for our start time offset @@ -237,28 +239,28 @@ void ExternalOutput::writeVideoData(char* buf, int len) { } void ExternalOutput::updateAudioCodec(RtpMap map) { - if (audio_codec_ != AV_CODEC_ID_NONE) { + if (audio_codec_id_ != AV_CODEC_ID_NONE) { return; } audio_map_ = map; if (map.encoding_name == "opus") { - audio_codec_ = AV_CODEC_ID_OPUS; + audio_codec_id_ = AV_CODEC_ID_OPUS; } else if (map.encoding_name == "PCMU") { - audio_codec_ = AV_CODEC_ID_PCM_MULAW; + audio_codec_id_ = AV_CODEC_ID_PCM_MULAW; } } void ExternalOutput::updateVideoCodec(RtpMap map) { - if (video_codec_ != AV_CODEC_ID_NONE) { + if (video_codec_id_ != AV_CODEC_ID_NONE) { return; } video_map_ = map; if (map.encoding_name == "VP8") { depacketizer_.reset(new Vp8Depacketizer()); - video_codec_ = AV_CODEC_ID_VP8; + video_codec_id_ = AV_CODEC_ID_VP8; } else if (map.encoding_name == "H264") { depacketizer_.reset(new H264Depacketizer()); - video_codec_ = AV_CODEC_ID_H264; + video_codec_id_ = AV_CODEC_ID_H264; } } @@ -369,48 +371,69 @@ int ExternalOutput::deliverEvent_(MediaEventPtr event) { } bool ExternalOutput::initContext() { - if (video_codec_ != AV_CODEC_ID_NONE && - audio_codec_ != AV_CODEC_ID_NONE && + if (video_codec_id_ != AV_CODEC_ID_NONE && + audio_codec_id_ != AV_CODEC_ID_NONE && video_stream_ == nullptr && audio_stream_ == nullptr) { - AVCodec* video_codec = avcodec_find_encoder(video_codec_); + video_codec = avcodec_find_encoder(video_codec_id_); + int ret = -1; if (video_codec == nullptr) { ELOG_ERROR("Could not find video codec"); return false; } need_to_send_fir_ = true; video_queue_.setTimebase(video_map_.clock_rate); + video_ctx_ = avcodec_alloc_context3(video_codec); + if (!video_ctx_) { + ELOG_ERROR("Could not alloc video codec context"); + return false; + } + video_ctx_->pix_fmt = AV_PIX_FMT_YUV420P; + video_ctx_->width = 640; + video_ctx_->height = 480; + video_ctx_->time_base = (AVRational) { 1, 30 }; + video_stream_ = avformat_new_stream(context_, video_codec); + ret = avcodec_parameters_from_context(video_stream_->codecpar, video_ctx_); + if (ret != 0) { + ELOG_ERROR("Could not copy video codec paramaters"); + return false; + } video_stream_->id = 0; - video_stream_->codec->codec_id = video_codec_; - video_stream_->codec->width = 640; - video_stream_->codec->height = 480; video_stream_->time_base = (AVRational) { 1, 30 }; video_stream_->metadata = genVideoMetadata(); - // A decent guess here suffices; if processing the file with ffmpeg, - // use -vsync 0 to force it not to duplicate frames. - video_stream_->codec->pix_fmt = PIX_FMT_YUV420P; if (context_->oformat->flags & AVFMT_GLOBALHEADER) { - video_stream_->codec->flags |= CODEC_FLAG_GLOBAL_HEADER; + video_ctx_->flags |= CODEC_FLAG_GLOBAL_HEADER; } context_->oformat->flags |= AVFMT_VARIABLE_FPS; - AVCodec* audio_codec = avcodec_find_encoder(audio_codec_); + audio_codec = avcodec_find_encoder(audio_codec_id_); if (audio_codec == nullptr) { ELOG_ERROR("Could not find audio codec"); return false; } - + audio_ctx_ = avcodec_alloc_context3(audio_codec); + if (!audio_ctx_) { + ELOG_ERROR("Could not alloc audio codec context"); + return false; + } + audio_ctx_->sample_rate = audio_map_.clock_rate; + audio_ctx_->time_base = (AVRational) { 1, audio_ctx_->sample_rate }; + audio_ctx_->channels = audio_map_.channels; audio_stream_ = avformat_new_stream(context_, audio_codec); + ret = avcodec_parameters_from_context(audio_stream_->codecpar, audio_ctx_); + if (ret != 0) { + ELOG_ERROR("Could not copy audio codec paramaters"); + return false; + } audio_stream_->id = 1; - audio_stream_->codec->codec_id = audio_codec_; - audio_stream_->codec->sample_rate = audio_map_.clock_rate; - audio_stream_->time_base = (AVRational) { 1, audio_stream_->codec->sample_rate }; - audio_stream_->codec->channels = audio_map_.channels; + audio_stream_->time_base = (AVRational) { 1, audio_ctx_->sample_rate }; + if (context_->oformat->flags & AVFMT_GLOBALHEADER) { - audio_stream_->codec->flags |= CODEC_FLAG_GLOBAL_HEADER; + video_ctx_->flags |= CODEC_FLAG_GLOBAL_HEADER; + audio_ctx_->flags |= CODEC_FLAG_GLOBAL_HEADER; } - + context_->oformat->flags |= AVFMT_VARIABLE_FPS; context_->streams[0] = video_stream_; context_->streams[1] = audio_stream_; if (avio_open(&context_->pb, context_->filename, AVIO_FLAG_WRITE) < 0) { @@ -442,7 +465,7 @@ void ExternalOutput::queueData(char* buffer, int length, packetType type) { if (getAudioSinkSSRC() == 0) { ELOG_DEBUG("No audio detected"); audio_map_ = RtpMap{0, "PCMU", 8000, AUDIO_TYPE, 1}; - audio_codec_ = AV_CODEC_ID_PCM_MULAW; + audio_codec_id_ = AV_CODEC_ID_PCM_MULAW; } } if (need_to_send_fir_ && video_source_ssrc_) { diff --git a/erizo/src/erizo/media/ExternalOutput.h b/erizo/src/erizo/media/ExternalOutput.h index 7e0c106e38..ba107c3782 100644 --- a/erizo/src/erizo/media/ExternalOutput.h +++ b/erizo/src/erizo/media/ExternalOutput.h @@ -107,8 +107,12 @@ class ExternalOutput : public MediaSink, public RawDataReceiver, public Feedback // so the second scheme seems not applicable. Too bad. bool need_to_send_fir_; std::vector rtp_mappings_; - enum AVCodecID video_codec_; - enum AVCodecID audio_codec_; + enum AVCodecID video_codec_id_; + enum AVCodecID audio_codec_id_; + AVCodec *video_codec; + AVCodec *audio_codec; + AVCodecContext *video_ctx_; + AVCodecContext *audio_ctx_; std::map video_maps_; std::map audio_maps_; RtpMap video_map_; diff --git a/erizo/src/erizo/media/codecs/AudioCodec.h b/erizo/src/erizo/media/codecs/AudioCodec.h index 2e75320d35..855776a502 100644 --- a/erizo/src/erizo/media/codecs/AudioCodec.h +++ b/erizo/src/erizo/media/codecs/AudioCodec.h @@ -4,6 +4,7 @@ #ifndef ERIZO_SRC_ERIZO_MEDIA_CODECS_AUDIOCODEC_H_ #define ERIZO_SRC_ERIZO_MEDIA_CODECS_AUDIOCODEC_H_ +#define __STDC_CONSTANT_MACROS extern "C" { #include #include diff --git a/erizo/src/erizo/media/codecs/VideoCodec.cpp b/erizo/src/erizo/media/codecs/VideoCodec.cpp index ea1b5c9618..96680c7870 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.cpp +++ b/erizo/src/erizo/media/codecs/VideoCodec.cpp @@ -63,7 +63,7 @@ int VideoEncoder::initEncoder(const VideoCodecInfo& info) { vCoderContext->width = info.width; vCoderContext->height = info.height; - vCoderContext->pix_fmt = PIX_FMT_YUV420P; + vCoderContext->pix_fmt = AV_PIX_FMT_YUV420P; vCoderContext->time_base = (AVRational) {1, 90000}; vCoderContext->sample_aspect_ratio = (AVRational) { info.width, info.height }; @@ -277,7 +277,7 @@ int VideoDecoder::decodeVideo(unsigned char* inBuff, int inBuffLen, cromV += dst_linesize; src += src_linesize; } - av_free_packet(&avpkt); + av_packet_unref(&avpkt); return outSize * 3 / 2; } diff --git a/scripts/installMacDeps.sh b/scripts/installMacDeps.sh index 27661f06f9..09afe5b375 100755 --- a/scripts/installMacDeps.sh +++ b/scripts/installMacDeps.sh @@ -173,11 +173,9 @@ install_mediadeps(){ brew install opus libvpx x264 if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://github.com/libav/libav/archive/v11.6.tar.gz - tar -zxvf v11.6.tar.gz - cd libav-11.6 - curl -OL https://github.com/libav/libav/commit/4d05e9392f84702e3c833efa86e84c7f1cf5f612.patch - patch libavcodec/libvpxenc.c 4d05e9392f84702e3c833efa86e84c7f1cf5f612.patch && \ + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 + tar -zxvf ffmpeg-3.4.2.tar.bz2 + cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc && \ make $FAST_MAKE -s V=0 && \ make install @@ -193,11 +191,9 @@ install_mediadeps_nogpl(){ brew install opus libvpx if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://github.com/libav/libav/archive/v11.6.tar.gz - tar -zxvf v11.6.tar.gz - cd libav-11.6 - curl -OL https://github.com/libav/libav/commit/4d05e9392f84702e3c833efa86e84c7f1cf5f612.patch - patch libavcodec/libvpxenc.c 4d05e9392f84702e3c833efa86e84c7f1cf5f612.patch && \ + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 + tar -zxvf ffmpeg-3.4.2.tar.bz2 + cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-libvpx --enable-libopus --disable-doc && \ make $FAST_MAKE -s V=0 && \ make install diff --git a/scripts/installUbuntuDeps.sh b/scripts/installUbuntuDeps.sh index 2d27e22933..242714539f 100755 --- a/scripts/installUbuntuDeps.sh +++ b/scripts/installUbuntuDeps.sh @@ -157,16 +157,13 @@ install_mediadeps(){ sudo apt-get -qq install yasm libvpx. libx264. if [ -d $LIB_DIR ]; then cd $LIB_DIR - if [ ! -f ./v11.1.tar.gz ]; then - curl -O -L https://github.com/libav/libav/archive/v11.1.tar.gz - tar -zxvf v11.1.tar.gz - cd libav-11.1 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc - make $FAST_MAKE -s V=0 - make install - else - echo "libav already installed" - fi + rm -r libav* + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 + tar -zxvf ffmpeg-3.4.2.tar.bz2 + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc + make $FAST_MAKE -s V=0 + make install cd $CURRENT_DIR else mkdir -p $LIB_DIR @@ -180,16 +177,13 @@ install_mediadeps_nogpl(){ sudo apt-get -qq install yasm libvpx. if [ -d $LIB_DIR ]; then cd $LIB_DIR - if [ ! -f ./v11.1.tar.gz ]; then - curl -O -L https://github.com/libav/libav/archive/v11.1.tar.gz - tar -zxvf v11.1.tar.gz - cd libav-11.1 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc - make $FAST_MAKE -s V=0 - make install - else - echo "libav already installed" - fi + rm -r libav* + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 + tar -zxvf ffmpeg-3.4.2.tar.bz2 + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc + make $FAST_MAKE -s V=0 + make install cd $CURRENT_DIR else mkdir -p $LIB_DIR @@ -217,7 +211,7 @@ cleanup(){ cd $LIB_DIR rm -r libnice* rm -r libsrtp* - rm -r libav* + rm -r fmpeg* rm -r v11* rm -r openssl* rm -r opus* diff --git a/scripts/travisInstallDeps.sh b/scripts/travisInstallDeps.sh index a7ebe9dcb0..868c8571e1 100755 --- a/scripts/travisInstallDeps.sh +++ b/scripts/travisInstallDeps.sh @@ -132,16 +132,13 @@ install_mediadeps(){ sudo apt-get -qq install yasm libvpx. libx264. if [ -d $LIB_DIR ]; then cd $LIB_DIR - if [ ! -f ./v11.1.tar.gz ]; then - curl -O -L https://github.com/libav/libav/archive/v11.1.tar.gz - tar -zxvf v11.1.tar.gz - cd libav-11.1 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc - make -s V=0 - make install - else - echo "libav already installed" - fi + rm -r libav* + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 + tar -zxvf ffmpeg-3.4.2.tar.bz2 + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc + make -s V=0 + make install cd $CURRENT_DIR else mkdir -p $LIB_DIR @@ -154,16 +151,13 @@ install_mediadeps_nogpl(){ sudo apt-get -qq install yasm libvpx. if [ -d $LIB_DIR ]; then cd $LIB_DIR - if [ ! -f ./v11.1.tar.gz ]; then - curl -O -L https://github.com/libav/libav/archive/v11.1.tar.gz - tar -zxvf v11.1.tar.gz - cd libav-11.1 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc - make -s V=0 - make install - else - echo "libav already installed" - fi + rm -r libav* + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 + tar -zxvf ffmpeg-3.4.2.tar.bz2 + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc + make -s V=0 + make install cd $CURRENT_DIR else mkdir -p $LIB_DIR From 8aafb1a021cc26f92dc18e12c4a23705b0242491 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Tue, 15 May 2018 12:49:39 -0300 Subject: [PATCH 02/32] Fix install dep scripts for ffmpeg --- scripts/installMacDeps.sh | 8 ++++---- scripts/installUbuntuDeps.sh | 14 ++++++-------- scripts/travisInstallDeps.sh | 10 ++++------ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/scripts/installMacDeps.sh b/scripts/installMacDeps.sh index 09afe5b375..52fbe47219 100755 --- a/scripts/installMacDeps.sh +++ b/scripts/installMacDeps.sh @@ -173,8 +173,8 @@ install_mediadeps(){ brew install opus libvpx x264 if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 - tar -zxvf ffmpeg-3.4.2.tar.bz2 + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc && \ make $FAST_MAKE -s V=0 && \ @@ -191,8 +191,8 @@ install_mediadeps_nogpl(){ brew install opus libvpx if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 - tar -zxvf ffmpeg-3.4.2.tar.bz2 + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-libvpx --enable-libopus --disable-doc && \ make $FAST_MAKE -s V=0 && \ diff --git a/scripts/installUbuntuDeps.sh b/scripts/installUbuntuDeps.sh index 242714539f..29c9f630eb 100755 --- a/scripts/installUbuntuDeps.sh +++ b/scripts/installUbuntuDeps.sh @@ -157,9 +157,8 @@ install_mediadeps(){ sudo apt-get -qq install yasm libvpx. libx264. if [ -d $LIB_DIR ]; then cd $LIB_DIR - rm -r libav* - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 - tar -zxvf ffmpeg-3.4.2.tar.bz2 + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc make $FAST_MAKE -s V=0 @@ -177,9 +176,8 @@ install_mediadeps_nogpl(){ sudo apt-get -qq install yasm libvpx. if [ -d $LIB_DIR ]; then cd $LIB_DIR - rm -r libav* - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 - tar -zxvf ffmpeg-3.4.2.tar.bz2 + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc make $FAST_MAKE -s V=0 @@ -211,8 +209,8 @@ cleanup(){ cd $LIB_DIR rm -r libnice* rm -r libsrtp* - rm -r fmpeg* - rm -r v11* + rm -r ffmpeg* + #rm -r v11* rm -r openssl* rm -r opus* cd $CURRENT_DIR diff --git a/scripts/travisInstallDeps.sh b/scripts/travisInstallDeps.sh index 868c8571e1..6c787965fc 100755 --- a/scripts/travisInstallDeps.sh +++ b/scripts/travisInstallDeps.sh @@ -132,9 +132,8 @@ install_mediadeps(){ sudo apt-get -qq install yasm libvpx. libx264. if [ -d $LIB_DIR ]; then cd $LIB_DIR - rm -r libav* - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 - tar -zxvf ffmpeg-3.4.2.tar.bz2 + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc make -s V=0 @@ -151,9 +150,8 @@ install_mediadeps_nogpl(){ sudo apt-get -qq install yasm libvpx. if [ -d $LIB_DIR ]; then cd $LIB_DIR - rm -r libav* - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.bz2 - tar -zxvf ffmpeg-3.4.2.tar.bz2 + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc make -s V=0 From 63c428fa65d742f86ee409e07e6d72425f42da01 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Wed, 16 May 2018 16:41:41 -0300 Subject: [PATCH 03/32] Update erizo cmakelists to enable -Werror Replicate Ubuntu behavior where Erizo is built with -Wall -Werror by default, not sure if it is inherited from another project or by differences between Ubuntu 14.04 CMake (2.8.x) and OSX Brew (3.x.x) --- erizo/src/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/erizo/src/CMakeLists.txt b/erizo/src/CMakeLists.txt index 51023b012a..b50af4092a 100644 --- a/erizo/src/CMakeLists.txt +++ b/erizo/src/CMakeLists.txt @@ -86,7 +86,7 @@ test_lib(${SSL}) find_library(CRYPTO crypto HINTS "${THIRD_PARTY_LIB}") test_lib(${CRYPTO}) -# Libav +# FFMPEG find_library(AVUTIL avutil HINTS "${THIRD_PARTY_LIB}") test_lib(${AVUTIL}) set (LIBS ${AVUTIL}) @@ -108,6 +108,9 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/third_party/webrtc.cmake") include("${CMAKE_CURRENT_SOURCE_DIR}/third_party/nicer.cmake") ## Erizo +if(APPLE) +add_definitions(-Wall -Werror) +endif() add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/erizo") ## Examples From fb5a4647ef28ff0cabd7b6be5d08b4fd39895c3e Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Thu, 17 May 2018 12:25:28 -0300 Subject: [PATCH 04/32] Update ExternalInput to ffmpeg API --- erizo/src/erizo/media/ExternalInput.cpp | 14 +-- erizo/src/erizo/media/ExternalInput.h | 1 + erizo/src/erizo/media/codecs/VideoCodec.cpp | 103 +++++++++++--------- erizo/src/erizo/media/codecs/VideoCodec.h | 2 +- 4 files changed, 68 insertions(+), 52 deletions(-) diff --git a/erizo/src/erizo/media/ExternalInput.cpp b/erizo/src/erizo/media/ExternalInput.cpp index 994703a15e..6ba0e82c86 100644 --- a/erizo/src/erizo/media/ExternalInput.cpp +++ b/erizo/src/erizo/media/ExternalInput.cpp @@ -86,12 +86,12 @@ int ExternalInput::init() { audio_stream_index_, audio_st->time_base.num, audio_st->time_base.den); audio_time_base_ = audio_st->time_base.den; ELOG_DEBUG("Audio Time base %d", audio_time_base_); - if (audio_st->codec->codec_id == AV_CODEC_ID_PCM_MULAW) { + if (audio_st->codecpar->codec_id == AV_CODEC_ID_PCM_MULAW) { ELOG_DEBUG("PCM U8"); om.audioCodec.sampleRate = 8000; om.audioCodec.codec = AUDIO_CODEC_PCM_U8; om.rtpAudioInfo.PT = PCMU_8000_PT; - } else if (audio_st->codec->codec_id == AV_CODEC_ID_OPUS) { + } else if (audio_st->codecpar->codec_id == AV_CODEC_ID_OPUS) { ELOG_DEBUG("OPUS"); om.audioCodec.sampleRate = 48000; om.audioCodec.codec = AUDIO_CODEC_OPUS; @@ -102,7 +102,7 @@ int ExternalInput::init() { } - if (st->codec->codec_id == AV_CODEC_ID_VP8 || !om.hasVideo) { + if (st->codecpar->codec_id == AV_CODEC_ID_VP8 || !om.hasVideo) { ELOG_DEBUG("No need for video transcoding, already VP8"); video_time_base_ = st->time_base.den; needTranscoding_ = false; @@ -110,12 +110,12 @@ int ExternalInput::init() { MediaInfo om; om.processorType = PACKAGE_ONLY; if (audio_st) { - if (audio_st->codec->codec_id == AV_CODEC_ID_PCM_MULAW) { + if (audio_st->codecpar->codec_id == AV_CODEC_ID_PCM_MULAW) { ELOG_DEBUG("PCM U8"); om.audioCodec.sampleRate = 8000; om.audioCodec.codec = AUDIO_CODEC_PCM_U8; om.rtpAudioInfo.PT = PCMU_8000_PT; - } else if (audio_st->codec->codec_id == AV_CODEC_ID_OPUS) { + } else if (audio_st->codecpar->codec_id == AV_CODEC_ID_OPUS) { ELOG_DEBUG("OPUS"); om.audioCodec.sampleRate = 48000; om.audioCodec.codec = AUDIO_CODEC_OPUS; @@ -126,9 +126,9 @@ int ExternalInput::init() { op_->init(om, this); } else { needTranscoding_ = true; - inCodec_.initDecoder(st->codec); + inCodec_.initDecoder(&video_codec_ctx_, st->codecpar); - bufflen_ = st->codec->width*st->codec->height*3/2; + bufflen_ = video_codec_ctx_->width*video_codec_ctx_->height*3/2; decodedBuffer_.reset((unsigned char*) malloc(bufflen_)); diff --git a/erizo/src/erizo/media/ExternalInput.h b/erizo/src/erizo/media/ExternalInput.h index a69f162545..93d2b5fb1b 100644 --- a/erizo/src/erizo/media/ExternalInput.h +++ b/erizo/src/erizo/media/ExternalInput.h @@ -52,6 +52,7 @@ class ExternalInput : public MediaSource, public RTPDataReceiver { std::queue packetQueue_; AVFormatContext* context_; AVPacket avpacket_; + AVCodecContext *video_codec_ctx_; int video_stream_index_, video_time_base_; int audio_stream_index_, audio_time_base_; int bufflen_; diff --git a/erizo/src/erizo/media/codecs/VideoCodec.cpp b/erizo/src/erizo/media/codecs/VideoCodec.cpp index 96680c7870..04a999cdc5 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.cpp +++ b/erizo/src/erizo/media/codecs/VideoCodec.cpp @@ -53,10 +53,6 @@ int VideoEncoder::initEncoder(const VideoCodecInfo& info) { vCoderContext->qmin = 0; vCoderContext->qmax = 40; // rc_quantifiers vCoderContext->profile = 3; - // vCoderContext->frame_skip_threshold = 30; - vCoderContext->rc_buffer_aggressivity = 0.95; - // vCoderContext->rc_buffer_size = vCoderContext->bit_rate; - // vCoderContext->rc_initial_buffer_occupancy = vCoderContext->bit_rate / 2; vCoderContext->rc_initial_buffer_occupancy = 500; vCoderContext->rc_buffer_size = 1000; @@ -86,8 +82,8 @@ int VideoEncoder::initEncoder(const VideoCodecInfo& info) { } int VideoEncoder::encodeVideo(unsigned char* inBuffer, int inLength, unsigned char* outBuffer, int outLength) { + int ret; int size = vCoderContext->width * vCoderContext->height; - // ELOG_DEBUG("vCoderContext width %d", vCoderContext->width); cPicture->pts = AV_NOPTS_VALUE; cPicture->data[0] = inBuffer; @@ -102,20 +98,26 @@ int VideoEncoder::encodeVideo(unsigned char* inBuffer, int inLength, unsigned ch pkt.data = outBuffer; pkt.size = outLength; - int ret = 0; - int got_packet = 0; - // ELOG_DEBUG( - // "Before encoding inBufflen %d, size %d, codecontext width %d pkt->size%d", - // inLength, size, vCoderContext->width, pkt.size); - ret = avcodec_encode_video2(vCoderContext, &pkt, cPicture, &got_packet); - // ELOG_DEBUG("Encoded video size %u, ret %d, got_packet %d, pts %lld, dts %lld", - // pkt.size, ret, got_packet, pkt.pts, pkt.dts); - if (!ret && got_packet && vCoderContext->coded_frame) { - vCoderContext->coded_frame->pts = pkt.pts; - vCoderContext->coded_frame->key_frame = - !!(pkt.flags & AV_PKT_FLAG_KEY); + ret = avcodec_send_frame(vCoderContext, cPicture); + if (ret == 0) { + while (ret >= 0) { + ret = avcodec_receive_packet(vCoderContext, &pkt); + if (ret == AVERROR(EAGAIN)) { + ELOG_WARN("Encoder request more input. EAGAIN"); + return ret; + } else if (ret == AVERROR_EOF) { + ELOG_WARN("Encoder AVERROR_EOF."); + return ret; + } else if (ret < 0) { + ELOG_ERROR("Error encoding frame."); + return ret; + } + // ELOG_DEBUG("Encoded video size %u, ret %d, pts %lld, dts %lld", + // pkt.size, ret, pkt.pts, pkt.dts); + av_packet_unref(&pkt); + } } - return ret ? ret : pkt.size; + return pkt.size > 0 ? pkt.size : ret; } int VideoEncoder::closeEncoder() { @@ -171,21 +173,39 @@ int VideoDecoder::initDecoder(const VideoCodecInfo& info) { return 0; } -int VideoDecoder::initDecoder(AVCodecContext* context) { - ELOG_DEBUG("Init Decoder with context"); +int VideoDecoder::initDecoder(AVCodecContext** context, AVCodecParameters *codecpar) { + int error; + AVCodecContext *c; + + ELOG_DEBUG("Init Decoder context"); initWithContext_ = true; - vDecoder = avcodec_find_decoder(context->codec_id); + + vDecoder = avcodec_find_decoder(codecpar->codec_id); if (!vDecoder) { ELOG_DEBUG("Error getting video decoder"); return -1; } - vDecoderContext = context; - if (avcodec_open2(vDecoderContext, vDecoder, NULL) < 0) { + c = avcodec_alloc_context3(vDecoder); + if (!c) { + ELOG_ERROR("Could not allocate video codec context."); + return -1; + } + + error = avcodec_parameters_to_context(c, codecpar); + if (error < 0) { + ELOG_ERROR("Could copy parameters to context."); + return -1; + } + + if (avcodec_open2(c, vDecoder, NULL) < 0) { ELOG_DEBUG("Error opening video decoder"); return -1; } + *context = c; + vDecoderContext = *context; + dPicture = av_frame_alloc(); if (!dPicture) { ELOG_DEBUG("Error allocating video frame"); @@ -210,28 +230,23 @@ int VideoDecoder::decodeVideo(unsigned char* inBuff, int inBuffLen, avpkt.data = inBuff; avpkt.size = inBuffLen; - int got_picture; - int len; - - while (avpkt.size > 0) { - len = avcodec_decode_video2(vDecoderContext, dPicture, &got_picture, - &avpkt); - - if (len < 0) { - ELOG_DEBUG("Error decoding video frame"); - return -1; + int ret; + + if (avpkt.size > 0) { + ret = avcodec_send_packet(vDecoderContext, &avpkt); + if (ret == 0) { + while (ret >= 0) { + ret = avcodec_receive_frame(vDecoderContext, dPicture); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + return -1; + } else if (ret < 0) { + ELOG_DEBUG("Error decoding video frame"); + return -1; + } + *gotFrame = 1; + goto decoding; + } } - - if (got_picture) { - *gotFrame = 1; - goto decoding; - } - avpkt.size -= len; - avpkt.data += len; - } - - if (!got_picture) { - return -1; } decoding: diff --git a/erizo/src/erizo/media/codecs/VideoCodec.h b/erizo/src/erizo/media/codecs/VideoCodec.h index ca3d38af7b..abd83feebf 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.h +++ b/erizo/src/erizo/media/codecs/VideoCodec.h @@ -47,7 +47,7 @@ class VideoDecoder { VideoDecoder(); virtual ~VideoDecoder(); int initDecoder(const VideoCodecInfo& info); - int initDecoder(AVCodecContext* context); + int initDecoder(AVCodecContext** context, AVCodecParameters *codecpar); int decodeVideo(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, int outBuffLen, int* gotFrame); int closeDecoder(); From 6d2129523f75fe8ceb66a067d61d8b759cd46690 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Tue, 22 May 2018 18:19:17 -0300 Subject: [PATCH 05/32] Refactor encoding/decoding pipeline. * Add sample format and channels to MediaInfo struct. * Move MediaInfo, RTPInfo, ProcessorType structs into a single file. * Abstract audio/video enc/dec from Input/OutputProcessor. * Abstract basic enc/dec FFmpeg API into Coder class. * Rename variables to follow cpp code guideline. --- erizo/src/erizo/media/ExternalInput.cpp | 2 +- erizo/src/erizo/media/MediaInfo.h | 36 +++ erizo/src/erizo/media/MediaProcessor.cpp | 277 +++--------------- erizo/src/erizo/media/MediaProcessor.h | 49 +--- erizo/src/erizo/media/codecs/AudioCodec.cpp | 251 +++++----------- erizo/src/erizo/media/codecs/AudioCodec.h | 24 +- erizo/src/erizo/media/codecs/Codecs.h | 2 + erizo/src/erizo/media/codecs/Coder.cpp | 115 ++++++++ erizo/src/erizo/media/codecs/Coder.h | 49 ++++ erizo/src/erizo/media/codecs/VideoCodec.cpp | 206 +++++-------- erizo/src/erizo/media/codecs/VideoCodec.h | 27 +- .../erizoAgent/log4cxx.properties | 2 +- 12 files changed, 442 insertions(+), 598 deletions(-) create mode 100644 erizo/src/erizo/media/MediaInfo.h create mode 100644 erizo/src/erizo/media/codecs/Coder.cpp create mode 100644 erizo/src/erizo/media/codecs/Coder.h diff --git a/erizo/src/erizo/media/ExternalInput.cpp b/erizo/src/erizo/media/ExternalInput.cpp index 6ba0e82c86..351e5f707c 100644 --- a/erizo/src/erizo/media/ExternalInput.cpp +++ b/erizo/src/erizo/media/ExternalInput.cpp @@ -184,7 +184,7 @@ void ExternalInput::receiveLoop() { AVPacket orig_pkt = avpacket_; if (needTranscoding_) { if (avpacket_.stream_index == video_stream_index_) { // packet is video - inCodec_.decodeVideo(avpacket_.data, avpacket_.size, decodedBuffer_.get(), bufflen_, &gotDecodedFrame); + inCodec_.decodeVideoBuffer(avpacket_.data, avpacket_.size, decodedBuffer_.get(), bufflen_, &gotDecodedFrame); RawDataPacket packetR; if (gotDecodedFrame) { packetR.data = decodedBuffer_.get(); diff --git a/erizo/src/erizo/media/MediaInfo.h b/erizo/src/erizo/media/MediaInfo.h new file mode 100644 index 0000000000..934121f078 --- /dev/null +++ b/erizo/src/erizo/media/MediaInfo.h @@ -0,0 +1,36 @@ +#ifndef ERIZO_SRC_ERIZO_MEDIA_MEDIAINFO_H_ +#define ERIZO_SRC_ERIZO_MEDIA_MEDIAINFO_H_ + +extern "C" { +#include +} + +#include +#include "codecs/Codecs.h" + +namespace erizo { + +struct RTPInfo { + enum AVCodecID codec; + unsigned int ssrc; + unsigned int PT; +}; + +enum ProcessorType { + RTP_ONLY, AVF, PACKAGE_ONLY +}; + +struct MediaInfo { + std::string url; + bool hasVideo; + bool hasAudio; + ProcessorType processorType; + RTPInfo rtpVideoInfo; + RTPInfo rtpAudioInfo; + VideoCodecInfo videoCodec; + AudioCodecInfo audioCodec; +}; + +} // namespace erizo + +#endif // ERIZO_SRC_ERIZO_MEDIA_MEDIAINFO_H_ diff --git a/erizo/src/erizo/media/MediaProcessor.cpp b/erizo/src/erizo/media/MediaProcessor.cpp index ce37218fe3..575ab8cb74 100644 --- a/erizo/src/erizo/media/MediaProcessor.cpp +++ b/erizo/src/erizo/media/MediaProcessor.cpp @@ -10,6 +10,7 @@ extern "C" { #include "rtp/RtpVP8Fragmenter.h" #include "rtp/RtpHeaders.h" #include "media/codecs/VideoCodec.h" +#include "media/codecs/AudioCodec.h" #include "lib/Clock.h" #include "lib/ClockUtils.h" @@ -21,8 +22,6 @@ DEFINE_LOGGER(InputProcessor, "media.InputProcessor"); DEFINE_LOGGER(OutputProcessor, "media.OutputProcessor"); InputProcessor::InputProcessor() { - audioDecoder = 0; - videoDecoder = 0; lastVideoTs_ = 0; audioUnpackager = 1; @@ -50,10 +49,9 @@ int InputProcessor::init(const MediaInfo& info, RawDataReceiver* receiver) { decodedBuffer_ = (unsigned char*) malloc( info.videoCodec.width * info.videoCodec.height * 3 / 2); unpackagedBufferPtr_ = unpackagedBuffer_ = (unsigned char*) malloc(UNPACKAGED_BUFFER_SIZE); - if (!vDecoder.initDecoder(mediaInfo.videoCodec)) { + if (!video_decoder_.initDecoder(mediaInfo.videoCodec)) { // TODO(javier) check this condition } - videoDecoder = 1; if (!this->initVideoUnpackager()) { // TODO(javier) check this condition } @@ -61,34 +59,39 @@ int InputProcessor::init(const MediaInfo& info, RawDataReceiver* receiver) { if (mediaInfo.hasAudio) { ELOG_DEBUG("Init AUDIO processor"); mediaInfo.audioCodec.codec = AUDIO_CODEC_PCM_U8; + if (!mediaInfo.audioCodec.sampleFmt) { + mediaInfo.audioCodec.sampleFmt = AV_SAMPLE_FMT_S16; + } + if (!mediaInfo.audioCodec.channels) { + mediaInfo.audioCodec.channels = 1; + } decodedAudioBuffer_ = (unsigned char*) malloc(UNPACKAGED_BUFFER_SIZE); - unpackagedAudioBuffer_ = (unsigned char*) malloc( - UNPACKAGED_BUFFER_SIZE); - this->initAudioDecoder(); + unpackagedAudioBuffer_ = (unsigned char*) malloc(UNPACKAGED_BUFFER_SIZE); + audio_decoder_.initDecoder(mediaInfo.audioCodec); this->initAudioUnpackager(); } return 0; } int InputProcessor::deliverAudioData_(std::shared_ptr audio_packet) { - if (audioDecoder && audioUnpackager) { + if (audioUnpackager && audio_decoder_.initialized) { std::shared_ptr copied_packet = std::make_shared(*audio_packet); - ELOG_DEBUG("Decoding audio"); int unp = unpackageAudio((unsigned char*) copied_packet->data, copied_packet->length, unpackagedAudioBuffer_); - int a = decodeAudio(unpackagedAudioBuffer_, unp, decodedAudioBuffer_); - ELOG_DEBUG("DECODED AUDIO a %d", a); - RawDataPacket p; - p.data = decodedAudioBuffer_; - p.type = AUDIO; - p.length = a; - rawReceiver_->receiveRawData(p); - return a; + int len = audio_decoder_.decodeAudio(unpackagedAudioBuffer_, unp, decodedAudioBuffer_, UNPACKAGED_BUFFER_SIZE); + if (len > 0) { + RawDataPacket p; + p.data = decodedAudioBuffer_; + p.type = AUDIO; + p.length = len; + rawReceiver_->receiveRawData(p); + } + return len; } return 0; } int InputProcessor::deliverVideoData_(std::shared_ptr video_packet) { - if (videoUnpackager && videoDecoder) { + if (videoUnpackager && video_decoder_.initialized) { std::shared_ptr copied_packet = std::make_shared(*video_packet); int ret = unpackageVideo(reinterpret_cast(copied_packet->data), copied_packet->length, unpackagedBufferPtr_, &gotUnpackagedFrame_); @@ -102,7 +105,7 @@ int InputProcessor::deliverVideoData_(std::shared_ptr video_packet) int c; int gotDecodedFrame = 0; - c = vDecoder.decodeVideo(unpackagedBufferPtr_, upackagedSize_, + c = video_decoder_.decodeVideoBuffer(unpackagedBufferPtr_, upackagedSize_, decodedBuffer_, mediaInfo.videoCodec.width * mediaInfo.videoCodec.height * 3 / 2, &gotDecodedFrame); @@ -130,32 +133,6 @@ int InputProcessor::deliverEvent_(MediaEventPtr event) { return 0; } -bool InputProcessor::initAudioDecoder() { - aDecoder = avcodec_find_decoder(static_cast(mediaInfo.audioCodec.codec)); - if (!aDecoder) { - ELOG_DEBUG("Decoder de audio no encontrado"); - return false; - } - - aDecoderContext = avcodec_alloc_context3(aDecoder); - if (!aDecoderContext) { - ELOG_DEBUG("Error de memoria en decoder de audio"); - return false; - } - - aDecoderContext->sample_fmt = AV_SAMPLE_FMT_S16; - aDecoderContext->bit_rate = mediaInfo.audioCodec.bitRate; - aDecoderContext->sample_rate = mediaInfo.audioCodec.sampleRate; - aDecoderContext->channels = 1; - - if (avcodec_open2(aDecoderContext, aDecoder, NULL) < 0) { - ELOG_DEBUG("Error al abrir el decoder de audio"); - return false; - } - audioDecoder = 1; - return true; -} - bool InputProcessor::initAudioUnpackager() { audioUnpackager = 1; return true; @@ -166,92 +143,6 @@ bool InputProcessor::initVideoUnpackager() { return true; } -int InputProcessor::decodeAudio(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff) { - if (audioDecoder == 0) { - ELOG_DEBUG("No se han inicializado los parámetros del audioDecoder"); - return -1; - } - - AVPacket avpkt; - int outSize = 0; - int decSize = 0; - int len = -1; - uint8_t *decBuff = reinterpret_cast(malloc(16000)); - - av_init_packet(&avpkt); - avpkt.data = (unsigned char*) inBuff; - avpkt.size = inBuffLen; - - while (avpkt.size > 0) { - outSize = 16000; - - // Puede fallar. Cogido de libavcodec/utils.c del paso de avcodec_decode_audio3 a avcodec_decode_audio4 - // avcodec_decode_audio3(aDecoderContext, (short*)decBuff, &outSize, &avpkt); - - AVFrame frame; - int got_frame = 0; - - // aDecoderContext->get_buffer = avcodec_default_get_buffer; - // aDecoderContext->release_buffer = avcodec_default_release_buffer; - - len = avcodec_decode_audio4(aDecoderContext, &frame, &got_frame, - &avpkt); - if (len >= 0 && got_frame) { - int plane_size; - // int planar = av_sample_fmt_is_planar(aDecoderContext->sample_fmt); - int data_size = av_samples_get_buffer_size(&plane_size, - aDecoderContext->channels, frame.nb_samples, - aDecoderContext->sample_fmt, 1); - if (outSize < data_size) { - ELOG_DEBUG("output buffer size is too small for the current frame"); - free(decBuff); - return AVERROR(EINVAL); - } - - memcpy(decBuff, frame.extended_data[0], plane_size); - - /* Si hay más de un canal - if (planar && aDecoderContext->channels > 1) { - uint8_t *out = ((uint8_t *)decBuff) + plane_size; - for (int ch = 1; ch < aDecoderContext->channels; ch++) { - memcpy(out, frame.extended_data[ch], plane_size); - out += plane_size; - } - } - */ - outSize = data_size; - } else { - outSize = 0; - } - - if (len < 0) { - ELOG_DEBUG("Error al decodificar audio"); - free(decBuff); - return -1; - } - - avpkt.size -= len; - avpkt.data += len; - - if (outSize <= 0) { - continue; - } - - memcpy(outBuff, decBuff, outSize); - outBuff += outSize; - decSize += outSize; - } - - free(decBuff); - - if (outSize <= 0) { - ELOG_DEBUG("Error de decodificación de audio debido a tamaño incorrecto"); - return -1; - } - - return decSize; -} - int InputProcessor::unpackageAudio(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff) { int l = inBuffLen - RtpHeader::MIN_SIZE; if (l < 0) { @@ -294,16 +185,8 @@ void InputProcessor::closeSink() { } void InputProcessor::close() { - if (audioDecoder == 1) { - avcodec_close(aDecoderContext); - av_free(aDecoderContext); - audioDecoder = 0; - } - - if (videoDecoder == 1) { - vDecoder.closeDecoder(); - videoDecoder = 0; - } + audio_decoder_.closeDecoder(); + video_decoder_.closeDecoder(); free(decodedBuffer_); decodedBuffer_ = NULL; free(unpackagedBuffer_); unpackagedBuffer_ = NULL; free(unpackagedAudioBuffer_); unpackagedAudioBuffer_ = NULL; @@ -311,9 +194,6 @@ void InputProcessor::close() { } OutputProcessor::OutputProcessor() { - audioCoder = 0; - videoCoder = 0; - audioPackager = 0; videoPackager = 0; timestamp_ = 0; @@ -346,7 +226,7 @@ int OutputProcessor::init(const MediaInfo& info, RTPDataReceiver* rtpReceiver) { } if (mediaInfo.hasVideo) { this->mediaInfo.videoCodec.codec = VIDEO_CODEC_VP8; - if (vCoder.initEncoder(mediaInfo.videoCodec)) { + if (video_encoder_.initEncoder(mediaInfo.videoCodec)) { ELOG_DEBUG("Error initing encoder"); } this->initVideoPackager(); @@ -356,9 +236,11 @@ int OutputProcessor::init(const MediaInfo& info, RTPDataReceiver* rtpReceiver) { mediaInfo.audioCodec.codec = AUDIO_CODEC_PCM_U8; mediaInfo.audioCodec.sampleRate = 44100; mediaInfo.audioCodec.bitRate = 64000; + mediaInfo.audioCodec.channels = 1; + mediaInfo.audioCodec.sampleFmt = AV_SAMPLE_FMT_S16; + audio_encoder_.initEncoder(mediaInfo.audioCodec); encodedAudioBuffer_ = (unsigned char*) malloc(UNPACKAGED_BUFFER_SIZE); packagedAudioBuffer_ = (unsigned char*) malloc(UNPACKAGED_BUFFER_SIZE); - this->initAudioCoder(); this->initAudioPackager(); } @@ -366,17 +248,8 @@ int OutputProcessor::init(const MediaInfo& info, RTPDataReceiver* rtpReceiver) { } void OutputProcessor::close() { - if (audioCoder == 1) { - avcodec_close(aCoderContext); - av_free(aCoderContext); - audioCoder = 0; - } - - if (videoCoder == 1) { - vCoder.closeEncoder(); - videoCoder = 0; - } - + audio_encoder_.closeEncoder(); + video_encoder_.closeEncoder(); free(encodedBuffer_); encodedBuffer_ = NULL; free(packagedBuffer_); packagedBuffer_ = NULL; free(rtpBuffer_); rtpBuffer_ = NULL; @@ -387,12 +260,14 @@ void OutputProcessor::close() { void OutputProcessor::receiveRawData(const RawDataPacket& packet) { if (packet.type == VIDEO) { - // ELOG_DEBUG("Encoding video: size %d", packet.length); - int a = vCoder.encodeVideo(packet.data, packet.length, encodedBuffer_, UNPACKAGED_BUFFER_SIZE); - if (a > 0) - this->packageVideo(encodedBuffer_, a, packagedBuffer_); + EncodeVideoBufferCB done_callback = [this](bool fill_buffer, int size) { + if (fill_buffer) { + this->packageVideo(encodedBuffer_, size, packagedBuffer_); + } + }; + video_encoder_.encodeVideoBuffer(packet.data, packet.length, encodedBuffer_, done_callback); } else { - // int a = this->encodeAudio(packet.data, packet.length, &pkt); + // int a = audio_encoder_.encodeAudio(packet.data, packet.length, &pkt); // if (a > 0) { // ELOG_DEBUG("GUAY a %d", a); // } @@ -400,33 +275,6 @@ void OutputProcessor::receiveRawData(const RawDataPacket& packet) { // av_free_packet(&pkt); } -bool OutputProcessor::initAudioCoder() { - aCoder = avcodec_find_encoder(static_cast(mediaInfo.audioCodec.codec)); - if (!aCoder) { - ELOG_DEBUG("Encoder de audio no encontrado"); - return false; - } - - aCoderContext = avcodec_alloc_context3(aCoder); - if (!aCoderContext) { - ELOG_DEBUG("Error de memoria en coder de audio"); - return false; - } - - aCoderContext->sample_fmt = AV_SAMPLE_FMT_S16; - aCoderContext->bit_rate = mediaInfo.audioCodec.bitRate; - aCoderContext->sample_rate = mediaInfo.audioCodec.sampleRate; - aCoderContext->channels = 1; - - if (avcodec_open2(aCoderContext, aCoder, NULL) < 0) { - ELOG_DEBUG("Error al abrir el coder de audio"); - return false; - } - - audioCoder = 1; - return true; -} - bool OutputProcessor::initAudioPackager() { audioPackager = 1; audioSeqnum_ = 0; @@ -511,55 +359,4 @@ int OutputProcessor::packageVideo(unsigned char* inBuff, int buffSize, unsigned return 0; } -int OutputProcessor::encodeAudio(unsigned char* inBuff, int nSamples, AVPacket* pkt) { - if (audioCoder == 0) { - ELOG_DEBUG("No se han inicializado los parámetros del audioCoder"); - return -1; - } - - AVFrame *frame = av_frame_alloc(); - if (!frame) { - ELOG_ERROR("could not allocate audio frame"); - return -1; - } - int ret, got_output, buffer_size; - // float t, tincr; - - frame->nb_samples = aCoderContext->frame_size; - frame->format = aCoderContext->sample_fmt; - // frame->channel_layout = aCoderContext->channel_layout; - - /* the codec gives us the frame size, in samples, - * we calculate the size of the samples buffer in bytes */ - ELOG_DEBUG("channels %d, frame_size %d, sample_fmt %d", - aCoderContext->channels, aCoderContext->frame_size, - aCoderContext->sample_fmt); - buffer_size = av_samples_get_buffer_size(NULL, aCoderContext->channels, - aCoderContext->frame_size, aCoderContext->sample_fmt, 0); - uint16_t* samples = reinterpret_cast(av_malloc(buffer_size)); - if (!samples) { - ELOG_ERROR("could not allocate %d bytes for samples buffer", - buffer_size); - return -1; - } - /* setup the data pointers in the AVFrame */ - ret = avcodec_fill_audio_frame(frame, aCoderContext->channels, - aCoderContext->sample_fmt, (const uint8_t*) samples, buffer_size, - 0); - if (ret < 0) { - ELOG_ERROR("could not setup audio frame"); - return ret; - } - - ret = avcodec_encode_audio2(aCoderContext, pkt, frame, &got_output); - if (ret < 0) { - ELOG_ERROR("error encoding audio frame"); - return ret; - } - if (got_output) { - // fwrite(pkt.data, 1, pkt.size, f); - ELOG_DEBUG("Got OUTPUT"); - } - return ret; -} } // namespace erizo diff --git a/erizo/src/erizo/media/MediaProcessor.h b/erizo/src/erizo/media/MediaProcessor.h index 19cf12598b..7ec3eba1ab 100644 --- a/erizo/src/erizo/media/MediaProcessor.h +++ b/erizo/src/erizo/media/MediaProcessor.h @@ -14,21 +14,13 @@ extern "C" { #include "rtp/RtpVP8Parser.h" #include "./MediaDefinitions.h" +#include "media/MediaInfo.h" #include "codecs/Codecs.h" #include "codecs/VideoCodec.h" +#include "codecs/AudioCodec.h" #include "./logger.h" namespace erizo { -struct RTPInfo { - enum AVCodecID codec; - unsigned int ssrc; - unsigned int PT; -}; - -enum ProcessorType { - RTP_ONLY, AVF, PACKAGE_ONLY -}; - enum DataType { VIDEO, AUDIO @@ -40,17 +32,6 @@ struct RawDataPacket { DataType type; }; -struct MediaInfo { - std::string url; - bool hasVideo; - bool hasAudio; - ProcessorType processorType; - RTPInfo rtpVideoInfo; - RTPInfo rtpAudioInfo; - VideoCodecInfo videoCodec; - AudioCodecInfo audioCodec; -}; - #define UNPACKAGED_BUFFER_SIZE 150000 #define PACKAGED_BUFFER_SIZE 2000 // class MediaProcessor{ @@ -93,9 +74,6 @@ class InputProcessor: public MediaSink { void close() override; private: - int audioDecoder; - int videoDecoder; - double lastVideoTs_; MediaInfo mediaInfo; @@ -113,24 +91,18 @@ class InputProcessor: public MediaSink { unsigned char* decodedAudioBuffer_; unsigned char* unpackagedAudioBuffer_; - AVCodec* aDecoder; - AVCodecContext* aDecoderContext; - - VideoDecoder vDecoder; + VideoDecoder video_decoder_; + AudioDecoder audio_decoder_; RawDataReceiver* rawReceiver_; erizo::RtpVP8Parser pars; - bool initAudioDecoder(); - bool initAudioUnpackager(); bool initVideoUnpackager(); int deliverAudioData_(std::shared_ptr audio_packet) override; int deliverVideoData_(std::shared_ptr video_packet) override; int deliverEvent_(MediaEventPtr event) override; - - int decodeAudio(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff); }; class OutputProcessor: public RawDataReceiver { DECLARE_LOGGER(); @@ -147,9 +119,6 @@ class OutputProcessor: public RawDataReceiver { int packageVideo(unsigned char* inBuff, int buffSize, unsigned char* outBuff, long int pts = 0); // NOLINT private: - int audioCoder; - int videoCoder; - int audioPackager; int videoPackager; @@ -169,19 +138,13 @@ class OutputProcessor: public RawDataReceiver { RTPDataReceiver* rtpReceiver_; - AVCodec* aCoder; - AVCodecContext* aCoderContext; - - VideoEncoder vCoder; + VideoEncoder video_encoder_; + AudioEncoder audio_encoder_; RtpVP8Parser pars; - bool initAudioCoder(); - bool initAudioPackager(); bool initVideoPackager(); - - int encodeAudio(unsigned char* inBuff, int nSamples, AVPacket* pkt); }; } // namespace erizo diff --git a/erizo/src/erizo/media/codecs/AudioCodec.cpp b/erizo/src/erizo/media/codecs/AudioCodec.cpp index 526e38e356..a990ef6243 100644 --- a/erizo/src/erizo/media/codecs/AudioCodec.cpp +++ b/erizo/src/erizo/media/codecs/AudioCodec.cpp @@ -24,9 +24,10 @@ inline AVCodecID AudioCodecID2ffmpegDecoderID(AudioCodecID codec) { } AudioEncoder::AudioEncoder() { - aCoder_ = NULL; - aCoderContext_ = NULL; - aFrame_ = NULL; + encode_codec_ = NULL; + encode_context_ = NULL; + encoded_frame_ = NULL; + initialized = false; } AudioEncoder::~AudioEncoder() { @@ -35,97 +36,64 @@ AudioEncoder::~AudioEncoder() { } int AudioEncoder::initEncoder(const AudioCodecInfo& mediaInfo) { - ELOG_DEBUG("Init audioEncoder begin"); - aCoder_ = avcodec_find_encoder(AudioCodecID2ffmpegDecoderID(mediaInfo.codec)); - if (!aCoder_) { - ELOG_DEBUG("Audio Codec not found"); - return false; + if (!coder_.allocCodecContext(&encode_context_, &encode_codec_, + AudioCodecID2ffmpegDecoderID(mediaInfo.codec), OPERATION_ENCODE)) { + return -1; } - aCoderContext_ = avcodec_alloc_context3(aCoder_); - if (!aCoderContext_) { - ELOG_DEBUG("Memory error allocating audio coder context"); - return false; - } + encode_context_->sample_fmt = AV_SAMPLE_FMT_FLT; + // encode_context_->bit_rate = mediaInfo.bitRate; + encode_context_->sample_rate = 8 /*mediaInfo.sampleRate*/; + encode_context_->channels = 1; - aCoderContext_->sample_fmt = AV_SAMPLE_FMT_FLT; - // aCoderContext_->bit_rate = mediaInfo.bitRate; - aCoderContext_->sample_rate = 8 /*mediaInfo.sampleRate*/; - aCoderContext_->channels = 1; - char errbuff[500]; - int res = avcodec_open2(aCoderContext_, aCoder_, NULL); - if (res != 0) { - av_strerror(res, reinterpret_cast(&errbuff), 500); - ELOG_DEBUG("fail when opening input %s", errbuff); - return -1; + initialized = coder_.openCodecContext(encode_context_, encode_codec_, NULL); + return initialized; +} + +int AudioEncoder::closeEncoder() { + if (encode_context_ != nullptr) { + avcodec_close(encode_context_); + avcodec_free_context(&encode_context_); } - ELOG_DEBUG("Init audioEncoder end"); - return true; + if (encoded_frame_ != nullptr) { + av_frame_free(&encoded_frame_); + } + return 0; } -int AudioEncoder::encodeAudio(unsigned char* inBuffer, int nSamples, AVPacket* pkt) { +void AudioEncoder::encodeAudioBuffer(unsigned char* inBuffer, int nSamples, AVPacket* pkt, + EncodeAudioBufferCB &done) { AVFrame *frame = av_frame_alloc(); if (!frame) { - ELOG_ERROR("could not allocate audio frame"); - return 0; + ELOG_ERROR("Could not allocate audio frame for encoding."); + return; } - int ret, got_output, buffer_size; - - frame->nb_samples = aCoderContext_->frame_size; - frame->format = aCoderContext_->sample_fmt; - // frame->channel_layout = aCoderContext_->channel_layout; + int ret; - /* the codec gives us the frame size, in samples, - * we calculate the size of the samples buffer in bytes */ - ELOG_DEBUG("channels %d, frame_size %d, sample_fmt %d", - aCoderContext_->channels, aCoderContext_->frame_size, - aCoderContext_->sample_fmt); - buffer_size = av_samples_get_buffer_size(NULL, aCoderContext_->channels, - aCoderContext_->frame_size, aCoderContext_->sample_fmt, 0); - uint16_t* samples = reinterpret_cast(malloc(buffer_size)); - if (!samples) { - ELOG_ERROR("could not allocate %d bytes for samples buffer", buffer_size); - return 0; - } - /* setup the data pointers in the AVFrame */ - ret = avcodec_fill_audio_frame(frame, aCoderContext_->channels, - aCoderContext_->sample_fmt, (const uint8_t*) samples, buffer_size, - 0); - if (ret < 0) { - free(samples); - ELOG_ERROR("could not setup audio frame"); - return 0; - } + frame->nb_samples = encode_context_->frame_size; - ret = avcodec_encode_audio2(aCoderContext_, pkt, frame, &got_output); - if (ret < 0) { - ELOG_ERROR("error encoding audio frame"); - free(samples); - return 0; - } - if (got_output) { - // fwrite(pkt.data, 1, pkt.size, f); - ELOG_DEBUG("Got OUTPUT"); + if ((ret = avcodec_fill_audio_frame(frame, + encode_context_->channels, + encode_context_->sample_fmt, + (const uint8_t*)inBuffer, + nSamples * 2, + 0)) < 0) { + ELOG_ERROR("avcodec_fill_audio_frame failed: %s", av_err2str(ret)); + return; } - return ret; + EncodeCB done_callback = [done](AVPacket *pkt, AVFrame *frame, bool got_packet) { + av_frame_free(&frame); + done(pkt, got_packet); + }; + coder_.encode(encode_context_, frame, pkt, done_callback); } -int AudioEncoder::closeEncoder() { - if (aCoderContext_ != NULL) { - avcodec_close(aCoderContext_); - } - if (aFrame_ != NULL) { - av_frame_free(&aFrame_); - } - return 0; -} - - AudioDecoder::AudioDecoder() { - aDecoder_ = NULL; - aDecoderContext_ = NULL; + decode_codec_ = NULL; + decode_context_ = NULL; dFrame_ = NULL; + initialized = false; } AudioDecoder::~AudioDecoder() { @@ -134,123 +102,64 @@ AudioDecoder::~AudioDecoder() { } int AudioDecoder::initDecoder(const AudioCodecInfo& info) { - aDecoder_ = avcodec_find_decoder(static_cast(info.codec)); - if (!aDecoder_) { - ELOG_DEBUG("Audio decoder not found"); - return false; - } - - aDecoderContext_ = avcodec_alloc_context3(aDecoder_); - if (!aDecoderContext_) { - ELOG_DEBUG("Error allocating audio decoder context"); - return false; + if (!coder_.allocCodecContext(&decode_context_, &decode_codec_, + AudioCodecID2ffmpegDecoderID(info.codec), OPERATION_DECODE)) { + return -1; } - aDecoderContext_->sample_fmt = AV_SAMPLE_FMT_S16; - aDecoderContext_->bit_rate = info.bitRate; - aDecoderContext_->sample_rate = info.sampleRate; - aDecoderContext_->channels = 1; + decode_context_->sample_fmt = AV_SAMPLE_FMT_S16; + decode_context_->bit_rate = info.bitRate; + decode_context_->sample_rate = info.sampleRate; + decode_context_->channels = 1; - if (avcodec_open2(aDecoderContext_, aDecoder_, NULL) < 0) { - ELOG_DEBUG("Error opening audio decoder"); - return false; - } - return true; -} - -int AudioDecoder::initDecoder(AVCodecContext* context) { - return 0; + initialized = coder_.openCodecContext(decode_context_, decode_codec_, NULL); + return initialized; } int AudioDecoder::decodeAudio(unsigned char* inBuff, int inBuffLen, - unsigned char* outBuff, int outBuffLen, int* gotFrame) { - AVPacket avpkt; - int outSize = 0; - int decSize = 0; - int len = -1; - uint8_t *decBuff = reinterpret_cast(malloc(16000)); + unsigned char* outBuff, int outBuffLen) { + int out_size = 0; + AVPacket avpkt; av_init_packet(&avpkt); avpkt.data = (unsigned char*) inBuff; avpkt.size = inBuffLen; - while (avpkt.size > 0) { - outSize = 16000; - - // Puede fallar. Cogido de libavcodec/utils.c del paso de avcodec_decode_audio3 a avcodec_decode_audio4 - // avcodec_decode_audio3(aDecoderContext, (short*)decBuff, &outSize, &avpkt); - - AVFrame frame; - int got_frame = 0; - - // aDecoderContext->get_buffer = avcodec_default_get_buffer; - // aDecoderContext->release_buffer = avcodec_default_release_buffer; - - len = avcodec_decode_audio4(aDecoderContext_, &frame, &got_frame, - &avpkt); - if (len >= 0 && got_frame) { - int plane_size; - // int planar = av_sample_fmt_is_planar(aDecoderContext->sample_fmt); - int data_size = av_samples_get_buffer_size(&plane_size, - aDecoderContext_->channels, frame.nb_samples, - aDecoderContext_->sample_fmt, 1); - if (outSize < data_size) { - ELOG_DEBUG("output buffer size is too small for the current frame"); - free(decBuff); - return AVERROR(EINVAL); + AVFrame *frame; + frame = av_frame_alloc(); + + if (coder_.decode(decode_context_, frame, &avpkt)) { + // Asume S16 Non-Planar Audio for now. + int planar = av_sample_fmt_is_planar(decode_context_->sample_fmt); + if (planar == 0) { + out_size = av_samples_get_buffer_size(NULL, + av_frame_get_channels(frame), + frame->nb_samples, + (AVSampleFormat)frame->format, 1); + if (out_size > outBuffLen) { + ELOG_ERROR("Data size bigger than buffer size."); + out_size = 0; + } else { + memcpy(outBuff, frame->data[0], out_size); } - - memcpy(decBuff, frame.extended_data[0], plane_size); - - /* Si hay más de un canal - if (planar && aDecoderContext->channels > 1) { - uint8_t *out = ((uint8_t *)decBuff) + plane_size; - for (int ch = 1; ch < aDecoderContext->channels; ch++) { - memcpy(out, frame.extended_data[ch], plane_size); - out += plane_size; - } - } - */ - outSize = data_size; } else { - outSize = 0; - } - - if (len < 0) { - ELOG_DEBUG("Error al decodificar audio"); - free(decBuff); - return -1; - } - - avpkt.size -= len; - avpkt.data += len; - - if (outSize <= 0) { - continue; + ELOG_ERROR("Audio planar buffer is not handled yet."); } - - memcpy(outBuff, decBuff, outSize); - outBuff += outSize; - decSize += outSize; - } - - free(decBuff); - - if (outSize <= 0) { - ELOG_DEBUG("Error de decodificación de audio debido a tamaño incorrecto"); - return -1; + av_packet_unref(&avpkt); } - return decSize; + av_frame_free(&frame); + return out_size; } int AudioDecoder::closeDecoder() { - if (aDecoderContext_ != NULL) { - avcodec_close(aDecoderContext_); + if (decode_context_ != NULL) { + avcodec_close(decode_context_); } if (dFrame_ != NULL) { av_frame_free(&dFrame_); } + initialized = false; return 0; } } // namespace erizo diff --git a/erizo/src/erizo/media/codecs/AudioCodec.h b/erizo/src/erizo/media/codecs/AudioCodec.h index 855776a502..be92121290 100644 --- a/erizo/src/erizo/media/codecs/AudioCodec.h +++ b/erizo/src/erizo/media/codecs/AudioCodec.h @@ -10,11 +10,15 @@ extern "C" { #include } +#include "media/MediaInfo.h" #include "./Codecs.h" +#include "./Coder.h" #include "./logger.h" namespace erizo { +typedef std::function EncodeAudioBufferCB; + class AudioEncoder { DECLARE_LOGGER(); @@ -22,13 +26,16 @@ class AudioEncoder { AudioEncoder(); virtual ~AudioEncoder(); int initEncoder(const AudioCodecInfo& info); - int encodeAudio(unsigned char* inBuffer, int nSamples, AVPacket* pkt); + void encodeAudioBuffer(unsigned char* inBuffer, int nSamples, AVPacket* pkt, + EncodeAudioBufferCB &done); int closeEncoder(); + bool initialized; private: - AVCodec* aCoder_; - AVCodecContext* aCoderContext_; - AVFrame* aFrame_; + Coder coder_; + AVCodec *encode_codec_; + AVCodecContext *encode_context_; + AVFrame *encoded_frame_; }; class AudioDecoder { @@ -38,13 +45,14 @@ class AudioDecoder { AudioDecoder(); virtual ~AudioDecoder(); int initDecoder(const AudioCodecInfo& info); - int initDecoder(AVCodecContext* context); - int decodeAudio(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, int outBuffLen, int* gotFrame); + int decodeAudio(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, int outBuffLen); int closeDecoder(); + bool initialized; private: - AVCodec* aDecoder_; - AVCodecContext* aDecoderContext_; + Coder coder_; + AVCodec* decode_codec_; + AVCodecContext* decode_context_; AVFrame* dFrame_; }; diff --git a/erizo/src/erizo/media/codecs/Codecs.h b/erizo/src/erizo/media/codecs/Codecs.h index 2a0173b40a..7b07ed4070 100644 --- a/erizo/src/erizo/media/codecs/Codecs.h +++ b/erizo/src/erizo/media/codecs/Codecs.h @@ -29,6 +29,8 @@ struct AudioCodecInfo { AudioCodecID codec; int bitRate; int sampleRate; + int sampleFmt; + int channels; }; } // namespace erizo #endif // ERIZO_SRC_ERIZO_MEDIA_CODECS_CODECS_H_ diff --git a/erizo/src/erizo/media/codecs/Coder.cpp b/erizo/src/erizo/media/codecs/Coder.cpp new file mode 100644 index 0000000000..0d05e6ba21 --- /dev/null +++ b/erizo/src/erizo/media/codecs/Coder.cpp @@ -0,0 +1,115 @@ +#include "media/codecs/Coder.h" + +namespace erizo { + +DEFINE_LOGGER(Coder, "media.codecs.Coder"); + +Coder::Coder() { +} + +Coder::~Coder() { +} + +bool Coder::allocCodecContext(AVCodecContext **ctx, AVCodec **c, AVCodecID codec_id, + CoderOperationType operation) { + AVCodec *codec = nullptr; + AVCodecContext *codec_ctx = *ctx; + if (operation == OPERATION_ENCODE) { + codec = avcodec_find_encoder(codec_id); + } else { + codec = avcodec_find_decoder(codec_id); + } + if (codec == nullptr) { + ELOG_ERROR("Cannot find codec %d", codec_id); + return false; + } + codec_ctx = avcodec_alloc_context3(codec); + if (!codec_ctx) { + ELOG_ERROR("Could not allocate codec context for %s", codec->name); + return false; + } else { + ELOG_INFO("Successfully allocated context for codec %s.", codec->name); + } + *c = codec; + *ctx = codec_ctx; + return true; +} + +bool Coder::openCodecContext(AVCodecContext *codec_ctx, AVCodec *codec, AVDictionary *opt) { + const char *operation = (av_codec_is_encoder(codec_ctx->codec) ? "encoder" : "decoder"); + int ret = avcodec_open2(codec_ctx, NULL, &opt); + if (ret < 0) { + static char error_str[255]; + av_strerror(ret, error_str, sizeof(error_str)); + ELOG_ERROR("Could not open %s codec of %s: %s", operation, + codec->name, + error_str); + } + av_dict_free(&opt); + this->logCodecContext(codec_ctx); + return ret < 0 ? false : true; +} + +void Coder::logCodecContext(AVCodecContext *codec_ctx) { + const char *operation = (av_codec_is_encoder(codec_ctx->codec) ? "encoder" : "decoder"); + if (codec_ctx->codec->type == AVMEDIA_TYPE_AUDIO) { + ELOG_DEBUG("\nAudio %s codec: %s \nchannel_layout: %d\nchannels: %d\nframe_size: %d\nsample_rate: %d\nsample_fmt: %s\nbits per sample: %d", + operation, codec_ctx->codec->name, codec_ctx->channel_layout, codec_ctx->channels, codec_ctx->frame_size, + codec_ctx->sample_rate, av_get_sample_fmt_name(codec_ctx->sample_fmt), + av_get_bytes_per_sample (codec_ctx->sample_fmt)); + } else if (codec_ctx->codec->type == AVMEDIA_TYPE_VIDEO) { + ELOG_DEBUG("\nVideo %s codec: %s\n framerate: {%d/%d}\n time_base: {%d/%d}\n", + operation, codec_ctx->codec->name, codec_ctx->framerate.num, codec_ctx->framerate.den, + codec_ctx->time_base.num, codec_ctx->time_base.den); + } +} + +bool Coder::decode(AVCodecContext *decode_ctx, AVFrame *frame, AVPacket *av_packet) { + int ret; + ret = avcodec_send_packet(decode_ctx, av_packet); + if (ret < 0) { + ELOG_ERROR("avcodec_send_packet failed. %d %s", ret, av_err2str(ret)); + return false; + } + ret = avcodec_receive_frame(decode_ctx, frame); + if (ret != 0) { + ELOG_ERROR("avcodec_receive_frame. %d %s", ret, av_err2str(ret)); + return false; + } + ELOG_DEBUG("decoded %s, nb_samples: %d, size: %d", decode_ctx->codec->name, + frame->nb_samples, av_packet->size); + return true; +} + +void Coder::encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_packet, const EncodeCB &done) { + auto t_started = std::chrono::high_resolution_clock::now(); + int ret = avcodec_send_frame(encode_ctx, frame); + if (ret < 0) { + ELOG_ERROR("avcodec_send_frame failed for %s. %d %s", encode_ctx->codec->name, + ret, av_err2str(ret)); + done(av_packet, frame, false); + } + while(ret >= 0) { + ret = avcodec_receive_packet(encode_ctx, av_packet); + if (ret == AVERROR_EOF) { + ELOG_DEBUG("avcodec_receive_packet AVERROR_EOF, %s, ret: %d, %s", encode_ctx->codec->name, + ret, av_err2str(ret)) + done(av_packet, frame, false); + return; + } else if (ret == AVERROR(EAGAIN)) { + done(av_packet, frame, false); + return; + } else if (ret == 0) { + auto t_done = std::chrono::high_resolution_clock::now(); + ELOG_DEBUG("Encoding %s time: %d milliseconds", encode_ctx->codec->name, + std::chrono::duration_cast(t_done-t_started).count()); + done(av_packet, frame, true); + } else { + ELOG_ERROR("avcodec_receive_packet failed. %s, %d %s", encode_ctx->codec->name, + ret, av_err2str(ret)); + done(av_packet, frame, false); + } + } +} + +} // namespace erizo diff --git a/erizo/src/erizo/media/codecs/Coder.h b/erizo/src/erizo/media/codecs/Coder.h new file mode 100644 index 0000000000..b2aa1f4ca9 --- /dev/null +++ b/erizo/src/erizo/media/codecs/Coder.h @@ -0,0 +1,49 @@ +#ifndef ERIZO_SRC_ERIZO_MEDIA_CODECS_CODER_H_ +#define ERIZO_SRC_ERIZO_MEDIA_CODECS_CODER_H_ + +#include +#include "./logger.h" + +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +} + +namespace erizo { + +typedef std::function EncodeCB; + +enum CoderOperationType { + OPERATION_ENCODE, + OPERATION_DECODE +}; + +class Coder { + DECLARE_LOGGER(); + + public: + Coder(); + virtual ~Coder(); + bool allocCodecContext(AVCodecContext **ctx, AVCodec **c, AVCodecID codec_id, + CoderOperationType operation); + bool openCodecContext(AVCodecContext *codec_ctx, AVCodec *codec, AVDictionary *opt); + void logCodecContext(AVCodecContext *codec_ctx); + bool decode(AVCodecContext *decode_ctx, AVFrame *frame, AVPacket *av_packet); + void encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_packet, + const EncodeCB &done); +}; + +} // namespace erizo +#endif // ERIZO_SRC_ERIZO_MEDIA_CODECS_CODER_H_ diff --git a/erizo/src/erizo/media/codecs/VideoCodec.cpp b/erizo/src/erizo/media/codecs/VideoCodec.cpp index 04a999cdc5..07dfa4cf2e 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.cpp +++ b/erizo/src/erizo/media/codecs/VideoCodec.cpp @@ -25,9 +25,10 @@ inline AVCodecID VideoCodecID2ffmpegDecoderID(VideoCodecID codec) { VideoEncoder::VideoEncoder() { avcodec_register_all(); - vCoder = NULL; - vCoderContext = NULL; + av_codec = NULL; + encode_context_ = NULL; cPicture = NULL; + initialized = false; } VideoEncoder::~VideoEncoder() { @@ -35,107 +36,85 @@ VideoEncoder::~VideoEncoder() { } int VideoEncoder::initEncoder(const VideoCodecInfo& info) { - vCoder = avcodec_find_encoder(VideoCodecID2ffmpegDecoderID(info.codec)); - if (!vCoder) { - ELOG_DEBUG("Video codec not found for encoder"); + if (!coder_.allocCodecContext(&encode_context_, &av_codec, + VideoCodecID2ffmpegDecoderID(info.codec), OPERATION_ENCODE)) { return -1; } + encode_context_->bit_rate = info.bitRate; + encode_context_->rc_min_rate = info.bitRate; + encode_context_->rc_max_rate = info.bitRate; // VPX_CBR + encode_context_->qmin = 0; + encode_context_->qmax = 40; // rc_quantifiers + encode_context_->profile = 3; + encode_context_->rc_initial_buffer_occupancy = 500; - vCoderContext = avcodec_alloc_context3(vCoder); - if (!vCoderContext) { - ELOG_DEBUG("Error allocating vCoderContext"); - return -2; - } - - vCoderContext->bit_rate = info.bitRate; - vCoderContext->rc_min_rate = info.bitRate; - vCoderContext->rc_max_rate = info.bitRate; // VPX_CBR - vCoderContext->qmin = 0; - vCoderContext->qmax = 40; // rc_quantifiers - vCoderContext->profile = 3; - vCoderContext->rc_initial_buffer_occupancy = 500; - - vCoderContext->rc_buffer_size = 1000; + encode_context_->rc_buffer_size = 1000; - vCoderContext->width = info.width; - vCoderContext->height = info.height; - vCoderContext->pix_fmt = AV_PIX_FMT_YUV420P; - vCoderContext->time_base = (AVRational) {1, 90000}; + encode_context_->width = info.width; + encode_context_->height = info.height; + encode_context_->pix_fmt = AV_PIX_FMT_YUV420P; + encode_context_->time_base = (AVRational) {1, 90000}; - vCoderContext->sample_aspect_ratio = (AVRational) { info.width, info.height }; - vCoderContext->thread_count = 4; + encode_context_->sample_aspect_ratio = (AVRational) { info.width, info.height }; + encode_context_->thread_count = 4; - if (avcodec_open2(vCoderContext, vCoder, NULL) < 0) { - ELOG_DEBUG("Error opening video decoder"); - return -3; + if (!coder_.openCodecContext(encode_context_, av_codec, NULL)) { + return -2; } cPicture = av_frame_alloc(); if (!cPicture) { ELOG_DEBUG("Error allocating video frame"); - return -4; + return -3; } - ELOG_DEBUG("videoCoder configured successfully %d x %d", vCoderContext->width, - vCoderContext->height); + ELOG_DEBUG("VideoEncoder configured successfully %d x %d", + encode_context_->width, encode_context_->height); + initialized = true; return 0; } -int VideoEncoder::encodeVideo(unsigned char* inBuffer, int inLength, unsigned char* outBuffer, int outLength) { - int ret; - int size = vCoderContext->width * vCoderContext->height; +void VideoEncoder::encodeVideoBuffer(unsigned char* inBuffer, int len, unsigned char* outBuffer, const EncodeVideoBufferCB &done) { + int size = encode_context_->width * encode_context_->height; cPicture->pts = AV_NOPTS_VALUE; cPicture->data[0] = inBuffer; cPicture->data[1] = inBuffer + size; cPicture->data[2] = inBuffer + size + size / 4; - cPicture->linesize[0] = vCoderContext->width; - cPicture->linesize[1] = vCoderContext->width / 2; - cPicture->linesize[2] = vCoderContext->width / 2; - - AVPacket pkt; - av_init_packet(&pkt); - pkt.data = outBuffer; - pkt.size = outLength; - - ret = avcodec_send_frame(vCoderContext, cPicture); - if (ret == 0) { - while (ret >= 0) { - ret = avcodec_receive_packet(vCoderContext, &pkt); - if (ret == AVERROR(EAGAIN)) { - ELOG_WARN("Encoder request more input. EAGAIN"); - return ret; - } else if (ret == AVERROR_EOF) { - ELOG_WARN("Encoder AVERROR_EOF."); - return ret; - } else if (ret < 0) { - ELOG_ERROR("Error encoding frame."); - return ret; - } - // ELOG_DEBUG("Encoded video size %u, ret %d, pts %lld, dts %lld", - // pkt.size, ret, pkt.pts, pkt.dts); - av_packet_unref(&pkt); - } - } - return pkt.size > 0 ? pkt.size : ret; + cPicture->linesize[0] = encode_context_->width; + cPicture->linesize[1] = encode_context_->width / 2; + cPicture->linesize[2] = encode_context_->width / 2; + + AVPacket *av_packet = av_packet_alloc(); + av_init_packet(av_packet); + av_packet->data = outBuffer; + + EncodeCB done_callback = [done](AVPacket *pkt, AVFrame *frame, bool got_packet) { + int len = pkt->size; + av_frame_free(&frame); + av_packet_free(&pkt); + done(got_packet, len); + }; + coder_.encode(encode_context_, cPicture, av_packet, done_callback); } int VideoEncoder::closeEncoder() { - if (vCoderContext != NULL) - avcodec_close(vCoderContext); + if (encode_context_ != NULL) + avcodec_close(encode_context_); if (cPicture != NULL) av_frame_free(&cPicture); - + initialized = false; return 0; } VideoDecoder::VideoDecoder() { avcodec_register_all(); - vDecoder = NULL; - vDecoderContext = NULL; + av_codec = NULL; + decode_context_ = NULL; dPicture = NULL; initWithContext_ = false; + initialized = false; } VideoDecoder::~VideoDecoder() { @@ -144,32 +123,25 @@ VideoDecoder::~VideoDecoder() { int VideoDecoder::initDecoder(const VideoCodecInfo& info) { ELOG_DEBUG("Init Decoder"); - vDecoder = avcodec_find_decoder(VideoCodecID2ffmpegDecoderID(info.codec)); - if (!vDecoder) { - ELOG_DEBUG("Error getting video decoder"); - return -1; - } - - vDecoderContext = avcodec_alloc_context3(vDecoder); - if (!vDecoderContext) { - ELOG_DEBUG("Error getting allocating decoder context"); + if (!coder_.allocCodecContext(&decode_context_, &av_codec, + VideoCodecID2ffmpegDecoderID(info.codec), OPERATION_DECODE)) { return -1; } - vDecoderContext->width = info.width; - vDecoderContext->height = info.height; + decode_context_->width = info.width; + decode_context_->height = info.height; - if (avcodec_open2(vDecoderContext, vDecoder, NULL) < 0) { - ELOG_DEBUG("Error opening video decoder"); - return -1; + if (!coder_.openCodecContext(decode_context_, av_codec, NULL)) { + return -2; } dPicture = av_frame_alloc(); if (!dPicture) { ELOG_DEBUG("Error allocating video frame"); - return -1; + return -3; } + initialized = true; return 0; } @@ -180,44 +152,37 @@ int VideoDecoder::initDecoder(AVCodecContext** context, AVCodecParameters *codec ELOG_DEBUG("Init Decoder context"); initWithContext_ = true; - vDecoder = avcodec_find_decoder(codecpar->codec_id); - if (!vDecoder) { - ELOG_DEBUG("Error getting video decoder"); - return -1; - } - - c = avcodec_alloc_context3(vDecoder); - if (!c) { - ELOG_ERROR("Could not allocate video codec context."); + if (!coder_.allocCodecContext(&c, &av_codec, + codecpar->codec_id, OPERATION_DECODE)) { return -1; } error = avcodec_parameters_to_context(c, codecpar); if (error < 0) { - ELOG_ERROR("Could copy parameters to context."); - return -1; + ELOG_ERROR("Could not copy parameters to context."); + return -2; } - if (avcodec_open2(c, vDecoder, NULL) < 0) { - ELOG_DEBUG("Error opening video decoder"); - return -1; + if (!coder_.openCodecContext(decode_context_, av_codec, NULL)) { + return -3; } *context = c; - vDecoderContext = *context; + decode_context_ = *context; dPicture = av_frame_alloc(); if (!dPicture) { ELOG_DEBUG("Error allocating video frame"); - return -1; + return -4; } + initialized = true; return 0; } -int VideoDecoder::decodeVideo(unsigned char* inBuff, int inBuffLen, +int VideoDecoder::decodeVideoBuffer(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, int outBuffLen, int* gotFrame) { - if (vDecoder == 0 || vDecoderContext == 0) { + if (av_codec == 0 || decode_context_ == 0) { ELOG_DEBUG("Init Codec First"); return -1; } @@ -230,27 +195,15 @@ int VideoDecoder::decodeVideo(unsigned char* inBuff, int inBuffLen, avpkt.data = inBuff; avpkt.size = inBuffLen; - int ret; - if (avpkt.size > 0) { - ret = avcodec_send_packet(vDecoderContext, &avpkt); - if (ret == 0) { - while (ret >= 0) { - ret = avcodec_receive_frame(vDecoderContext, dPicture); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { - return -1; - } else if (ret < 0) { - ELOG_DEBUG("Error decoding video frame"); - return -1; - } - *gotFrame = 1; - goto decoding; - } + if (coder_.decode(decode_context_, dPicture, &avpkt)) { + *gotFrame = 1; + goto decoding; } } decoding: - int outSize = vDecoderContext->height * vDecoderContext->width; + int outSize = decode_context_->height * decode_context_->width; if (outBuffLen < (outSize * 3 / 2)) { return outSize * 3 / 2; @@ -264,30 +217,30 @@ int VideoDecoder::decodeVideo(unsigned char* inBuff, int inBuffLen, int src_linesize, dst_linesize; src_linesize = dPicture->linesize[0]; - dst_linesize = vDecoderContext->width; + dst_linesize = decode_context_->width; src = dPicture->data[0]; - for (int i = vDecoderContext->height; i > 0; i--) { + for (int i = decode_context_->height; i > 0; i--) { memcpy(lum, src, dst_linesize); lum += dst_linesize; src += src_linesize; } src_linesize = dPicture->linesize[1]; - dst_linesize = vDecoderContext->width / 2; + dst_linesize = decode_context_->width / 2; src = dPicture->data[1]; - for (int i = vDecoderContext->height / 2; i > 0; i--) { + for (int i = decode_context_->height / 2; i > 0; i--) { memcpy(cromU, src, dst_linesize); cromU += dst_linesize; src += src_linesize; } src_linesize = dPicture->linesize[2]; - dst_linesize = vDecoderContext->width / 2; + dst_linesize = decode_context_->width / 2; src = dPicture->data[2]; - for (int i = vDecoderContext->height / 2; i > 0; i--) { + for (int i = decode_context_->height / 2; i > 0; i--) { memcpy(cromV, src, dst_linesize); cromV += dst_linesize; src += src_linesize; @@ -298,10 +251,11 @@ int VideoDecoder::decodeVideo(unsigned char* inBuff, int inBuffLen, } int VideoDecoder::closeDecoder() { - if (!initWithContext_ && vDecoderContext != NULL) - avcodec_close(vDecoderContext); + if (!initWithContext_ && decode_context_ != NULL) + avcodec_close(decode_context_); if (dPicture != NULL) av_frame_free(&dPicture); + initialized = false; return 0; } diff --git a/erizo/src/erizo/media/codecs/VideoCodec.h b/erizo/src/erizo/media/codecs/VideoCodec.h index abd83feebf..77e1db7580 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.h +++ b/erizo/src/erizo/media/codecs/VideoCodec.h @@ -5,7 +5,11 @@ #ifndef ERIZO_SRC_ERIZO_MEDIA_CODECS_VIDEOCODEC_H_ #define ERIZO_SRC_ERIZO_MEDIA_CODECS_VIDEOCODEC_H_ +#include + +#include "media/MediaInfo.h" #include "media/codecs/Codecs.h" +#include "media/codecs/Coder.h" #include "./logger.h" extern "C" { @@ -24,6 +28,8 @@ extern "C" { namespace erizo { +typedef std::function EncodeVideoBufferCB; + class VideoEncoder { DECLARE_LOGGER(); @@ -31,12 +37,15 @@ class VideoEncoder { VideoEncoder(); virtual ~VideoEncoder(); int initEncoder(const VideoCodecInfo& info); - int encodeVideo(unsigned char* inBuffer, int length, unsigned char* outBuffer, int outLength); + void encodeVideoBuffer(unsigned char* inBuffer, int len, unsigned char* outBuffer, + const EncodeVideoBufferCB &done); int closeEncoder(); + bool initialized; private: - AVCodec* vCoder; - AVCodecContext* vCoderContext; + Coder coder_; + AVCodec* av_codec; + AVCodecContext* encode_context_; AVFrame* cPicture; }; @@ -48,15 +57,17 @@ class VideoDecoder { virtual ~VideoDecoder(); int initDecoder(const VideoCodecInfo& info); int initDecoder(AVCodecContext** context, AVCodecParameters *codecpar); - int decodeVideo(unsigned char* inBuff, int inBuffLen, - unsigned char* outBuff, int outBuffLen, int* gotFrame); + int decodeVideoBuffer(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, + int outBuffLen, int* gotFrame); int closeDecoder(); + bool initialized; private: - AVCodec* vDecoder; - bool initWithContext_; - AVCodecContext* vDecoderContext; + Coder coder_; + AVCodec* av_codec; + AVCodecContext* decode_context_; AVFrame* dPicture; + bool initWithContext_; }; } // namespace erizo diff --git a/erizo_controller/erizoAgent/log4cxx.properties b/erizo_controller/erizoAgent/log4cxx.properties index 61bd827ac3..760706b2d0 100644 --- a/erizo_controller/erizoAgent/log4cxx.properties +++ b/erizo_controller/erizoAgent/log4cxx.properties @@ -34,7 +34,7 @@ log4j.logger.media.InputProcessor=WARN log4j.logger.media.OneToManyTranscoder=WARN log4j.logger.media.OutputProcessor=WARN - +log4j.logger.media.codecs.Coder=WARN log4j.logger.media.codecs.VideoEncoder=WARN log4j.logger.media.codecs.VideoDecoder=WARN log4j.logger.media.codecs.AudioEncoder=WARN From 62ba1e02636b34fa837bc9da5146617c3c65a944 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Wed, 23 May 2018 10:36:33 -0300 Subject: [PATCH 06/32] Add avresample lib to cmakelists and enable build --- erizo/src/CMakeLists.txt | 3 +++ scripts/installMacDeps.sh | 4 ++-- scripts/installUbuntuDeps.sh | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/erizo/src/CMakeLists.txt b/erizo/src/CMakeLists.txt index b50af4092a..1bb712b2bd 100644 --- a/erizo/src/CMakeLists.txt +++ b/erizo/src/CMakeLists.txt @@ -96,6 +96,9 @@ set (LIBS ${LIBS} ${AVCODEC}) find_library(AVFORMAT avformat HINTS "${THIRD_PARTY_LIB}") test_lib(${AVFORMAT}) set (LIBS ${LIBS} ${AVFORMAT}) +find_library(AVRESAMPLE avresample HINTS "${THIRD_PARTY_LIB}") +test_lib(${AVRESAMPLE}) +set (LIBS ${LIBS} ${AVRESAMPLE}) # LOG4CXX find_library(LOG log4cxx) diff --git a/scripts/installMacDeps.sh b/scripts/installMacDeps.sh index 52fbe47219..f805912d94 100755 --- a/scripts/installMacDeps.sh +++ b/scripts/installMacDeps.sh @@ -176,7 +176,7 @@ install_mediadeps(){ curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc && \ + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc --enable-avresample && \ make $FAST_MAKE -s V=0 && \ make install check_result $? @@ -194,7 +194,7 @@ install_mediadeps_nogpl(){ curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-libvpx --enable-libopus --disable-doc && \ + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-libvpx --enable-libopus --disable-doc --enable-avresample && \ make $FAST_MAKE -s V=0 && \ make install check_result $? diff --git a/scripts/installUbuntuDeps.sh b/scripts/installUbuntuDeps.sh index 29c9f630eb..e32c92c1be 100755 --- a/scripts/installUbuntuDeps.sh +++ b/scripts/installUbuntuDeps.sh @@ -160,7 +160,7 @@ install_mediadeps(){ curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc --enable-avresample make $FAST_MAKE -s V=0 make install cd $CURRENT_DIR @@ -179,7 +179,7 @@ install_mediadeps_nogpl(){ curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz tar -zxvf ffmpeg-3.4.2.tar.gz cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc --enable-avresample make $FAST_MAKE -s V=0 make install cd $CURRENT_DIR From 6fac4e7f980e690fbc9b94df1973339cd1d945c5 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Wed, 23 May 2018 12:36:40 -0300 Subject: [PATCH 07/32] Add av_str_2_err_cpp macro --- erizo/src/erizo/media/codecs/AudioCodec.cpp | 2 +- erizo/src/erizo/media/codecs/Coder.cpp | 10 +++++----- erizo/src/erizo/media/codecs/Coder.h | 8 ++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/erizo/src/erizo/media/codecs/AudioCodec.cpp b/erizo/src/erizo/media/codecs/AudioCodec.cpp index a990ef6243..d880aebcc5 100644 --- a/erizo/src/erizo/media/codecs/AudioCodec.cpp +++ b/erizo/src/erizo/media/codecs/AudioCodec.cpp @@ -78,7 +78,7 @@ void AudioEncoder::encodeAudioBuffer(unsigned char* inBuffer, int nSamples, AVPa (const uint8_t*)inBuffer, nSamples * 2, 0)) < 0) { - ELOG_ERROR("avcodec_fill_audio_frame failed: %s", av_err2str(ret)); + ELOG_ERROR("avcodec_fill_audio_frame failed: %s", av_err2str_cpp(ret)); return; } diff --git a/erizo/src/erizo/media/codecs/Coder.cpp b/erizo/src/erizo/media/codecs/Coder.cpp index 0d05e6ba21..f891e523ad 100644 --- a/erizo/src/erizo/media/codecs/Coder.cpp +++ b/erizo/src/erizo/media/codecs/Coder.cpp @@ -68,12 +68,12 @@ bool Coder::decode(AVCodecContext *decode_ctx, AVFrame *frame, AVPacket *av_pack int ret; ret = avcodec_send_packet(decode_ctx, av_packet); if (ret < 0) { - ELOG_ERROR("avcodec_send_packet failed. %d %s", ret, av_err2str(ret)); + ELOG_ERROR("avcodec_send_packet failed. %d %s", ret, av_err2str_cpp(ret)); return false; } ret = avcodec_receive_frame(decode_ctx, frame); if (ret != 0) { - ELOG_ERROR("avcodec_receive_frame. %d %s", ret, av_err2str(ret)); + ELOG_ERROR("avcodec_receive_frame. %d %s", ret, av_err2str_cpp(ret)); return false; } ELOG_DEBUG("decoded %s, nb_samples: %d, size: %d", decode_ctx->codec->name, @@ -86,14 +86,14 @@ void Coder::encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_pack int ret = avcodec_send_frame(encode_ctx, frame); if (ret < 0) { ELOG_ERROR("avcodec_send_frame failed for %s. %d %s", encode_ctx->codec->name, - ret, av_err2str(ret)); + ret, av_err2str_cpp(ret)); done(av_packet, frame, false); } while(ret >= 0) { ret = avcodec_receive_packet(encode_ctx, av_packet); if (ret == AVERROR_EOF) { ELOG_DEBUG("avcodec_receive_packet AVERROR_EOF, %s, ret: %d, %s", encode_ctx->codec->name, - ret, av_err2str(ret)) + ret, av_err2str_cpp(ret)) done(av_packet, frame, false); return; } else if (ret == AVERROR(EAGAIN)) { @@ -106,7 +106,7 @@ void Coder::encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_pack done(av_packet, frame, true); } else { ELOG_ERROR("avcodec_receive_packet failed. %s, %d %s", encode_ctx->codec->name, - ret, av_err2str(ret)); + ret, av_err2str_cpp(ret)); done(av_packet, frame, false); } } diff --git a/erizo/src/erizo/media/codecs/Coder.h b/erizo/src/erizo/media/codecs/Coder.h index b2aa1f4ca9..3ae0e9121a 100644 --- a/erizo/src/erizo/media/codecs/Coder.h +++ b/erizo/src/erizo/media/codecs/Coder.h @@ -21,6 +21,14 @@ extern "C" { #include } +inline static const std::string av_make_error_string_cpp(int errnum) { + char errbuf[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(errnum, errbuf, AV_ERROR_MAX_STRING_SIZE); + return (std::string)errbuf; +} + +#define av_err2str_cpp(errnum) av_make_error_string_cpp(errnum).c_str() + namespace erizo { typedef std::function EncodeCB; From 59ff231f3229fe4d91fd5c3140e53f22e6b95113 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Wed, 23 May 2018 14:57:47 -0300 Subject: [PATCH 08/32] Include chrono in Coder class --- erizo/src/erizo/media/codecs/Coder.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erizo/src/erizo/media/codecs/Coder.h b/erizo/src/erizo/media/codecs/Coder.h index 3ae0e9121a..97bd6deab4 100644 --- a/erizo/src/erizo/media/codecs/Coder.h +++ b/erizo/src/erizo/media/codecs/Coder.h @@ -2,9 +2,8 @@ #define ERIZO_SRC_ERIZO_MEDIA_CODECS_CODER_H_ #include -#include "./logger.h" - #include +#include extern "C" { #include @@ -21,6 +20,8 @@ extern "C" { #include } +#include "./logger.h" + inline static const std::string av_make_error_string_cpp(int errnum) { char errbuf[AV_ERROR_MAX_STRING_SIZE]; av_strerror(errnum, errbuf, AV_ERROR_MAX_STRING_SIZE); From 7700db04319ff74ef4861599189e24d736b96888 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Thu, 24 May 2018 10:38:49 -0300 Subject: [PATCH 09/32] ExternalOutput uses AV Coder classes * ExternalOutput transcode AV for required output. * AV encoder/decoder inherits from CoderEncoder, CoderDecoder. * CoderCodec is base class for CoderEncoder, CoderDecoder --- erizo/src/erizo/media/ExternalInput.cpp | 4 +- erizo/src/erizo/media/ExternalInput.h | 1 - erizo/src/erizo/media/ExternalOutput.cpp | 413 ++++++++++++------ erizo/src/erizo/media/ExternalOutput.h | 44 +- erizo/src/erizo/media/MediaProcessor.cpp | 8 +- erizo/src/erizo/media/codecs/AudioCodec.cpp | 228 +++++++--- erizo/src/erizo/media/codecs/AudioCodec.h | 36 +- erizo/src/erizo/media/codecs/Coder.cpp | 155 ++++++- erizo/src/erizo/media/codecs/Coder.h | 48 +- erizo/src/erizo/media/codecs/VideoCodec.cpp | 221 +++------- erizo/src/erizo/media/codecs/VideoCodec.h | 35 +- .../erizoAgent/log4cxx.properties | 7 +- 12 files changed, 761 insertions(+), 439 deletions(-) diff --git a/erizo/src/erizo/media/ExternalInput.cpp b/erizo/src/erizo/media/ExternalInput.cpp index 351e5f707c..6eb9f5007e 100644 --- a/erizo/src/erizo/media/ExternalInput.cpp +++ b/erizo/src/erizo/media/ExternalInput.cpp @@ -126,9 +126,9 @@ int ExternalInput::init() { op_->init(om, this); } else { needTranscoding_ = true; - inCodec_.initDecoder(&video_codec_ctx_, st->codecpar); + inCodec_.initDecoder(st->codecpar); - bufflen_ = video_codec_ctx_->width*video_codec_ctx_->height*3/2; + bufflen_ = inCodec_.codec_context_->width*inCodec_.codec_context_->height*3/2; decodedBuffer_.reset((unsigned char*) malloc(bufflen_)); diff --git a/erizo/src/erizo/media/ExternalInput.h b/erizo/src/erizo/media/ExternalInput.h index 93d2b5fb1b..a69f162545 100644 --- a/erizo/src/erizo/media/ExternalInput.h +++ b/erizo/src/erizo/media/ExternalInput.h @@ -52,7 +52,6 @@ class ExternalInput : public MediaSource, public RTPDataReceiver { std::queue packetQueue_; AVFormatContext* context_; AVPacket avpacket_; - AVCodecContext *video_codec_ctx_; int video_stream_index_, video_time_base_; int audio_stream_index_, audio_time_base_; int bufflen_; diff --git a/erizo/src/erizo/media/ExternalOutput.cpp b/erizo/src/erizo/media/ExternalOutput.cpp index 121fa65f98..32ce1b1808 100644 --- a/erizo/src/erizo/media/ExternalOutput.cpp +++ b/erizo/src/erizo/media/ExternalOutput.cpp @@ -23,12 +23,12 @@ ExternalOutput::ExternalOutput(std::shared_ptr worker, const std::string const std::vector rtp_mappings, const std::vector ext_mappings) : worker_{worker}, pipeline_{Pipeline::create()}, audio_queue_{5.0, 10.0}, video_queue_{5.0, 10.0}, - inited_{false}, video_stream_{nullptr}, - audio_stream_{nullptr}, video_source_ssrc_{0}, + initialized_context_{false}, video_source_ssrc_{0}, first_video_timestamp_{-1}, first_audio_timestamp_{-1}, first_data_received_{}, video_offset_ms_{-1}, audio_offset_ms_{-1}, - need_to_send_fir_{true}, rtp_mappings_{rtp_mappings}, video_codec_id_{AV_CODEC_ID_NONE}, - audio_codec_id_{AV_CODEC_ID_NONE}, pipeline_initialized_{false}, ext_processor_{ext_mappings} { + need_to_send_fir_{true}, rtp_mappings_{rtp_mappings}, input_video_codec_id_{AV_CODEC_ID_NONE}, + input_audio_codec_id_{AV_CODEC_ID_NONE}, need_video_transcode_{false}, need_audio_transcode_{false}, + need_audio_resample_{false}, ext_processor_{ext_mappings}, pipeline_initialized_{false} { ELOG_DEBUG("Creating output to %s", output_url.c_str()); fb_sink_ = nullptr; @@ -37,6 +37,7 @@ ExternalOutput::ExternalOutput(std::shared_ptr worker, const std::string // TODO(pedro): these should really only be called once per application run av_register_all(); avcodec_register_all(); + avformat_network_init(); fec_receiver_.reset(webrtc::UlpfecReceiver::Create(this)); stats_ = std::make_shared(); @@ -61,7 +62,7 @@ ExternalOutput::ExternalOutput(std::shared_ptr worker, const std::string } else { output_url.copy(context_->filename, sizeof(context_->filename), 0); - context_->oformat = av_guess_format(nullptr, context_->filename, nullptr); + context_->oformat = av_guess_format(nullptr, context_->filename, nullptr); if (!context_->oformat) { ELOG_ERROR("Error guessing format %s", context_->filename); } @@ -102,38 +103,33 @@ void ExternalOutput::close() { } void ExternalOutput::syncClose() { - if (!recording_) { - return; - } - // Stop our thread so we can safely nuke libav stuff and close our - // our file. cond_.notify_one(); thread_.join(); - if (audio_stream_ != nullptr && video_stream_ != nullptr && context_ != nullptr) { - av_write_trailer(context_); + if (context_ != nullptr) { + av_write_trailer(context_); } - - if (video_ctx_ != nullptr) { - avcodec_close(video_ctx_); - avcodec_free_context(&video_ctx_); + if (video_decoder_.initialized) { + video_decoder_.closeCodec(); } - - if (audio_ctx_ != nullptr) { - avcodec_close(audio_ctx_); - avcodec_free_context(&audio_ctx_); + if (video_encoder_.initialized) { + video_encoder_.closeCodec(); + } + if (audio_decoder_.initialized) { + audio_decoder_.closeCodec(); + } + if (audio_encoder_.initialized) { + audio_encoder_.closeCodec(); } - if (context_ != nullptr) { - avio_close(context_->pb); - avformat_free_context(context_); - context_ = nullptr; + avio_close(context_->pb); } + avformat_free_context(context_); pipeline_initialized_ = false; recording_ = false; - ELOG_DEBUG("Closed Successfully"); + ELOG_INFO("Closed Successfully"); } void ExternalOutput::asyncTask(std::function)> f) { @@ -179,27 +175,13 @@ void ExternalOutput::writeAudioData(char* buf, int len) { updateAudioCodec(map_iterator->second); } - initContext(); - - if (audio_stream_ == nullptr) { - // not yet. - return; - } - - long long current_timestamp = head->getTimestamp(); // NOLINT - if (current_timestamp - first_audio_timestamp_ < 0) { - // we wrapped. add 2^32 to correct this. We only handle a single wrap around - // since that's 13 hours of recording, minimum. - current_timestamp += 0xFFFFFFFF; + if (!initContext()) { + return; } - long long timestamp_to_write = (current_timestamp - first_audio_timestamp_) / // NOLINT - (audio_ctx_->sample_rate / audio_stream_->time_base.den); - // generally 48000 / 1000 for the denominator portion, at least for opus - // Adjust for our start time offset - - // in practice, our timebase den is 1000, so this operation is a no-op. - timestamp_to_write += audio_offset_ms_ / (1000 / audio_stream_->time_base.den); + int64_t current_timestamp = head->getTimestamp(); // NOLINT + int64_t timestamp_to_write = (current_timestamp - first_audio_timestamp_); + timestamp_to_write += av_rescale(audio_offset_ms_, 1000, audio_map_.clock_rate); AVPacket av_packet; av_init_packet(&av_packet); @@ -207,7 +189,27 @@ void ExternalOutput::writeAudioData(char* buf, int len) { av_packet.size = len - head->getHeaderLength(); av_packet.pts = timestamp_to_write; av_packet.stream_index = 1; - av_interleaved_write_frame(context_, &av_packet); // takes ownership of the packet + + if (av_packet.size > 0 && need_audio_transcode_) { + if (audio_decoder_.decode(audio_decoder_.frame_, &av_packet)) { + EncodeCB encode_callback = [this](AVPacket *pkt, AVFrame *frame, bool should_write){ + this->writePacket(pkt, audio_st_, should_write); + }; + if (need_audio_resample_) { + if (audio_encoder_.resampleAudioFrame(audio_decoder_.frame_, &audio_encoder_.resampled_frame_)) { + static int64_t i = 0; + audio_encoder_.resampled_frame_->pts = i; + i += audio_encoder_.resampled_frame_->nb_samples; + audio_encoder_.encode(audio_encoder_.resampled_frame_, &av_packet, encode_callback); + } + } else { + audio_encoder_.encode(audio_decoder_.frame_, &av_packet, encode_callback); + } + } + } else if (av_packet.size > 0) { + this->writePacket(&av_packet, audio_st_, true); + } + av_packet_unref(&av_packet); } void ExternalOutput::writeVideoData(char* buf, int len) { @@ -239,28 +241,28 @@ void ExternalOutput::writeVideoData(char* buf, int len) { } void ExternalOutput::updateAudioCodec(RtpMap map) { - if (audio_codec_id_ != AV_CODEC_ID_NONE) { + if (input_audio_codec_id_ != AV_CODEC_ID_NONE) { return; } audio_map_ = map; if (map.encoding_name == "opus") { - audio_codec_id_ = AV_CODEC_ID_OPUS; + input_audio_codec_id_ = AV_CODEC_ID_OPUS; } else if (map.encoding_name == "PCMU") { - audio_codec_id_ = AV_CODEC_ID_PCM_MULAW; + input_audio_codec_id_ = AV_CODEC_ID_PCM_MULAW; } } void ExternalOutput::updateVideoCodec(RtpMap map) { - if (video_codec_id_ != AV_CODEC_ID_NONE) { + if (input_video_codec_id_ != AV_CODEC_ID_NONE) { return; } video_map_ = map; if (map.encoding_name == "VP8") { depacketizer_.reset(new Vp8Depacketizer()); - video_codec_id_ = AV_CODEC_ID_VP8; + input_video_codec_id_ = AV_CODEC_ID_VP8; } else if (map.encoding_name == "H264") { depacketizer_.reset(new H264Depacketizer()); - video_codec_id_ = AV_CODEC_ID_H264; + input_video_codec_id_ = AV_CODEC_ID_H264; } } @@ -269,9 +271,8 @@ void ExternalOutput::maybeWriteVideoPacket(char* buf, int len) { depacketizer_->fetchPacket((unsigned char*)buf, len); bool deliver = depacketizer_->processPacket(); - initContext(); - if (video_stream_ == nullptr) { - // could not init our context yet. + if (!initContext()) { + depacketizer_->reset(); return; } @@ -283,14 +284,8 @@ void ExternalOutput::maybeWriteVideoPacket(char* buf, int len) { current_timestamp += 0xFFFFFFFF; } - // All of our video offerings are using a 90khz clock. - long long timestamp_to_write = (current_timestamp - first_video_timestamp_) / // NOLINT - (video_map_.clock_rate / video_stream_->time_base.den); - - // Adjust for our start time offset - - // in practice, our timebase den is 1000, so this operation is a no-op. - timestamp_to_write += video_offset_ms_ / (1000 / video_stream_->time_base.den); + long long timestamp_to_write = (current_timestamp - first_video_timestamp_);// NOLINT + timestamp_to_write += av_rescale(video_offset_ms_, 1000, video_map_.clock_rate); AVPacket av_packet; av_init_packet(&av_packet); @@ -298,11 +293,54 @@ void ExternalOutput::maybeWriteVideoPacket(char* buf, int len) { av_packet.size = depacketizer_->frameSize(); av_packet.pts = timestamp_to_write; av_packet.stream_index = 0; - av_interleaved_write_frame(context_, &av_packet); // takes ownership of the packet + if (depacketizer_->isKeyframe()) { + av_packet.flags |= AV_PKT_FLAG_KEY; + } + + if (av_packet.size > 0 && need_video_transcode_) { + if (video_decoder_.decode(video_decoder_.frame_, &av_packet)) { + auto done_callback = [this](AVPacket *pkt, AVFrame *frame, bool should_write){ + this->writePacket(pkt, video_st_, should_write); + }; + video_encoder_.encode(video_decoder_.frame_, &av_packet, done_callback); + } + } else { + this->writePacket(&av_packet, video_st_, av_packet.size > 0); + } + av_packet_unref(&av_packet); depacketizer_->reset(); } } +void ExternalOutput::writePacket(AVPacket *pkt, AVStream *st, bool should_write) { + const char *media_type; + if (st->id == 0) { + media_type = "video"; + av_packet_rescale_ts(pkt, AVRational{1, static_cast(video_map_.clock_rate)}, + video_st_->time_base); + } else { + media_type = "audio"; + av_packet_rescale_ts(pkt, audio_encoder_.codec_context_->time_base, audio_st_->time_base); + } + pkt->stream_index = st->id; + + int64_t pts = pkt->pts; + int64_t dts = pkt->dts; + int64_t dl = pkt->dts; + + if (should_write) { + int ret = av_interleaved_write_frame(context_, pkt); + if (ret != 0) { + ELOG_ERROR("av_interleaved_write_frame pts: %d failed with: %d %s", + pts, ret, av_err2str_cpp(ret)); + } else { + ELOG_DEBUG("Writed %s packet with pts: %lld, dts: %lld, duration: %lld", + media_type, pts, dts, dl); + } + } + av_packet_unref(pkt); +} + void ExternalOutput::notifyUpdateToHandlers() { asyncTask([] (std::shared_ptr output) { output->pipeline_->notifyUpdate(); @@ -371,83 +409,69 @@ int ExternalOutput::deliverEvent_(MediaEventPtr event) { } bool ExternalOutput::initContext() { - if (video_codec_id_ != AV_CODEC_ID_NONE && - audio_codec_id_ != AV_CODEC_ID_NONE && - video_stream_ == nullptr && - audio_stream_ == nullptr) { - video_codec = avcodec_find_encoder(video_codec_id_); - int ret = -1; - if (video_codec == nullptr) { - ELOG_ERROR("Could not find video codec"); - return false; - } + if (input_video_codec_id_ != AV_CODEC_ID_NONE && + input_audio_codec_id_ != AV_CODEC_ID_NONE && + initialized_context_ == false) { need_to_send_fir_ = true; video_queue_.setTimebase(video_map_.clock_rate); - video_ctx_ = avcodec_alloc_context3(video_codec); - if (!video_ctx_) { - ELOG_ERROR("Could not alloc video codec context"); - return false; - } - video_ctx_->pix_fmt = AV_PIX_FMT_YUV420P; - video_ctx_->width = 640; - video_ctx_->height = 480; - video_ctx_->time_base = (AVRational) { 1, 30 }; + if (avio_open(&context_->pb, context_->filename, AVIO_FLAG_WRITE) < 0) { + ELOG_ERROR("Error opening output context."); + exit(1); + } else { + AVCodecID output_video_codec_id = this->bestMatchOutputCodecId(input_video_codec_id_); + AVCodecID output_audio_codec_id = this->bestMatchOutputCodecId(input_audio_codec_id_); - video_stream_ = avformat_new_stream(context_, video_codec); - ret = avcodec_parameters_from_context(video_stream_->codecpar, video_ctx_); - if (ret != 0) { - ELOG_ERROR("Could not copy video codec paramaters"); - return false; - } - video_stream_->id = 0; - video_stream_->time_base = (AVRational) { 1, 30 }; - video_stream_->metadata = genVideoMetadata(); - if (context_->oformat->flags & AVFMT_GLOBALHEADER) { - video_ctx_->flags |= CODEC_FLAG_GLOBAL_HEADER; - } - context_->oformat->flags |= AVFMT_VARIABLE_FPS; + if (input_video_codec_id_ == output_video_codec_id) { + ELOG_INFO("Video input codec matches output codec, don't need to transcode."); + } else { + need_video_transcode_ = true; + } + if (input_audio_codec_id_ == output_audio_codec_id) { + ELOG_INFO("Audio input codec matches output codec, don't need to transcode."); + } else { + need_audio_transcode_ = true; + } - audio_codec = avcodec_find_encoder(audio_codec_id_); - if (audio_codec == nullptr) { - ELOG_ERROR("Could not find audio codec"); - return false; - } - audio_ctx_ = avcodec_alloc_context3(audio_codec); - if (!audio_ctx_) { - ELOG_ERROR("Could not alloc audio codec context"); - return false; - } - audio_ctx_->sample_rate = audio_map_.clock_rate; - audio_ctx_->time_base = (AVRational) { 1, audio_ctx_->sample_rate }; - audio_ctx_->channels = audio_map_.channels; - audio_stream_ = avformat_new_stream(context_, audio_codec); - ret = avcodec_parameters_from_context(audio_stream_->codecpar, audio_ctx_); - if (ret != 0) { - ELOG_ERROR("Could not copy audio codec paramaters"); - return false; - } - audio_stream_->id = 1; - audio_stream_->time_base = (AVRational) { 1, audio_ctx_->sample_rate }; + video_decoder_.initDecoder(input_video_codec_id_, + (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + this->setupVideoDecodingParams(context, dict); + }); - if (context_->oformat->flags & AVFMT_GLOBALHEADER) { - video_ctx_->flags |= CODEC_FLAG_GLOBAL_HEADER; - audio_ctx_->flags |= CODEC_FLAG_GLOBAL_HEADER; - } - context_->oformat->flags |= AVFMT_VARIABLE_FPS; - context_->streams[0] = video_stream_; - context_->streams[1] = audio_stream_; - if (avio_open(&context_->pb, context_->filename, AVIO_FLAG_WRITE) < 0) { - ELOG_ERROR("Error opening output file"); - return false; - } + audio_decoder_.initDecoder(input_audio_codec_id_, + (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + this->setupAudioDecodingParams(context, dict); + }); + + video_encoder_.initEncoder(output_video_codec_id, + (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + this->setupVideoEncodingParams(context, dict); + }); + + audio_encoder_.initEncoder(output_audio_codec_id, + (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + this->setupAudioEncodingParams(context, dict); + }); + + video_st_ = this->addOutputStream(0, &video_encoder_); + audio_st_ = this->addOutputStream(1, &audio_encoder_); - if (avformat_write_header(context_, nullptr) < 0) { - ELOG_ERROR("Error writing header"); - return false; + if (this->audioNeedsResample()) { + ELOG_INFO("Encode Audio needs resampling."); + need_audio_resample_ = true; + } + if (need_audio_resample_ && + !audio_encoder_.initAudioResampler(audio_decoder_.codec_context_)) { + exit(1); + } + + if (!this->writeContextHeader()) { + this->close(); + exit(1); + } + initialized_context_ = true; } } - - return true; + return initialized_context_; } void ExternalOutput::queueData(char* buffer, int length, packetType type) { @@ -465,7 +489,7 @@ void ExternalOutput::queueData(char* buffer, int length, packetType type) { if (getAudioSinkSSRC() == 0) { ELOG_DEBUG("No audio detected"); audio_map_ = RtpMap{0, "PCMU", 8000, AUDIO_TYPE, 1}; - audio_codec_id_ = AV_CODEC_ID_PCM_MULAW; + input_audio_codec_id_ = AV_CODEC_ID_PCM_MULAW; } } if (need_to_send_fir_ && video_source_ssrc_) { @@ -554,9 +578,6 @@ void ExternalOutput::sendLoop() { boost::shared_ptr video_packet = video_queue_.popPacket(); writeVideoData(video_packet->data, video_packet->length); } - if (!inited_ && first_data_received_ != time_point()) { - inited_ = true; - } } // Since we're bailing, let's completely drain our queues of all data. @@ -570,6 +591,125 @@ void ExternalOutput::sendLoop() { } } +AVStream * ExternalOutput::addOutputStream(int index, CoderCodec *coder_codec) { + AVStream *st; + const char *codec_name = coder_codec->av_codec_->name; + st = avformat_new_stream(context_, coder_codec->av_codec_); + if (!st) { + ELOG_ERROR("Could not create stream for %s.", codec_name); + return nullptr; + } else { + int ret = avcodec_parameters_from_context(st->codecpar, coder_codec->codec_context_); + if (ret != 0) { + ELOG_ERROR("Could not copy codec paramaters for %s.", codec_name); + return nullptr; + } + st->id = index; + st->time_base = coder_codec->codec_context_->time_base; + context_->streams[index] = st; + ELOG_INFO("Created stream for codec: %s with index: %d", codec_name, index); + return st; + } +} + +bool ExternalOutput::writeContextHeader() { + AVDictionary *opt = NULL; + int ret = avformat_write_header(context_, &opt); + av_dict_free(&opt); + if (ret < 0) { + static char error_str[255]; + av_strerror(ret, error_str, sizeof(error_str)); + ELOG_ERROR("Error writing header: %s", error_str); + return false; + } + return true; +} + +void ExternalOutput::setupAudioEncodingParams(AVCodecContext *context, AVDictionary *dict) { + AVSampleFormat sample_fmt = context->codec->sample_fmts ? + context->codec->sample_fmts[0] : AV_SAMPLE_FMT_S16; + context->sample_fmt = sample_fmt; + context->bit_rate = 64000; + context->sample_rate = audio_encoder_.getBestSampleRate(context->codec); + context->time_base = (AVRational) { 1, context->sample_rate }; + context->channel_layout = audio_encoder_.getChannelLayout(context->codec); + context->channels = av_get_channel_layout_nb_channels(context->channel_layout); + + if (context->codec->id == AV_CODEC_ID_AAC) { + context->sample_rate = 48000; + context->time_base = (AVRational) { 1, context->sample_rate }; +#ifdef FF_PROFILE_AAC_MAIN + context->profile = FF_PROFILE_AAC_MAIN; +#endif + av_dict_set(&dict, "strict", "experimental", 0); + } + if (context_->oformat->flags & AVFMT_GLOBALHEADER) { + context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + } +} + +void ExternalOutput::setupAudioDecodingParams(AVCodecContext *context, AVDictionary *dict) { + context->sample_rate = audio_map_.clock_rate; + context->time_base = (AVRational) { 1, context->sample_rate }; + context->channels = audio_map_.channels; + context->channel_layout = AV_CH_LAYOUT_STEREO; +} + +void ExternalOutput::setupVideoEncodingParams(AVCodecContext *context, AVDictionary *dict) { + context->gop_size = 45; + context->bit_rate = 400 * 1000; + context->width = 640; + context->height = 480; + context->framerate = (AVRational){25, 1}; + context->time_base = (AVRational){1, 25}; + context->pix_fmt = AV_PIX_FMT_YUV420P; + + if (context->codec->id == AV_CODEC_ID_VP8 || context->codec->id == AV_CODEC_ID_VP9) { + ELOG_DEBUG("Setting VPX params"); + } + if (context_->oformat->flags & AVFMT_GLOBALHEADER) { + context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + } +} + +void ExternalOutput::setupVideoDecodingParams(AVCodecContext *context, AVDictionary *dict) { + context->width = 640; + context->height = 480; + context->framerate = (AVRational){0, 1}; + context->pix_fmt = AV_PIX_FMT_YUV420P; +} + +AVCodecID ExternalOutput::bestMatchOutputCodecId(AVCodecID input_codec_id) { + AVMediaType type = avcodec_get_type(input_codec_id); + if (av_codec_get_tag(context_->oformat->codec_tag, input_codec_id) != 0) { + return input_codec_id; + } else { + if (type == AVMEDIA_TYPE_VIDEO) { + return context_->oformat->video_codec; + } else if (type == AVMEDIA_TYPE_AUDIO) { + return context_->oformat->audio_codec; + } else { + return AV_CODEC_ID_NONE; + } + } +} + +bool ExternalOutput::audioNeedsResample() { + if (AV_SAMPLE_FMT_S16 != audio_encoder_.codec_context_->sample_fmt) { + return true; + } + if (audio_decoder_.codec_context_->sample_rate != audio_encoder_.codec_context_->sample_rate) { + return true; + } + if (audio_decoder_.codec_context_->channels != audio_encoder_.codec_context_->channels) { + return true; + } + if (audio_decoder_.codec_context_->channel_layout != audio_encoder_.codec_context_->channel_layout) { + return true; + } + return false; +} + AVDictionary* ExternalOutput::genVideoMetadata() { AVDictionary* dict = NULL; switch (ext_processor_.getVideoRotation()) { @@ -591,4 +731,5 @@ AVDictionary* ExternalOutput::genVideoMetadata() { } return dict; } + } // namespace erizo diff --git a/erizo/src/erizo/media/ExternalOutput.h b/erizo/src/erizo/media/ExternalOutput.h index ba107c3782..41cd7954ab 100644 --- a/erizo/src/erizo/media/ExternalOutput.h +++ b/erizo/src/erizo/media/ExternalOutput.h @@ -6,6 +6,15 @@ extern "C" { #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include } #include @@ -23,6 +32,8 @@ extern "C" { #include "rtp/QualityManager.h" #include "pipeline/Handler.h" #include "pipeline/HandlerManager.h" +#include "media/codecs/AudioCodec.h" +#include "media/codecs/VideoCodec.h" #include "./logger.h" @@ -62,13 +73,17 @@ class ExternalOutput : public MediaSink, public RawDataReceiver, public Feedback Pipeline::Ptr pipeline_; std::unique_ptr fec_receiver_; RtpPacketQueue audio_queue_, video_queue_; - std::atomic recording_, inited_; + std::atomic recording_, initialized_context_; boost::mutex mtx_; // a mutex we use to signal our writer thread that data is waiting. boost::thread thread_; boost::condition_variable cond_; - AVStream *video_stream_, *audio_stream_; + AVStream *video_st_; + AVStream *audio_st_; AVFormatContext *context_; - + VideoDecoder video_decoder_; + VideoEncoder video_encoder_; + AudioDecoder audio_decoder_; + AudioEncoder audio_encoder_; uint32_t video_source_ssrc_; std::unique_ptr depacketizer_; @@ -107,21 +122,20 @@ class ExternalOutput : public MediaSink, public RawDataReceiver, public Feedback // so the second scheme seems not applicable. Too bad. bool need_to_send_fir_; std::vector rtp_mappings_; - enum AVCodecID video_codec_id_; - enum AVCodecID audio_codec_id_; - AVCodec *video_codec; - AVCodec *audio_codec; - AVCodecContext *video_ctx_; - AVCodecContext *audio_ctx_; + enum AVCodecID input_video_codec_id_; + enum AVCodecID input_audio_codec_id_; + bool need_video_transcode_; + bool need_audio_transcode_; + bool need_audio_resample_; std::map video_maps_; std::map audio_maps_; RtpMap video_map_; RtpMap audio_map_; + RtpExtensionProcessor ext_processor_; bool pipeline_initialized_; std::shared_ptr stats_; std::shared_ptr quality_manager_; std::shared_ptr handler_manager_; - RtpExtensionProcessor ext_processor_; bool initContext(); int sendFirPacket(); @@ -140,6 +154,15 @@ class ExternalOutput : public MediaSink, public RawDataReceiver, public Feedback void initializePipeline(); void syncClose(); AVDictionary* genVideoMetadata(); + void writePacket(AVPacket *av_packet, AVStream *st, bool should_write); + bool writeContextHeader(); + void setupVideoDecodingParams(AVCodecContext *context, AVDictionary *dict); + void setupAudioDecodingParams(AVCodecContext *context, AVDictionary *dict); + void setupVideoEncodingParams(AVCodecContext *context, AVDictionary *dict); + void setupAudioEncodingParams(AVCodecContext *context, AVDictionary *dict); + AVStream * addOutputStream(int index, CoderCodec *coder_codec); + AVCodecID bestMatchOutputCodecId(AVCodecID input_codec_id); + bool audioNeedsResample(); }; class ExternalOuputWriter : public OutboundHandler { @@ -165,6 +188,5 @@ class ExternalOuputWriter : public OutboundHandler { private: std::weak_ptr output_; }; - } // namespace erizo #endif // ERIZO_SRC_ERIZO_MEDIA_EXTERNALOUTPUT_H_ diff --git a/erizo/src/erizo/media/MediaProcessor.cpp b/erizo/src/erizo/media/MediaProcessor.cpp index 575ab8cb74..c0fbd466fe 100644 --- a/erizo/src/erizo/media/MediaProcessor.cpp +++ b/erizo/src/erizo/media/MediaProcessor.cpp @@ -185,8 +185,8 @@ void InputProcessor::closeSink() { } void InputProcessor::close() { - audio_decoder_.closeDecoder(); - video_decoder_.closeDecoder(); + audio_decoder_.closeCodec(); + video_decoder_.closeCodec(); free(decodedBuffer_); decodedBuffer_ = NULL; free(unpackagedBuffer_); unpackagedBuffer_ = NULL; free(unpackagedAudioBuffer_); unpackagedAudioBuffer_ = NULL; @@ -248,8 +248,8 @@ int OutputProcessor::init(const MediaInfo& info, RTPDataReceiver* rtpReceiver) { } void OutputProcessor::close() { - audio_encoder_.closeEncoder(); - video_encoder_.closeEncoder(); + audio_encoder_.closeCodec(); + video_encoder_.closeCodec(); free(encodedBuffer_); encodedBuffer_ = NULL; free(packagedBuffer_); packagedBuffer_ = NULL; free(rtpBuffer_); rtpBuffer_ = NULL; diff --git a/erizo/src/erizo/media/codecs/AudioCodec.cpp b/erizo/src/erizo/media/codecs/AudioCodec.cpp index d880aebcc5..6fb1caabc0 100644 --- a/erizo/src/erizo/media/codecs/AudioCodec.cpp +++ b/erizo/src/erizo/media/codecs/AudioCodec.cpp @@ -24,45 +24,41 @@ inline AVCodecID AudioCodecID2ffmpegDecoderID(AudioCodecID codec) { } AudioEncoder::AudioEncoder() { - encode_codec_ = NULL; - encode_context_ = NULL; - encoded_frame_ = NULL; - initialized = false; + resampled_frame_ = av_frame_alloc(); + if (!resampled_frame_) + ELOG_ERROR("Could not allocate a frame for resampling."); } AudioEncoder::~AudioEncoder() { ELOG_DEBUG("AudioEncoder Destructor"); - this->closeEncoder(); + this->closeCodec(); } int AudioEncoder::initEncoder(const AudioCodecInfo& mediaInfo) { - if (!coder_.allocCodecContext(&encode_context_, &encode_codec_, - AudioCodecID2ffmpegDecoderID(mediaInfo.codec), OPERATION_ENCODE)) { - return -1; - } - - encode_context_->sample_fmt = AV_SAMPLE_FMT_FLT; - // encode_context_->bit_rate = mediaInfo.bitRate; - encode_context_->sample_rate = 8 /*mediaInfo.sampleRate*/; - encode_context_->channels = 1; + const InitContextCB callback = [mediaInfo](AVCodecContext *context, AVDictionary *dict) { + context->sample_fmt = AV_SAMPLE_FMT_FLT; + // codec_context_->bit_rate = mediaInfo.bitRate; + context->sample_rate = 8 /*mediaInfo.sampleRate*/; + context->channels = 1; + }; - initialized = coder_.openCodecContext(encode_context_, encode_codec_, NULL); - return initialized; + AVCodecID av_codec_id = AudioCodecID2ffmpegDecoderID(mediaInfo.codec); + return this->initEncoder(av_codec_id, callback); } -int AudioEncoder::closeEncoder() { - if (encode_context_ != nullptr) { - avcodec_close(encode_context_); - avcodec_free_context(&encode_context_); +int AudioEncoder::closeCodec() { + CoderEncoder::closeCodec(); + if (resampled_frame_ != nullptr) { + av_frame_free(&resampled_frame_); } - if (encoded_frame_ != nullptr) { - av_frame_free(&encoded_frame_); + if (audio_fifo_ != nullptr) { + av_audio_fifo_free(audio_fifo_); } return 0; } void AudioEncoder::encodeAudioBuffer(unsigned char* inBuffer, int nSamples, AVPacket* pkt, - EncodeAudioBufferCB &done) { + const EncodeAudioBufferCB &done) { AVFrame *frame = av_frame_alloc(); if (!frame) { ELOG_ERROR("Could not allocate audio frame for encoding."); @@ -70,11 +66,11 @@ void AudioEncoder::encodeAudioBuffer(unsigned char* inBuffer, int nSamples, AVPa } int ret; - frame->nb_samples = encode_context_->frame_size; + frame->nb_samples = codec_context_->frame_size; if ((ret = avcodec_fill_audio_frame(frame, - encode_context_->channels, - encode_context_->sample_fmt, + codec_context_->channels, + codec_context_->sample_fmt, (const uint8_t*)inBuffer, nSamples * 2, 0)) < 0) { @@ -86,34 +82,161 @@ void AudioEncoder::encodeAudioBuffer(unsigned char* inBuffer, int nSamples, AVPa av_frame_free(&frame); done(pkt, got_packet); }; - coder_.encode(encode_context_, frame, pkt, done_callback); + this->encode(frame, pkt, done_callback); } -AudioDecoder::AudioDecoder() { - decode_codec_ = NULL; - decode_context_ = NULL; - dFrame_ = NULL; - initialized = false; +bool AudioEncoder::resampleAudioFrame(AVFrame *frame, AVFrame **o_frame) { + uint8_t **converted_input_samples = NULL; + int ret; + int out_linesize; + bool got_output = false; + AVFrame *out_frame = resampled_frame_; + + if (this->initConvertedSamples(&converted_input_samples, frame->nb_samples, &out_linesize)) { + if ((ret = avresample_convert(avr_context_, + converted_input_samples, + out_linesize, + codec_context_->frame_size, + frame->extended_data, + frame->linesize[0], + frame->nb_samples)) < 0) { + ELOG_ERROR("Could not convert input samples (error '%s')", av_err2str_cpp(ret)); + } else if (ret == 0) { + ELOG_WARN("0 Audio samples converted."); + } else if (ret > 0) { + ELOG_DEBUG("%d Audio samples converted.", ret); + this->addSamplesToFifo(converted_input_samples, ret); + if (av_audio_fifo_size(audio_fifo_) >= codec_context_->frame_size) { + out_frame->nb_samples = codec_context_->frame_size; + out_frame->channel_layout = codec_context_->channel_layout; + out_frame->format = codec_context_->sample_fmt; + out_frame->sample_rate = codec_context_->sample_rate; + if (av_frame_get_buffer(out_frame, 0) < 0) { + ELOG_ERROR("Could not allocate output frame samples"); + } else { + if (av_audio_fifo_read(audio_fifo_, reinterpret_cast(out_frame->data), + codec_context_->frame_size) != codec_context_->frame_size) { + ELOG_ERROR("Could not read data from audio FIFO."); + } else { + if (av_frame_make_writable(out_frame) != 0) { + ELOG_ERROR("Audio frame not writeable!"); + } else { + got_output = true; + } + } + } + } + } + } + *o_frame = out_frame; + return got_output; } -AudioDecoder::~AudioDecoder() { - ELOG_DEBUG("AudioDecoder Destructor"); - this->closeDecoder(); +bool AudioEncoder::initAudioResampler(AVCodecContext *decode_ctx) { + avr_context_ = avresample_alloc_context(); + if (!avr_context_) { + ELOG_ERROR("Could not allocate resampler context"); + return false; + } + av_opt_set_int(avr_context_, "in_channel_layout", decode_ctx->channel_layout, 0); + av_opt_set_int(avr_context_, "in_sample_rate", decode_ctx->sample_rate, 0); + av_opt_set_int(avr_context_, "in_sample_fmt", decode_ctx->sample_fmt, 0); + av_opt_set_int(avr_context_, "out_channel_layout", codec_context_->channel_layout, 0); + av_opt_set_int(avr_context_, "out_sample_rate", codec_context_->sample_rate, 0); + av_opt_set_int(avr_context_, "out_sample_fmt", codec_context_->sample_fmt, 0); + + if (avresample_open(avr_context_) < 0) { + ELOG_ERROR("Failed to initialize resampling context"); + return false; + } + + if (!(audio_fifo_ = av_audio_fifo_alloc(codec_context_->sample_fmt, + codec_context_->channels, 1))) { + ELOG_ERROR("Could not allocate audio fifo"); + return false; + } + ELOG_DEBUG("Initialized Audio resample context."); + return true; } -int AudioDecoder::initDecoder(const AudioCodecInfo& info) { - if (!coder_.allocCodecContext(&decode_context_, &decode_codec_, - AudioCodecID2ffmpegDecoderID(info.codec), OPERATION_DECODE)) { - return -1; +bool AudioEncoder::initConvertedSamples(uint8_t ***converted_input_samples, int frame_size, int *out_linesize) { + int ret; + static char error_str[255]; + + if (!(*converted_input_samples = reinterpret_cast(calloc(codec_context_->channels, + sizeof(**converted_input_samples))))) { + ELOG_ERROR("Could not allocate converted input sample pointers"); + return false; + } + + if ((ret = av_samples_alloc(*converted_input_samples, out_linesize, + codec_context_->channels, + frame_size, + codec_context_->sample_fmt, 0)) < 0) { + ELOG_ERROR("Could not allocate converted input samples: %s", + av_strerror(ret, error_str, sizeof(error_str))); + av_freep(&(*converted_input_samples)[0]); + free(*converted_input_samples); + return false; + } + return true; +} + +bool AudioEncoder::addSamplesToFifo(uint8_t **converted_input_samples, const int frame_size) { + if (av_audio_fifo_realloc(audio_fifo_, av_audio_fifo_size(audio_fifo_) + frame_size) < 0) { + ELOG_ERROR("Could not reallocate FIFO."); + return false; + } + if (av_audio_fifo_write(audio_fifo_, reinterpret_cast(converted_input_samples), + frame_size) < frame_size) { + ELOG_ERROR("Could not write data to FIFO"); + return false; + } + return true; +} + +int AudioEncoder::getBestSampleRate(const AVCodec *codec) { + const int *p; + int best_samplerate = 44100; + if (!codec->supported_samplerates) { + return best_samplerate; + } + p = codec->supported_samplerates; + while (*p) { + best_samplerate = FFMAX(*p, best_samplerate); + p++; + } + return best_samplerate; +} + +uint64_t AudioEncoder::getChannelLayout(const AVCodec *codec) { + const uint64_t *p; + uint64_t best_ch_layout = 0; + uint64_t best_nb_channels = 0; + if (!codec->channel_layouts) + return AV_CH_LAYOUT_STEREO; + p = codec->channel_layouts; + while (*p) { + uint64_t nb_channels = av_get_channel_layout_nb_channels(*p); + if (nb_channels > best_nb_channels) { + best_ch_layout = *p; + best_nb_channels = nb_channels; + } + p++; } + return best_ch_layout; +} - decode_context_->sample_fmt = AV_SAMPLE_FMT_S16; - decode_context_->bit_rate = info.bitRate; - decode_context_->sample_rate = info.sampleRate; - decode_context_->channels = 1; +int AudioDecoder::initDecoder(const AudioCodecInfo& info) { + const InitContextCB callback = [info](AVCodecContext *context, AVDictionary *dict) { + context->sample_fmt = AV_SAMPLE_FMT_S16; + context->bit_rate = info.bitRate; + context->sample_rate = info.sampleRate; + context->channels = 1; + }; - initialized = coder_.openCodecContext(decode_context_, decode_codec_, NULL); - return initialized; + AVCodecID av_codec_id = AudioCodecID2ffmpegDecoderID(info.codec); + return this->initDecoder(av_codec_id, callback); } int AudioDecoder::decodeAudio(unsigned char* inBuff, int inBuffLen, @@ -128,9 +251,9 @@ int AudioDecoder::decodeAudio(unsigned char* inBuff, int inBuffLen, AVFrame *frame; frame = av_frame_alloc(); - if (coder_.decode(decode_context_, frame, &avpkt)) { + if (this->decode(frame, &avpkt)) { // Asume S16 Non-Planar Audio for now. - int planar = av_sample_fmt_is_planar(decode_context_->sample_fmt); + int planar = av_sample_fmt_is_planar(codec_context_->sample_fmt); if (planar == 0) { out_size = av_samples_get_buffer_size(NULL, av_frame_get_channels(frame), @@ -151,15 +274,4 @@ int AudioDecoder::decodeAudio(unsigned char* inBuff, int inBuffLen, av_frame_free(&frame); return out_size; } - -int AudioDecoder::closeDecoder() { - if (decode_context_ != NULL) { - avcodec_close(decode_context_); - } - if (dFrame_ != NULL) { - av_frame_free(&dFrame_); - } - initialized = false; - return 0; -} } // namespace erizo diff --git a/erizo/src/erizo/media/codecs/AudioCodec.h b/erizo/src/erizo/media/codecs/AudioCodec.h index be92121290..5d9bdf0aa2 100644 --- a/erizo/src/erizo/media/codecs/AudioCodec.h +++ b/erizo/src/erizo/media/codecs/AudioCodec.h @@ -19,41 +19,39 @@ namespace erizo { typedef std::function EncodeAudioBufferCB; -class AudioEncoder { +class AudioEncoder : public CoderEncoder { DECLARE_LOGGER(); public: AudioEncoder(); virtual ~AudioEncoder(); + using CoderEncoder::initEncoder; int initEncoder(const AudioCodecInfo& info); + int closeCodec() override; + bool initAudioResampler(AVCodecContext *decode_ctx); void encodeAudioBuffer(unsigned char* inBuffer, int nSamples, AVPacket* pkt, - EncodeAudioBufferCB &done); - int closeEncoder(); - bool initialized; + const EncodeAudioBufferCB &done); + bool resampleAudioFrame(AVFrame *frame, AVFrame **out_frame); + bool initConvertedSamples(uint8_t ***converted_input_samples, int frame_size, int *out_linesize); + bool addSamplesToFifo(uint8_t **converted_input_samples, const int frame_size); + uint64_t getChannelLayout(const AVCodec *codec); + int getBestSampleRate(const AVCodec *codec); + + public: + AVFrame *resampled_frame_; private: - Coder coder_; - AVCodec *encode_codec_; - AVCodecContext *encode_context_; - AVFrame *encoded_frame_; + AVAudioFifo *audio_fifo_; + AVAudioResampleContext *avr_context_; }; -class AudioDecoder { +class AudioDecoder : public CoderDecoder { DECLARE_LOGGER(); public: - AudioDecoder(); - virtual ~AudioDecoder(); + using CoderDecoder::initDecoder; int initDecoder(const AudioCodecInfo& info); int decodeAudio(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, int outBuffLen); - int closeDecoder(); - bool initialized; - - private: - Coder coder_; - AVCodec* decode_codec_; - AVCodecContext* decode_context_; - AVFrame* dFrame_; }; } // namespace erizo diff --git a/erizo/src/erizo/media/codecs/Coder.cpp b/erizo/src/erizo/media/codecs/Coder.cpp index f891e523ad..d01ffb7596 100644 --- a/erizo/src/erizo/media/codecs/Coder.cpp +++ b/erizo/src/erizo/media/codecs/Coder.cpp @@ -2,33 +2,50 @@ namespace erizo { -DEFINE_LOGGER(Coder, "media.codecs.Coder"); +DEFINE_LOGGER(Coder, "media.codecs.Coder"); +DEFINE_LOGGER(CoderCodec, "media.codecs.CoderCodec"); +DEFINE_LOGGER(CoderEncoder, "media.codecs.CoderEncoder"); +DEFINE_LOGGER(CoderDecoder, "media.codecs.CoderDecoder"); -Coder::Coder() { -} - -Coder::~Coder() { +bool Coder::initContext(AVCodecContext **codec_ctx, AVCodec **av_codec, AVCodecID codec_id, + CoderOperationType operation, InitContextCB callback) { + bool success = false; + AVDictionary *opt = NULL; + const char *codec_name = avcodec_get_name(codec_id); + const char *codec_for = operation == ENCODE_AV ? "encode" : "decode"; + if (this->allocCodecContext(codec_ctx, av_codec, codec_id, operation)) { + callback(*codec_ctx, opt); + bool success = this->openCodecContext(*codec_ctx, *av_codec, opt); + if (success) { + ELOG_INFO("Successfully initialized %s context for %s.", codec_for, codec_name); + } else { + ELOG_ERROR("Could not initialize %s context for %s.", codec_for, codec_name); + } + } + av_dict_free(&opt); + return success; } bool Coder::allocCodecContext(AVCodecContext **ctx, AVCodec **c, AVCodecID codec_id, CoderOperationType operation) { - AVCodec *codec = nullptr; - AVCodecContext *codec_ctx = *ctx; - if (operation == OPERATION_ENCODE) { + AVCodec *codec; + AVCodecContext *codec_ctx; + const char *codec_for; + if (operation == ENCODE_AV) { codec = avcodec_find_encoder(codec_id); + codec_for = "encoding"; } else { codec = avcodec_find_decoder(codec_id); + codec_for = "decoding"; } - if (codec == nullptr) { - ELOG_ERROR("Cannot find codec %d", codec_id); + if (!codec) { + ELOG_ERROR("Cannot find codec %d for %s", codec_id, codec_for); return false; } codec_ctx = avcodec_alloc_context3(codec); if (!codec_ctx) { ELOG_ERROR("Could not allocate codec context for %s", codec->name); return false; - } else { - ELOG_INFO("Successfully allocated context for codec %s.", codec->name); } *c = codec; *ctx = codec_ctx; @@ -53,10 +70,11 @@ bool Coder::openCodecContext(AVCodecContext *codec_ctx, AVCodec *codec, AVDictio void Coder::logCodecContext(AVCodecContext *codec_ctx) { const char *operation = (av_codec_is_encoder(codec_ctx->codec) ? "encoder" : "decoder"); if (codec_ctx->codec->type == AVMEDIA_TYPE_AUDIO) { - ELOG_DEBUG("\nAudio %s codec: %s \nchannel_layout: %d\nchannels: %d\nframe_size: %d\nsample_rate: %d\nsample_fmt: %s\nbits per sample: %d", + ELOG_DEBUG("\nAudio %s codec: %s \nchannel_layout: %d\nchannels: %d\nframe_size: %d\ntime_base: %d/%d\n, sample_rate: " + "%d\nsample_fmt: %s\nbits per sample: %d", operation, codec_ctx->codec->name, codec_ctx->channel_layout, codec_ctx->channels, codec_ctx->frame_size, - codec_ctx->sample_rate, av_get_sample_fmt_name(codec_ctx->sample_fmt), - av_get_bytes_per_sample (codec_ctx->sample_fmt)); + codec_ctx->time_base.num, codec_ctx->time_base.den, codec_ctx->sample_rate, + av_get_sample_fmt_name(codec_ctx->sample_fmt), av_get_bytes_per_sample(codec_ctx->sample_fmt)); } else if (codec_ctx->codec->type == AVMEDIA_TYPE_VIDEO) { ELOG_DEBUG("\nVideo %s codec: %s\n framerate: {%d/%d}\n time_base: {%d/%d}\n", operation, codec_ctx->codec->name, codec_ctx->framerate.num, codec_ctx->framerate.den, @@ -64,6 +82,12 @@ void Coder::logCodecContext(AVCodecContext *codec_ctx) { } } +void Coder::logAVStream(AVStream *st) { + ELOG_DEBUG("Stream idx: %d id: %d: codec: %s, start: %d, time_base: %d/%d", + st->index, st->id, avcodec_get_name(st->codecpar->codec_id), st->start_time, + st->time_base.num, st->time_base.den); +} + bool Coder::decode(AVCodecContext *decode_ctx, AVFrame *frame, AVPacket *av_packet) { int ret; ret = avcodec_send_packet(decode_ctx, av_packet); @@ -89,7 +113,7 @@ void Coder::encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_pack ret, av_err2str_cpp(ret)); done(av_packet, frame, false); } - while(ret >= 0) { + while (ret >= 0) { ret = avcodec_receive_packet(encode_ctx, av_packet); if (ret == AVERROR_EOF) { ELOG_DEBUG("avcodec_receive_packet AVERROR_EOF, %s, ret: %d, %s", encode_ctx->codec->name, @@ -112,4 +136,103 @@ void Coder::encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_pack } } +void Coder::saveFrameAsJPEG(AVFrame *frame) { + AVCodec *codec; + AVCodecContext *context; + AVPacket packet; + FILE *file; + char name[256]; + static int frame_number = 0; + + const InitContextCB context_callback = [frame](AVCodecContext *context, AVDictionary *dict) { + context->pix_fmt = AV_PIX_FMT_YUVJ420P; + context->height = frame->height; + context->width = frame->width; + context->color_range = AVCOL_RANGE_JPEG; + context->time_base = (AVRational) {1, 30}; + }; + if (!this->initContext(&context, &codec, AV_CODEC_ID_MJPEG, ENCODE_AV, context_callback)) { + return; + } + + av_init_packet(&packet); + EncodeCB encode_callback = [&name, &file, &context](AVPacket *pkt, AVFrame *frame, bool should_write) { + if (should_write) { + int len = 255; + snprintf(name, len, "/tmp/test/dvr-%06d.jpg", frame_number); + file = fopen(name, "wb"); + int saved = fwrite(pkt->data, 1, pkt->size, file); + if (saved != pkt->size || ferror(file)) { + ELOG_ERROR("Could not save JPEG frame number: %d, size: %d", frame_number, pkt->size); + } else { + ELOG_DEBUG("Save JPEG frame with number: %d, size: %d at: %s", frame_number, pkt->size, name); + } + fclose(file); + } + avcodec_close(context); + avcodec_free_context(&context); + frame_number++; + }; + + this->encode(context, frame, &packet, encode_callback); +} + +CoderCodec::CoderCodec() { + av_register_all(); + avcodec_register_all(); + av_codec_ = nullptr; + codec_context_ = nullptr; + initialized = false; + frame_ = av_frame_alloc(); + if (!frame_) { + ELOG_ERROR("Error allocating encode frame"); + } +} + +CoderCodec::~CoderCodec() { + this->closeCodec(); +} + +void CoderCodec::logCodecContext() { + coder_.logCodecContext(codec_context_); +} + +int CoderCodec::closeCodec() { + ELOG_DEBUG("Closing CoderCodec."); + if (codec_context_ != nullptr) { + avcodec_close(codec_context_); + avcodec_free_context(&codec_context_); + } + if (frame_ != nullptr) { + av_frame_free(&frame_); + } + initialized = false; + return 0; +} + +int CoderEncoder::initEncoder(const AVCodecID codec_id, const InitContextCB callback) { + if (coder_.initContext(&codec_context_, &av_codec_, codec_id, ENCODE_AV, callback)) { + initialized = true; + return 0; + } else { + return -1; + } +} + +void CoderEncoder::encode(AVFrame *frame, AVPacket *av_packet, const EncodeCB &done) { + coder_.encode(codec_context_, frame, av_packet, done); +} + +int CoderDecoder::initDecoder(const AVCodecID codec_id, InitContextCB callback) { + if (coder_.initContext(&codec_context_, &av_codec_, codec_id, DECODE_AV, callback)) { + initialized = true; + return 0; + } else { + return -1; + } +} + +bool CoderDecoder::decode(AVFrame *frame, AVPacket *av_packet) { + return coder_.decode(codec_context_, frame, av_packet); +} } // namespace erizo diff --git a/erizo/src/erizo/media/codecs/Coder.h b/erizo/src/erizo/media/codecs/Coder.h index 97bd6deab4..ebb27f1934 100644 --- a/erizo/src/erizo/media/codecs/Coder.h +++ b/erizo/src/erizo/media/codecs/Coder.h @@ -3,7 +3,7 @@ #include #include -#include +#include // NOLINT extern "C" { #include @@ -33,18 +33,19 @@ inline static const std::string av_make_error_string_cpp(int errnum) { namespace erizo { typedef std::function EncodeCB; +typedef std::function InitContextCB; enum CoderOperationType { - OPERATION_ENCODE, - OPERATION_DECODE + ENCODE_AV, + DECODE_AV }; class Coder { DECLARE_LOGGER(); public: - Coder(); - virtual ~Coder(); + bool initContext(AVCodecContext **codec_ctx, AVCodec **av_codec, AVCodecID codec_id, + CoderOperationType operation, const InitContextCB callback); bool allocCodecContext(AVCodecContext **ctx, AVCodec **c, AVCodecID codec_id, CoderOperationType operation); bool openCodecContext(AVCodecContext *codec_ctx, AVCodec *codec, AVDictionary *opt); @@ -52,6 +53,43 @@ class Coder { bool decode(AVCodecContext *decode_ctx, AVFrame *frame, AVPacket *av_packet); void encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_packet, const EncodeCB &done); + void saveFrameAsJPEG(AVFrame *frame); + static void logAVStream(AVStream *av_stream); +}; + +class CoderCodec { + DECLARE_LOGGER(); + + public: + CoderCodec(); + virtual ~CoderCodec(); + virtual int closeCodec(); + void logCodecContext(); + + public: + bool initialized; + AVCodec* av_codec_; + AVCodecContext* codec_context_; + AVFrame* frame_; + + protected: + Coder coder_; +}; + +class CoderEncoder : public CoderCodec { + DECLARE_LOGGER(); + + public: + int initEncoder(const AVCodecID codec_id, const InitContextCB callback); + void encode(AVFrame *frame, AVPacket *av_packet, const EncodeCB &done); +}; + +class CoderDecoder : public CoderCodec { + DECLARE_LOGGER(); + + public: + int initDecoder(const AVCodecID codec_id, const InitContextCB callback); + bool decode(AVFrame *frame, AVPacket *av_packet); }; } // namespace erizo diff --git a/erizo/src/erizo/media/codecs/VideoCodec.cpp b/erizo/src/erizo/media/codecs/VideoCodec.cpp index 07dfa4cf2e..69f27dfec9 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.cpp +++ b/erizo/src/erizo/media/codecs/VideoCodec.cpp @@ -23,67 +23,39 @@ inline AVCodecID VideoCodecID2ffmpegDecoderID(VideoCodecID codec) { } } -VideoEncoder::VideoEncoder() { - avcodec_register_all(); - av_codec = NULL; - encode_context_ = NULL; - cPicture = NULL; - initialized = false; -} - -VideoEncoder::~VideoEncoder() { - this->closeEncoder(); -} - int VideoEncoder::initEncoder(const VideoCodecInfo& info) { - if (!coder_.allocCodecContext(&encode_context_, &av_codec, - VideoCodecID2ffmpegDecoderID(info.codec), OPERATION_ENCODE)) { - return -1; - } - encode_context_->bit_rate = info.bitRate; - encode_context_->rc_min_rate = info.bitRate; - encode_context_->rc_max_rate = info.bitRate; // VPX_CBR - encode_context_->qmin = 0; - encode_context_->qmax = 40; // rc_quantifiers - encode_context_->profile = 3; - encode_context_->rc_initial_buffer_occupancy = 500; - - encode_context_->rc_buffer_size = 1000; - - encode_context_->width = info.width; - encode_context_->height = info.height; - encode_context_->pix_fmt = AV_PIX_FMT_YUV420P; - encode_context_->time_base = (AVRational) {1, 90000}; - - encode_context_->sample_aspect_ratio = (AVRational) { info.width, info.height }; - encode_context_->thread_count = 4; - - if (!coder_.openCodecContext(encode_context_, av_codec, NULL)) { - return -2; - } - - cPicture = av_frame_alloc(); - if (!cPicture) { - ELOG_DEBUG("Error allocating video frame"); - return -3; - } + const InitContextCB callback = [info](AVCodecContext *context, AVDictionary *dict) { + context->bit_rate = info.bitRate; + context->rc_min_rate = info.bitRate; + context->rc_max_rate = info.bitRate; // VPX_CBR + context->qmin = 0; + context->qmax = 40; // rc_quantifiers + context->profile = 3; + context->rc_initial_buffer_occupancy = 500; + context->rc_buffer_size = 1000; + context->width = info.width; + context->height = info.height; + context->pix_fmt = AV_PIX_FMT_YUV420P; + context->time_base = (AVRational) {1, 90000}; + context->sample_aspect_ratio = (AVRational) { info.width, info.height }; + context->thread_count = 4; + }; - ELOG_DEBUG("VideoEncoder configured successfully %d x %d", - encode_context_->width, encode_context_->height); - initialized = true; - return 0; + AVCodecID av_codec_id = VideoCodecID2ffmpegDecoderID(info.codec); + return this->initEncoder(av_codec_id, callback); } -void VideoEncoder::encodeVideoBuffer(unsigned char* inBuffer, int len, unsigned char* outBuffer, const EncodeVideoBufferCB &done) { - int size = encode_context_->width * encode_context_->height; +void VideoEncoder::encodeVideoBuffer(unsigned char* inBuffer, int len, unsigned char* outBuffer, + const EncodeVideoBufferCB &done) { + int size = codec_context_->width * codec_context_->height; - cPicture->pts = AV_NOPTS_VALUE; - cPicture->data[0] = inBuffer; - cPicture->data[1] = inBuffer + size; - cPicture->data[2] = inBuffer + size + size / 4; - cPicture->linesize[0] = encode_context_->width; - cPicture->linesize[1] = encode_context_->width / 2; - cPicture->linesize[2] = encode_context_->width / 2; + frame_->pts = AV_NOPTS_VALUE; + frame_->data[0] = inBuffer; + frame_->data[1] = inBuffer + size; + frame_->data[2] = inBuffer + size + size / 4; + frame_->linesize[0] = codec_context_->width; + frame_->linesize[1] = codec_context_->width / 2; + frame_->linesize[2] = codec_context_->width / 2; AVPacket *av_packet = av_packet_alloc(); av_init_packet(av_packet); @@ -95,94 +67,41 @@ void VideoEncoder::encodeVideoBuffer(unsigned char* inBuffer, int len, unsigned av_packet_free(&pkt); done(got_packet, len); }; - coder_.encode(encode_context_, cPicture, av_packet, done_callback); -} - -int VideoEncoder::closeEncoder() { - if (encode_context_ != NULL) - avcodec_close(encode_context_); - if (cPicture != NULL) - av_frame_free(&cPicture); - initialized = false; - return 0; -} - - -VideoDecoder::VideoDecoder() { - avcodec_register_all(); - av_codec = NULL; - decode_context_ = NULL; - dPicture = NULL; - initWithContext_ = false; - initialized = false; -} - -VideoDecoder::~VideoDecoder() { - this->closeDecoder(); + this->encode(frame_, av_packet, done_callback); } int VideoDecoder::initDecoder(const VideoCodecInfo& info) { - ELOG_DEBUG("Init Decoder"); - if (!coder_.allocCodecContext(&decode_context_, &av_codec, - VideoCodecID2ffmpegDecoderID(info.codec), OPERATION_DECODE)) { - return -1; - } - - decode_context_->width = info.width; - decode_context_->height = info.height; - - if (!coder_.openCodecContext(decode_context_, av_codec, NULL)) { - return -2; - } - - dPicture = av_frame_alloc(); - if (!dPicture) { - ELOG_DEBUG("Error allocating video frame"); - return -3; - } + const InitContextCB callback = [info](AVCodecContext *context, AVDictionary *dict) { + context->width = info.width; + context->height = info.height; + }; - initialized = true; - return 0; + AVCodecID av_codec_id = VideoCodecID2ffmpegDecoderID(info.codec); + return this->initDecoder(av_codec_id, callback); } -int VideoDecoder::initDecoder(AVCodecContext** context, AVCodecParameters *codecpar) { - int error; - AVCodecContext *c; - - ELOG_DEBUG("Init Decoder context"); - initWithContext_ = true; - - if (!coder_.allocCodecContext(&c, &av_codec, - codecpar->codec_id, OPERATION_DECODE)) { - return -1; - } - - error = avcodec_parameters_to_context(c, codecpar); - if (error < 0) { - ELOG_ERROR("Could not copy parameters to context."); - return -2; - } - - if (!coder_.openCodecContext(decode_context_, av_codec, NULL)) { - return -3; - } - - *context = c; - decode_context_ = *context; +int VideoDecoder::initDecoder(AVCodecParameters *codecpar) { + const InitContextCB callback = [codecpar](AVCodecContext *context, AVDictionary *dict) { + int error = avcodec_parameters_to_context(context, codecpar); + if (error < 0) { + ELOG_ERROR("Could not copy parameters to context."); + } + }; - dPicture = av_frame_alloc(); - if (!dPicture) { - ELOG_DEBUG("Error allocating video frame"); - return -4; - } + return this->initDecoder(codecpar->codec_id, callback); +} - initialized = true; - return 0; +bool VideoDecoder::decode(AVFrame *frame, AVPacket *av_packet) { + frame_->format = codec_context_->pix_fmt; + frame_->width = codec_context_->width; + frame_->height = codec_context_->height; + frame_->pts = av_packet->pts; + return CoderDecoder::decode(frame, av_packet); } int VideoDecoder::decodeVideoBuffer(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, int outBuffLen, int* gotFrame) { - if (av_codec == 0 || decode_context_ == 0) { + if (av_codec_ == 0 || codec_context_ == 0) { ELOG_DEBUG("Init Codec First"); return -1; } @@ -196,14 +115,14 @@ int VideoDecoder::decodeVideoBuffer(unsigned char* inBuff, int inBuffLen, avpkt.size = inBuffLen; if (avpkt.size > 0) { - if (coder_.decode(decode_context_, dPicture, &avpkt)) { + if (this->decode(frame_, &avpkt)) { *gotFrame = 1; goto decoding; } } decoding: - int outSize = decode_context_->height * decode_context_->width; + int outSize = codec_context_->height * codec_context_->width; if (outBuffLen < (outSize * 3 / 2)) { return outSize * 3 / 2; @@ -216,31 +135,31 @@ int VideoDecoder::decodeVideoBuffer(unsigned char* inBuff, int inBuffLen, unsigned char *src = NULL; int src_linesize, dst_linesize; - src_linesize = dPicture->linesize[0]; - dst_linesize = decode_context_->width; - src = dPicture->data[0]; + src_linesize = frame_->linesize[0]; + dst_linesize = codec_context_->width; + src = frame_->data[0]; - for (int i = decode_context_->height; i > 0; i--) { + for (int i = codec_context_->height; i > 0; i--) { memcpy(lum, src, dst_linesize); lum += dst_linesize; src += src_linesize; } - src_linesize = dPicture->linesize[1]; - dst_linesize = decode_context_->width / 2; - src = dPicture->data[1]; + src_linesize = frame_->linesize[1]; + dst_linesize = codec_context_->width / 2; + src = frame_->data[1]; - for (int i = decode_context_->height / 2; i > 0; i--) { + for (int i = codec_context_->height / 2; i > 0; i--) { memcpy(cromU, src, dst_linesize); cromU += dst_linesize; src += src_linesize; } - src_linesize = dPicture->linesize[2]; - dst_linesize = decode_context_->width / 2; - src = dPicture->data[2]; + src_linesize = frame_->linesize[2]; + dst_linesize = codec_context_->width / 2; + src = frame_->data[2]; - for (int i = decode_context_->height / 2; i > 0; i--) { + for (int i = codec_context_->height / 2; i > 0; i--) { memcpy(cromV, src, dst_linesize); cromV += dst_linesize; src += src_linesize; @@ -249,14 +168,4 @@ int VideoDecoder::decodeVideoBuffer(unsigned char* inBuff, int inBuffLen, return outSize * 3 / 2; } - -int VideoDecoder::closeDecoder() { - if (!initWithContext_ && decode_context_ != NULL) - avcodec_close(decode_context_); - if (dPicture != NULL) - av_frame_free(&dPicture); - initialized = false; - return 0; -} - } // namespace erizo diff --git a/erizo/src/erizo/media/codecs/VideoCodec.h b/erizo/src/erizo/media/codecs/VideoCodec.h index 77e1db7580..039ec4bd7f 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.h +++ b/erizo/src/erizo/media/codecs/VideoCodec.h @@ -20,54 +20,31 @@ extern "C" { #include #include } -// Forward Declarations - -// struct AVCodec; -// struct AVCodecContext; -// struct AVFrame; namespace erizo { typedef std::function EncodeVideoBufferCB; -class VideoEncoder { +class VideoEncoder : public CoderEncoder { DECLARE_LOGGER(); public: - VideoEncoder(); - virtual ~VideoEncoder(); + using CoderEncoder::initEncoder; int initEncoder(const VideoCodecInfo& info); void encodeVideoBuffer(unsigned char* inBuffer, int len, unsigned char* outBuffer, const EncodeVideoBufferCB &done); - int closeEncoder(); - bool initialized; - - private: - Coder coder_; - AVCodec* av_codec; - AVCodecContext* encode_context_; - AVFrame* cPicture; }; -class VideoDecoder { +class VideoDecoder : public CoderDecoder { DECLARE_LOGGER(); public: - VideoDecoder(); - virtual ~VideoDecoder(); + using CoderDecoder::initDecoder; int initDecoder(const VideoCodecInfo& info); - int initDecoder(AVCodecContext** context, AVCodecParameters *codecpar); + int initDecoder(AVCodecParameters *codecpar); + bool decode(AVFrame *frame, AVPacket *av_packet); int decodeVideoBuffer(unsigned char* inBuff, int inBuffLen, unsigned char* outBuff, int outBuffLen, int* gotFrame); - int closeDecoder(); - bool initialized; - - private: - Coder coder_; - AVCodec* av_codec; - AVCodecContext* decode_context_; - AVFrame* dPicture; - bool initWithContext_; }; } // namespace erizo diff --git a/erizo_controller/erizoAgent/log4cxx.properties b/erizo_controller/erizoAgent/log4cxx.properties index 760706b2d0..c086d4740b 100644 --- a/erizo_controller/erizoAgent/log4cxx.properties +++ b/erizo_controller/erizoAgent/log4cxx.properties @@ -28,13 +28,15 @@ log4j.logger.dtls.DtlsFactory=WARN log4j.logger.dtls.DtlsSocketContext=WARN log4j.logger.dtls.SSL=WARN -log4j.logger.media.ExternalInput=WARN -log4j.logger.media.ExternalOutput=WARN +log4j.logger.media.ExternalInput=DEBUG +log4j.logger.media.ExternalOutput=DEBUG log4j.logger.media.InputProcessor=WARN log4j.logger.media.OneToManyTranscoder=WARN log4j.logger.media.OutputProcessor=WARN + log4j.logger.media.codecs.Coder=WARN +log4j.logger.media.codecs.CoderCodec=WARN log4j.logger.media.codecs.VideoEncoder=WARN log4j.logger.media.codecs.VideoDecoder=WARN log4j.logger.media.codecs.AudioEncoder=WARN @@ -70,3 +72,4 @@ log4j.logger.rtp.LayerDetectorHandler=WARN log4j.logger.rtp.PliPacerHandler=WARN log4j.logger.rtp.RtpPaddingGeneratorHandler=WARN log4j.logger.rtp.PacketCodecParser=WARN + From 5b085ff2e3e2bdf644535ba97e3abb78791ffd2a Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Tue, 29 May 2018 18:38:01 -0300 Subject: [PATCH 10/32] Update Erizo log verbosity --- erizo_controller/erizoAgent/log4cxx.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erizo_controller/erizoAgent/log4cxx.properties b/erizo_controller/erizoAgent/log4cxx.properties index c086d4740b..bf9937e4fc 100644 --- a/erizo_controller/erizoAgent/log4cxx.properties +++ b/erizo_controller/erizoAgent/log4cxx.properties @@ -28,8 +28,8 @@ log4j.logger.dtls.DtlsFactory=WARN log4j.logger.dtls.DtlsSocketContext=WARN log4j.logger.dtls.SSL=WARN -log4j.logger.media.ExternalInput=DEBUG -log4j.logger.media.ExternalOutput=DEBUG +log4j.logger.media.ExternalInput=WARN +log4j.logger.media.ExternalOutput=WARN log4j.logger.media.InputProcessor=WARN log4j.logger.media.OneToManyTranscoder=WARN log4j.logger.media.OutputProcessor=WARN From 8f476c0fd5439890f1a66c37d3ad77d8aa4e9f04 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Tue, 29 May 2018 19:09:50 -0300 Subject: [PATCH 11/32] Make cpp lint pass --- erizo/src/erizo/media/codecs/Coder.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erizo/src/erizo/media/codecs/Coder.cpp b/erizo/src/erizo/media/codecs/Coder.cpp index d01ffb7596..519ef33a32 100644 --- a/erizo/src/erizo/media/codecs/Coder.cpp +++ b/erizo/src/erizo/media/codecs/Coder.cpp @@ -70,8 +70,8 @@ bool Coder::openCodecContext(AVCodecContext *codec_ctx, AVCodec *codec, AVDictio void Coder::logCodecContext(AVCodecContext *codec_ctx) { const char *operation = (av_codec_is_encoder(codec_ctx->codec) ? "encoder" : "decoder"); if (codec_ctx->codec->type == AVMEDIA_TYPE_AUDIO) { - ELOG_DEBUG("\nAudio %s codec: %s \nchannel_layout: %d\nchannels: %d\nframe_size: %d\ntime_base: %d/%d\n, sample_rate: " - "%d\nsample_fmt: %s\nbits per sample: %d", + ELOG_DEBUG("\nAudio %s codec: %s \nchannel_layout: %d\nchannels: %d\nframe_size: %d\ntime_base: %d/%d\n," + "sample_rate: %d\nsample_fmt: %s\nbits per sample: %d", operation, codec_ctx->codec->name, codec_ctx->channel_layout, codec_ctx->channels, codec_ctx->frame_size, codec_ctx->time_base.num, codec_ctx->time_base.den, codec_ctx->sample_rate, av_get_sample_fmt_name(codec_ctx->sample_fmt), av_get_bytes_per_sample(codec_ctx->sample_fmt)); From f6494e1195c8830987dfdb658034151cdf0d4f48 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Thu, 31 May 2018 11:44:13 -0300 Subject: [PATCH 12/32] Fix unref frames after decoding/encoding external output * Fix no bother rescale ts if packet will not be written. * Set context it will be variable fps. --- erizo/src/erizo/media/ExternalOutput.cpp | 35 +++++++++++++++--------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/erizo/src/erizo/media/ExternalOutput.cpp b/erizo/src/erizo/media/ExternalOutput.cpp index 32ce1b1808..f60d58e239 100644 --- a/erizo/src/erizo/media/ExternalOutput.cpp +++ b/erizo/src/erizo/media/ExternalOutput.cpp @@ -193,6 +193,7 @@ void ExternalOutput::writeAudioData(char* buf, int len) { if (av_packet.size > 0 && need_audio_transcode_) { if (audio_decoder_.decode(audio_decoder_.frame_, &av_packet)) { EncodeCB encode_callback = [this](AVPacket *pkt, AVFrame *frame, bool should_write){ + av_frame_unref(frame); this->writePacket(pkt, audio_st_, should_write); }; if (need_audio_resample_) { @@ -202,6 +203,7 @@ void ExternalOutput::writeAudioData(char* buf, int len) { i += audio_encoder_.resampled_frame_->nb_samples; audio_encoder_.encode(audio_encoder_.resampled_frame_, &av_packet, encode_callback); } + av_frame_unref(audio_decoder_.frame_); } else { audio_encoder_.encode(audio_decoder_.frame_, &av_packet, encode_callback); } @@ -300,6 +302,7 @@ void ExternalOutput::maybeWriteVideoPacket(char* buf, int len) { if (av_packet.size > 0 && need_video_transcode_) { if (video_decoder_.decode(video_decoder_.frame_, &av_packet)) { auto done_callback = [this](AVPacket *pkt, AVFrame *frame, bool should_write){ + av_frame_unref(frame); this->writePacket(pkt, video_st_, should_write); }; video_encoder_.encode(video_decoder_.frame_, &av_packet, done_callback); @@ -313,22 +316,23 @@ void ExternalOutput::maybeWriteVideoPacket(char* buf, int len) { } void ExternalOutput::writePacket(AVPacket *pkt, AVStream *st, bool should_write) { - const char *media_type; - if (st->id == 0) { - media_type = "video"; + if (should_write) { + const char *media_type; + + if (st->id == 0) { + media_type = "video"; av_packet_rescale_ts(pkt, AVRational{1, static_cast(video_map_.clock_rate)}, - video_st_->time_base); - } else { - media_type = "audio"; - av_packet_rescale_ts(pkt, audio_encoder_.codec_context_->time_base, audio_st_->time_base); - } - pkt->stream_index = st->id; + video_st_->time_base); + } else { + media_type = "audio"; + av_packet_rescale_ts(pkt, audio_encoder_.codec_context_->time_base, audio_st_->time_base); + } + pkt->stream_index = st->id; - int64_t pts = pkt->pts; - int64_t dts = pkt->dts; - int64_t dl = pkt->dts; + int64_t pts = pkt->pts; + int64_t dts = pkt->dts; + int64_t dl = pkt->dts; - if (should_write) { int ret = av_interleaved_write_frame(context_, pkt); if (ret != 0) { ELOG_ERROR("av_interleaved_write_frame pts: %d failed with: %d %s", @@ -614,6 +618,7 @@ AVStream * ExternalOutput::addOutputStream(int index, CoderCodec *coder_codec) { bool ExternalOutput::writeContextHeader() { AVDictionary *opt = NULL; + context_->oformat->flags |= AVFMT_VARIABLE_FPS; int ret = avformat_write_header(context_, &opt); av_dict_free(&opt); if (ret < 0) { @@ -664,6 +669,10 @@ void ExternalOutput::setupVideoEncodingParams(AVCodecContext *context, AVDiction context->time_base = (AVRational){1, 25}; context->pix_fmt = AV_PIX_FMT_YUV420P; + if (context->codec->id == AV_CODEC_ID_H264) { + av_opt_set(context->priv_data, "profile", "main", 0); + } + if (context->codec->id == AV_CODEC_ID_VP8 || context->codec->id == AV_CODEC_ID_VP9) { ELOG_DEBUG("Setting VPX params"); } From 865db5b5968977b61382915d9c1dbb1c8732621a Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Fri, 1 Jun 2018 09:35:20 -0300 Subject: [PATCH 13/32] Move Erizo build flag -Werror to the right place --- erizo/src/CMakeLists.txt | 3 --- erizo/src/erizo/CMakeLists.txt | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/erizo/src/CMakeLists.txt b/erizo/src/CMakeLists.txt index 1bb712b2bd..2f58aadcde 100644 --- a/erizo/src/CMakeLists.txt +++ b/erizo/src/CMakeLists.txt @@ -111,9 +111,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/third_party/webrtc.cmake") include("${CMAKE_CURRENT_SOURCE_DIR}/third_party/nicer.cmake") ## Erizo -if(APPLE) -add_definitions(-Wall -Werror) -endif() add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/erizo") ## Examples diff --git a/erizo/src/erizo/CMakeLists.txt b/erizo/src/erizo/CMakeLists.txt index 8b5d18b5f9..9c98421d8e 100644 --- a/erizo/src/erizo/CMakeLists.txt +++ b/erizo/src/erizo/CMakeLists.txt @@ -6,10 +6,10 @@ set(ERIZO_VERSION_MAJOR 0) set(ERIZO_VERSION_MINOR 1) if(${ERIZO_BUILD_TYPE} STREQUAL "debug") message("Generating DEBUG project") - set(CMAKE_CXX_FLAGS "-g -Wall -std=c++11 ${ERIZO_CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "-g -Werror -Wall -std=c++11 ${ERIZO_CMAKE_CXX_FLAGS}") else() message("Generating RELEASE project") - set(CMAKE_CXX_FLAGS "-g -Wall -O3 -std=c++11 ${ERIZO_CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "-g -Werror -Wall -O3 -std=c++11 ${ERIZO_CMAKE_CXX_FLAGS}") endif() From 595c9003cc17798cc45111abd3d935606671a149 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Fri, 1 Jun 2018 09:58:16 -0300 Subject: [PATCH 14/32] Fixes around ExternalOutput refactor * Function syncClose returns if not recording. * Properly check if context_ isn't null before free it. * Do not exit if context couldn't be initialized. * Remove not used save jpeg function from Coder class. * Add CoderCodec isInitialized getter. * Add CoderCodec width/height accessors. * Add CoderCodec abitlity to initialize AVStream. * Rename InitContextCB to InitContextBeforeOpenCB. * Fix Coder::initContext return variable was overrided. * InitEncoder / InitDecord now returns bool. * Protect AVCodec in CoderCodec. * Check ffmpeg is installed before download and compile. --- erizo/src/erizo/media/ExternalInput.cpp | 2 +- erizo/src/erizo/media/ExternalOutput.cpp | 74 ++++++-------- erizo/src/erizo/media/ExternalOutput.h | 1 - erizo/src/erizo/media/MediaProcessor.cpp | 4 +- erizo/src/erizo/media/codecs/AudioCodec.cpp | 4 +- erizo/src/erizo/media/codecs/Coder.cpp | 102 +++++++++----------- erizo/src/erizo/media/codecs/Coder.h | 19 ++-- erizo/src/erizo/media/codecs/VideoCodec.cpp | 6 +- scripts/installUbuntuDeps.sh | 32 +++--- scripts/travisInstallDeps.sh | 32 +++--- 10 files changed, 135 insertions(+), 141 deletions(-) diff --git a/erizo/src/erizo/media/ExternalInput.cpp b/erizo/src/erizo/media/ExternalInput.cpp index 6eb9f5007e..c001cea88f 100644 --- a/erizo/src/erizo/media/ExternalInput.cpp +++ b/erizo/src/erizo/media/ExternalInput.cpp @@ -128,7 +128,7 @@ int ExternalInput::init() { needTranscoding_ = true; inCodec_.initDecoder(st->codecpar); - bufflen_ = inCodec_.codec_context_->width*inCodec_.codec_context_->height*3/2; + bufflen_ = inCodec_.getContextWidth()*inCodec_.getContextHeight()*3/2; decodedBuffer_.reset((unsigned char*) malloc(bufflen_)); diff --git a/erizo/src/erizo/media/ExternalOutput.cpp b/erizo/src/erizo/media/ExternalOutput.cpp index f60d58e239..dfe64eb70f 100644 --- a/erizo/src/erizo/media/ExternalOutput.cpp +++ b/erizo/src/erizo/media/ExternalOutput.cpp @@ -103,28 +103,32 @@ void ExternalOutput::close() { } void ExternalOutput::syncClose() { + if (!recording_) { + return; + } + cond_.notify_one(); thread_.join(); if (context_ != nullptr) { av_write_trailer(context_); } - if (video_decoder_.initialized) { + if (video_decoder_.isInitialized()) { video_decoder_.closeCodec(); } - if (video_encoder_.initialized) { + if (video_encoder_.isInitialized()) { video_encoder_.closeCodec(); } - if (audio_decoder_.initialized) { + if (audio_decoder_.isInitialized()) { audio_decoder_.closeCodec(); } - if (audio_encoder_.initialized) { + if (audio_encoder_.isInitialized()) { audio_encoder_.closeCodec(); } if (context_ != nullptr) { avio_close(context_->pb); + avformat_free_context(context_); } - avformat_free_context(context_); pipeline_initialized_ = false; recording_ = false; @@ -420,7 +424,6 @@ bool ExternalOutput::initContext() { video_queue_.setTimebase(video_map_.clock_rate); if (avio_open(&context_->pb, context_->filename, AVIO_FLAG_WRITE) < 0) { ELOG_ERROR("Error opening output context."); - exit(1); } else { AVCodecID output_video_codec_id = this->bestMatchOutputCodecId(input_video_codec_id_); AVCodecID output_audio_codec_id = this->bestMatchOutputCodecId(input_audio_codec_id_); @@ -437,35 +440,37 @@ bool ExternalOutput::initContext() { } video_decoder_.initDecoder(input_video_codec_id_, - (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + (InitContextBeforeOpenCB)[this](AVCodecContext *context, AVDictionary *dict) { this->setupVideoDecodingParams(context, dict); }); audio_decoder_.initDecoder(input_audio_codec_id_, - (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + (InitContextBeforeOpenCB)[this](AVCodecContext *context, AVDictionary *dict) { this->setupAudioDecodingParams(context, dict); }); - video_encoder_.initEncoder(output_video_codec_id, - (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + if (video_encoder_.initEncoder(output_video_codec_id, + (InitContextBeforeOpenCB)[this](AVCodecContext *context, AVDictionary *dict) { this->setupVideoEncodingParams(context, dict); - }); + })) { + if (video_encoder_.initializeStream(0, context_, &video_st_)) { + context_->streams[0] = video_st_; + } + } - audio_encoder_.initEncoder(output_audio_codec_id, - (InitContextCB)[this](AVCodecContext *context, AVDictionary *dict) { + if (audio_encoder_.initEncoder(output_audio_codec_id, + (InitContextBeforeOpenCB)[this](AVCodecContext *context, AVDictionary *dict) { this->setupAudioEncodingParams(context, dict); - }); - - video_st_ = this->addOutputStream(0, &video_encoder_); - audio_st_ = this->addOutputStream(1, &audio_encoder_); + })) { + if (audio_encoder_.initializeStream(1, context_, &audio_st_)) { + context_->streams[1] = audio_st_; + } - if (this->audioNeedsResample()) { - ELOG_INFO("Encode Audio needs resampling."); - need_audio_resample_ = true; - } - if (need_audio_resample_ && - !audio_encoder_.initAudioResampler(audio_decoder_.codec_context_)) { - exit(1); + if (this->audioNeedsResample()) { + ELOG_INFO("Encode Audio needs resampling."); + need_audio_resample_ = true; + audio_encoder_.initAudioResampler(audio_decoder_.codec_context_); + } } if (!this->writeContextHeader()) { @@ -595,27 +600,6 @@ void ExternalOutput::sendLoop() { } } -AVStream * ExternalOutput::addOutputStream(int index, CoderCodec *coder_codec) { - AVStream *st; - const char *codec_name = coder_codec->av_codec_->name; - st = avformat_new_stream(context_, coder_codec->av_codec_); - if (!st) { - ELOG_ERROR("Could not create stream for %s.", codec_name); - return nullptr; - } else { - int ret = avcodec_parameters_from_context(st->codecpar, coder_codec->codec_context_); - if (ret != 0) { - ELOG_ERROR("Could not copy codec paramaters for %s.", codec_name); - return nullptr; - } - st->id = index; - st->time_base = coder_codec->codec_context_->time_base; - context_->streams[index] = st; - ELOG_INFO("Created stream for codec: %s with index: %d", codec_name, index); - return st; - } -} - bool ExternalOutput::writeContextHeader() { AVDictionary *opt = NULL; context_->oformat->flags |= AVFMT_VARIABLE_FPS; diff --git a/erizo/src/erizo/media/ExternalOutput.h b/erizo/src/erizo/media/ExternalOutput.h index 41cd7954ab..fb494d00bf 100644 --- a/erizo/src/erizo/media/ExternalOutput.h +++ b/erizo/src/erizo/media/ExternalOutput.h @@ -160,7 +160,6 @@ class ExternalOutput : public MediaSink, public RawDataReceiver, public Feedback void setupAudioDecodingParams(AVCodecContext *context, AVDictionary *dict); void setupVideoEncodingParams(AVCodecContext *context, AVDictionary *dict); void setupAudioEncodingParams(AVCodecContext *context, AVDictionary *dict); - AVStream * addOutputStream(int index, CoderCodec *coder_codec); AVCodecID bestMatchOutputCodecId(AVCodecID input_codec_id); bool audioNeedsResample(); }; diff --git a/erizo/src/erizo/media/MediaProcessor.cpp b/erizo/src/erizo/media/MediaProcessor.cpp index c0fbd466fe..0bbb41dd56 100644 --- a/erizo/src/erizo/media/MediaProcessor.cpp +++ b/erizo/src/erizo/media/MediaProcessor.cpp @@ -74,7 +74,7 @@ int InputProcessor::init(const MediaInfo& info, RawDataReceiver* receiver) { } int InputProcessor::deliverAudioData_(std::shared_ptr audio_packet) { - if (audioUnpackager && audio_decoder_.initialized) { + if (audioUnpackager && audio_decoder_.isInitialized()) { std::shared_ptr copied_packet = std::make_shared(*audio_packet); int unp = unpackageAudio((unsigned char*) copied_packet->data, copied_packet->length, unpackagedAudioBuffer_); @@ -91,7 +91,7 @@ int InputProcessor::deliverAudioData_(std::shared_ptr audio_packet) return 0; } int InputProcessor::deliverVideoData_(std::shared_ptr video_packet) { - if (videoUnpackager && video_decoder_.initialized) { + if (videoUnpackager && video_decoder_.isInitialized()) { std::shared_ptr copied_packet = std::make_shared(*video_packet); int ret = unpackageVideo(reinterpret_cast(copied_packet->data), copied_packet->length, unpackagedBufferPtr_, &gotUnpackagedFrame_); diff --git a/erizo/src/erizo/media/codecs/AudioCodec.cpp b/erizo/src/erizo/media/codecs/AudioCodec.cpp index 6fb1caabc0..e64b29ebd1 100644 --- a/erizo/src/erizo/media/codecs/AudioCodec.cpp +++ b/erizo/src/erizo/media/codecs/AudioCodec.cpp @@ -35,7 +35,7 @@ AudioEncoder::~AudioEncoder() { } int AudioEncoder::initEncoder(const AudioCodecInfo& mediaInfo) { - const InitContextCB callback = [mediaInfo](AVCodecContext *context, AVDictionary *dict) { + const InitContextBeforeOpenCB callback = [mediaInfo](AVCodecContext *context, AVDictionary *dict) { context->sample_fmt = AV_SAMPLE_FMT_FLT; // codec_context_->bit_rate = mediaInfo.bitRate; context->sample_rate = 8 /*mediaInfo.sampleRate*/; @@ -228,7 +228,7 @@ uint64_t AudioEncoder::getChannelLayout(const AVCodec *codec) { } int AudioDecoder::initDecoder(const AudioCodecInfo& info) { - const InitContextCB callback = [info](AVCodecContext *context, AVDictionary *dict) { + const InitContextBeforeOpenCB callback = [info](AVCodecContext *context, AVDictionary *dict) { context->sample_fmt = AV_SAMPLE_FMT_S16; context->bit_rate = info.bitRate; context->sample_rate = info.sampleRate; diff --git a/erizo/src/erizo/media/codecs/Coder.cpp b/erizo/src/erizo/media/codecs/Coder.cpp index 519ef33a32..b545566610 100644 --- a/erizo/src/erizo/media/codecs/Coder.cpp +++ b/erizo/src/erizo/media/codecs/Coder.cpp @@ -8,14 +8,14 @@ DEFINE_LOGGER(CoderEncoder, "media.codecs.CoderEncoder"); DEFINE_LOGGER(CoderDecoder, "media.codecs.CoderDecoder"); bool Coder::initContext(AVCodecContext **codec_ctx, AVCodec **av_codec, AVCodecID codec_id, - CoderOperationType operation, InitContextCB callback) { + CoderOperationType operation, InitContextBeforeOpenCB callback) { bool success = false; AVDictionary *opt = NULL; const char *codec_name = avcodec_get_name(codec_id); const char *codec_for = operation == ENCODE_AV ? "encode" : "decode"; if (this->allocCodecContext(codec_ctx, av_codec, codec_id, operation)) { callback(*codec_ctx, opt); - bool success = this->openCodecContext(*codec_ctx, *av_codec, opt); + success = this->openCodecContext(*codec_ctx, *av_codec, opt); if (success) { ELOG_INFO("Successfully initialized %s context for %s.", codec_for, codec_name); } else { @@ -136,53 +136,12 @@ void Coder::encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_pack } } -void Coder::saveFrameAsJPEG(AVFrame *frame) { - AVCodec *codec; - AVCodecContext *context; - AVPacket packet; - FILE *file; - char name[256]; - static int frame_number = 0; - - const InitContextCB context_callback = [frame](AVCodecContext *context, AVDictionary *dict) { - context->pix_fmt = AV_PIX_FMT_YUVJ420P; - context->height = frame->height; - context->width = frame->width; - context->color_range = AVCOL_RANGE_JPEG; - context->time_base = (AVRational) {1, 30}; - }; - if (!this->initContext(&context, &codec, AV_CODEC_ID_MJPEG, ENCODE_AV, context_callback)) { - return; - } - - av_init_packet(&packet); - EncodeCB encode_callback = [&name, &file, &context](AVPacket *pkt, AVFrame *frame, bool should_write) { - if (should_write) { - int len = 255; - snprintf(name, len, "/tmp/test/dvr-%06d.jpg", frame_number); - file = fopen(name, "wb"); - int saved = fwrite(pkt->data, 1, pkt->size, file); - if (saved != pkt->size || ferror(file)) { - ELOG_ERROR("Could not save JPEG frame number: %d, size: %d", frame_number, pkt->size); - } else { - ELOG_DEBUG("Save JPEG frame with number: %d, size: %d at: %s", frame_number, pkt->size, name); - } - fclose(file); - } - avcodec_close(context); - avcodec_free_context(&context); - frame_number++; - }; - - this->encode(context, frame, &packet, encode_callback); -} - CoderCodec::CoderCodec() { av_register_all(); avcodec_register_all(); av_codec_ = nullptr; codec_context_ = nullptr; - initialized = false; + initialized_ = false; frame_ = av_frame_alloc(); if (!frame_) { ELOG_ERROR("Error allocating encode frame"); @@ -197,6 +156,43 @@ void CoderCodec::logCodecContext() { coder_.logCodecContext(codec_context_); } +bool CoderCodec::isInitialized() { + return initialized_; +} + +bool CoderCodec::initializeStream(int index, AVFormatContext *f_context, AVStream **out_st) { + if (!this->isInitialized()) { + ELOG_ERROR("You are trying to initialize a stream with an uninitialized codec."); + return false; + } + AVStream *st; + const char *codec_name = av_codec_->name; + st = avformat_new_stream(f_context, av_codec_); + if (!st) { + ELOG_ERROR("Could not create stream for %s.", codec_name); + } else { + int ret = avcodec_parameters_from_context(st->codecpar, codec_context_); + if (ret != 0) { + ELOG_ERROR("Could not copy codec paramaters for %s.", codec_name); + } else { + st->id = index; + st->time_base = codec_context_->time_base; + ELOG_INFO("Created stream for codec: %s with index: %d", codec_name, index); + *out_st = st; + return true; + } + } + return false; +} + +int CoderCodec::getContextWidth() { + return codec_context_->width; +} + +int CoderCodec::getContextHeight() { + return codec_context_->height; +} + int CoderCodec::closeCodec() { ELOG_DEBUG("Closing CoderCodec."); if (codec_context_ != nullptr) { @@ -206,30 +202,26 @@ int CoderCodec::closeCodec() { if (frame_ != nullptr) { av_frame_free(&frame_); } - initialized = false; + initialized_ = false; return 0; } -int CoderEncoder::initEncoder(const AVCodecID codec_id, const InitContextCB callback) { +bool CoderEncoder::initEncoder(const AVCodecID codec_id, const InitContextBeforeOpenCB callback) { if (coder_.initContext(&codec_context_, &av_codec_, codec_id, ENCODE_AV, callback)) { - initialized = true; - return 0; - } else { - return -1; + initialized_ = true; } + return initialized_; } void CoderEncoder::encode(AVFrame *frame, AVPacket *av_packet, const EncodeCB &done) { coder_.encode(codec_context_, frame, av_packet, done); } -int CoderDecoder::initDecoder(const AVCodecID codec_id, InitContextCB callback) { +bool CoderDecoder::initDecoder(const AVCodecID codec_id, const InitContextBeforeOpenCB callback) { if (coder_.initContext(&codec_context_, &av_codec_, codec_id, DECODE_AV, callback)) { - initialized = true; - return 0; - } else { - return -1; + initialized_ = true; } + return initialized_; } bool CoderDecoder::decode(AVFrame *frame, AVPacket *av_packet) { diff --git a/erizo/src/erizo/media/codecs/Coder.h b/erizo/src/erizo/media/codecs/Coder.h index ebb27f1934..b47e638d30 100644 --- a/erizo/src/erizo/media/codecs/Coder.h +++ b/erizo/src/erizo/media/codecs/Coder.h @@ -33,7 +33,7 @@ inline static const std::string av_make_error_string_cpp(int errnum) { namespace erizo { typedef std::function EncodeCB; -typedef std::function InitContextCB; +typedef std::function InitContextBeforeOpenCB; enum CoderOperationType { ENCODE_AV, @@ -45,7 +45,7 @@ class Coder { public: bool initContext(AVCodecContext **codec_ctx, AVCodec **av_codec, AVCodecID codec_id, - CoderOperationType operation, const InitContextCB callback); + CoderOperationType operation, const InitContextBeforeOpenCB callback); bool allocCodecContext(AVCodecContext **ctx, AVCodec **c, AVCodecID codec_id, CoderOperationType operation); bool openCodecContext(AVCodecContext *codec_ctx, AVCodec *codec, AVDictionary *opt); @@ -53,7 +53,6 @@ class Coder { bool decode(AVCodecContext *decode_ctx, AVFrame *frame, AVPacket *av_packet); void encode(AVCodecContext *encode_ctx, AVFrame *frame, AVPacket *av_packet, const EncodeCB &done); - void saveFrameAsJPEG(AVFrame *frame); static void logAVStream(AVStream *av_stream); }; @@ -65,22 +64,26 @@ class CoderCodec { virtual ~CoderCodec(); virtual int closeCodec(); void logCodecContext(); + bool isInitialized(); + bool initializeStream(int index, AVFormatContext *f_context, AVStream **out_st); + int getContextWidth(); + int getContextHeight(); public: - bool initialized; - AVCodec* av_codec_; - AVCodecContext* codec_context_; AVFrame* frame_; + AVCodecContext* codec_context_; protected: Coder coder_; + AVCodec* av_codec_; + bool initialized_; }; class CoderEncoder : public CoderCodec { DECLARE_LOGGER(); public: - int initEncoder(const AVCodecID codec_id, const InitContextCB callback); + bool initEncoder(const AVCodecID codec_id, const InitContextBeforeOpenCB callback); void encode(AVFrame *frame, AVPacket *av_packet, const EncodeCB &done); }; @@ -88,7 +91,7 @@ class CoderDecoder : public CoderCodec { DECLARE_LOGGER(); public: - int initDecoder(const AVCodecID codec_id, const InitContextCB callback); + bool initDecoder(const AVCodecID codec_id, const InitContextBeforeOpenCB callback); bool decode(AVFrame *frame, AVPacket *av_packet); }; diff --git a/erizo/src/erizo/media/codecs/VideoCodec.cpp b/erizo/src/erizo/media/codecs/VideoCodec.cpp index 69f27dfec9..0bffcd9df8 100644 --- a/erizo/src/erizo/media/codecs/VideoCodec.cpp +++ b/erizo/src/erizo/media/codecs/VideoCodec.cpp @@ -24,7 +24,7 @@ inline AVCodecID VideoCodecID2ffmpegDecoderID(VideoCodecID codec) { } int VideoEncoder::initEncoder(const VideoCodecInfo& info) { - const InitContextCB callback = [info](AVCodecContext *context, AVDictionary *dict) { + const InitContextBeforeOpenCB callback = [info](AVCodecContext *context, AVDictionary *dict) { context->bit_rate = info.bitRate; context->rc_min_rate = info.bitRate; context->rc_max_rate = info.bitRate; // VPX_CBR @@ -71,7 +71,7 @@ void VideoEncoder::encodeVideoBuffer(unsigned char* inBuffer, int len, unsigned } int VideoDecoder::initDecoder(const VideoCodecInfo& info) { - const InitContextCB callback = [info](AVCodecContext *context, AVDictionary *dict) { + const InitContextBeforeOpenCB callback = [info](AVCodecContext *context, AVDictionary *dict) { context->width = info.width; context->height = info.height; }; @@ -81,7 +81,7 @@ int VideoDecoder::initDecoder(const VideoCodecInfo& info) { } int VideoDecoder::initDecoder(AVCodecParameters *codecpar) { - const InitContextCB callback = [codecpar](AVCodecContext *context, AVDictionary *dict) { + const InitContextBeforeOpenCB callback = [codecpar](AVCodecContext *context, AVDictionary *dict) { int error = avcodec_parameters_to_context(context, codecpar); if (error < 0) { ELOG_ERROR("Could not copy parameters to context."); diff --git a/scripts/installUbuntuDeps.sh b/scripts/installUbuntuDeps.sh index e32c92c1be..afffb1580d 100755 --- a/scripts/installUbuntuDeps.sh +++ b/scripts/installUbuntuDeps.sh @@ -157,12 +157,16 @@ install_mediadeps(){ sudo apt-get -qq install yasm libvpx. libx264. if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz - tar -zxvf ffmpeg-3.4.2.tar.gz - cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc --enable-avresample - make $FAST_MAKE -s V=0 - make install + if [ ! -f ./ffmpeg-3.4.2.tar.gz ]; then + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc --enable-avresample + make $FAST_MAKE -s V=0 + make install + else + echo "ffmpeg already installed" + fi cd $CURRENT_DIR else mkdir -p $LIB_DIR @@ -176,12 +180,16 @@ install_mediadeps_nogpl(){ sudo apt-get -qq install yasm libvpx. if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz - tar -zxvf ffmpeg-3.4.2.tar.gz - cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc --enable-avresample - make $FAST_MAKE -s V=0 - make install + if [ ! -f ./ffmpeg-3.4.2.tar.gz ]; then + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc --enable-avresample + make $FAST_MAKE -s V=0 + make install + else + echo "ffmpeg already installed" + fi cd $CURRENT_DIR else mkdir -p $LIB_DIR diff --git a/scripts/travisInstallDeps.sh b/scripts/travisInstallDeps.sh index 6c787965fc..71b20ee205 100755 --- a/scripts/travisInstallDeps.sh +++ b/scripts/travisInstallDeps.sh @@ -132,12 +132,16 @@ install_mediadeps(){ sudo apt-get -qq install yasm libvpx. libx264. if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz - tar -zxvf ffmpeg-3.4.2.tar.gz - cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc - make -s V=0 - make install + if [ ! -f ./ffmpeg-3.4.2.tar.gz ]; then + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libx264 --enable-libopus --disable-doc + make -s V=0 + make install + else + echo "ffmpeg already installed" + fi cd $CURRENT_DIR else mkdir -p $LIB_DIR @@ -150,12 +154,16 @@ install_mediadeps_nogpl(){ sudo apt-get -qq install yasm libvpx. if [ -d $LIB_DIR ]; then cd $LIB_DIR - curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz - tar -zxvf ffmpeg-3.4.2.tar.gz - cd ffmpeg-3.4.2 - PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc - make -s V=0 - make install + if [ ! -f ./ffmpeg-3.4.2.tar.gz ]; then + curl -O -L https://ffmpeg.org/releases/ffmpeg-3.4.2.tar.gz + tar -zxvf ffmpeg-3.4.2.tar.gz + cd ffmpeg-3.4.2 + PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig ./configure --prefix=$PREFIX_DIR --enable-shared --enable-gpl --enable-libvpx --enable-libopus --disable-doc + make -s V=0 + make install + else + echo "ffmpeg already installed" + fi cd $CURRENT_DIR else mkdir -p $LIB_DIR From b43063e843b471dc8a2248f7efb8e4d5979ac4a7 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Wed, 30 May 2018 10:00:46 -0300 Subject: [PATCH 15/32] Start recording accepts format extension --- erizo_controller/erizoClient/src/Room.js | 12 ++++++++-- .../erizoController/models/Client.js | 23 +++++++------------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index 7fb227fa13..7b634bf8ef 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -682,14 +682,22 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { }; // Returns callback(id, error) - that.startRecording = (stream, callback = () => {}) => { + that.startRecording = (stream, callbackOrOptions) => { + let callback = () => {}; + let extension = 'mkv'; + if (typeof callbackOrOptions === 'function') { + callback = callbackOrOptions; + } else if (typeof callbackOrOptions === 'object') { + callback = callbackOrOptions.callback || callback; + extension = callbackOrOptions.extension || extension; + } if (stream === undefined) { Logger.error('Trying to start recording on an invalid stream', stream); callback(undefined, 'Invalid Stream'); return; } Logger.debug(`Start Recording stream: ${stream.getID()}`); - socket.sendMessage('startRecorder', { to: stream.getID() }, (id, error) => { + socket.sendMessage('startRecorder', { to: stream.getID(), extension }, (id, error) => { if (id === null) { Logger.error('Error on start recording', error); callback(undefined, error); diff --git a/erizo_controller/erizoController/models/Client.js b/erizo_controller/erizoController/models/Client.js index 07fe038ced..47cdab031d 100644 --- a/erizo_controller/erizoController/models/Client.js +++ b/erizo_controller/erizoController/models/Client.js @@ -31,7 +31,7 @@ class Client extends events.EventEmitter { this.id = uuidv4(); this.options = options; listenToSocketEvents(this); - this.user = {name: token.userName, role: token.role, permissions: {}}; + this.user = {name: token.userName, role: token.role, permissions: {}, recordings: {}}; const permissions = global.config.erizoController.roles[token.role] || []; for (const right in permissions) { this.user.permissions[right] = permissions[right]; @@ -151,11 +151,7 @@ class Client extends events.EventEmitter { let url = sdp; if (options.state === 'recording') { const recordingId = sdp; - if (global.config.erizoController.recording_path) { // jshint ignore:line - url = global.config.erizoController.recording_path + recordingId + '.mkv'; // jshint ignore:line - } else { - url = '/tmp/' + recordingId + '.mkv'; - } + url = this.user.recordings[recordingId]; } this.room.controller.addExternalInput(id, url, (result) => { if (result === 'success') { @@ -357,16 +353,19 @@ class Client extends events.EventEmitter { callback(null, 'Unauthorized'); return; } + var extension = options.extension || 'mkv'; var streamId = options.to; var recordingId = Math.random() * 1000000000000000000; var url; if (global.config.erizoController.recording_path) { // jshint ignore:line - url = global.config.erizoController.recording_path + recordingId + '.mkv'; // jshint ignore:line + url = global.config.erizoController.recording_path + recordingId + '.' + extension; // jshint ignore:line } else { - url = '/tmp/' + recordingId + '.mkv'; + url = '/tmp/' + recordingId + '.' + extension; } + this.user.recordings[recordingId] = url; + log.info('message: startRecorder, ' + 'state: RECORD_REQUESTED, ' + 'streamId: ' + streamId + ', ' + @@ -412,13 +411,7 @@ class Client extends events.EventEmitter { return; } var recordingId = options.id; - var url; - - if (global.config.erizoController.recording_path) { // jshint ignore:line - url = global.config.erizoController.recording_path + recordingId + '.mkv'; // jshint ignore:line - } else { - url = '/tmp/' + recordingId + '.mkv'; - } + var url = this.user.recordings[recordingId]; log.info('message: startRecorder, ' + 'state: RECORD_STOPPED, ' + From 13c2c37f8d97a0c73743f64296948241cac210ac Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Wed, 30 May 2018 10:19:58 -0300 Subject: [PATCH 16/32] Start recording accepts urls --- erizo_controller/erizoClient/src/Room.js | 4 +++- erizo_controller/erizoController/models/Client.js | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index 7b634bf8ef..71b2644c7f 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -685,11 +685,13 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { that.startRecording = (stream, callbackOrOptions) => { let callback = () => {}; let extension = 'mkv'; + let url = null; if (typeof callbackOrOptions === 'function') { callback = callbackOrOptions; } else if (typeof callbackOrOptions === 'object') { callback = callbackOrOptions.callback || callback; extension = callbackOrOptions.extension || extension; + url = callbackOrOptions.url || url; } if (stream === undefined) { Logger.error('Trying to start recording on an invalid stream', stream); @@ -697,7 +699,7 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { return; } Logger.debug(`Start Recording stream: ${stream.getID()}`); - socket.sendMessage('startRecorder', { to: stream.getID(), extension }, (id, error) => { + socket.sendMessage('startRecorder', { to: stream.getID(), extension, url }, (id, error) => { if (id === null) { Logger.error('Error on start recording', error); callback(undefined, error); diff --git a/erizo_controller/erizoController/models/Client.js b/erizo_controller/erizoController/models/Client.js index 47cdab031d..809bec81ac 100644 --- a/erizo_controller/erizoController/models/Client.js +++ b/erizo_controller/erizoController/models/Client.js @@ -356,9 +356,11 @@ class Client extends events.EventEmitter { var extension = options.extension || 'mkv'; var streamId = options.to; var recordingId = Math.random() * 1000000000000000000; - var url; + var url = null; - if (global.config.erizoController.recording_path) { // jshint ignore:line + if (options.url != null && options.url != undefined) { + url = options.url.replace('{RECORDING_ID}', recordingId); + } else if (global.config.erizoController.recording_path) { // jshint ignore:line url = global.config.erizoController.recording_path + recordingId + '.' + extension; // jshint ignore:line } else { url = '/tmp/' + recordingId + '.' + extension; From a22fb8db1d8177d0666eb16dc48dadef3e933150 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Fri, 1 Jun 2018 15:11:41 -0300 Subject: [PATCH 17/32] Fix lint for erizoController --- erizo_controller/erizoController/models/Client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erizo_controller/erizoController/models/Client.js b/erizo_controller/erizoController/models/Client.js index 809bec81ac..d384ee79a1 100644 --- a/erizo_controller/erizoController/models/Client.js +++ b/erizo_controller/erizoController/models/Client.js @@ -358,7 +358,7 @@ class Client extends events.EventEmitter { var recordingId = Math.random() * 1000000000000000000; var url = null; - if (options.url != null && options.url != undefined) { + if (options.url !== null && options.url !== undefined) { url = options.url.replace('{RECORDING_ID}', recordingId); } else if (global.config.erizoController.recording_path) { // jshint ignore:line url = global.config.erizoController.recording_path + recordingId + '.' + extension; // jshint ignore:line From 68ad15b9b998b767f6185d1be122f5d11f847067 Mon Sep 17 00:00:00 2001 From: Francesco Durighetto Date: Wed, 30 May 2018 22:11:01 +0200 Subject: [PATCH 18/32] Fix the way ICE propagates connection state events (#1232) --- .../erizoClient/src/ErizoConnectionManager.js | 3 ++- erizo_controller/erizoClient/src/Room.js | 12 ++++++++---- erizo_controller/erizoClient/src/Stream.js | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/erizo_controller/erizoClient/src/ErizoConnectionManager.js b/erizo_controller/erizoClient/src/ErizoConnectionManager.js index d6d0b1f373..ae0a68face 100644 --- a/erizo_controller/erizoClient/src/ErizoConnectionManager.js +++ b/erizo_controller/erizoClient/src/ErizoConnectionManager.js @@ -65,7 +65,8 @@ class ErizoConnection extends EventEmitterConst { this.emit(ConnectionEvent({ type: 'remove-stream', stream: evt.stream })); }; - this.stack.peerConnection.oniceconnectionstatechange = (state) => { + this.stack.peerConnection.oniceconnectionstatechange = () => { + const state = this.stack.peerConnection.iceConnectionState; this.emit(ConnectionEvent({ type: 'ice-state-change', state })); }; } diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index 71b2644c7f..6f99196469 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -125,7 +125,8 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { getP2PConnectionOptions(stream, peerSocket))); stream.on('added', dispatchStreamSubscribed.bind(null, stream)); stream.on('icestatechanged', (evt) => { - if (evt.state === 'failed') { + Logger.info(`${stream.getID()} - iceConnectionState: ${evt.msg.state}`); + if (evt.msg.state === 'failed') { onStreamFailed(stream); } }); @@ -139,7 +140,8 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { stream.addPC(connection, peerSocket); stream.on('icestatechanged', (evt) => { - if (evt.state === 'failed') { + Logger.info(`${stream.getID()} - iceConnectionState: ${evt.msg.state}`); + if (evt.msg.state === 'failed') { stream.pc.get(peerSocket).close(); stream.pc.remove(peerSocket); } @@ -193,7 +195,8 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { getErizoConnectionOptions(stream, options, true), erizoId, spec.singlePC)); stream.on('added', dispatchStreamSubscribed.bind(null, stream)); stream.on('icestatechanged', (evt) => { - if (evt.state === 'failed') { + Logger.info(`${stream.getID()} - iceConnectionState: ${evt.msg.state}`); + if (evt.msg.state === 'failed') { onStreamFailed(stream); } }); @@ -206,7 +209,8 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { getErizoConnectionOptions(stream, options), erizoId, spec.singlePC)); stream.on('icestatechanged', (evt) => { - if (evt.state === 'failed') { + Logger.info(`${stream.getID()} - iceConnectionState: ${evt.msg.state}`); + if (evt.msg.state === 'failed') { onStreamFailed(stream); } }); diff --git a/erizo_controller/erizoClient/src/Stream.js b/erizo_controller/erizoClient/src/Stream.js index 08def71d9d..2e524e11ba 100644 --- a/erizo_controller/erizoClient/src/Stream.js +++ b/erizo_controller/erizoClient/src/Stream.js @@ -45,8 +45,8 @@ const Stream = (altConnectionHelpers, specInput) => { } }; - const onICEConnectionStateChange = (state) => { - that.emit(StreamEvent({ type: 'icestatechanged', msg: state })); + const onICEConnectionStateChange = (msg) => { + that.emit(StreamEvent({ type: 'icestatechanged', msg })); }; if (that.videoSize !== undefined && From ce6f84bb6c598f24d9c66bd769f4a5dbe8adc289 Mon Sep 17 00:00:00 2001 From: Guang-De Lin Date: Fri, 1 Jun 2018 15:35:54 +0800 Subject: [PATCH 19/32] Do not check if room param is NaN in basic example server. (#1231) --- extras/basic_example/basicServer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/basic_example/basicServer.js b/extras/basic_example/basicServer.js index 6b50bc91c5..c1fdff95b1 100644 --- a/extras/basic_example/basicServer.js +++ b/extras/basic_example/basicServer.js @@ -154,7 +154,7 @@ app.post('/createToken/', function(req, res) { let room = defaultRoomName, type, roomId, mediaConfiguration; - if (req.body.room && !isNaN(req.body.room)) room = req.body.room; + if (req.body.room) room = req.body.room; if (req.body.type) type = req.body.type; if (req.body.roomId) roomId = req.body.roomId; if (req.body.mediaConfiguration) mediaConfiguration = req.body.mediaConfiguration; From beecac07718d72804c1b59739e91b498b927c9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E5=B0=91=E5=8D=8E?= Date: Mon, 4 Jun 2018 16:00:58 +0800 Subject: [PATCH 20/32] Fix crashes in addRemoteCandidate when subscriber is not removed from publisher (#1227) --- erizo_controller/erizoJS/erizoJSController.js | 1 + 1 file changed, 1 insertion(+) diff --git a/erizo_controller/erizoJS/erizoJSController.js b/erizo_controller/erizoJS/erizoJSController.js index 6685f4cc14..5e47ca6bd9 100644 --- a/erizo_controller/erizoJS/erizoJSController.js +++ b/erizo_controller/erizoJS/erizoJSController.js @@ -223,6 +223,7 @@ exports.ErizoJSController = function (threadPool, ioThreadPool) { let subscriber = publisher.getSubscriber(subscriberKey); log.info('message: Removing subscriber, id: ' + subscriberKey); closeNode(subscriber); + publisher.removeSubscriber(subscriberKey); } publisher.removeExternalOutputs().then(function() { closeNode(publisher); From e6767767ff6c15caf92a7e6e787558a487b3fcf9 Mon Sep 17 00:00:00 2001 From: Equod Date: Thu, 7 Jun 2018 09:45:14 +0200 Subject: [PATCH 21/32] Added pipeline shared_ptr check before dereferencing it (#1238) --- erizo/src/erizo/MediaStream.cpp | 35 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/erizo/src/erizo/MediaStream.cpp b/erizo/src/erizo/MediaStream.cpp index 362f29206e..884c01f1b4 100644 --- a/erizo/src/erizo/MediaStream.cpp +++ b/erizo/src/erizo/MediaStream.cpp @@ -95,7 +95,9 @@ void MediaStream::setMaxVideoBW(uint32_t max_video_bw) { asyncTask([max_video_bw] (std::shared_ptr stream) { if (stream->rtcp_processor_) { stream->rtcp_processor_->setMaxVideoBW(max_video_bw * 1000); - stream->pipeline_->notifyUpdate(); + if (stream->pipeline_) { + stream->pipeline_->notifyUpdate(); + } } }); } @@ -146,7 +148,7 @@ bool MediaStream::setRemoteSdp(std::shared_ptr sdp) { this->rtcp_processor_->setMaxVideoBW(remote_sdp_->videoBandwidth*1000); } - if (pipeline_initialized_) { + if (pipeline_initialized_ && pipeline_) { pipeline_->notifyUpdate(); return true; } @@ -272,7 +274,10 @@ int MediaStream::deliverEvent_(MediaEventPtr event) { if (!stream_ptr->pipeline_initialized_) { return; } - stream_ptr->pipeline_->notifyEvent(event); + + if (stream_ptr->pipeline_) { + stream_ptr->pipeline_->notifyEvent(event); + } }); return 1; } @@ -309,7 +314,9 @@ void MediaStream::onTransportData(std::shared_ptr incoming_packet, T } } - stream_ptr->pipeline_->read(std::move(packet)); + if (stream_ptr->pipeline_) { + stream_ptr->pipeline_->read(std::move(packet)); + } }); } @@ -448,7 +455,9 @@ void MediaStream::muteStream(bool mute_video, bool mute_audio) { CumulativeStat{mute_audio}); media_stream->stats_->getNode()[media_stream->getAudioSinkSSRC()].insertStat("erizoVideoMute", CumulativeStat{mute_video}); - media_stream->pipeline_->notifyUpdate(); + if (media_stream && media_stream->pipeline_) { + media_stream->pipeline_->notifyUpdate(); + } }); } @@ -557,19 +566,25 @@ void MediaStream::write(std::shared_ptr packet) { void MediaStream::enableHandler(const std::string &name) { asyncTask([name] (std::shared_ptr conn) { - conn->pipeline_->enable(name); + if (conn && conn->pipeline_) { + conn->pipeline_->enable(name); + } }); } void MediaStream::disableHandler(const std::string &name) { asyncTask([name] (std::shared_ptr conn) { - conn->pipeline_->disable(name); + if (conn && conn->pipeline_) { + conn->pipeline_->disable(name); + } }); } void MediaStream::notifyUpdateToHandlers() { asyncTask([] (std::shared_ptr conn) { - conn->pipeline_->notifyUpdate(); + if (conn && conn->pipeline_) { + conn->pipeline_->notifyUpdate(); + } }); } @@ -612,7 +627,9 @@ void MediaStream::sendPacket(std::shared_ptr p) { return; } - pipeline_->write(std::move(p)); + if (pipeline_) { + pipeline_->write(std::move(p)); + } } void MediaStream::setQualityLayer(int spatial_layer, int temporal_layer) { From 38c845a12ddffaf2b08556553b599c319cd4d231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Andr=C3=A9s?= Date: Thu, 7 Jun 2018 10:24:30 +0200 Subject: [PATCH 22/32] Sanitise token signature checking (#1235) --- erizo_controller/erizoController/models/Channel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erizo_controller/erizoController/models/Channel.js b/erizo_controller/erizoController/models/Channel.js index a4702ee983..2a03a8e9c0 100644 --- a/erizo_controller/erizoController/models/Channel.js +++ b/erizo_controller/erizoController/models/Channel.js @@ -63,7 +63,7 @@ class Channel extends events.EventEmitter { onToken(options, callback) { const token = options.token; log.debug('message: token received'); - if (checkSignature(token, NUVE_KEY)) { + if (token && checkSignature(token, NUVE_KEY)) { this.nuve.deleteToken(token.tokenId).then(tokenDB => { if (token.host === tokenDB.host) { this.state = CONNECTED; From ee4eb15fdd7b69304d42b28335e23c78f626f138 Mon Sep 17 00:00:00 2001 From: Francesco Durighetto Date: Thu, 7 Jun 2018 11:22:52 +0200 Subject: [PATCH 23/32] Fix unpublish in p2p (#1236) --- erizo_controller/erizoClient/src/Stream.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/erizo_controller/erizoClient/src/Stream.js b/erizo_controller/erizoClient/src/Stream.js index 2e524e11ba..06f35d9cee 100644 --- a/erizo_controller/erizoClient/src/Stream.js +++ b/erizo_controller/erizoClient/src/Stream.js @@ -30,6 +30,7 @@ const Stream = (altConnectionHelpers, specInput) => { that.desktopStreamId = spec.desktopStreamId; that.audioMuted = false; that.videoMuted = false; + that.p2p = false; that.ConnectionHelpers = altConnectionHelpers === undefined ? ConnectionHelpers : altConnectionHelpers; @@ -111,6 +112,7 @@ const Stream = (altConnectionHelpers, specInput) => { that.addPC = (pc, p2pKey = undefined) => { if (p2pKey) { + that.p2p = true; if (that.pc === undefined) { that.pc = ErizoMap(); } @@ -229,10 +231,16 @@ const Stream = (altConnectionHelpers, specInput) => { } that.stream = undefined; } - if (that.pc) { + if (that.pc && !that.p2p) { that.pc.off('add-stream', spec.onStreamAddedToPC); that.pc.off('remove-stream', spec.onStreamRemovedFroPC); that.pc.off('ice-state-change', spec.onICEConnectionStateChange); + } else if (that.pc && that.p2p) { + that.pc.forEach((pc) => { + pc.off('add-stream', spec.onStreamAddedToPC); + pc.off('remove-stream', spec.onStreamRemovedFroPC); + pc.off('ice-state-change', spec.onICEConnectionStateChange); + }); } }; From a4e5330a9b5d4528f9acbd0b4c16f4d7dacba912 Mon Sep 17 00:00:00 2001 From: Francesco Durighetto Date: Thu, 7 Jun 2018 11:52:08 +0200 Subject: [PATCH 24/32] Fix concurrency in Nuve create room(#1225) --- nuve/nuveAPI/mdb/serviceRegistry.js | 14 +++++++++++++- nuve/nuveAPI/resource/roomsResource.js | 4 ++-- nuve/nuveAPI/test/mdb/serviceRegistry.js | 7 +++++++ nuve/nuveAPI/test/resource/roomsResource.js | 4 ++-- nuve/nuveAPI/test/utils.js | 1 + 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/nuve/nuveAPI/mdb/serviceRegistry.js b/nuve/nuveAPI/mdb/serviceRegistry.js index e1a97b484b..d8163f786e 100644 --- a/nuve/nuveAPI/mdb/serviceRegistry.js +++ b/nuve/nuveAPI/mdb/serviceRegistry.js @@ -55,9 +55,21 @@ exports.addService = function (service, callback) { /* * Updates a determined service in the data base. */ -exports.updateService = function (service) { +exports.updateService = function (service, callback) { db.services.save(service, function (error) { if (error) log.info('message: updateService error, ' + logger.objectToLog(error)); + if (callback) callback(); + }); +}; + +/* + * Updates a determined service in the data base with a new room. + */ +exports.addRoomToService = function (service, room, callback) { + db.services.update({_id: db.ObjectId(service._id)}, {$addToSet: {rooms: room}}, + function (error) { + if (error) log.info('message: updateService error, ' + logger.objectToLog(error)); + if (callback) callback(); }); }; diff --git a/nuve/nuveAPI/resource/roomsResource.js b/nuve/nuveAPI/resource/roomsResource.js index b42eb31a6a..c30f28b953 100644 --- a/nuve/nuveAPI/resource/roomsResource.js +++ b/nuve/nuveAPI/resource/roomsResource.js @@ -37,7 +37,7 @@ exports.createRoom = function (req, res) { roomRegistry.addRoom(room, function (result) { currentService.testRoom = result; currentService.rooms.push(result); - serviceRegistry.updateService(currentService); + serviceRegistry.addRoomToService(currentService, result); log.info('message: testRoom created, serviceId: ' + currentService.name); res.send(result); }); @@ -56,7 +56,7 @@ exports.createRoom = function (req, res) { } roomRegistry.addRoom(room, function (result) { currentService.rooms.push(result); - serviceRegistry.updateService(currentService); + serviceRegistry.addRoomToService(currentService, result); log.info('message: createRoom success, roomName:' + req.body.name + ', serviceId: ' + currentService.name + ', p2p: ' + room.p2p); res.send(result); diff --git a/nuve/nuveAPI/test/mdb/serviceRegistry.js b/nuve/nuveAPI/test/mdb/serviceRegistry.js index 46c2a99362..9d220c2cc4 100644 --- a/nuve/nuveAPI/test/mdb/serviceRegistry.js +++ b/nuve/nuveAPI/test/mdb/serviceRegistry.js @@ -6,6 +6,7 @@ var expect = require('chai').expect; var kArbitraryService = {name: 'arbitraryService'}; var kArbitraryServiceId = '1'; +var kArbitraryRoom = {name:'', options: {test: true}}; describe('Service Registry', function() { var serviceRegistry, @@ -82,6 +83,12 @@ describe('Service Registry', function() { expect(dataBase.db.services.save.calledOnce).to.be.true; // jshint ignore:line }); + it('should call update on Database when calling addRoomToService', function() { + serviceRegistry.addRoomToService(kArbitraryService, kArbitraryRoom); + + expect(dataBase.db.services.update.calledOnce).to.be.true; // jshint ignore:line + }); + it('should call remove on Database when removeService is called and it exists', function() { dataBase.db.services.findOne.callsArgWith(1, null, undefined); serviceRegistry.removeService(kArbitraryServiceId); diff --git a/nuve/nuveAPI/test/resource/roomsResource.js b/nuve/nuveAPI/test/resource/roomsResource.js index 71e2399895..86486d9ae1 100644 --- a/nuve/nuveAPI/test/resource/roomsResource.js +++ b/nuve/nuveAPI/test/resource/roomsResource.js @@ -107,7 +107,7 @@ describe('Rooms Resource', function() { .end(function(err) { if (err) throw err; expect(roomRegistryMock.addRoom.called).to.be.true; // jshint ignore:line - expect(serviceRegistryMock.updateService.called).to.be.true; // jshint ignore:line + expect(serviceRegistryMock.addRoomToService.called).to.be.true; // jshint ignore:line done(); }); }); @@ -136,7 +136,7 @@ describe('Rooms Resource', function() { .end(function(err) { if (err) throw err; expect(roomRegistryMock.addRoom.called).to.be.true; // jshint ignore:line - expect(serviceRegistryMock.updateService.called).to.be.true; // jshint ignore:line + expect(serviceRegistryMock.addRoomToService.called).to.be.true; // jshint ignore:line done(); }); }); diff --git a/nuve/nuveAPI/test/utils.js b/nuve/nuveAPI/test/utils.js index b6aadff4a0..510a4f432b 100644 --- a/nuve/nuveAPI/test/utils.js +++ b/nuve/nuveAPI/test/utils.js @@ -50,6 +50,7 @@ var reset = module.exports.reset = function() { getService: sinon.stub(), addService: sinon.stub(), updateService: sinon.stub(), + addRoomToService: sinon.stub(), removeService: sinon.stub(), getRoomForService: sinon.stub(), getList: sinon.stub().returns() From 17eebd46841ff33ba4b8f143fbd187c9909bb2e2 Mon Sep 17 00:00:00 2001 From: Francesco Durighetto Date: Thu, 7 Jun 2018 12:13:15 +0200 Subject: [PATCH 25/32] Update build script to automatically detect the number of CPUs (#1226) --- scripts/installErizo.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/installErizo.sh b/scripts/installErizo.sh index 76c3d33a8a..07c581044d 100755 --- a/scripts/installErizo.sh +++ b/scripts/installErizo.sh @@ -13,6 +13,12 @@ PREFIX_DIR=$LIB_DIR/build/ NVM_CHECK="$PATHNAME"/checkNvm.sh FAST_MAKE='' +NUM_CORES=1; +if [ "$(uname)" == "Darwin" ]; then + NUM_CORES=$(sysctl -n hw.ncpu); +elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + NUM_CORES=$(grep -c ^processor /proc/cpuinfo); +fi export ERIZO_HOME=$ROOT/erizo @@ -128,8 +134,9 @@ else execute_tests ;; f) - FAST_MAKE='-j4' - FAST_BUILD='env JOBS=4' + FAST_MAKE="-j$NUM_CORES" + FAST_BUILD="env JOBS=$NUM_CORES" + echo "Compiling using $NUM_CORES threads" ;; d) DELETE_OBJECT_FILES='true' From 5ca71eb34fd1cf7e0063bfd1ace480e2163054cf Mon Sep 17 00:00:00 2001 From: Francesco Durighetto Date: Wed, 13 Jun 2018 13:41:22 +0200 Subject: [PATCH 26/32] Assign room to stream (#1243) --- erizo_controller/erizoClient/src/Room.js | 1 + 1 file changed, 1 insertion(+) diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index 6f99196469..49ca184544 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -610,6 +610,7 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { label: arg.label, screen: arg.screen, attributes: arg.attributes }); + stream.room = that; streamList.push(stream); remoteStreams.add(arg.id, stream); } From 4b4fe08c753b42183cbc3649b4c31d2341350691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Cervi=C3=B1o?= Date: Wed, 13 Jun 2018 14:19:17 +0200 Subject: [PATCH 27/32] Fix circleci scripts (#1248) --- .circleci/config.yml | 60 +++++--------------------------------------- 1 file changed, 6 insertions(+), 54 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9bc6328a03..3bed9dd830 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,31 +3,15 @@ version: 2.0 jobs: build: docker: - - image: buildpack-deps:trusty-curl + - image: circleci/buildpack-deps:trusty-curl-browsers working_directory: /tmp/licode steps: - - run: - name: Install Git client - command: | - set -x - sudo apt-get update - sudo apt-get install -y git - - checkout - setup_remote_docker - - run: - name: Install Docker client - command: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - mv /tmp/docker/* /usr/bin - - run: name: Pull latest docker image command: | @@ -65,7 +49,7 @@ jobs: - deploy: name: Push Licode Docker image command: | - if [ "${CIRCLE_BRANCH}" == "master" ]; then + if [ "${CIRCLE_BRANCH}" == "master" ] && [ "${CIRCLE_PROJECT_USERNAME}" == "lynckia" ]; then SHORT_GIT_HASH="$(echo ${CIRCLE_SHA1} | cut -c -7)" docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} echo Tagging to lynckia/licode:${SHORT_GIT_HASH} @@ -76,36 +60,20 @@ jobs: prerelease: docker: - - image: buildpack-deps:trusty-curl + - image: circleci/buildpack-deps:trusty-curl-browsers working_directory: /tmp/licode steps: - - run: - name: Install Git client - command: | - set -x - sudo apt-get update - sudo apt-get install -y git - - checkout - setup_remote_docker - - run: - name: Install Docker client - command: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - mv /tmp/docker/* /usr/bin - - run: name: Install deps command: | sudo apt-get update - sudo apt-get install -y git jq + sudo apt-get install -y jq - run: name: Create Prerelease @@ -114,36 +82,20 @@ jobs: release: docker: - - image: buildpack-deps:trusty-curl + - image: circleci/buildpack-deps:trusty-curl-browsers working_directory: /tmp/licode steps: - - run: - name: Install Git client - command: | - set -x - sudo apt-get update - sudo apt-get install -y git - - checkout - setup_remote_docker - - run: - name: Install Docker client - command: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - mv /tmp/docker/* /usr/bin - - run: name: Install deps command: | sudo apt-get update - sudo apt-get install -y git jq + sudo apt-get install -y jq - run: name: Create release From 013ca008245a89a8624b3a60140e60d5387da80e Mon Sep 17 00:00:00 2001 From: Equod Date: Wed, 13 Jun 2018 16:07:38 +0200 Subject: [PATCH 28/32] Small code review like avoiding redundant objects copy. (#1247) --- erizo/src/erizo/DtlsTransport.cpp | 14 +++--- erizo/src/erizo/DtlsTransport.h | 4 +- erizo/src/erizo/IceConnection.cpp | 4 +- erizo/src/erizo/IceConnection.h | 6 +-- erizo/src/erizo/MediaStream.cpp | 56 ++++++++++++------------ erizo/src/erizo/WebRtcConnection.cpp | 14 +++--- erizo/src/erizo/dtls/DtlsClient.cpp | 6 +-- erizo/src/erizo/dtls/DtlsSocket.h | 4 +- erizo/src/erizo/logger.h | 12 ++--- erizo/src/erizo/media/ExternalOutput.cpp | 6 +-- 10 files changed, 63 insertions(+), 63 deletions(-) diff --git a/erizo/src/erizo/DtlsTransport.cpp b/erizo/src/erizo/DtlsTransport.cpp index 0b9dc92032..a725dc7515 100644 --- a/erizo/src/erizo/DtlsTransport.cpp +++ b/erizo/src/erizo/DtlsTransport.cpp @@ -127,7 +127,7 @@ DtlsTransport::~DtlsTransport() { void DtlsTransport::start() { ice_->setIceListener(shared_from_this()); - ice_->copyLogContextFrom(this); + ice_->copyLogContextFrom(*this); ELOG_DEBUG("%s message: starting ice", toLog()); ice_->start(); } @@ -279,10 +279,10 @@ void DtlsTransport::onHandshakeCompleted(DtlsSocketContext *ctx, std::string cli boost::mutex::scoped_lock lock(sessionMutex_); std::string temp; - if (rtp_timeout_checker_.get() != NULL) { + if (rtp_timeout_checker_) { rtp_timeout_checker_->cancel(); } - if (rtcp_timeout_checker_.get() != NULL) { + if (rtcp_timeout_checker_) { rtcp_timeout_checker_->cancel(); } @@ -316,14 +316,14 @@ void DtlsTransport::onHandshakeCompleted(DtlsSocketContext *ctx, std::string cli } } -void DtlsTransport::onHandshakeFailed(DtlsSocketContext *ctx, const std::string error) { +void DtlsTransport::onHandshakeFailed(DtlsSocketContext *ctx, const std::string& error) { ELOG_WARN("%s message: Handshake failed, transportName:%s, openSSLerror: %s", toLog(), transport_name.c_str(), error.c_str()); running_ = false; updateTransportState(TRANSPORT_FAILED); } -std::string DtlsTransport::getMyFingerprint() { +std::string DtlsTransport::getMyFingerprint() const { return dtlsRtp->getFingerprint(); } @@ -368,8 +368,8 @@ void DtlsTransport::processLocalSdp(SdpInfo *localSdp_) { ELOG_DEBUG("%s message: processing local sdp, transportName: %s", toLog(), transport_name.c_str()); localSdp_->isFingerprint = true; localSdp_->fingerprint = getMyFingerprint(); - std::string username = ice_->getLocalUsername(); - std::string password = ice_->getLocalPassword(); + std::string username(ice_->getLocalUsername()); + std::string password(ice_->getLocalPassword()); if (bundle_) { localSdp_->setCredentials(username, password, VIDEO_TYPE); localSdp_->setCredentials(username, password, AUDIO_TYPE); diff --git a/erizo/src/erizo/DtlsTransport.h b/erizo/src/erizo/DtlsTransport.h index c5fd6d8769..a764d56af2 100644 --- a/erizo/src/erizo/DtlsTransport.h +++ b/erizo/src/erizo/DtlsTransport.h @@ -26,7 +26,7 @@ class DtlsTransport : dtls::DtlsReceiver, public Transport { std::shared_ptr io_worker); virtual ~DtlsTransport(); void connectionStateChanged(IceState newState); - std::string getMyFingerprint(); + std::string getMyFingerprint() const; static bool isDtlsPacket(const char* buf, int len); void start() override; void close() override; @@ -37,7 +37,7 @@ class DtlsTransport : dtls::DtlsReceiver, public Transport { void writeDtlsPacket(dtls::DtlsSocketContext *ctx, packetPtr packet); void onHandshakeCompleted(dtls::DtlsSocketContext *ctx, std::string clientKey, std::string serverKey, std::string srtp_profile) override; - void onHandshakeFailed(dtls::DtlsSocketContext *ctx, const std::string error) override; + void onHandshakeFailed(dtls::DtlsSocketContext *ctx, const std::string& error) override; void updateIceState(IceState state, IceConnection *conn) override; void processLocalSdp(SdpInfo *localSdp_) override; diff --git a/erizo/src/erizo/IceConnection.cpp b/erizo/src/erizo/IceConnection.cpp index 656410bb30..51822e5a37 100644 --- a/erizo/src/erizo/IceConnection.cpp +++ b/erizo/src/erizo/IceConnection.cpp @@ -31,11 +31,11 @@ std::weak_ptr IceConnection::getIceListener() { return listener_; } -std::string IceConnection::getLocalUsername() { +const std::string& IceConnection::getLocalUsername() const { return ufrag_; } -std::string IceConnection::getLocalPassword() { +const std::string& IceConnection::getLocalPassword() const { return upass_; } diff --git a/erizo/src/erizo/IceConnection.h b/erizo/src/erizo/IceConnection.h index 2b0edddb57..3e7d22acce 100644 --- a/erizo/src/erizo/IceConnection.h +++ b/erizo/src/erizo/IceConnection.h @@ -110,14 +110,14 @@ class IceConnection : public LogContext { virtual void setIceListener(std::weak_ptr listener); virtual std::weak_ptr getIceListener(); - virtual std::string getLocalUsername(); - virtual std::string getLocalPassword(); + virtual const std::string& getLocalUsername() const; + virtual const std::string& getLocalPassword() const; private: virtual std::string iceStateToString(IceState state) const; protected: - inline std::string toLog() { + inline std::string toLog() const { return "id: " + ice_config_.connection_id + ", " + printLogContext(); } diff --git a/erizo/src/erizo/MediaStream.cpp b/erizo/src/erizo/MediaStream.cpp index 884c01f1b4..35d6662b88 100644 --- a/erizo/src/erizo/MediaStream.cpp +++ b/erizo/src/erizo/MediaStream.cpp @@ -48,12 +48,12 @@ MediaStream::MediaStream(std::shared_ptr worker, const std::string& media_stream_label, bool is_publisher) : audio_enabled_{false}, video_enabled_{false}, - connection_{connection}, + connection_{std::move(connection)}, stream_id_{media_stream_id}, mslabel_ {media_stream_label}, bundle_{false}, pipeline_{Pipeline::create()}, - worker_{worker}, + worker_{std::move(worker)}, audio_muted_{false}, video_muted_{false}, pipeline_initialized_{false}, is_publisher_{is_publisher} { @@ -190,7 +190,7 @@ bool MediaStream::setRemoteSdp(std::shared_ptr sdp) { } bool MediaStream::setLocalSdp(std::shared_ptr sdp) { - local_sdp_ = sdp; + local_sdp_ = std::move(sdp); return true; } @@ -203,41 +203,41 @@ void MediaStream::initializePipeline() { pipeline_->addService(quality_manager_); pipeline_->addService(packet_buffer_); - pipeline_->addFront(PacketReader(this)); - - pipeline_->addFront(RtcpProcessorHandler()); - pipeline_->addFront(FecReceiverHandler()); - pipeline_->addFront(LayerBitrateCalculationHandler()); - pipeline_->addFront(QualityFilterHandler()); - pipeline_->addFront(IncomingStatsHandler()); - pipeline_->addFront(RtpTrackMuteHandler()); - pipeline_->addFront(RtpSlideShowHandler()); - pipeline_->addFront(RtpPaddingGeneratorHandler()); - pipeline_->addFront(PliPacerHandler()); - pipeline_->addFront(BandwidthEstimationHandler()); - pipeline_->addFront(RtpPaddingRemovalHandler()); - pipeline_->addFront(RtcpFeedbackGenerationHandler()); - pipeline_->addFront(RtpRetransmissionHandler()); - pipeline_->addFront(SRPacketHandler()); - pipeline_->addFront(SenderBandwidthEstimationHandler()); - pipeline_->addFront(LayerDetectorHandler()); - pipeline_->addFront(OutgoingStatsHandler()); - pipeline_->addFront(PacketCodecParser()); - - pipeline_->addFront(PacketWriter(this)); + pipeline_->addFront(std::make_shared(this)); + + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); + + pipeline_->addFront(std::make_shared(this)); pipeline_->finalize(); pipeline_initialized_ = true; } int MediaStream::deliverAudioData_(std::shared_ptr audio_packet) { - if (audio_enabled_ == true) { + if (audio_enabled_) { sendPacketAsync(std::make_shared(*audio_packet)); } return audio_packet->length; } int MediaStream::deliverVideoData_(std::shared_ptr video_packet) { - if (video_enabled_ == true) { + if (video_enabled_) { sendPacketAsync(std::make_shared(*video_packet)); } return video_packet->length; @@ -388,7 +388,7 @@ void MediaStream::notifyMediaStreamEvent(const std::string& type, const std::str } void MediaStream::notifyToEventSink(MediaEventPtr event) { - event_sink_->deliverEvent(event); + event_sink_->deliverEvent(std::move(event)); } int MediaStream::sendPLI() { diff --git a/erizo/src/erizo/WebRtcConnection.cpp b/erizo/src/erizo/WebRtcConnection.cpp index 30927d1133..e4c7f9904d 100644 --- a/erizo/src/erizo/WebRtcConnection.cpp +++ b/erizo/src/erizo/WebRtcConnection.cpp @@ -128,20 +128,20 @@ bool WebRtcConnection::createOffer(bool video_enabled, bool audioEnabled, bool b if (bundle_) { video_transport_.reset(new DtlsTransport(VIDEO_TYPE, "video", connection_id_, bundle_, true, listener, ice_config_ , "", "", true, worker_, io_worker_)); - video_transport_->copyLogContextFrom(this); + video_transport_->copyLogContextFrom(*this); video_transport_->start(); } else { if (video_transport_.get() == nullptr && video_enabled_) { // For now we don't re/check transports, if they are already created we leave them there video_transport_.reset(new DtlsTransport(VIDEO_TYPE, "video", connection_id_, bundle_, true, listener, ice_config_ , "", "", true, worker_, io_worker_)); - video_transport_->copyLogContextFrom(this); + video_transport_->copyLogContextFrom(*this); video_transport_->start(); } if (audio_transport_.get() == nullptr && audio_enabled_) { audio_transport_.reset(new DtlsTransport(AUDIO_TYPE, "audio", connection_id_, bundle_, true, listener, ice_config_, "", "", true, worker_, io_worker_)); - audio_transport_->copyLogContextFrom(this); + audio_transport_->copyLogContextFrom(*this); audio_transport_->start(); } } @@ -339,7 +339,7 @@ bool WebRtcConnection::processRemoteSdp(std::string stream_id) { video_transport_.reset(new DtlsTransport(VIDEO_TYPE, "video", connection_id_, bundle_, remote_sdp_->isRtcpMux, listener, ice_config_ , username, password, false, worker_, io_worker_)); - video_transport_->copyLogContextFrom(this); + video_transport_->copyLogContextFrom(*this); video_transport_->start(); } else { ELOG_DEBUG("%s message: Updating videoTransport, ufrag: %s, pass: %s", @@ -356,7 +356,7 @@ bool WebRtcConnection::processRemoteSdp(std::string stream_id) { audio_transport_.reset(new DtlsTransport(AUDIO_TYPE, "audio", connection_id_, bundle_, remote_sdp_->isRtcpMux, listener, ice_config_, username, password, false, worker_, io_worker_)); - audio_transport_->copyLogContextFrom(this); + audio_transport_->copyLogContextFrom(*this); audio_transport_->start(); } else { ELOG_DEBUG("%s message: Update audioTransport, ufrag: %s, pass: %s", @@ -455,7 +455,7 @@ std::string WebRtcConnection::getJSONCandidate(const std::string& mid, const std std::ostringstream theString; theString << "{"; - for (std::map::iterator it = object.begin(); it != object.end(); ++it) { + for (std::map::const_iterator it = object.begin(); it != object.end(); ++it) { theString << "\"" << it->first << "\":\"" << it->second << "\""; if (++it != object.end()) { theString << ","; @@ -737,7 +737,7 @@ void WebRtcConnection::syncWrite(std::shared_ptr packet) { } void WebRtcConnection::setTransport(std::shared_ptr transport) { // Only for Testing purposes - video_transport_ = transport; + video_transport_ = std::move(transport); bundle_ = true; } diff --git a/erizo/src/erizo/dtls/DtlsClient.cpp b/erizo/src/erizo/dtls/DtlsClient.cpp index 1cd50cd034..a6ec7ac33b 100644 --- a/erizo/src/erizo/dtls/DtlsClient.cpp +++ b/erizo/src/erizo/dtls/DtlsClient.cpp @@ -309,10 +309,10 @@ int createCert(const std::string& pAor, int expireDays, int keyLen, X509*& outCe } - std::string DtlsSocketContext::getFingerprint() { - char fprint[100]; + std::string DtlsSocketContext::getFingerprint() const { + char fprint[100] = {}; mSocket->getMyCertFingerprint(fprint); - return std::string(fprint, strlen(fprint)); + return std::string(fprint); } void DtlsSocketContext::start() { diff --git a/erizo/src/erizo/dtls/DtlsSocket.h b/erizo/src/erizo/dtls/DtlsSocket.h index a6d7115eaa..2ea47952a2 100644 --- a/erizo/src/erizo/dtls/DtlsSocket.h +++ b/erizo/src/erizo/dtls/DtlsSocket.h @@ -131,7 +131,7 @@ class DtlsReceiver { virtual void onDtlsPacket(DtlsSocketContext *ctx, const unsigned char* data, unsigned int len) = 0; virtual void onHandshakeCompleted(DtlsSocketContext *ctx, std::string clientKey, std::string serverKey, std::string srtp_profile) = 0; - virtual void onHandshakeFailed(DtlsSocketContext *ctx, const std::string error) = 0; + virtual void onHandshakeFailed(DtlsSocketContext *ctx, const std::string& error) = 0; }; class DtlsSocketContext { @@ -153,7 +153,7 @@ class DtlsSocketContext { void handshakeFailed(const char *err); void setDtlsReceiver(DtlsReceiver *recv); void setDtlsSocket(DtlsSocket *sock) {mSocket = sock;} - std::string getFingerprint(); + std::string getFingerprint() const; void handleTimeout(); diff --git a/erizo/src/erizo/logger.h b/erizo/src/erizo/logger.h index 426654a6d8..d13460517d 100644 --- a/erizo/src/erizo/logger.h +++ b/erizo/src/erizo/logger.h @@ -35,21 +35,21 @@ class LogContext { LogContext() : context_log_{""} { } - virtual ~LogContext() {} + virtual ~LogContext() = default; - void setLogContext(std::map context) { + void setLogContext(const std::map& context) { context_ = context; context_log_ = ""; - for (const std::pair &item : context) { + for (const auto &item : context) { context_log_ += item.first + ": " + item.second + ", "; } } - void copyLogContextFrom(LogContext *log_context) { - setLogContext(log_context->context_); + void copyLogContextFrom(const LogContext& log_context) { + setLogContext(log_context.context_); } - std::string printLogContext() { + const std::string& printLogContext() const { return context_log_; } diff --git a/erizo/src/erizo/media/ExternalOutput.cpp b/erizo/src/erizo/media/ExternalOutput.cpp index dfe64eb70f..064b9baca2 100644 --- a/erizo/src/erizo/media/ExternalOutput.cpp +++ b/erizo/src/erizo/media/ExternalOutput.cpp @@ -364,10 +364,10 @@ void ExternalOutput::initializePipeline() { pipeline_->addService(quality_manager_); pipeline_->addService(stats_); - pipeline_->addFront(LayerBitrateCalculationHandler()); - pipeline_->addFront(QualityFilterHandler()); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); - pipeline_->addFront(ExternalOuputWriter(shared_from_this())); + pipeline_->addFront(std::make_shared(shared_from_this())); pipeline_->finalize(); pipeline_initialized_ = true; } From b733cebc5992a056bb50752e816cc425da228fdb Mon Sep 17 00:00:00 2001 From: Equod Date: Wed, 13 Jun 2018 17:22:02 +0200 Subject: [PATCH 29/32] Added clion configs to gitignore (.idea folder) (#1246) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 96a34a8f5a..d6045dfbe9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ erizo_controller/erizoAgent/out.log erizo_controller/erizoAgent/erizo-*.log .project .cproject +.idea spine/erizofc.js spine/testResult.json extras/basic_example/public/assets/ From d1ae92ef8d51cbb59d5958adbbc3afcae6c7abf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E5=B0=91=E5=8D=8E?= Date: Mon, 18 Jun 2018 15:51:12 +0800 Subject: [PATCH 30/32] Fix WebRtcConnection close by updating publishers immediately (#1237) --- erizo_controller/erizoJS/erizoJSController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erizo_controller/erizoJS/erizoJSController.js b/erizo_controller/erizoJS/erizoJSController.js index 5e47ca6bd9..c6a63d7bb6 100644 --- a/erizo_controller/erizoJS/erizoJSController.js +++ b/erizo_controller/erizoJS/erizoJSController.js @@ -227,11 +227,11 @@ exports.ErizoJSController = function (threadPool, ioThreadPool) { } publisher.removeExternalOutputs().then(function() { closeNode(publisher); + delete publishers[streamId]; publisher.muxer.close(function(message) { log.info('message: muxer closed succesfully, ' + 'id: ' + streamId + ', ' + logger.objectToLog(message)); - delete publishers[streamId]; var count = 0; for (var k in publishers) { if (publishers.hasOwnProperty(k)) { From a1fa1f5717140e5cd96580918721d37d73ec5d26 Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Wed, 4 Jul 2018 12:19:21 -0300 Subject: [PATCH 31/32] Improve startRecording advancedOptions --- erizo_controller/erizoClient/src/Room.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index 49ca184544..14b9515519 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -687,16 +687,12 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { }; // Returns callback(id, error) - that.startRecording = (stream, callbackOrOptions) => { - let callback = () => {}; + that.startRecording = (stream, callback, advancedOptions=undefined) => { let extension = 'mkv'; let url = null; - if (typeof callbackOrOptions === 'function') { - callback = callbackOrOptions; - } else if (typeof callbackOrOptions === 'object') { - callback = callbackOrOptions.callback || callback; - extension = callbackOrOptions.extension || extension; - url = callbackOrOptions.url || url; + if (typeof advancedOptions === 'object') { + extension = advancedOptions.extension || extension; + url = advancedOptions.url || url; } if (stream === undefined) { Logger.error('Trying to start recording on an invalid stream', stream); From 8dcf31b2ba5a004cc45890b662e7baa781ecb54f Mon Sep 17 00:00:00 2001 From: Alvaro Gil Date: Tue, 10 Jul 2018 12:06:22 -0300 Subject: [PATCH 32/32] fix js client lint --- erizo_controller/erizoClient/src/Room.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index 14b9515519..2ff6f7767d 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -687,7 +687,7 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { }; // Returns callback(id, error) - that.startRecording = (stream, callback, advancedOptions=undefined) => { + that.startRecording = (stream, callback, advancedOptions = undefined) => { let extension = 'mkv'; let url = null; if (typeof advancedOptions === 'object') {