From 5a90e0064e4cbe30f8e7fd74bc8a220d4eccec24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Cervi=C3=B1o?= Date: Thu, 16 Feb 2017 16:54:56 +0100 Subject: [PATCH 1/5] First version --- erizo/src/erizo/rtp/QualityFilterHandler.cpp | 92 ++++++++++++++++++++ erizo/src/erizo/rtp/QualityFilterHandler.h | 48 ++++++++++ 2 files changed, 140 insertions(+) create mode 100644 erizo/src/erizo/rtp/QualityFilterHandler.cpp create mode 100644 erizo/src/erizo/rtp/QualityFilterHandler.h diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.cpp b/erizo/src/erizo/rtp/QualityFilterHandler.cpp new file mode 100644 index 0000000000..0491affcae --- /dev/null +++ b/erizo/src/erizo/rtp/QualityFilterHandler.cpp @@ -0,0 +1,92 @@ +#include "rtp/QualityFilterHandler.h" + +#include "./WebRtcConnection.h" +#include "lib/ClockUtils.h" + +namespace erizo { + +DEFINE_LOGGER(QualityFilterHandler, "rtp.QualityFilterHandler"); + +QualityFilterHandler::QualityFilterHandler() + : connection_{nullptr}, enabled_{true}, initialized_{false}, + last_seq_number_{0}, + target_spatial_layer_{0}, target_temporal_layer_{0}, video_sink_ssrc_{0} {} + +void QualityFilterHandler::enable() { + enabled_ = true; +} + +void QualityFilterHandler::disable() { + enabled_ = false; +} + +void QualityFilterHandler::read(Context *ctx, std::shared_ptr packet) { + RtcpHeader *chead = reinterpret_cast(packet->data); + int len = packet->length; + if (chead->isFeedback()) { + char* movingBuf = packet->data; + int rtcpLength = 0; + int totalLength = 0; + int currentBlock = 0; + + do { + movingBuf+=rtcpLength; + chead = reinterpret_cast(movingBuf); + rtcpLength = (ntohs(chead->length) + 1) * 4; + totalLength += rtcpLength; + switch (chead->packettype) { + case RTCP_PS_Feedback_PT: + switch (chead->getBlockCount()) { + case RTCP_AFB: + { + char *uniqueId = reinterpret_cast(&chead->report.rembPacket.uniqueid); + if (!strncmp(uniqueId, "REMB", 4)) { + chead->setREMBBitRate(300000); + } + break; + } + default: + break; + } + default: + break; + } + currentBlock++; + } while (totalLength < len); + } + ctx->fireRead(packet); +} + +void QualityFilterHandler::write(Context *ctx, std::shared_ptr packet) { + RtcpHeader *chead = reinterpret_cast(packet->data); + if (!chead->isRtcp() && enabled_ && packet->type == VIDEO_PACKET) { + if (!packet->belongsToSpatialLayer(target_spatial_layer_) || + !packet->belongsToTemporalLayer(target_temporal_layer_)) { + return; + } + RtpHeader *rtp_header = reinterpret_cast(packet->data); + rtp_header->setSSRC(video_sink_ssrc_); + rtp_header->setSeqNumber(++last_seq_number_); + } + + ctx->fireWrite(packet); +} + +void QualityFilterHandler::notifyUpdate() { + if (initialized_) { + return; + } + + auto pipeline = getContext()->getPipelineShared(); + if (!pipeline) { + return; + } + + connection_ = pipeline->getService().get(); + if (!connection_) { + return; + } + + video_sink_ssrc_ = connection_->getVideoSinkSSRC(); +} +} // namespace erizo diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.h b/erizo/src/erizo/rtp/QualityFilterHandler.h new file mode 100644 index 0000000000..5c202ef9bc --- /dev/null +++ b/erizo/src/erizo/rtp/QualityFilterHandler.h @@ -0,0 +1,48 @@ +#ifndef ERIZO_SRC_ERIZO_RTP_QUALITYFILTERHANDLER_H_ +#define ERIZO_SRC_ERIZO_RTP_QUALITYFILTERHANDLER_H_ + +#include +#include +#include +#include + +#include "./logger.h" +#include "pipeline/Handler.h" + +#define MAX_DELAY 450000 + +namespace erizo { + +class WebRtcConnection; + + +class QualityFilterHandler: public Handler, public std::enable_shared_from_this { + DECLARE_LOGGER(); + + + public: + QualityFilterHandler(); + + void enable() override; + void disable() override; + + std::string getName() override { + return "quality_filter"; + } + + void read(Context *ctx, std::shared_ptr packet) override; + void write(Context *ctx, std::shared_ptr packet) override; + void notifyUpdate() override; + + private: + WebRtcConnection *connection_; + bool enabled_; + bool initialized_; + uint16_t last_seq_number_; + int target_spatial_layer_; + int target_temporal_layer_; + uint video_sink_ssrc_; +}; +} // namespace erizo + +#endif // ERIZO_SRC_ERIZO_RTP_QUALITYFILTERHANDLER_H_ From 45443fe5cbae43a89f75afbd0d484eb5ccd19601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Cervi=C3=B1o?= Date: Tue, 21 Feb 2017 14:49:45 +0100 Subject: [PATCH 2/5] Adapt filter to SequenceNumberTranslator --- erizo/src/erizo/rtp/QualityFilterHandler.cpp | 32 ++++++++++++++++---- erizo/src/erizo/rtp/QualityFilterHandler.h | 3 ++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.cpp b/erizo/src/erizo/rtp/QualityFilterHandler.cpp index 0491affcae..63b5d7c3f9 100644 --- a/erizo/src/erizo/rtp/QualityFilterHandler.cpp +++ b/erizo/src/erizo/rtp/QualityFilterHandler.cpp @@ -20,7 +20,7 @@ void QualityFilterHandler::disable() { enabled_ = false; } -void QualityFilterHandler::read(Context *ctx, std::shared_ptr packet) { +void QualityFilterHandler::applyMaxRembMaybe(std::shared_ptr packet) { RtcpHeader *chead = reinterpret_cast(packet->data); int len = packet->length; if (chead->isFeedback()) { @@ -54,21 +54,41 @@ void QualityFilterHandler::read(Context *ctx, std::shared_ptr packet currentBlock++; } while (totalLength < len); } +} + +void QualityFilterHandler::read(Context *ctx, std::shared_ptr packet) { + applyMaxRembMaybe(packet); // TODO(javier) remove this line when RTCP termination is enabled + + // TODO(javier): Handle RRs and NACKs and translate Sequence Numbers? + ctx->fireRead(packet); } void QualityFilterHandler::write(Context *ctx, std::shared_ptr packet) { RtcpHeader *chead = reinterpret_cast(packet->data); if (!chead->isRtcp() && enabled_ && packet->type == VIDEO_PACKET) { - if (!packet->belongsToSpatialLayer(target_spatial_layer_) || - !packet->belongsToTemporalLayer(target_temporal_layer_)) { - return; - } RtpHeader *rtp_header = reinterpret_cast(packet->data); + + if (!packet->belongsToSpatialLayer(target_spatial_layer_)) { + return; + } rtp_header->setSSRC(video_sink_ssrc_); - rtp_header->setSeqNumber(++last_seq_number_); + + uint16_t sequence_number = rtp_header->getSeqNumber(); + if (!packet->belongsToTemporalLayer(target_temporal_layer_)) { + sequence_number_translator.get(sequence_number, true); + return; + } + SequenceNumber sequence_number_info = sequence_number_translator.get(sequence_number, false); + if (sequence_number_info.type != SequenceNumberType::Valid) { + return; + } + + rtp_header->setSeqNumber(sequence_number_info.output); } + // TODO(javier): Handle SRs and translate Sequence Numbers? + ctx->fireWrite(packet); } diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.h b/erizo/src/erizo/rtp/QualityFilterHandler.h index 5c202ef9bc..b65baa1e52 100644 --- a/erizo/src/erizo/rtp/QualityFilterHandler.h +++ b/erizo/src/erizo/rtp/QualityFilterHandler.h @@ -8,6 +8,7 @@ #include "./logger.h" #include "pipeline/Handler.h" +#include "rtp/SequenceNumberTranslator.h" #define MAX_DELAY 450000 @@ -33,8 +34,10 @@ class QualityFilterHandler: public Handler, public std::enable_shared_from_this< void read(Context *ctx, std::shared_ptr packet) override; void write(Context *ctx, std::shared_ptr packet) override; void notifyUpdate() override; + void applyMaxRembMaybe(std::shared_ptr packet); private: + SequenceNumberTranslator translator; WebRtcConnection *connection_; bool enabled_; bool initialized_; From e8be7c8845b7a2320b5a497b00a2e98b70d97f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Cervi=C3=B1o?= Date: Wed, 22 Feb 2017 16:24:10 +0100 Subject: [PATCH 3/5] Make it compatible with Simulcast and VP9 SVC --- erizo/src/erizo/MediaDefinitions.h | 8 +- erizo/src/erizo/WebRtcConnection.cpp | 14 ++- erizo/src/erizo/WebRtcConnection.h | 5 + erizo/src/erizo/rtp/LayerDetectorHandler.cpp | 4 +- erizo/src/erizo/rtp/QualityFilterHandler.cpp | 98 +++++++++++-------- erizo/src/erizo/rtp/QualityFilterHandler.h | 19 ++-- erizo/src/erizo/rtp/QualityManager.cpp | 10 ++ erizo/src/erizo/rtp/QualityManager.h | 27 +++++ erizo/src/erizo/rtp/RtpUtils.cpp | 50 ++++++++++ erizo/src/erizo/rtp/RtpUtils.h | 14 +++ erizoAPI/WebRtcConnection.cc | 13 +++ erizoAPI/WebRtcConnection.h | 2 + erizo_controller/erizoClient/src/Stream.js | 15 ++- .../src/webrtc-stacks/ChromeStableStack.js | 2 +- .../src/webrtc-stacks/FirefoxStack.js | 2 +- erizo_controller/erizoJS/erizoJSController.js | 10 ++ erizo_controller/erizoJS/models/Publisher.js | 7 ++ 17 files changed, 243 insertions(+), 57 deletions(-) create mode 100644 erizo/src/erizo/rtp/QualityManager.cpp create mode 100644 erizo/src/erizo/rtp/QualityManager.h diff --git a/erizo/src/erizo/MediaDefinitions.h b/erizo/src/erizo/MediaDefinitions.h index d47ea74987..2dbd852a5f 100644 --- a/erizo/src/erizo/MediaDefinitions.h +++ b/erizo/src/erizo/MediaDefinitions.h @@ -25,19 +25,20 @@ struct dataPacket { dataPacket() = default; dataPacket(int comp_, const char *data_, int length_, packetType type_, uint64_t received_time_ms_) : - comp{comp_}, length{length_}, type{type_}, received_time_ms{received_time_ms_}, is_keyframe{false} { + comp{comp_}, length{length_}, type{type_}, received_time_ms{received_time_ms_}, is_keyframe{false}, + ending_of_layer_frame{false} { memcpy(data, data_, length_); } dataPacket(int comp_, const char *data_, int length_, packetType type_) : comp{comp_}, length{length_}, type{type_}, received_time_ms{ClockUtils::timePointToMs(clock::now())}, - is_keyframe{false} { + is_keyframe{false}, ending_of_layer_frame{false} { memcpy(data, data_, length_); } dataPacket(int comp_, const unsigned char *data_, int length_) : comp{comp_}, length{length_}, type{VIDEO_PACKET}, received_time_ms{ClockUtils::timePointToMs(clock::now())}, - is_keyframe{false} { + is_keyframe{false}, ending_of_layer_frame{false} { memcpy(data, data_, length_); } @@ -65,6 +66,7 @@ struct dataPacket { std::vector compatible_spatial_layers; std::vector compatible_temporal_layers; bool is_keyframe; // Note: It can be just a keyframe first packet in VP8 + bool ending_of_layer_frame; }; class Monitor { diff --git a/erizo/src/erizo/WebRtcConnection.cpp b/erizo/src/erizo/WebRtcConnection.cpp index a1ba9d4fbd..c04a543e36 100644 --- a/erizo/src/erizo/WebRtcConnection.cpp +++ b/erizo/src/erizo/WebRtcConnection.cpp @@ -27,6 +27,8 @@ #include "rtp/SRPacketHandler.h" #include "rtp/SenderBandwidthEstimationHandler.h" #include "rtp/LayerDetectorHandler.h" +#include "rtp/QualityFilterHandler.h" +#include "rtp/QualityManager.h" namespace erizo { DEFINE_LOGGER(WebRtcConnection, "WebRtcConnection"); @@ -45,6 +47,7 @@ WebRtcConnection::WebRtcConnection(std::shared_ptr worker, const std::st source_fb_sink_ = this; sink_fb_source_ = this; stats_ = std::make_shared(); + quality_manager_ = std::make_shared(); globalState_ = CONN_INITIAL; rtcp_processor_ = std::make_shared(static_cast(this), static_cast(this)); @@ -230,8 +233,6 @@ bool WebRtcConnection::setRemoteSdp(const std::string &sdp) { } } - - initializePipeline(); return true; @@ -241,6 +242,7 @@ void WebRtcConnection::initializePipeline() { pipeline_->addService(shared_from_this()); pipeline_->addService(rtcp_processor_); pipeline_->addService(stats_); + pipeline_->addService(quality_manager_); pipeline_->addFront(PacketReader(this)); @@ -248,6 +250,7 @@ void WebRtcConnection::initializePipeline() { pipeline_->addFront(RtcpProcessorHandler()); pipeline_->addFront(IncomingStatsHandler()); pipeline_->addFront(FecReceiverHandler()); + pipeline_->addFront(QualityFilterHandler()); pipeline_->addFront(RtpAudioMuteHandler()); pipeline_->addFront(RtpSlideShowHandler()); pipeline_->addFront(BandwidthEstimationHandler()); @@ -777,4 +780,11 @@ void WebRtcConnection::sendPacket(std::shared_ptr p) { pipeline_->write(p); } +void WebRtcConnection::setQualityLayer(int spatial_layer, int temporal_layer) { + asyncTask([spatial_layer, temporal_layer] (std::shared_ptr connection) { + connection->quality_manager_->setSpatialLayer(spatial_layer); + connection->quality_manager_->setTemporalLayer(temporal_layer); + }); +} + } // namespace erizo diff --git a/erizo/src/erizo/WebRtcConnection.h b/erizo/src/erizo/WebRtcConnection.h index 2890299572..f5601bb049 100644 --- a/erizo/src/erizo/WebRtcConnection.h +++ b/erizo/src/erizo/WebRtcConnection.h @@ -19,6 +19,7 @@ #include "lib/Clock.h" #include "pipeline/Handler.h" #include "pipeline/Service.h" +#include "rtp/QualityManager.h" namespace erizo { @@ -105,6 +106,9 @@ class WebRtcConnection: public MediaSink, public MediaSource, public FeedbackSin * @return the size of the data sent */ int sendPLI() override; + + void setQualityLayer(int spatial_layer, int temporal_layer); + /** * Sets the Event Listener for this WebRtcConnection */ @@ -210,6 +214,7 @@ class WebRtcConnection: public MediaSink, public MediaSource, public FeedbackSin std::shared_ptr videoTransport_, audioTransport_; std::shared_ptr stats_; + std::shared_ptr quality_manager_; WebRTCEvent globalState_; boost::mutex updateStateMutex_; // , slideShowMutex_; diff --git a/erizo/src/erizo/rtp/LayerDetectorHandler.cpp b/erizo/src/erizo/rtp/LayerDetectorHandler.cpp index 2254c3c274..1f3233a14f 100644 --- a/erizo/src/erizo/rtp/LayerDetectorHandler.cpp +++ b/erizo/src/erizo/rtp/LayerDetectorHandler.cpp @@ -85,7 +85,7 @@ void LayerDetectorHandler::parseLayerInfoFromVP9(std::shared_ptr pac int spatial_layer = payload->spatialID; packet->compatible_spatial_layers = {}; - for (int i = 0; i <= spatial_layer; i++) { + for (int i = 5; i >= spatial_layer; i--) { packet->compatible_spatial_layers.push_back(i); } @@ -110,6 +110,8 @@ void LayerDetectorHandler::parseLayerInfoFromVP9(std::shared_ptr pac } else { packet->is_keyframe = false; } + + packet->ending_of_layer_frame = payload->endingOfLayerFrame; delete payload; } diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.cpp b/erizo/src/erizo/rtp/QualityFilterHandler.cpp index 63b5d7c3f9..ff156dda22 100644 --- a/erizo/src/erizo/rtp/QualityFilterHandler.cpp +++ b/erizo/src/erizo/rtp/QualityFilterHandler.cpp @@ -2,6 +2,7 @@ #include "./WebRtcConnection.h" #include "lib/ClockUtils.h" +#include "rtp/RtpUtils.h" namespace erizo { @@ -9,8 +10,8 @@ DEFINE_LOGGER(QualityFilterHandler, "rtp.QualityFilterHandler"); QualityFilterHandler::QualityFilterHandler() : connection_{nullptr}, enabled_{true}, initialized_{false}, - last_seq_number_{0}, - target_spatial_layer_{0}, target_temporal_layer_{0}, video_sink_ssrc_{0} {} + target_spatial_layer_{0}, target_temporal_layer_{0}, + video_sink_ssrc_{0}, video_source_ssrc_{0}, last_ssrc_received_{0} {} void QualityFilterHandler::enable() { enabled_ = true; @@ -20,66 +21,80 @@ void QualityFilterHandler::disable() { enabled_ = false; } -void QualityFilterHandler::applyMaxRembMaybe(std::shared_ptr packet) { - RtcpHeader *chead = reinterpret_cast(packet->data); - int len = packet->length; - if (chead->isFeedback()) { - char* movingBuf = packet->data; - int rtcpLength = 0; - int totalLength = 0; - int currentBlock = 0; - - do { - movingBuf+=rtcpLength; - chead = reinterpret_cast(movingBuf); - rtcpLength = (ntohs(chead->length) + 1) * 4; - totalLength += rtcpLength; - switch (chead->packettype) { - case RTCP_PS_Feedback_PT: - switch (chead->getBlockCount()) { - case RTCP_AFB: - { - char *uniqueId = reinterpret_cast(&chead->report.rembPacket.uniqueid); - if (!strncmp(uniqueId, "REMB", 4)) { - chead->setREMBBitRate(300000); - } - break; - } - default: - break; - } - default: - break; +void QualityFilterHandler::handleFeedbackPackets(std::shared_ptr packet) { + RtpUtils::forEachRRBlock(packet, [this](RtcpHeader *chead) { + RtpUtils::updateREMB(chead, 400000); + + RtpUtils::forEachNack(chead, [this, chead](uint16_t seq_num, uint16_t plb) { + SequenceNumber result = translator_.reverse(seq_num); + if (result.type == SequenceNumberType::Valid) { + chead->setNackPid(result.input); } - currentBlock++; - } while (totalLength < len); - } + }); + }); } void QualityFilterHandler::read(Context *ctx, std::shared_ptr packet) { - applyMaxRembMaybe(packet); // TODO(javier) remove this line when RTCP termination is enabled + if (enabled_) { + handleFeedbackPackets(packet); // TODO(javier) remove this line when RTCP termination is enabled - // TODO(javier): Handle RRs and NACKs and translate Sequence Numbers? + // TODO(javier): Handle RRs and NACKs and translate Sequence Numbers? + } ctx->fireRead(packet); } +void QualityFilterHandler::checkLayers() { + int new_spatial_layer = quality_manager_->getSpatialLayer(); + if (new_spatial_layer != target_spatial_layer_) { + sendPLI(); + target_spatial_layer_ = new_spatial_layer; + } + int new_temporal_layer = quality_manager_->getTemporalLayer(); + target_temporal_layer_ = new_temporal_layer; +} + +void QualityFilterHandler::checkSSRCChange(uint32_t ssrc) { + if (last_ssrc_received_ != ssrc) { + translator_.reset(); + } + last_ssrc_received_ = ssrc; +} + +void QualityFilterHandler::sendPLI() { + getContext()->fireRead(RtpUtils::createPLI(video_sink_ssrc_, video_source_ssrc_)); +} + void QualityFilterHandler::write(Context *ctx, std::shared_ptr packet) { RtcpHeader *chead = reinterpret_cast(packet->data); if (!chead->isRtcp() && enabled_ && packet->type == VIDEO_PACKET) { RtpHeader *rtp_header = reinterpret_cast(packet->data); + checkLayers(); + + uint32_t ssrc = rtp_header->getSSRC(); + uint16_t sequence_number = rtp_header->getSeqNumber(); + if (!packet->belongsToSpatialLayer(target_spatial_layer_)) { + if (ssrc == video_sink_ssrc_) { + translator_.get(sequence_number, true); + } return; } + + checkSSRCChange(ssrc); rtp_header->setSSRC(video_sink_ssrc_); - uint16_t sequence_number = rtp_header->getSeqNumber(); + if (packet->compatible_spatial_layers.back() == target_spatial_layer_ && packet->ending_of_layer_frame) { + rtp_header->setMarker(1); + } + if (!packet->belongsToTemporalLayer(target_temporal_layer_)) { - sequence_number_translator.get(sequence_number, true); + translator_.get(sequence_number, true); return; } - SequenceNumber sequence_number_info = sequence_number_translator.get(sequence_number, false); + + SequenceNumber sequence_number_info = translator_.get(sequence_number, false); if (sequence_number_info.type != SequenceNumberType::Valid) { return; } @@ -107,6 +122,9 @@ void QualityFilterHandler::notifyUpdate() { return; } + quality_manager_ = pipeline->getService(); + video_sink_ssrc_ = connection_->getVideoSinkSSRC(); + video_source_ssrc_ = connection_->getVideoSourceSSRC(); } } // namespace erizo diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.h b/erizo/src/erizo/rtp/QualityFilterHandler.h index b65baa1e52..795d069b44 100644 --- a/erizo/src/erizo/rtp/QualityFilterHandler.h +++ b/erizo/src/erizo/rtp/QualityFilterHandler.h @@ -9,14 +9,12 @@ #include "./logger.h" #include "pipeline/Handler.h" #include "rtp/SequenceNumberTranslator.h" - -#define MAX_DELAY 450000 +#include "rtp/QualityManager.h" namespace erizo { class WebRtcConnection; - class QualityFilterHandler: public Handler, public std::enable_shared_from_this { DECLARE_LOGGER(); @@ -34,17 +32,24 @@ class QualityFilterHandler: public Handler, public std::enable_shared_from_this< void read(Context *ctx, std::shared_ptr packet) override; void write(Context *ctx, std::shared_ptr packet) override; void notifyUpdate() override; - void applyMaxRembMaybe(std::shared_ptr packet); private: - SequenceNumberTranslator translator; + void sendPLI(); + void checkLayers(); + void handleFeedbackPackets(std::shared_ptr packet); + void checkSSRCChange(uint32_t ssrc); + + private: + std::shared_ptr quality_manager_; + SequenceNumberTranslator translator_; WebRtcConnection *connection_; bool enabled_; bool initialized_; - uint16_t last_seq_number_; int target_spatial_layer_; int target_temporal_layer_; - uint video_sink_ssrc_; + uint32_t video_sink_ssrc_; + uint32_t video_source_ssrc_; + uint32_t last_ssrc_received_; }; } // namespace erizo diff --git a/erizo/src/erizo/rtp/QualityManager.cpp b/erizo/src/erizo/rtp/QualityManager.cpp new file mode 100644 index 0000000000..2920e7e322 --- /dev/null +++ b/erizo/src/erizo/rtp/QualityManager.cpp @@ -0,0 +1,10 @@ +#include "rtp/QualityManager.h" + +namespace erizo { + +DEFINE_LOGGER(QualityManager, "rtp.QualityManager"); + +QualityManager::QualityManager() + : spatial_layer_{0}, temporal_layer_{0} {} + +} // namespace erizo diff --git a/erizo/src/erizo/rtp/QualityManager.h b/erizo/src/erizo/rtp/QualityManager.h new file mode 100644 index 0000000000..99ff7e1c2a --- /dev/null +++ b/erizo/src/erizo/rtp/QualityManager.h @@ -0,0 +1,27 @@ +#ifndef ERIZO_SRC_ERIZO_RTP_QUALITYMANAGER_H_ +#define ERIZO_SRC_ERIZO_RTP_QUALITYMANAGER_H_ + +#include "./logger.h" +#include "pipeline/Service.h" + +namespace erizo { + +class QualityManager: public Service, public std::enable_shared_from_this { + DECLARE_LOGGER(); + + public: + QualityManager(); + + int getSpatialLayer() const { return spatial_layer_; } + int getTemporalLayer() const { return temporal_layer_; } + + void setSpatialLayer(int spatial_layer) { spatial_layer_ = spatial_layer; } + void setTemporalLayer(int temporal_layer) { temporal_layer_ = temporal_layer; } + + private: + int spatial_layer_; + int temporal_layer_; +}; +} // namespace erizo + +#endif // ERIZO_SRC_ERIZO_RTP_QUALITYMANAGER_H_ diff --git a/erizo/src/erizo/rtp/RtpUtils.cpp b/erizo/src/erizo/rtp/RtpUtils.cpp index e270d5b388..4591028c2f 100644 --- a/erizo/src/erizo/rtp/RtpUtils.cpp +++ b/erizo/src/erizo/rtp/RtpUtils.cpp @@ -1,5 +1,6 @@ #include "rtp/RtpUtils.h" +#include namespace erizo { @@ -8,4 +9,53 @@ bool RtpUtils::sequenceNumberLessThan(uint16_t first, uint16_t last) { return result > 0xF000; } +void RtpUtils::updateREMB(RtcpHeader *chead, uint bitrate) { + if (chead->packettype == RTCP_PS_Feedback_PT && chead->getBlockCount() == RTCP_AFB) { + char *uniqueId = reinterpret_cast(&chead->report.rembPacket.uniqueid); + if (!strncmp(uniqueId, "REMB", 4)) { + chead->setREMBBitRate(bitrate); + } + } } + +void RtpUtils::forEachNack(RtcpHeader *chead, std::function f) { + if (chead->packettype == RTCP_RTP_Feedback_PT) { + uint16_t initial_seq_num = chead->getNackPid(); + uint16_t plb = chead->getNackBlp(); + f(initial_seq_num, plb); + } +} + +std::shared_ptr RtpUtils::createPLI(uint32_t source_ssrc, uint32_t sink_ssrc) { + RtcpHeader pli; + pli.setPacketType(RTCP_PS_Feedback_PT); + pli.setBlockCount(1); + pli.setSSRC(sink_ssrc); + pli.setSourceSSRC(source_ssrc); + pli.setLength(2); + char *buf = reinterpret_cast(&pli); + int len = (pli.getLength() + 1) * 4; + return std::make_shared(0, buf, len, VIDEO_PACKET); +} + +void RtpUtils::forEachRRBlock(std::shared_ptr packet, std::function f) { + RtcpHeader *chead = reinterpret_cast(packet->data); + int len = packet->length; + if (chead->isFeedback()) { + char* moving_buffer = packet->data; + int rtcp_length = 0; + int total_length = 0; + int currentBlock = 0; + + do { + moving_buffer += rtcp_length; + chead = reinterpret_cast(moving_buffer); + rtcp_length = (ntohs(chead->length) + 1) * 4; + total_length += rtcp_length; + f(chead); + currentBlock++; + } while (total_length < len); + } +} + +} // namespace erizo diff --git a/erizo/src/erizo/rtp/RtpUtils.h b/erizo/src/erizo/rtp/RtpUtils.h index 123b8ffef4..9f3b6d051d 100644 --- a/erizo/src/erizo/rtp/RtpUtils.h +++ b/erizo/src/erizo/rtp/RtpUtils.h @@ -1,13 +1,27 @@ #ifndef ERIZO_SRC_ERIZO_RTP_RTPUTILS_H_ #define ERIZO_SRC_ERIZO_RTP_RTPUTILS_H_ +#include "rtp/RtpHeaders.h" + +#include "./MediaDefinitions.h" + #include +#include + namespace erizo { class RtpUtils { public: static bool sequenceNumberLessThan(uint16_t first, uint16_t second); + + static void forEachRRBlock(std::shared_ptr packet, std::function f); + + static void updateREMB(RtcpHeader *chead, uint bitrate); + + static void forEachNack(RtcpHeader *chead, std::function f); + + static std::shared_ptr createPLI(uint32_t source_ssrc, uint32_t sink_ssrc); }; } // namespace erizo diff --git a/erizoAPI/WebRtcConnection.cc b/erizoAPI/WebRtcConnection.cc index 9f659d6aa2..76f3edaae5 100644 --- a/erizoAPI/WebRtcConnection.cc +++ b/erizoAPI/WebRtcConnection.cc @@ -75,6 +75,7 @@ NAN_MODULE_INIT(WebRtcConnection::Init) { Nan::SetPrototypeMethod(tpl, "createOffer", createOffer); Nan::SetPrototypeMethod(tpl, "setSlideShowMode", setSlideShowMode); Nan::SetPrototypeMethod(tpl, "muteStream", muteStream); + Nan::SetPrototypeMethod(tpl, "setQualityLayer", setQualityLayer); Nan::SetPrototypeMethod(tpl, "setMetadata", setMetadata); Nan::SetPrototypeMethod(tpl, "enableHandler", enableHandler); Nan::SetPrototypeMethod(tpl, "disableHandler", disableHandler); @@ -409,6 +410,18 @@ NAN_METHOD(WebRtcConnection::disableHandler) { return; } +NAN_METHOD(WebRtcConnection::setQualityLayer) { + WebRtcConnection* obj = Nan::ObjectWrap::Unwrap(info.Holder()); + std::shared_ptr me = obj->me; + + int spatial_layer = info[0]->IntegerValue(); + int temporal_layer = info[1]->IntegerValue(); + + me->setQualityLayer(spatial_layer, temporal_layer); + + return; +} + NAN_METHOD(WebRtcConnection::setFeedbackReports) { WebRtcConnection* obj = Nan::ObjectWrap::Unwrap(info.Holder()); std::shared_ptr me = obj->me; diff --git a/erizoAPI/WebRtcConnection.h b/erizoAPI/WebRtcConnection.h index bf3507914b..8ddceeec68 100644 --- a/erizoAPI/WebRtcConnection.h +++ b/erizoAPI/WebRtcConnection.h @@ -156,6 +156,8 @@ class WebRtcConnection : public MediaSink, public erizo::WebRtcConnectionEventLi */ static NAN_METHOD(disableHandler); + static NAN_METHOD(setQualityLayer); + static Nan::Persistent constructor; static NAUV_WORK_CB(eventsCallback); diff --git a/erizo_controller/erizoClient/src/Stream.js b/erizo_controller/erizoClient/src/Stream.js index 5800f53cad..7e02ecc467 100644 --- a/erizo_controller/erizoClient/src/Stream.js +++ b/erizo_controller/erizoClient/src/Stream.js @@ -101,13 +101,13 @@ Erizo.Stream = function (spec) { videoOpt.mandatory.maxWidth = that.videoSize[2]; videoOpt.mandatory.maxHeight = that.videoSize[3]; } - + if (that.videoFrameRate !== undefined) { videoOpt.optional = [] videoOpt.optional.push({minFrameRate: that.videoFrameRate[0]}); videoOpt.optional.push({maxFrameRate: that.videoFrameRate[1]}); } - + } else if (spec.screen === true && videoOpt === undefined) { videoOpt = true; } @@ -302,6 +302,17 @@ Erizo.Stream = function (spec) { that.pc.updateSpec(config, callback); }; + that.setQualityLayer = function(spatialLayer, temporalLayer, callback) { + if (that.room && that.room.p2p){ + L.Logger.warning('setQualityLayer is not implemented in p2p streams'); + callback ('error'); + return; + } + var config = {qualityLayer : {spatialLayer: spatialLayer, temporalLayer: temporalLayer}}; + that.checkOptions(config, true); + that.pc.updateSpec(config, callback); + }; + controlHandler = function (handlers, publisherSide, enable) { publisherSide = !(publisherSide !== true); var handlers = (typeof handlers === 'string') ? [handlers] : handlers; diff --git a/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js b/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js index 9e5cd4a4c8..0e0f3db974 100644 --- a/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js +++ b/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js @@ -289,7 +289,7 @@ Erizo.ChromeStableStack = function (spec) { } } if (config.minVideoBW || (config.slideShowMode!==undefined) || - (config.muteStream !== undefined)){ + (config.muteStream !== undefined) || (config.qualityLayer !== undefined)){ L.Logger.debug ('MinVideo Changed to ', config.minVideoBW); L.Logger.debug ('SlideShowMode Changed to ', config.slideShowMode); L.Logger.debug ('muteStream changed to ', config.muteStream); diff --git a/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js b/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js index d205362c19..61245e077e 100644 --- a/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js +++ b/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js @@ -173,7 +173,7 @@ Erizo.FirefoxStack = function (spec) { } } if (config.minVideoBW || (config.slideShowMode!==undefined) || - (config.muteStream !== undefined)){ + (config.muteStream !== undefined) || (config.qualityLayer !== undefined)){ L.Logger.debug ('MinVideo Changed to ', config.minVideoBW); L.Logger.debug ('SlideShowMode Changed to ', config.slideShowMode); L.Logger.debug ('muteStream changed to ', config.muteStream); diff --git a/erizo_controller/erizoJS/erizoJSController.js b/erizo_controller/erizoJS/erizoJSController.js index 878f52f39f..a630a070e7 100644 --- a/erizo_controller/erizoJS/erizoJSController.js +++ b/erizo_controller/erizoJS/erizoJSController.js @@ -258,6 +258,9 @@ exports.ErizoJSController = function (threadPool) { if (msg.config.muteStream !== undefined) { that.muteStream (msg.config.muteStream, peerId, streamId); } + if (msg.config.qualityLayer !== undefined) { + that.setQualityLayer (msg.config.qualityLayer, peerId, streamId); + } } } else if (msg.type === 'control') { processControlMessage(publisher, peerId, msg.action); @@ -517,6 +520,13 @@ exports.ErizoJSController = function (threadPool) { } }; + that.setQualityLayer = function (qualityLayer, from, to) { + var publisher = this.publishers[to]; + if (publisher.hasSubscriber(from)) { + publisher.setQualityLayer(from, qualityLayer.spatialLayer, qualityLayer.temporalLayer); + } + }; + /* eslint no-param-reassign: ["error", { "props": false }] */ const getWrtcStats = (label, stats, wrtc) => { const promise = new Promise((resolve) => { diff --git a/erizo_controller/erizoJS/models/Publisher.js b/erizo_controller/erizoJS/models/Publisher.js index 62893ef979..43b7a41b8d 100644 --- a/erizo_controller/erizoJS/models/Publisher.js +++ b/erizo_controller/erizoJS/models/Publisher.js @@ -104,6 +104,13 @@ class Source { } } + setQualityLayer(id, spatialLayer, temporalLayer) { + var subscriber = this.getSubscriber(id); + log.info('message: setQualityLayer, spatialLayer: ', spatialLayer, + ', temporalLayer: ', temporalLayer); + subscriber.setQualityLayer(spatialLayer, temporalLayer); + } + muteSubscriberStream(id, muteVideo, muteAudio) { var subscriber = this.getSubscriber(id); subscriber.muteVideo = muteVideo; From 8e7d7d10cd6c0d6b4f8ee6a2b044fec14abf04cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Cervi=C3=B1o?= Date: Wed, 22 Feb 2017 16:29:26 +0100 Subject: [PATCH 4/5] Make API private --- erizo_controller/erizoClient/src/Stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erizo_controller/erizoClient/src/Stream.js b/erizo_controller/erizoClient/src/Stream.js index 7e02ecc467..d723030aed 100644 --- a/erizo_controller/erizoClient/src/Stream.js +++ b/erizo_controller/erizoClient/src/Stream.js @@ -302,7 +302,7 @@ Erizo.Stream = function (spec) { that.pc.updateSpec(config, callback); }; - that.setQualityLayer = function(spatialLayer, temporalLayer, callback) { + that._setQualityLayer = function(spatialLayer, temporalLayer, callback) { if (that.room && that.room.p2p){ L.Logger.warning('setQualityLayer is not implemented in p2p streams'); callback ('error'); From 42aa996363cabc46e30b6e6e8356a1fb4d8ed7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Cervi=C3=B1o?= Date: Thu, 23 Feb 2017 10:10:58 +0100 Subject: [PATCH 5/5] Fix with comments from PR and disabled by default --- erizo/src/erizo/rtp/QualityFilterHandler.cpp | 31 +++++++++++++------- erizo/src/erizo/rtp/QualityFilterHandler.h | 2 ++ erizo/src/erizo/rtp/RtcpAggregator.cpp | 4 --- erizo/src/erizo/rtp/RtcpAggregator.h | 1 - erizo/src/erizo/rtp/RtcpForwarder.cpp | 4 --- erizo/src/erizo/rtp/RtcpForwarder.h | 1 - erizo/src/erizo/rtp/RtcpProcessor.h | 4 ++- scripts/licode_default.js | 2 +- 8 files changed, 27 insertions(+), 22 deletions(-) diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.cpp b/erizo/src/erizo/rtp/QualityFilterHandler.cpp index ff156dda22..d4cb9855a7 100644 --- a/erizo/src/erizo/rtp/QualityFilterHandler.cpp +++ b/erizo/src/erizo/rtp/QualityFilterHandler.cpp @@ -10,8 +10,9 @@ DEFINE_LOGGER(QualityFilterHandler, "rtp.QualityFilterHandler"); QualityFilterHandler::QualityFilterHandler() : connection_{nullptr}, enabled_{true}, initialized_{false}, - target_spatial_layer_{0}, target_temporal_layer_{0}, - video_sink_ssrc_{0}, video_source_ssrc_{0}, last_ssrc_received_{0} {} + receiving_multiple_ssrc_{false}, target_spatial_layer_{0}, target_temporal_layer_{0}, + video_sink_ssrc_{0}, video_source_ssrc_{0}, last_ssrc_received_{0}, + max_video_bw_{0} {} void QualityFilterHandler::enable() { enabled_ = true; @@ -23,7 +24,7 @@ void QualityFilterHandler::disable() { void QualityFilterHandler::handleFeedbackPackets(std::shared_ptr packet) { RtpUtils::forEachRRBlock(packet, [this](RtcpHeader *chead) { - RtpUtils::updateREMB(chead, 400000); + RtpUtils::updateREMB(chead, max_video_bw_); RtpUtils::forEachNack(chead, [this, chead](uint16_t seq_num, uint16_t plb) { SequenceNumber result = translator_.reverse(seq_num); @@ -75,6 +76,10 @@ void QualityFilterHandler::write(Context *ctx, std::shared_ptr packe uint32_t ssrc = rtp_header->getSSRC(); uint16_t sequence_number = rtp_header->getSeqNumber(); + if (ssrc != last_ssrc_received_) { + receiving_multiple_ssrc_ = true; + } + if (!packet->belongsToSpatialLayer(target_spatial_layer_)) { if (ssrc == video_sink_ssrc_) { translator_.get(sequence_number, true); @@ -85,10 +90,6 @@ void QualityFilterHandler::write(Context *ctx, std::shared_ptr packe checkSSRCChange(ssrc); rtp_header->setSSRC(video_sink_ssrc_); - if (packet->compatible_spatial_layers.back() == target_spatial_layer_ && packet->ending_of_layer_frame) { - rtp_header->setMarker(1); - } - if (!packet->belongsToTemporalLayer(target_temporal_layer_)) { translator_.get(sequence_number, true); return; @@ -99,6 +100,10 @@ void QualityFilterHandler::write(Context *ctx, std::shared_ptr packe return; } + if (packet->compatible_spatial_layers.back() == target_spatial_layer_ && packet->ending_of_layer_frame) { + rtp_header->setMarker(1); + } + rtp_header->setSeqNumber(sequence_number_info.output); } @@ -108,12 +113,18 @@ void QualityFilterHandler::write(Context *ctx, std::shared_ptr packe } void QualityFilterHandler::notifyUpdate() { - if (initialized_) { + auto pipeline = getContext()->getPipelineShared(); + if (!pipeline) { return; } - auto pipeline = getContext()->getPipelineShared(); - if (!pipeline) { + auto processor = pipeline->getService(); + + if (processor) { + max_video_bw_ = processor->getMaxVideoBW(); + } + + if (initialized_) { return; } diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.h b/erizo/src/erizo/rtp/QualityFilterHandler.h index 795d069b44..4a82e9ad3c 100644 --- a/erizo/src/erizo/rtp/QualityFilterHandler.h +++ b/erizo/src/erizo/rtp/QualityFilterHandler.h @@ -45,11 +45,13 @@ class QualityFilterHandler: public Handler, public std::enable_shared_from_this< WebRtcConnection *connection_; bool enabled_; bool initialized_; + bool receiving_multiple_ssrc_; int target_spatial_layer_; int target_temporal_layer_; uint32_t video_sink_ssrc_; uint32_t video_source_ssrc_; uint32_t last_ssrc_received_; + uint32_t max_video_bw_; }; } // namespace erizo diff --git a/erizo/src/erizo/rtp/RtcpAggregator.cpp b/erizo/src/erizo/rtp/RtcpAggregator.cpp index de8d0696b1..32b5b0df06 100644 --- a/erizo/src/erizo/rtp/RtcpAggregator.cpp +++ b/erizo/src/erizo/rtp/RtcpAggregator.cpp @@ -42,10 +42,6 @@ void RtcpAggregator::addSourceSsrc(uint32_t ssrc) { } } -void RtcpAggregator::setMaxVideoBW(uint32_t bandwidth) { - this->maxVideoBw_ = bandwidth; -} - void RtcpAggregator::setPublisherBW(uint32_t bandwidth) { defaultVideoBw_ = (bandwidth*1.2) > maxVideoBw_? maxVideoBw_:(bandwidth*1.2); } diff --git a/erizo/src/erizo/rtp/RtcpAggregator.h b/erizo/src/erizo/rtp/RtcpAggregator.h index 238346578e..8976815354 100644 --- a/erizo/src/erizo/rtp/RtcpAggregator.h +++ b/erizo/src/erizo/rtp/RtcpAggregator.h @@ -23,7 +23,6 @@ class RtcpAggregator: public RtcpProcessor{ RtcpAggregator(MediaSink* msink, MediaSource* msource, uint32_t maxVideoBw = 300000); virtual ~RtcpAggregator() {} void addSourceSsrc(uint32_t ssrc); - void setMaxVideoBW(uint32_t bandwidth); void setPublisherBW(uint32_t bandwidth); void analyzeSr(RtcpHeader* chead); int analyzeFeedback(char* buf, int len); diff --git a/erizo/src/erizo/rtp/RtcpForwarder.cpp b/erizo/src/erizo/rtp/RtcpForwarder.cpp index 0eff91dc9e..15858f9627 100644 --- a/erizo/src/erizo/rtp/RtcpForwarder.cpp +++ b/erizo/src/erizo/rtp/RtcpForwarder.cpp @@ -33,10 +33,6 @@ void RtcpForwarder::addSourceSsrc(uint32_t ssrc) { } } -void RtcpForwarder::setMaxVideoBW(uint32_t bandwidth) { - this->maxVideoBw_ = bandwidth; -} - void RtcpForwarder::setPublisherBW(uint32_t bandwidth) { } diff --git a/erizo/src/erizo/rtp/RtcpForwarder.h b/erizo/src/erizo/rtp/RtcpForwarder.h index dceeaee9cc..577c99513c 100644 --- a/erizo/src/erizo/rtp/RtcpForwarder.h +++ b/erizo/src/erizo/rtp/RtcpForwarder.h @@ -22,7 +22,6 @@ class RtcpForwarder: public RtcpProcessor{ RtcpForwarder(MediaSink* msink, MediaSource* msource, uint32_t maxVideoBw = 300000); virtual ~RtcpForwarder() {} void addSourceSsrc(uint32_t ssrc); - void setMaxVideoBW(uint32_t bandwidth); void setPublisherBW(uint32_t bandwidth); void analyzeSr(RtcpHeader* chead); int analyzeFeedback(char* buf, int len); diff --git a/erizo/src/erizo/rtp/RtcpProcessor.h b/erizo/src/erizo/rtp/RtcpProcessor.h index 87abe04cc3..b822868b2c 100644 --- a/erizo/src/erizo/rtp/RtcpProcessor.h +++ b/erizo/src/erizo/rtp/RtcpProcessor.h @@ -113,12 +113,14 @@ class RtcpProcessor : public Service { rtcpSink_(msink), rtcpSource_(msource) {} virtual ~RtcpProcessor() {} virtual void addSourceSsrc(uint32_t ssrc) = 0; - virtual void setMaxVideoBW(uint32_t bandwidth) = 0; virtual void setPublisherBW(uint32_t bandwidth) = 0; virtual void analyzeSr(RtcpHeader* chead) = 0; virtual int analyzeFeedback(char* buf, int len) = 0; virtual void checkRtcpFb() = 0; + virtual void setMaxVideoBW(uint32_t bandwidth) { maxVideoBw_ = bandwidth; } + virtual uint32_t getMaxVideoBW() { return maxVideoBw_; } + protected: MediaSink* rtcpSink_; // The sink to send RRs MediaSource* rtcpSource_; // The source of SRs diff --git a/scripts/licode_default.js b/scripts/licode_default.js index b93bac0451..0b00d89413 100644 --- a/scripts/licode_default.js +++ b/scripts/licode_default.js @@ -162,7 +162,7 @@ config.erizo.networkinterface = ''; //default value: '' config.erizo.minport = 0; // default value: 0 config.erizo.maxport = 0; // default value: 0 -config.erizo['disabled_handlers'] = []; // there are no handlers disabled by default +config.erizo['disabled_handlers'] = ['quality_filter']; // there are no handlers disabled by default /***** END *****/ // Following lines are always needed.