From 6dd75470a402596f9cd22a61a661f9bfda6746ad Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Thu, 20 Sep 2018 15:33:51 +0200 Subject: [PATCH] API to enable slideshow keyframes below a spatial layer (#1297) --- erizo/src/erizo/MediaStream.cpp | 6 +- erizo/src/erizo/MediaStream.h | 2 +- erizo/src/erizo/rtp/QualityManager.cpp | 63 +++++++++++-------- erizo/src/erizo/rtp/QualityManager.h | 9 +-- erizo/src/erizo/rtp/RtpSlideShowHandler.cpp | 6 +- erizo/src/test/rtp/QualityManagerTest.cpp | 4 +- erizoAPI/MediaStream.cc | 12 ++-- erizoAPI/MediaStream.h | 2 +- erizo_controller/erizoClient/src/Stream.js | 10 ++- .../src/webrtc-stacks/BaseStack.js | 3 +- erizo_controller/erizoJS/models/Publisher.js | 8 ++- erizo_controller/erizoJS/models/Subscriber.js | 5 +- 12 files changed, 75 insertions(+), 55 deletions(-) diff --git a/erizo/src/erizo/MediaStream.cpp b/erizo/src/erizo/MediaStream.cpp index eff9cdfc26..ea953f6622 100644 --- a/erizo/src/erizo/MediaStream.cpp +++ b/erizo/src/erizo/MediaStream.cpp @@ -796,9 +796,9 @@ void MediaStream::setQualityLayer(int spatial_layer, int temporal_layer) { }); } -void MediaStream::setMinDesiredSpatialLayer(int spatial_layer) { - asyncTask([spatial_layer] (std::shared_ptr media_stream) { - media_stream->quality_manager_->setMinDesiredSpatialLayer(spatial_layer); +void MediaStream::enableSlideShowBelowSpatialLayer(bool enabled, int spatial_layer) { + asyncTask([enabled, spatial_layer] (std::shared_ptr media_stream) { + media_stream->quality_manager_->enableSlideShowBelowSpatialLayer(enabled, spatial_layer); }); } diff --git a/erizo/src/erizo/MediaStream.h b/erizo/src/erizo/MediaStream.h index 95d0b69a30..742c728212 100644 --- a/erizo/src/erizo/MediaStream.h +++ b/erizo/src/erizo/MediaStream.h @@ -84,7 +84,7 @@ class MediaStream: public MediaSink, public MediaSource, public FeedbackSink, int sendPLI() override; void sendPLIToFeedback(); void setQualityLayer(int spatial_layer, int temporal_layer); - void setMinDesiredSpatialLayer(int spatial_layer); + void enableSlideShowBelowSpatialLayer(bool enabled, int spatial_layer); WebRTCEvent getCurrentState(); diff --git a/erizo/src/erizo/rtp/QualityManager.cpp b/erizo/src/erizo/rtp/QualityManager.cpp index 4b3d3486e2..20fe2d2f56 100644 --- a/erizo/src/erizo/rtp/QualityManager.cpp +++ b/erizo/src/erizo/rtp/QualityManager.cpp @@ -15,9 +15,9 @@ constexpr float QualityManager::kIncreaseLayerBitrateThreshold; QualityManager::QualityManager(std::shared_ptr the_clock) : initialized_{false}, enabled_{false}, padding_enabled_{false}, forced_layers_{false}, - slideshow_fallback_active_{false}, spatial_layer_{0}, + freeze_fallback_active_{false}, enable_slideshow_below_spatial_layer_{false}, spatial_layer_{0}, temporal_layer_{0}, max_active_spatial_layer_{0}, - max_active_temporal_layer_{0}, min_desired_spatial_layer_{0}, max_video_width_{-1}, + max_active_temporal_layer_{0}, slideshow_below_spatial_layer_{-1}, max_video_width_{-1}, max_video_height_{-1}, max_video_frame_rate_{-1}, current_estimated_bitrate_{0}, last_quality_check_{the_clock->now()}, last_activity_check_{the_clock->now()}, clock_{the_clock} {} @@ -83,10 +83,10 @@ void QualityManager::notifyQualityUpdate() { bool layer_is_active = spatial_layer_ <= max_active_spatial_layer_; - if (!layer_is_active || (estimated_is_under_layer_bitrate && !slideshow_fallback_active_)) { + if (!layer_is_active || (estimated_is_under_layer_bitrate && !freeze_fallback_active_)) { ELOG_DEBUG("message: Forcing calculate new layer, " - "estimated_is_under_layer_bitrate: %d, layer_is_active: %d, slideshow_fallback_active_: %d", - estimated_is_under_layer_bitrate, layer_is_active, slideshow_fallback_active_); + "estimated_is_under_layer_bitrate: %d, layer_is_active: %d, freeze_fallback_active_: %d", + estimated_is_under_layer_bitrate, layer_is_active, freeze_fallback_active_); selectLayer(false); } else if (now - last_quality_check_ > kMinLayerSwitchInterval) { selectLayer(true); @@ -131,7 +131,9 @@ void QualityManager::selectLayer(bool try_higher_layers) { } stream_->setSimulcast(true); last_quality_check_ = clock_->now(); - int min_valid_spatial_layer = std::min(min_desired_spatial_layer_, max_active_spatial_layer_); + int min_requested_spatial_layer = + enable_slideshow_below_spatial_layer_ ? std::max(slideshow_below_spatial_layer_, 0) : 0; + int min_valid_spatial_layer = std::min(min_requested_spatial_layer, max_active_spatial_layer_); int aux_temporal_layer = 0; int aux_spatial_layer = 0; int next_temporal_layer = 0; @@ -139,8 +141,8 @@ void QualityManager::selectLayer(bool try_higher_layers) { float bitrate_margin = try_higher_layers ? kIncreaseLayerBitrateThreshold : 0; bool below_min_layer = true; bool layer_capped_by_constraints = false; - ELOG_DEBUG("message: Calculate best layer, estimated_bitrate: %lu, current layer %d/%d", - current_estimated_bitrate_, spatial_layer_, temporal_layer_); + ELOG_DEBUG("message: Calculate best layer, estimated_bitrate: %lu, current layer %d/%d, min_requested_spatial %d", + current_estimated_bitrate_, spatial_layer_, temporal_layer_, min_requested_spatial_layer); for (auto &spatial_layer_node : stats_->getNode()["qualityLayers"].getMap()) { if (aux_spatial_layer >= min_valid_spatial_layer) { for (auto &temporal_layer_node : spatial_layer_node.second->getMap()) { @@ -159,31 +161,36 @@ void QualityManager::selectLayer(bool try_higher_layers) { aux_temporal_layer++; } } else { - ELOG_DEBUG("message: Skipping below min spatial layer, aux_layer: %d, min_valid_spatial_layer", + ELOG_DEBUG("message: Skipping below min spatial layer, aux_layer: %d, min_valid_spatial_layer: %d", aux_spatial_layer, min_valid_spatial_layer); } aux_temporal_layer = 0; aux_spatial_layer++; } - if (below_min_layer != slideshow_fallback_active_) { + ELOG_DEBUG("message: below_min_layer %u, freeze_fallback_active_: %u", below_min_layer, freeze_fallback_active_); + if (below_min_layer != freeze_fallback_active_) { if (below_min_layer || try_higher_layers) { - slideshow_fallback_active_ = below_min_layer; + freeze_fallback_active_ = below_min_layer; ELOG_DEBUG("message: Setting slideshow fallback, below_min_layer %u, spatial_layer %d," - "next_spatial_layer %d slidehow_fallback_active_: %d", - below_min_layer, spatial_layer_, next_spatial_layer, slideshow_fallback_active_); + "next_spatial_layer %d freeze_fallback_active_: %d, min_requested_spatial_layer: %d," + "slideshow_below_spatial_layer_ %d", + below_min_layer, spatial_layer_, next_spatial_layer, freeze_fallback_active_, + min_requested_spatial_layer, slideshow_below_spatial_layer_); HandlerManager *manager = getContext()->getPipelineShared()->getService().get(); if (manager) { manager->notifyUpdateToHandlers(); } - if (below_min_layer && next_spatial_layer != 0) { - ELOG_DEBUG("message: Spatial layer is below minimum desired layer %d, activating keyframe resquests", - min_valid_spatial_layer); - stream_->notifyMediaStreamEvent("slideshow_fallback_update", "true"); - } else if (spatial_layer_ != 0) { - ELOG_DEBUG("message: Spatial layer has recovered %d, deactivating keyframe resquests", - next_spatial_layer); - stream_->notifyMediaStreamEvent("slideshow_fallback_update", "false"); + if (enable_slideshow_below_spatial_layer_ && slideshow_below_spatial_layer_ >= 0) { + if (below_min_layer) { + ELOG_DEBUG("message: Spatial layer is below min valid spatial layer %d and slideshow is requested " + ", activating keyframe requests", min_valid_spatial_layer); + stream_->notifyMediaStreamEvent("slideshow_fallback_update", "true"); + } else { + ELOG_DEBUG("message: Spatial layer has recovered %d, deactivating keyframe requests", + next_spatial_layer); + stream_->notifyMediaStreamEvent("slideshow_fallback_update", "false"); + } } } } @@ -196,10 +203,9 @@ void QualityManager::selectLayer(bool try_higher_layers) { // TODO(javier): should we wait for the actual spatial switch? // should we disable padding temporarily to avoid congestion (old padding + new bitrate)? - - ELOG_DEBUG("message: Is padding enabled, padding_enabled_: %d", padding_enabled_); } setPadding(!isInMaxLayer() && !layer_capped_by_constraints); + ELOG_DEBUG("message: Is padding enabled, padding_enabled_: %d", padding_enabled_); } void QualityManager::calculateMaxActiveLayer() { @@ -259,9 +265,14 @@ void QualityManager::forceLayers(int spatial_layer, int temporal_layer) { temporal_layer_ = temporal_layer; } -void QualityManager::setMinDesiredSpatialLayer(int spatial_layer) { - ELOG_DEBUG("message: setting min desired spatial layer, spatial_layer: %d", spatial_layer); - min_desired_spatial_layer_ = spatial_layer; +void QualityManager::enableSlideShowBelowSpatialLayer(bool enable, int spatial_layer) { + ELOG_DEBUG("message: enableSlideShowBelowSpatialLayer, enable %d, spatial_layer: %d", enable, spatial_layer); + enable_slideshow_below_spatial_layer_ = enable; + slideshow_below_spatial_layer_ = spatial_layer; + + stream_->notifyMediaStreamEvent("slideshow_fallback_update", "false"); + + freeze_fallback_active_ = false; selectLayer(true); } diff --git a/erizo/src/erizo/rtp/QualityManager.h b/erizo/src/erizo/rtp/QualityManager.h index b591aaaa0a..a6710ba704 100644 --- a/erizo/src/erizo/rtp/QualityManager.h +++ b/erizo/src/erizo/rtp/QualityManager.h @@ -25,13 +25,13 @@ class QualityManager: public Service, public std::enable_shared_from_thisgetService().get(); } - bool fallback_slideshow_enabled = pipeline->getService()->isFallbackSlideShowEnabled(); + bool fallback_slideshow_enabled = pipeline->getService()->isFallbackFreezeEnabled(); bool manual_slideshow_enabled = stream_->isSlideShowModeEnabled(); if (fallback_slideshow_enabled) { - ELOG_DEBUG("Slideshow fallback mode enabled"); + ELOG_DEBUG("SlideShow fallback mode enabled"); } else { - ELOG_DEBUG("Slideshow fallback mode disabled"); + ELOG_DEBUG("SlideShow fallback mode disabled"); } setSlideShowMode(fallback_slideshow_enabled || manual_slideshow_enabled); } diff --git a/erizo/src/test/rtp/QualityManagerTest.cpp b/erizo/src/test/rtp/QualityManagerTest.cpp index 4fd5ebba07..4390f6e8df 100644 --- a/erizo/src/test/rtp/QualityManagerTest.cpp +++ b/erizo/src/test/rtp/QualityManagerTest.cpp @@ -264,7 +264,7 @@ TEST_F(QualityManagerTest, shouldNotGoBelowMinDesiredSpatialLayerIfAvailable) { quality_manager->setSpatialLayer(kArbitrarySpatialLayer); quality_manager->setTemporalLayer(kArbitraryTemporalLayer); - quality_manager->setMinDesiredSpatialLayer(kArbitraryDesiredMinSpatialLayer); + quality_manager->enableSlideShowBelowSpatialLayer(true, kArbitraryDesiredMinSpatialLayer); addStatToLayer(kArbitrarySpatialLayer, kArbitraryTemporalLayer, 150); advanceClock(erizo::QualityManager::kMinLayerSwitchInterval + std::chrono::milliseconds(1)); @@ -283,7 +283,7 @@ TEST_F(QualityManagerTest, shouldGoBelowMinDesiredSpatialLayerIfNotAvailable) { quality_manager->setSpatialLayer(kArbitrarySpatialLayer); quality_manager->setTemporalLayer(kArbitraryTemporalLayer); - quality_manager->setMinDesiredSpatialLayer(kArbitraryDesiredMinSpatialLayer); + quality_manager->enableSlideShowBelowSpatialLayer(true, kArbitraryDesiredMinSpatialLayer); clearLayer(kArbitrarySpatialLayer, kArbitraryTemporalLayer); quality_manager->notifyQualityUpdate(); diff --git a/erizoAPI/MediaStream.cc b/erizoAPI/MediaStream.cc index 020cad8e89..6868abc0b6 100644 --- a/erizoAPI/MediaStream.cc +++ b/erizoAPI/MediaStream.cc @@ -114,7 +114,7 @@ NAN_MODULE_INIT(MediaStream::Init) { Nan::SetPrototypeMethod(tpl, "muteStream", muteStream); Nan::SetPrototypeMethod(tpl, "setMaxVideoBW", setMaxVideoBW); Nan::SetPrototypeMethod(tpl, "setQualityLayer", setQualityLayer); - Nan::SetPrototypeMethod(tpl, "setMinSpatialLayer", setMinSpatialLayer); + Nan::SetPrototypeMethod(tpl, "enableSlideShowBelowSpatialLayer", enableSlideShowBelowSpatialLayer); Nan::SetPrototypeMethod(tpl, "onMediaStreamEvent", onMediaStreamEvent); Nan::SetPrototypeMethod(tpl, "setVideoConstraints", setVideoConstraints); Nan::SetPrototypeMethod(tpl, "setMetadata", setMetadata); @@ -348,16 +348,16 @@ NAN_METHOD(MediaStream::setQualityLayer) { me->setQualityLayer(spatial_layer, temporal_layer); } -NAN_METHOD(MediaStream::setMinSpatialLayer) { +NAN_METHOD(MediaStream::enableSlideShowBelowSpatialLayer) { MediaStream* obj = Nan::ObjectWrap::Unwrap(info.Holder()); std::shared_ptr me = obj->me; if (!me) { return; } - - int spatial_layer = info[0]->IntegerValue(); - - me->setMinDesiredSpatialLayer(spatial_layer); + + bool enabled = info[0]->BooleanValue(); + int spatial_layer = info[1]->IntegerValue(); + me->enableSlideShowBelowSpatialLayer(enabled, spatial_layer); } NAN_METHOD(MediaStream::getStats) { diff --git a/erizoAPI/MediaStream.h b/erizoAPI/MediaStream.h index d8a67d9da3..24f284fbd6 100644 --- a/erizoAPI/MediaStream.h +++ b/erizoAPI/MediaStream.h @@ -153,7 +153,7 @@ class MediaStream : public MediaSink, public erizo::MediaStreamStatsListener, pu static NAN_METHOD(disableHandler); static NAN_METHOD(setQualityLayer); - static NAN_METHOD(setMinSpatialLayer); + static NAN_METHOD(enableSlideShowBelowSpatialLayer); static NAN_METHOD(onMediaStreamEvent); diff --git a/erizo_controller/erizoClient/src/Stream.js b/erizo_controller/erizoClient/src/Stream.js index 9551ce67fd..df3ee78cba 100644 --- a/erizo_controller/erizoClient/src/Stream.js +++ b/erizo_controller/erizoClient/src/Stream.js @@ -420,18 +420,22 @@ const Stream = (altConnectionHelpers, specInput) => { }; // eslint-disable-next-line no-underscore-dangle - that._setMinSpatialLayer = (spatialLayer, callback = () => {}) => { + that._enableSlideShowBelowSpatialLayer = (enabled, spatialLayer = 0, callback = () => {}) => { if (that.room && that.room.p2p) { - Logger.warning('setMinSpatialLayer is not implemented in p2p streams'); + Logger.warning('enableSlideShowBelowSpatialLayer is not implemented in p2p streams'); callback('error'); return; } - const config = { minLayer: { spatialLayer } }; + const config = { slideShowBelowLayer: { enabled, spatialLayer } }; that.checkOptions(config, true); Logger.debug('Calling updateSpec with config', config); that.pc.updateSpec(config, that.getID(), callback); }; + // This is an alias to keep backwards compatibility + // eslint-disable-next-line no-underscore-dangle + that._setMinSpatialLayer = that._enableSlideShowBelowSpatialLayer.bind(this, true); + const controlHandler = (handlersInput, publisherSideInput, enable) => { let publisherSide = publisherSideInput; let handlers = handlersInput; diff --git a/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js b/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js index 1a487ce375..bef4ab401c 100644 --- a/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js +++ b/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js @@ -312,13 +312,14 @@ const BaseStack = (specInput) => { (config.slideShowMode !== undefined) || (config.muteStream !== undefined) || (config.qualityLayer !== undefined) || - (config.minLayer !== undefined) || + (config.slideShowBelowLayer !== undefined) || (config.video !== undefined)) { Logger.debug('MaxVideoBW Changed to ', config.maxVideoBW); Logger.debug('MinVideo Changed to ', config.minVideoBW); Logger.debug('SlideShowMode Changed to ', config.slideShowMode); Logger.debug('muteStream changed to ', config.muteStream); Logger.debug('Video Constraints', config.video); + Logger.debug('Will activate slideshow when below layer', config.slideShowBelowLayer); specBase.callback({ type: 'updatestream', config }, streamId); } }; diff --git a/erizo_controller/erizoJS/models/Publisher.js b/erizo_controller/erizoJS/models/Publisher.js index 84f0ef4b69..10321c50e0 100644 --- a/erizo_controller/erizoJS/models/Publisher.js +++ b/erizo_controller/erizoJS/models/Publisher.js @@ -302,13 +302,15 @@ class Source extends NodeClass { subscriber.mediaStream.setQualityLayer(qualityLayer.spatialLayer, qualityLayer.temporalLayer); } - setMinSpatialLayer(qualityLayer, clientId) { + enableSlideShowBelowSpatialLayer(qualityLayer, clientId) { const subscriber = this.getSubscriber(clientId); if (!subscriber) { return; } - log.info('message: setMinSpatialLayer, spatialLayer: ', qualityLayer.spatialLayer); - subscriber.mediaStream.setMinSpatialLayer(qualityLayer.spatialLayer); + log.info('message: setMinSpatialLayer, enabled: ', qualityLayer.enabled, + ' spatialLayer: ', qualityLayer.spatialLayer); + subscriber.mediaStream.enableSlideShowBelowSpatialLayer(qualityLayer.enabled, + qualityLayer.spatialLayer); } muteSubscriberStream(clientId, muteVideo, muteAudio) { diff --git a/erizo_controller/erizoJS/models/Subscriber.js b/erizo_controller/erizoJS/models/Subscriber.js index 36b0427658..4adc15ada5 100644 --- a/erizo_controller/erizoJS/models/Subscriber.js +++ b/erizo_controller/erizoJS/models/Subscriber.js @@ -106,8 +106,9 @@ class Subscriber extends NodeClass { if (msg.config.qualityLayer !== undefined) { this.publisher.setQualityLayer(msg.config.qualityLayer, this.clientId); } - if (msg.config.minLayer !== undefined) { - this.publisher.setMinSpatialLayer(msg.config.minLayer, this.clientId); + if (msg.config.slideShowBelowLayer !== undefined) { + this.publisher.enableSlideShowBelowSpatialLayer( + msg.config.slideShowBelowLayer, this.clientId); } if (msg.config.video !== undefined) { this.publisher.setVideoConstraints(msg.config.video, this.clientId);