diff --git a/erizo/src/erizo/MediaStream.cpp b/erizo/src/erizo/MediaStream.cpp index 4b8342212..ff4fc08e5 100644 --- a/erizo/src/erizo/MediaStream.cpp +++ b/erizo/src/erizo/MediaStream.cpp @@ -21,7 +21,6 @@ #include "rtp/RtcpForwarder.h" #include "rtp/RtpSlideShowHandler.h" #include "rtp/RtpTrackMuteHandler.h" -#include "rtp/BandwidthEstimationHandler.h" #include "rtp/FecReceiverHandler.h" #include "rtp/RtcpProcessorHandler.h" #include "rtp/RtpRetransmissionHandler.h" @@ -470,7 +469,6 @@ void MediaStream::initializePipeline() { addHandlerInPosition(MIDDLE, handler_pointer_dic, handler_order); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); - pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); diff --git a/erizo/src/erizo/MediaStream.h b/erizo/src/erizo/MediaStream.h index 3a6b6f918..04b8c6e6d 100644 --- a/erizo/src/erizo/MediaStream.h +++ b/erizo/src/erizo/MediaStream.h @@ -184,7 +184,7 @@ class MediaStream: public MediaSink, public MediaSource, public FeedbackSink, bool isPipelineInitialized() { return pipeline_initialized_; } bool isRunning() { return pipeline_initialized_ && sending_; } - bool isReady() { return ready_; } + virtual bool isReady() { return ready_; } Pipeline::Ptr getPipeline() { return pipeline_; } bool isPublisher() { return is_publisher_; } void setBitrateFromMaxQualityLayer(uint64_t bitrate) { bitrate_from_max_quality_layer_ = bitrate; } diff --git a/erizo/src/erizo/WebRtcConnection.cpp b/erizo/src/erizo/WebRtcConnection.cpp index 8ab4bd7ad..21ab8e628 100644 --- a/erizo/src/erizo/WebRtcConnection.cpp +++ b/erizo/src/erizo/WebRtcConnection.cpp @@ -19,6 +19,7 @@ #include "bandwidth/TargetVideoBWDistributor.h" #include "rtp/RtpHeaders.h" #include "rtp/SenderBandwidthEstimationHandler.h" +#include "rtp/BandwidthEstimationHandler.h" #include "rtp/RtpPaddingManagerHandler.h" #include "rtp/RtpUtils.h" @@ -113,6 +114,8 @@ void WebRtcConnection::initializePipeline() { pipeline_->addFront(std::make_shared(this)); + pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); @@ -769,6 +772,7 @@ boost::future WebRtcConnection::processRemoteSdp() { local_sdp_->setOfferSdp(remote_sdp_); extension_processor_.setSdpInfo(local_sdp_); + notifyUpdateToHandlers(); local_sdp_->updateSupportedExtensionMap(extension_processor_.getSupportedExtensionMap()); if (first_remote_sdp_processed_) { diff --git a/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp b/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp index 5d561c070..03b08f074 100644 --- a/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp +++ b/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp @@ -2,6 +2,7 @@ #include +#include "./WebRtcConnection.h" #include "./MediaStream.h" #include "lib/Clock.h" #include "lib/ClockUtils.h" @@ -39,11 +40,11 @@ std::unique_ptr RemoteBitrateEstimatorPicker::pickEstima } BandwidthEstimationHandler::BandwidthEstimationHandler(std::shared_ptr picker) : - stream_{nullptr}, clock_{webrtc::Clock::GetRealTimeClock()}, + connection_{nullptr}, clock_{webrtc::Clock::GetRealTimeClock()}, picker_{picker}, using_absolute_send_time_{false}, packets_since_absolute_send_time_{0}, min_bitrate_bps_{kMinBitRateAllowed}, - bitrate_{0}, last_send_bitrate_{0}, max_video_bw_{kDefaultMaxVideoBWInKbps}, last_remb_time_{0}, + bitrate_{0}, last_send_bitrate_{0}, last_remb_time_{0}, running_{false}, active_{true}, initialized_{false} { rtc::LogMessage::SetLogToStderr(false); } @@ -58,31 +59,23 @@ void BandwidthEstimationHandler::disable() { void BandwidthEstimationHandler::notifyUpdate() { auto pipeline = getContext()->getPipelineShared(); - - if (pipeline) { - auto rtcp_processor = pipeline->getService(); - if (rtcp_processor) { - max_video_bw_ = rtcp_processor->getMaxVideoBW(); - } + if (pipeline && !connection_) { + connection_ = pipeline->getService().get(); } - if (initialized_) { + if (!connection_) { + ELOG_ERROR("Returning because there is no connection"); return; } - if (pipeline && !stream_) { - stream_ = pipeline->getService().get(); - } - if (!stream_) { + RtpExtensionProcessor& ext_processor = connection_->getRtpExtensionProcessor(); + updateExtensionMaps(ext_processor.getVideoExtensionMap(), ext_processor.getAudioExtensionMap()); + + if (initialized_) { return; } - worker_ = stream_->getWorker(); + worker_ = connection_->getWorker(); stats_ = pipeline->getService(); - RtpExtensionProcessor& ext_processor = stream_->getRtpExtensionProcessor(); - if (ext_processor.getVideoExtensionMap().size() == 0) { - return; - } - updateExtensionMaps(ext_processor.getVideoExtensionMap(), ext_processor.getAudioExtensionMap()); pickEstimator(); initialized_ = true; @@ -229,19 +222,40 @@ void BandwidthEstimationHandler::pickEstimator() { } void BandwidthEstimationHandler::sendREMBPacket() { + uint32_t sink_ssrc = 0; + source_ssrcs_.clear(); + ELOG_DEBUG("Update MediaStream SSRCs"); + connection_->forEachMediaStream([this, &sink_ssrc] (const std::shared_ptr &media_stream) { + ELOG_DEBUG("MediaStream %s, publisher %u, sink %u, source %u, isReady %d", media_stream->getId().c_str(), + media_stream->isPublisher(), media_stream->getVideoSinkSSRC(), media_stream->getVideoSourceSSRC(), + media_stream->isReady()); + if (media_stream->isReady() && media_stream->isPublisher()) { + sink_ssrc = media_stream->getVideoSinkSSRC(); + } + source_ssrcs_.push_back(media_stream->getVideoSourceSSRC()); + }); + + if (sink_ssrc == 0) { + ELOG_DEBUG("No SSRC available to send REMB"); + return; + } remb_packet_.setPacketType(RTCP_PS_Feedback_PT); remb_packet_.setBlockCount(RTCP_AFB); memcpy(&remb_packet_.report.rembPacket.uniqueid, "REMB", 4); - remb_packet_.setSSRC(stream_->getVideoSinkSSRC()); - // todo(pedro) figure out which sourceSSRC to use here - remb_packet_.setSourceSSRC(stream_->getVideoSourceSSRC()); - remb_packet_.setLength(5); - uint32_t capped_bitrate = max_video_bw_ > 0 ? std::min(max_video_bw_, bitrate_) : bitrate_; - ELOG_DEBUG("Bitrates min(%u,%u) = %u", bitrate_, max_video_bw_, capped_bitrate); + remb_packet_.setSSRC(sink_ssrc); + remb_packet_.setSourceSSRC(0); + remb_packet_.setLength(4 + source_ssrcs_.size()); + uint32_t capped_bitrate = bitrate_; + ELOG_DEBUG("Bitrates min(%u) = %u", bitrate_, capped_bitrate); remb_packet_.setREMBBitRate(capped_bitrate); - remb_packet_.setREMBNumSSRC(1); - remb_packet_.setREMBFeedSSRC(0, stream_->getVideoSourceSSRC()); + remb_packet_.setREMBNumSSRC(source_ssrcs_.size()); + + for (std::size_t i = 0; i < source_ssrcs_.size(); i++) { + ELOG_DEBUG("Setting REMBFeedSSRC %u to ssrc %u, size %u", i, source_ssrcs_[i], source_ssrcs_.size()); + remb_packet_.setREMBFeedSSRC(i, source_ssrcs_[i]); + } + int remb_length = (remb_packet_.getLength() + 1) * 4; if (active_) { ELOG_DEBUG("BWE Estimation is %d", last_send_bitrate_); @@ -274,8 +288,7 @@ void BandwidthEstimationHandler::OnReceiveBitrateChanged(const std::vectorgetNode() - [stream_->getVideoSourceSSRC()].insertStat("erizoBandwidth", CumulativeStat{last_send_bitrate_}); + stats_->getNode().insertStat("erizoBandwidth", CumulativeStat{last_send_bitrate_}); sendREMBPacket(); } diff --git a/erizo/src/erizo/rtp/BandwidthEstimationHandler.h b/erizo/src/erizo/rtp/BandwidthEstimationHandler.h index da3aae426..ca616b230 100644 --- a/erizo/src/erizo/rtp/BandwidthEstimationHandler.h +++ b/erizo/src/erizo/rtp/BandwidthEstimationHandler.h @@ -20,7 +20,7 @@ namespace erizo { -class MediaStream; +class WebRtcConnection; using webrtc::RemoteBitrateEstimator; using webrtc::RemoteBitrateObserver; using webrtc::RtpHeaderExtensionMap; @@ -68,7 +68,7 @@ class BandwidthEstimationHandler: public Handler, public RemoteBitrateObserver, void updateExtensionMap(bool video, std::array map); - MediaStream *stream_; + WebRtcConnection *connection_; std::shared_ptr worker_; std::shared_ptr stats_; webrtc::Clock* const clock_; @@ -82,8 +82,8 @@ class BandwidthEstimationHandler: public Handler, public RemoteBitrateObserver, RtpHeaderExtensionMap ext_map_audio_, ext_map_video_; uint32_t bitrate_; uint32_t last_send_bitrate_; - uint32_t max_video_bw_; uint64_t last_remb_time_; + std::vector source_ssrcs_; bool running_; bool active_; bool initialized_; diff --git a/erizo/src/test/rtp/BandwidthEstimationHandlerTest.cpp b/erizo/src/test/rtp/BandwidthEstimationHandlerTest.cpp index 1748316cb..6cb989190 100644 --- a/erizo/src/test/rtp/BandwidthEstimationHandlerTest.cpp +++ b/erizo/src/test/rtp/BandwidthEstimationHandlerTest.cpp @@ -20,6 +20,7 @@ using ::testing::_; using ::testing::IsNull; using ::testing::Args; using ::testing::Return; +using ::testing::AtLeast; using erizo::DataPacket; using erizo::packetType; using erizo::AUDIO_PACKET; @@ -46,14 +47,37 @@ class BandwidthEstimationHandlerTest : public erizo::HandlerTest { picker = std::make_shared(); EXPECT_CALL(*picker.get(), pickEstimatorProxy(_, _, _)) .WillRepeatedly(Return(new erizo::RemoteBitrateEstimatorProxy(&estimator))); - bwe_handler = std::make_shared(picker); pipeline->addBack(bwe_handler); } + void afterPipelineSetup() { + } + + std::shared_ptr addMediaStreamToConnection(std::string id, + bool is_publisher, bool ready) { + auto media_stream = + std::make_shared(simulated_worker, connection, id, id, rtp_maps, is_publisher); + std::shared_ptr stream_ptr = std::dynamic_pointer_cast(media_stream); + connection->addMediaStream(stream_ptr); + simulated_worker->executeTasks(); + EXPECT_CALL(*media_stream.get(), isReady()).WillRepeatedly(Return(ready)); + streams.push_back(media_stream); + return media_stream; + } + + void internalTearDown() { + std::for_each(streams.begin(), streams.end(), + [this](const std::shared_ptr &stream) { + connection->removeMediaStream(stream->getId()); + }); + simulated_worker->executeTasks(); + } + std::shared_ptr bwe_handler; std::shared_ptr picker; erizo::MockRemoteBitrateEstimator estimator; + std::vector> streams; }; TEST_F(BandwidthEstimationHandlerTest, basicBehaviourShouldWritePackets) { @@ -78,8 +102,11 @@ TEST_F(BandwidthEstimationHandlerTest, basicBehaviourShouldReadPackets) { pipeline->read(packet2); } -TEST_F(BandwidthEstimationHandlerTest, shouldSendRembPacketWithEstimatedBitrate) { +TEST_F(BandwidthEstimationHandlerTest, shouldSendRembPacketWithEstimatedBitrateIfThereIsAPublisherReady) { uint32_t kArbitraryBitrate = 100000; + + addMediaStreamToConnection("test1", true, true); + auto packet = erizo::PacketTools::createDataPacket(erizo::kArbitrarySeqNumber, VIDEO_PACKET); EXPECT_CALL(estimator, Process()); @@ -93,9 +120,11 @@ TEST_F(BandwidthEstimationHandlerTest, shouldSendRembPacketWithEstimatedBitrate) picker->observer_->OnReceiveBitrateChanged(std::vector(), kArbitraryBitrate); } -TEST_F(BandwidthEstimationHandlerTest, shouldSendRembPacketWithCappedBitrate) { +TEST_F(BandwidthEstimationHandlerTest, shouldNotSendRembPacketWithEstimatedBitrateIfThePublisherIsNotReady) { uint32_t kArbitraryBitrate = 100000; - uint32_t kArbitraryCappedBitrate = kArbitraryBitrate - 100; + + addMediaStreamToConnection("test1", true, false); + auto packet = erizo::PacketTools::createDataPacket(erizo::kArbitrarySeqNumber, VIDEO_PACKET); EXPECT_CALL(estimator, Process()); @@ -103,11 +132,8 @@ TEST_F(BandwidthEstimationHandlerTest, shouldSendRembPacketWithCappedBitrate) { EXPECT_CALL(estimator, IncomingPacket(_, _, _)); EXPECT_CALL(*reader.get(), read(_, _)). With(Args<1>(erizo::RtpHasSequenceNumber(erizo::kArbitrarySeqNumber))).Times(1); - EXPECT_CALL(*processor.get(), getMaxVideoBW()).WillRepeatedly(Return(kArbitraryCappedBitrate)); - pipeline->notifyUpdate(); pipeline->read(packet); - EXPECT_CALL(*writer.get(), write(_, _)).With(Args<1>(erizo::RembHasBitrateValue(kArbitraryCappedBitrate))).Times(1); - + EXPECT_CALL(*writer.get(), write(_, _)).With(Args<1>(erizo::RembHasBitrateValue(kArbitraryBitrate))).Times(0); picker->observer_->OnReceiveBitrateChanged(std::vector(), kArbitraryBitrate); } diff --git a/erizo/src/test/utils/Mocks.h b/erizo/src/test/utils/Mocks.h index a16f8afeb..8dcfb7c0e 100644 --- a/erizo/src/test/utils/Mocks.h +++ b/erizo/src/test/utils/Mocks.h @@ -117,6 +117,7 @@ class MockMediaStream: public MediaStream { MOCK_METHOD0(getBitrateFromMaxQualityLayer, uint32_t()); MOCK_METHOD0(isSlideShowModeEnabled, bool()); MOCK_METHOD0(isSimulcast, bool()); + MOCK_METHOD0(isReady, bool()); MOCK_METHOD2(onTransportData, void(std::shared_ptr, Transport*)); MOCK_METHOD1(deliverEventInternal, void(MediaEventPtr)); MOCK_METHOD0(getTargetPaddingBitrate, uint64_t()); diff --git a/erizo_controller/erizoClient/src/ErizoConnectionManager.js b/erizo_controller/erizoClient/src/ErizoConnectionManager.js index c5002e18c..780a4a41d 100644 --- a/erizo_controller/erizoClient/src/ErizoConnectionManager.js +++ b/erizo_controller/erizoClient/src/ErizoConnectionManager.js @@ -122,7 +122,7 @@ class ErizoConnection extends EventEmitterConst { log.debug(`message: Adding stream to Connection, ${this.toLog()}, ${stream.toLog()}`); this.streamsMap.add(stream.getID(), stream); if (stream.local) { - this.stack.addStream(stream.stream, stream.hasScreen()); + this.stack.addStream(stream); } } @@ -148,30 +148,10 @@ class ErizoConnection extends EventEmitterConst { this.stack.sendSignalingMessage(msg); } - setSimulcast(enable) { - this.stack.setSimulcast(enable); - } - - setVideo(video) { - this.stack.setVideo(video); - } - - setAudio(audio) { - this.stack.setAudio(audio); - } - updateSpec(configInput, streamId, callback) { this.stack.updateSpec(configInput, streamId, callback); } - updateSimulcastLayersBitrate(bitrates) { - this.stack.updateSimulcastLayersBitrate(bitrates); - } - - updateSimulcastActiveLayers(layersInfo) { - this.stack.updateSimulcastActiveLayers(layersInfo); - } - setQualityLevel(level) { this.qualityLevel = QUALITY_LEVELS[level]; } @@ -229,16 +209,6 @@ class ErizoConnectionManager { this.ErizoConnectionsMap.set(erizoId, connectionEntry); } } - if (specInput.simulcast) { - connection.setSimulcast(specInput.simulcast); - } - if (specInput.video) { - connection.setVideo(specInput.video); - } - if (specInput.audio) { - connection.setVideo(specInput.audio); - } - return connection; } diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index da3dff8f8..e4bf91412 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -165,9 +165,9 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { const createRemoteStreamP2PConnection = (streamInput, peerSocket) => { const stream = streamInput; - const connection = that.erizoConnectionManager.getOrBuildErizoConnection( - getP2PConnectionOptions(stream, peerSocket)); - stream.addPC(connection); + const connectionOptions = getP2PConnectionOptions(stream, peerSocket); + const connection = that.erizoConnectionManager.getOrBuildErizoConnection(connectionOptions); + stream.addPC(connection, false, connectionOptions); connection.on('connection-failed', that.dispatchEvent.bind(this)); stream.on('added', dispatchStreamSubscribed.bind(null, stream)); stream.on('icestatechanged', (evt) => { @@ -220,7 +220,6 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { const getErizoConnectionOptions = (stream, connectionId, erizoId, options, isRemote) => { const connectionOpts = { callback(message, streamId = stream.getID()) { - log.debug(`message: Sending message, data: ${JSON.stringify(message)}, ${stream.toLog()}, ${toLog()}`); if (message && message.type && message.type === 'updatestream') { socket.sendSDP('streamMessage', { streamId, @@ -241,6 +240,7 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { video: options.video && stream.hasVideo(), maxAudioBW: options.maxAudioBW, maxVideoBW: options.maxVideoBW, + simulcast: options.simulcast, limitMaxAudioBW: spec.maxAudioBW, limitMaxVideoBW: spec.maxVideoBW, label: stream.getLabel(), @@ -252,7 +252,6 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { isRemote, }; if (!isRemote) { - connectionOpts.simulcast = options.simulcast; connectionOpts.startVideoBW = options.startVideoBW; connectionOpts.hardMinVideoBW = options.hardMinVideoBW; } @@ -264,7 +263,7 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { const connectionOpts = getErizoConnectionOptions(stream, connectionId, erizoId, options, true); const connection = that.erizoConnectionManager .getOrBuildErizoConnection(connectionOpts, erizoId, spec.singlePC); - stream.addPC(connection); + stream.addPC(connection, false, connectionOpts); connection.on('connection-failed', that.dispatchEvent.bind(this)); stream.on('added', dispatchStreamSubscribed.bind(null, stream)); @@ -285,7 +284,7 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { const connectionOpts = getErizoConnectionOptions(stream, connectionId, erizoId, options); const connection = that.erizoConnectionManager .getOrBuildErizoConnection(connectionOpts, erizoId, spec.singlePC); - stream.addPC(connection); + stream.addPC(connection, false, options); connection.on('connection-failed', that.dispatchEvent.bind(this)); stream.on('icestatechanged', (evt) => { log.debug(`message: icestatechanged, ${stream.toLog()}, iceConnectionState: ${evt.msg.state}, ${toLog()}`); @@ -805,6 +804,8 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { log.info(`message: Publishing stream, ${stream.toLog()}, ${toLog()}`); options.maxVideoBW = options.maxVideoBW || spec.defaultVideoBW; + options.limitMaxVideoBW = spec.maxVideoBW; + options.limitMaxAudioBW = spec.maxAudioBW; if (options.maxVideoBW > spec.maxVideoBW) { options.maxVideoBW = spec.maxVideoBW; } diff --git a/erizo_controller/erizoClient/src/Stream.js b/erizo_controller/erizoClient/src/Stream.js index 43c708dca..7dde97c23 100644 --- a/erizo_controller/erizoClient/src/Stream.js +++ b/erizo_controller/erizoClient/src/Stream.js @@ -17,6 +17,12 @@ const log = Logger.module('Stream'); const Stream = (altConnectionHelpers, specInput) => { const spec = specInput; const that = EventDispatcher(spec); + let limitMaxVideoBW; + let limitMaxAudioBW; + + const defaultSimulcastSpatialLayers = 3; + const scaleResolutionDownBase = 2; + const scaleResolutionDownBaseScreenshare = 1; that.stream = spec.stream; that.url = spec.url; @@ -33,10 +39,13 @@ const Stream = (altConnectionHelpers, specInput) => { that.desktopStreamId = spec.desktopStreamId; that.audioMuted = false; that.videoMuted = false; + that.maxVideoBW = undefined; + that.maxAudioBW = undefined; that.unsubscribing = { callbackReceived: false, pcEventReceived: false, }; + const videoSenderLicodeParameters = {}; that.p2p = false; that.ConnectionHelpers = altConnectionHelpers === undefined ? ConnectionHelpers : altConnectionHelpers; @@ -68,6 +77,79 @@ const Stream = (altConnectionHelpers, specInput) => { that.local = true; } + const setMaxVideoBW = (maxVideoBW) => { + if (that.local) { + // Estimate codec bitrate from connection (with overhead) bitrate - source https://datatracker.ietf.org/doc/html/rfc8829 + // using 0.90 instead of 0.95 to allow more margin to our quality selection algorithms + const translated = (maxVideoBW * 1000 * 0.90) - (50 * 40 * 8); + log.info(`message: Setting maxVideoBW, streamId: ${that.getID()}, maxVideoBW: ${maxVideoBW}, translated: ${translated}`); + that.maxVideoBW = translated; + } else { + that.maxVideoBW = maxVideoBW; + } + }; + + const configureParameterForLayer = (layerParameters, layerConfig) => { + const newParameters = layerParameters; + newParameters.maxBitrate = layerConfig.maxBitrate; + if (layerConfig.active !== undefined) { + newParameters.active = layerConfig.active; + } + return newParameters; + }; + + that.applySenderEncoderParameters = () => { + that.stream.transceivers.forEach((transceiver) => { + if (transceiver.sender && transceiver.sender.track.kind === 'video') { + const parameters = transceiver.sender.getParameters(); + Object.keys(videoSenderLicodeParameters).forEach((layerId) => { + if (parameters.encodings[layerId] === undefined) { + log.warning(`message: Failed Configure parameters for layer, layer: ${layerId}, config: ${videoSenderLicodeParameters[layerId]}`); + } else { + parameters.encodings[layerId] = configureParameterForLayer( + parameters.encodings[layerId], + videoSenderLicodeParameters[layerId]); + } + }); + transceiver.sender.setParameters(parameters) + .then((result) => { + log.debug(`message: Success setting simulcast layer configs, result: ${result}`); + }) + .catch((e) => { + log.warning(`message: Error setting simulcast layer configs, error: ${e}`); + }); + } + }); + }; + + const initializeEncoderParameters = (simulcastConfig) => { + log.info('Initializing encoder simulcastConfig', simulcastConfig, 'MaxVideoBW is ', that.maxVideoBW); + if (!simulcastConfig) { + videoSenderLicodeParameters[0] = { maxBitrate: that.maxVideoBW }; // No simulcast + return; + } + const layersToConfigure = simulcastConfig.numSpatialLayers; + for (let index = 0; index < layersToConfigure; index += 1) { + videoSenderLicodeParameters[index] = {}; + } + if (that.maxVideoBW) { + log.debug('Setting maxVideoBW', that.maxVideoBW); + videoSenderLicodeParameters[layersToConfigure - 1].maxBitrate = that.maxVideoBW; + } + }; + + const configureVideoStream = (options) => { + log.debug('configureVideoStream', options); + limitMaxAudioBW = options.limitMaxAudioBW; + limitMaxVideoBW = options.limitMaxVideoBW; + if (options.maxVideoBW) { + setMaxVideoBW(options.maxVideoBW); + } + if (that.local) { + initializeEncoderParameters(options.simulcast); + } + }; + // Public functions that.getID = () => { let id; @@ -130,7 +212,26 @@ const Stream = (altConnectionHelpers, specInput) => { that.isExternal = () => that.url !== undefined || that.recording !== undefined; - that.addPC = (pc, p2pKey = undefined) => { + that.getMaxVideoBW = () => that.maxVideoBW; + that.hasSimulcast = () => Object.keys(videoSenderLicodeParameters).length > 1; + + that.generateEncoderParameters = () => { + const nativeSenderParameters = []; + const requestedLayers = Object.keys(videoSenderLicodeParameters).length || + defaultSimulcastSpatialLayers; + const isScreenshare = that.hasScreen(); + const base = isScreenshare ? scaleResolutionDownBaseScreenshare : scaleResolutionDownBase; + + for (let layer = 1; layer <= requestedLayers; layer += 1) { + const layerConfig = Object.assign({}, videoSenderLicodeParameters[layer - 1]); + layerConfig.rid = (layer).toString(); + layerConfig.scaleResolutionDownBy = base ** (requestedLayers - layer); + nativeSenderParameters.push(layerConfig); + } + return nativeSenderParameters; + }; + + that.addPC = (pc, p2pKey = undefined, options) => { if (p2pKey) { that.p2p = true; if (that.pc === undefined) { @@ -149,6 +250,9 @@ const Stream = (altConnectionHelpers, specInput) => { that.pc.on('add-stream', onStreamAddedToPC); that.pc.on('remove-stream', onStreamRemovedFromPC); that.pc.on('ice-state-change', onICEConnectionStateChange); + if (options) { + configureVideoStream(options); + } }; // Sends data through this stream. @@ -372,6 +476,12 @@ const Stream = (altConnectionHelpers, specInput) => { that.checkOptions = (configInput, isUpdate) => { const config = configInput; // TODO: Check for any incompatible options + if (config.maxVideoBW && config.maxVideoBW > limitMaxVideoBW) { + config.maxVideoBW = limitMaxVideoBW; + } + if (config.maxAudioBW && config.maxAudioBW > limitMaxAudioBW) { + config.maxAudioBW = limitMaxAudioBW; + } if (isUpdate === true) { // We are updating the stream if (config.audio || config.screen) { log.warning(`message: Cannot update type of subscription, ${that.toLog()}`); @@ -492,15 +602,41 @@ const Stream = (altConnectionHelpers, specInput) => { controlHandler(handlers, publisherSide, true); }; + const setEncodingConfig = (field, values, check = () => true) => { + Object.keys(values).forEach((layerId) => { + const value = values[layerId]; + if (!videoSenderLicodeParameters[layerId]) { + log.warning(`Cannot set parameter ${field} for layer ${layerId}, it does not exist`); + } + if (check(value)) { + videoSenderLicodeParameters[layerId][field] = value; + } + }); + }; + that.updateSimulcastLayersBitrate = (bitrates) => { if (that.pc && that.local) { - that.pc.updateSimulcastLayersBitrate(bitrates); + // limit with maxVideoBW + const limitedBitrates = bitrates; + Object.keys(limitedBitrates).forEach((key) => { + if (limitedBitrates[key] > that.maxVideoBW) { + log.warning(`Trying to set bitrate ${limitedBitrates[key]} higher that max ${that.maxVideoBW}` + + `in Layer ${key}`); + limitedBitrates[key] = that.maxVideoBW; + } + limitedBitrates[key] = + limitedBitrates[key] > that.maxVideoBW ? that.maxVideoBW : limitedBitrates[key]; + }); + setEncodingConfig('maxBitrate', limitedBitrates); + that.applySenderEncoderParameters(); } }; that.updateSimulcastActiveLayers = (layersInfo) => { if (that.pc && that.local) { - that.pc.updateSimulcastActiveLayers(layersInfo); + const ifIsBoolean = value => value === true || value === false; + setEncodingConfig('active', layersInfo, ifIsBoolean); + that.applySenderEncoderParameters(); } }; @@ -509,12 +645,19 @@ const Stream = (altConnectionHelpers, specInput) => { if (that.pc) { that.checkOptions(config, true); if (that.local) { + if (config.maxVideoBW) { + setMaxVideoBW(config.maxVideoBW); + that.applySenderEncoderParameters(); + } if (that.room.p2p) { for (let index = 0; index < that.pc.length; index += 1) { that.pc[index].updateSpec(config, that.getID(), callback); } } else { that.pc.updateSpec(config, that.getID(), callback); + if (config.maxVideoBW) { + setMaxVideoBW(config.maxVideoBW); + } } } else { that.pc.updateSpec(config, that.getID(), callback); diff --git a/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js b/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js index ae2c38247..f16dae11d 100644 --- a/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js +++ b/erizo_controller/erizoClient/src/webrtc-stacks/BaseStack.js @@ -29,14 +29,6 @@ const BaseStack = (specInput) => { that.pcConfig.iceTransportPolicy = 'relay'; } that.pcConfig.bundlePolicy = 'max-bundle'; - that.audio = specBase.audio; - that.video = specBase.video; - if (that.audio === undefined) { - that.audio = true; - } - if (that.video === undefined) { - that.video = true; - } that.peerConnection = new RTCPeerConnection(that.pcConfig, that.con); @@ -50,7 +42,6 @@ const BaseStack = (specInput) => { specBase.callback({ type: that.peerConnection.localDescription.type, sdp: that.peerConnection.localDescription.sdp, - config: { maxVideoBW: specBase.maxVideoBW }, }); } catch (e) { logSDP('onnegotiationneeded - error', e.message); @@ -96,75 +87,9 @@ const BaseStack = (specInput) => { return sdpInput; }; - that.enableSimulcast = (sdpInput) => { - log.error('message: Simulcast not implemented'); - return sdpInput; - }; - - const setSpatialLayersConfig = (field, values, check = () => true) => { - if (that.simulcast) { - Object.keys(values).forEach((layerId) => { - const value = values[layerId]; - if (!that.simulcast.spatialLayerConfigs) { - that.simulcast.spatialLayerConfigs = {}; - } - if (!that.simulcast.spatialLayerConfigs[layerId]) { - that.simulcast.spatialLayerConfigs[layerId] = {}; - } - if (check(value)) { - that.simulcast.spatialLayerConfigs[layerId][field] = value; - } - }); - that.setSimulcastLayersConfig(); - } - }; - - that.updateSimulcastLayersBitrate = (bitrates) => { - setSpatialLayersConfig('maxBitrate', bitrates); - }; - - that.updateSimulcastActiveLayers = (layersInfo) => { - const ifIsBoolean = value => value === true || value === false; - setSpatialLayersConfig('active', layersInfo, ifIsBoolean); - }; - - that.setSimulcastLayersConfig = () => { - log.error('message: Simulcast not implemented'); - }; - - that.setSimulcast = (enable) => { - that.simulcast = enable; - }; - - that.setVideo = (video) => { - that.video = video; - }; - - that.setAudio = (audio) => { - that.audio = audio; - }; - - that.updateSpec = (configInput, streamId, callback = () => {}) => { + that.updateSpec = (configInput, streamId) => { const config = configInput; - const shouldApplyMaxVideoBWToSdp = specBase.p2p && config.maxVideoBW; const shouldSendMaxVideoBWInOptions = !specBase.p2p && config.maxVideoBW; - if (config.maxVideoBW) { - log.debug(`message: Maxvideo Requested, value: ${config.maxVideoBW}, limit: ${specBase.limitMaxVideoBW}`); - if (config.maxVideoBW > specBase.limitMaxVideoBW) { - config.maxVideoBW = specBase.limitMaxVideoBW; - } - specBase.maxVideoBW = config.maxVideoBW; - log.debug(`message: Maxvideo Result, value: ${config.maxVideoBW}, limit: ${specBase.limitMaxVideoBW}`); - } - if (config.maxAudioBW) { - if (config.maxAudioBW > specBase.limitMaxAudioBW) { - config.maxAudioBW = specBase.limitMaxAudioBW; - } - specBase.maxAudioBW = config.maxAudioBW; - } - if (shouldApplyMaxVideoBWToSdp || config.maxAudioBW) { - that.enqueuedCalls.negotiationQueue.negotiateMaxBW(config, callback); - } if (shouldSendMaxVideoBWInOptions || config.minVideoBW || (config.slideShowMode !== undefined) || @@ -182,43 +107,24 @@ const BaseStack = (specInput) => { } }; - const defaultSimulcastSpatialLayers = 3; - - const scaleResolutionDownBase = 2; - const scaleResolutionDownBaseScreenshare = 1; - - const getSimulcastParameters = (isScreenshare) => { - const numSpatialLayers = that.simulcast.numSpatialLayers || defaultSimulcastSpatialLayers; - const parameters = []; - const base = isScreenshare ? scaleResolutionDownBaseScreenshare : scaleResolutionDownBase; - - for (let layer = 1; layer <= numSpatialLayers; layer += 1) { - parameters.push({ - rid: (layer).toString(), - scaleResolutionDownBy: base ** (numSpatialLayers - layer), - }); - } - return parameters; - }; - - that.addStream = (streamInput, isScreenshare) => { - const stream = streamInput; - stream.transceivers = []; - stream.getTracks().forEach(async (track) => { + that.addStream = (streamInput) => { + const nativeStream = streamInput.stream; + nativeStream.transceivers = []; + nativeStream.getTracks().forEach(async (track) => { let options = {}; - if (track.kind === 'video' && that.simulcast) { + if (track.kind === 'video') { options = { - sendEncodings: getSimulcastParameters(isScreenshare), + sendEncodings: streamInput.generateEncoderParameters(), }; } - options.streams = [stream]; + options.streams = [nativeStream]; const transceiver = that.peerConnection.addTransceiver(track, options); - stream.transceivers.push(transceiver); + nativeStream.transceivers.push(transceiver); }); }; - that.removeStream = (stream) => { - stream.transceivers.forEach((transceiver) => { + that.removeStream = (nativeStream) => { + nativeStream.transceivers.forEach((transceiver) => { log.debug('Stopping transceiver', transceiver); // Don't remove the tagged m section, which is the first one (mid=0). if (transceiver.mid === '0') { @@ -241,6 +147,7 @@ const BaseStack = (specInput) => { await that.peerConnection.setRemoteDescription(description); }; + that.processSignalingMessage = async (msgInput) => { const msg = msgInput; logSDP('processSignalingMessage, type: ', msgInput.type); @@ -269,7 +176,6 @@ const BaseStack = (specInput) => { specBase.callback({ type: that.peerConnection.localDescription.type, sdp: that.peerConnection.localDescription.sdp, - config: { maxVideoBW: specBase.maxVideoBW }, }); } } catch (e) { @@ -278,7 +184,6 @@ const BaseStack = (specInput) => { specBase.callback({ type: 'offer-error', sdp: that.peerConnection.localDescription.sdp, - config: { maxVideoBW: specBase.maxVideoBW }, }); } } diff --git a/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js b/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js index c0a45348f..1d10c2a66 100644 --- a/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js +++ b/erizo_controller/erizoClient/src/webrtc-stacks/ChromeStableStack.js @@ -8,117 +8,13 @@ const ChromeStableStack = (specInput) => { log.debug(`message: Starting Chrome stable stack, spec: ${JSON.stringify(specInput)}`); const spec = specInput; const that = BaseStack(specInput); - const defaultSimulcastSpatialLayers = 2; that.mediaConstraints = { offerToReceiveVideo: true, offerToReceiveAudio: true, }; - that.enableSimulcast = (sdpInput) => { - let result; - let sdp = sdpInput; - if (!that.simulcast) { - return sdp; - } - const hasAlreadySetSimulcast = sdp.match(new RegExp('a=ssrc-group:SIM', 'g')) !== null; - if (hasAlreadySetSimulcast) { - return sdp; - } - // TODO(javier): Improve the way we check for current video ssrcs - const matchGroup = sdp.match(/a=ssrc-group:FID ([0-9]*) ([0-9]*)\r?\n/); - if (!matchGroup || (matchGroup.length <= 0)) { - return sdp; - } - // TODO (pedro): Consider adding these to SdpHelpers - const numSpatialLayers = that.simulcast.numSpatialLayers || defaultSimulcastSpatialLayers; - const baseSsrc = parseInt(matchGroup[1], 10); - const baseSsrcRtx = parseInt(matchGroup[2], 10); - const cname = sdp.match(new RegExp(`a=ssrc:${matchGroup[1]} cname:(.*)\r?\n`))[1]; - const msid = sdp.match(new RegExp(`a=ssrc:${matchGroup[1]} msid:(.*)\r?\n`))[1]; - const mslabel = sdp.match(new RegExp(`a=ssrc:${matchGroup[1]} mslabel:(.*)\r?\n`))[1]; - const label = sdp.match(new RegExp(`a=ssrc:${matchGroup[1]} label:(.*)\r?\n`))[1]; - - sdp.match(new RegExp(`a=ssrc:${matchGroup[1]}.*\r?\n`, 'g')).forEach((line) => { - sdp = sdp.replace(line, ''); - }); - sdp.match(new RegExp(`a=ssrc:${matchGroup[2]}.*\r?\n`, 'g')).forEach((line) => { - sdp = sdp.replace(line, ''); - }); - - const spatialLayers = [baseSsrc]; - const spatialLayersRtx = [baseSsrcRtx]; - - for (let i = 1; i < numSpatialLayers; i += 1) { - spatialLayers.push(baseSsrc + (i * 1000)); - spatialLayersRtx.push(baseSsrcRtx + (i * 1000)); - } - - result = SdpHelpers.addSim(spatialLayers); - let spatialLayerId; - let spatialLayerIdRtx; - for (let i = 0; i < spatialLayers.length; i += 1) { - spatialLayerId = spatialLayers[i]; - spatialLayerIdRtx = spatialLayersRtx[i]; - result += SdpHelpers.addGroup(spatialLayerId, spatialLayerIdRtx); - } - - for (let i = 0; i < spatialLayers.length; i += 1) { - spatialLayerId = spatialLayers[i]; - spatialLayerIdRtx = spatialLayersRtx[i]; - result += SdpHelpers.addSpatialLayer(cname, - msid, mslabel, label, spatialLayerId, spatialLayerIdRtx); - } - result += 'a=x-google-flag:conference\r\n'; - return sdp.replace(matchGroup[0], result); - }; - - const configureParameter = (parameters, config, layerId) => { - if (parameters.encodings[layerId] === undefined || - config[layerId] === undefined) { - return parameters; - } - const newParameters = parameters; - newParameters.encodings[layerId].maxBitrate = config[layerId].maxBitrate; - if (config[layerId].active !== undefined) { - newParameters.encodings[layerId].active = config[layerId].active; - } - return newParameters; - }; - - const setBitrateForVideoLayers = (sender) => { - if (typeof sender.getParameters !== 'function' || typeof sender.setParameters !== 'function') { - log.warning('message: Cannot set simulcast layers bitrate, reason: get/setParameters not available'); - return; - } - let parameters = sender.getParameters(); - Object.keys(that.simulcast.spatialLayerConfigs).forEach((layerId) => { - if (parameters.encodings[layerId] !== undefined) { - log.debug(`message: Configure parameters for layer, layer: ${layerId}, config: ${that.simulcast.spatialLayerConfigs[layerId]}`); - parameters = configureParameter(parameters, that.simulcast.spatialLayerConfigs, layerId); - } - }); - sender.setParameters(parameters) - .then((result) => { - log.debug(`message: Success setting simulcast layer configs, result: ${result}`); - }) - .catch((e) => { - log.warning(`message: Error setting simulcast layer configs, error: ${e}`); - }); - }; - that.prepareCreateOffer = () => Promise.resolve(); - that.setSimulcastLayersConfig = () => { - log.debug(`message: Maybe set simulcast Layers config, simulcast: ${JSON.stringify(that.simulcast)}`); - if (that.simulcast && that.simulcast.spatialLayerConfigs) { - that.peerConnection.getSenders().forEach((sender) => { - if (sender.track.kind === 'video') { - setBitrateForVideoLayers(sender); - } - }); - } - }; - that.setStartVideoBW = (sdpInfo) => { if (that.video && spec.startVideoBW) { log.debug(`message: startVideoBW, requested: ${spec.startVideoBW}`); diff --git a/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js b/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js index 12040bdb5..14f2e6033 100644 --- a/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js +++ b/erizo_controller/erizoClient/src/webrtc-stacks/FirefoxStack.js @@ -3,54 +3,27 @@ import BaseStack from './BaseStack'; const log = Logger.module('FirefoxStack'); -const defaultSimulcastSpatialLayers = 3; - -const scaleResolutionDownBase = 2; -const scaleResolutionDownBaseScreenshare = 1; const FirefoxStack = (specInput) => { log.debug('message: Starting Firefox stack'); const that = BaseStack(specInput); - that.enableSimulcast = sdp => sdp; - - const getSimulcastParameters = (isScreenshare) => { - const numSpatialLayers = that.simulcast.numSpatialLayers || defaultSimulcastSpatialLayers; - const parameters = []; - const base = isScreenshare ? scaleResolutionDownBaseScreenshare : scaleResolutionDownBase; - - for (let layer = 1; layer <= numSpatialLayers; layer += 1) { - parameters.push({ - rid: (layer).toString(), - scaleResolutionDownBy: base ** (numSpatialLayers - layer), - }); - } - return parameters; - }; - - const getSimulcastParametersForFirefox = (sender, isScreenshare) => { - const parameters = sender.getParameters() || {}; - parameters.encodings = getSimulcastParameters(isScreenshare); - - return sender.setParameters(parameters); - }; - - that.addStream = (streamInput, isScreenshare) => { - const stream = streamInput; - stream.transceivers = []; - stream.getTracks().forEach(async (track) => { + that.addStream = (streamInput) => { + const nativeStream = streamInput.stream; + nativeStream.transceivers = []; + nativeStream.getTracks().forEach(async (track) => { let options = {}; - if (track.kind === 'video' && that.simulcast) { + if (track.kind === 'video' && streamInput.simulcast) { options = { sendEncodings: [], }; } - options.streams = [stream]; + options.streams = [nativeStream]; const transceiver = that.peerConnection.addTransceiver(track, options); - stream.transceivers.push(transceiver); - if (track.kind === 'video' && that.simulcast) { - getSimulcastParametersForFirefox(transceiver.sender, isScreenshare).catch(() => {}); - } + nativeStream.transceivers.push(transceiver); + const parameters = transceiver.sender.getParameters() || {}; + parameters.encodings = streamInput.generateEncoderParameters(); + return transceiver.sender.setParameters(parameters); }); }; diff --git a/erizo_controller/erizoJS/models/Publisher.js b/erizo_controller/erizoJS/models/Publisher.js index 4a43f2e16..6fa5f6c8a 100644 --- a/erizo_controller/erizoJS/models/Publisher.js +++ b/erizo_controller/erizoJS/models/Publisher.js @@ -421,6 +421,7 @@ class Publisher extends Source { } setMaxVideoBW(maxVideoBW) { + this.options.maxVideoBW = maxVideoBW; this.maxVideoBW = maxVideoBW; this.mediaStream.setMaxVideoBW(maxVideoBW); this.forEachSubscriber((id, subscriber) => {