From cfe9e81322a5fbc605659de796752706d8b8bf6a Mon Sep 17 00:00:00 2001 From: Pedro Rodriguez Date: Tue, 13 Nov 2018 13:57:47 +0100 Subject: [PATCH] Add fake keyframe handler (#1332) --- erizo/src/erizo/MediaStream.cpp | 2 + .../rtp/FakeKeyframeGeneratorHandler.cpp | 97 +++++++++++++ .../erizo/rtp/FakeKeyframeGeneratorHandler.h | 53 ++++++++ erizo/src/erizo/rtp/RtpUtils.cpp | 57 +++++++- erizo/src/erizo/rtp/RtpUtils.h | 1 + .../rtp/FakeKeyframeGeneratorHandlerTest.cpp | 128 ++++++++++++++++++ erizo/src/test/utils/Matchers.h | 4 +- erizo/src/test/utils/Tools.h | 6 + 8 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.cpp create mode 100644 erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.h create mode 100644 erizo/src/test/rtp/FakeKeyframeGeneratorHandlerTest.cpp diff --git a/erizo/src/erizo/MediaStream.cpp b/erizo/src/erizo/MediaStream.cpp index 15cbb7fefc..4f19c91ffd 100644 --- a/erizo/src/erizo/MediaStream.cpp +++ b/erizo/src/erizo/MediaStream.cpp @@ -27,6 +27,7 @@ #include "rtp/RtpRetransmissionHandler.h" #include "rtp/RtcpFeedbackGenerationHandler.h" #include "rtp/RtpPaddingRemovalHandler.h" +#include "rtp/FakeKeyframeGeneratorHandler.h" #include "rtp/StatsHandler.h" #include "rtp/SRPacketHandler.h" #include "rtp/SenderBandwidthEstimationHandler.h" @@ -353,6 +354,7 @@ void MediaStream::initializePipeline() { pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); diff --git a/erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.cpp b/erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.cpp new file mode 100644 index 0000000000..a2a178a23f --- /dev/null +++ b/erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.cpp @@ -0,0 +1,97 @@ +#include "rtp/FakeKeyframeGeneratorHandler.h" + +#include +#include + +#include "./MediaDefinitions.h" +#include "./MediaStream.h" +#include "./RtpUtils.h" + +namespace erizo { + +DEFINE_LOGGER(FakeKeyframeGeneratorHandler, "rtp.FakeKeyframeGeneratorHandler"); + +constexpr uint64_t kPliPeriodMs = 300; + +FakeKeyframeGeneratorHandler::FakeKeyframeGeneratorHandler() : + stream_{nullptr}, enabled_{true}, first_keyframe_received_{false}, plis_scheduled_{false}, + video_source_ssrc_{0}, video_sink_ssrc_{0} { + } + +void FakeKeyframeGeneratorHandler::enable() { + enabled_ = true; +} + +void FakeKeyframeGeneratorHandler::disable() { + enabled_ = false; +} + +void FakeKeyframeGeneratorHandler::notifyUpdate() { + auto pipeline = getContext()->getPipelineShared(); + if (pipeline && !stream_) { + stream_ = pipeline->getService().get(); + } + if (stream_) { + video_source_ssrc_ = stream_->getVideoSourceSSRC(); + video_sink_ssrc_ = stream_->getVideoSinkSSRC(); + } +} + +void FakeKeyframeGeneratorHandler::read(Context *ctx, std::shared_ptr packet) { + ctx->fireRead(std::move(packet)); +} + +void FakeKeyframeGeneratorHandler::write(Context *ctx, std::shared_ptr packet) { + RtcpHeader *chead = reinterpret_cast(packet->data); + if (enabled_) { + if (!first_keyframe_received_ && packet->type == VIDEO_PACKET && !chead->isRtcp()) { + if (!packet->is_keyframe) { + if (!plis_scheduled_) { + plis_scheduled_ = true; + ELOG_DEBUG("Scheduling PLIs"); + sendPLI(); + schedulePLI(); + } + ELOG_DEBUG("Building a black keyframe from packet"); + auto keyframe_packet = transformIntoKeyframePacket(packet); + ctx->fireWrite(keyframe_packet); + return; + } else { + ELOG_DEBUG("First part of a keyframe received, stop rewriting packets"); + first_keyframe_received_ = true; + } + } + } + ctx->fireWrite(std::move(packet)); +} + +std::shared_ptr FakeKeyframeGeneratorHandler::transformIntoKeyframePacket +(std::shared_ptr packet) { + if (packet->codec == "VP8") { + auto keyframe_packet = RtpUtils::makeVP8BlackKeyframePacket(packet); + return keyframe_packet; + } else { + ELOG_DEBUG("Generate keyframe packet is not available for codec %s", packet->codec); + return packet; + } +} +void FakeKeyframeGeneratorHandler::sendPLI() { + getContext()->fireRead(RtpUtils::createPLI(video_sink_ssrc_, video_source_ssrc_)); +} +void FakeKeyframeGeneratorHandler::schedulePLI() { + std::weak_ptr weak_this = shared_from_this(); + stream_->getWorker()->scheduleEvery([weak_this] { + if (auto this_ptr = weak_this.lock()) { + if (!this_ptr->first_keyframe_received_) { + ELOG_DEBUG("Sending PLI in FakeGenerator, scheduling another"); + this_ptr->sendPLI(); + return true; + } else { + ELOG_DEBUG("Stop sending scheduled PLI packets"); + return false; + } + } + return false; + }, std::chrono::milliseconds(kPliPeriodMs)); +} +} // namespace erizo diff --git a/erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.h b/erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.h new file mode 100644 index 0000000000..b14ec1e553 --- /dev/null +++ b/erizo/src/erizo/rtp/FakeKeyframeGeneratorHandler.h @@ -0,0 +1,53 @@ +#ifndef ERIZO_SRC_ERIZO_RTP_FAKEKEYFRAMEGENERATORHANDLER_H_ +#define ERIZO_SRC_ERIZO_RTP_FAKEKEYFRAMEGENERATORHANDLER_H_ + +#include + +#include "./logger.h" +#include "pipeline/Handler.h" +#include "lib/Clock.h" +#include "lib/TokenBucket.h" +#include "thread/Worker.h" +#include "rtp/SequenceNumberTranslator.h" +#include "rtp/RtpVP8Parser.h" +#include "./Stats.h" + +namespace erizo { + +class MediaStream; + +class FakeKeyframeGeneratorHandler: public Handler, public std::enable_shared_from_this { + DECLARE_LOGGER(); + + public: + FakeKeyframeGeneratorHandler(); + void enable() override; + void disable() override; + + std::string getName() override { + return "fake-keyframe-generator"; + } + + void read(Context *ctx, std::shared_ptr packet) override; + void write(Context *ctx, std::shared_ptr packet) override; + void notifyUpdate() override; + + private: + std::shared_ptr transformIntoKeyframePacket(std::shared_ptr packet); + void sendPLI(); + void schedulePLI(); + + private: + MediaStream* stream_; + bool enabled_; + bool first_keyframe_received_; + bool plis_scheduled_; + uint32_t video_source_ssrc_; + uint32_t video_sink_ssrc_; + RtpVP8Parser vp8_parser_; + std::shared_ptr scheduled_task_; +}; + +} // namespace erizo + +#endif // ERIZO_SRC_ERIZO_RTP_FAKEKEYFRAMEGENERATORHANDLER_H_ diff --git a/erizo/src/erizo/rtp/RtpUtils.cpp b/erizo/src/erizo/rtp/RtpUtils.cpp index da970c6bf7..99669f0cc1 100644 --- a/erizo/src/erizo/rtp/RtpUtils.cpp +++ b/erizo/src/erizo/rtp/RtpUtils.cpp @@ -7,7 +7,6 @@ namespace erizo { constexpr int kMaxPacketSize = 1500; - bool RtpUtils::sequenceNumberLessThan(uint16_t first, uint16_t last) { return RtpUtils::numberLessThan(first, last, 16); } @@ -160,4 +159,60 @@ std::shared_ptr RtpUtils::makePaddingPacket(std::shared_ptr(packet->comp, packet_buffer, packet_length, packet->type); } +std::shared_ptr RtpUtils::makeVP8BlackKeyframePacket(std::shared_ptr packet) { + uint8_t vp8_keyframe[] = { + (uint8_t) 0x90, (uint8_t) 0xe0, (uint8_t) 0x80, (uint8_t) 0x01, // payload header 1 + (uint8_t) 0x00, (uint8_t) 0x20, (uint8_t) 0x10, (uint8_t) 0x0f, // payload header 2 + (uint8_t) 0x00, (uint8_t) 0x9d, (uint8_t) 0x01, (uint8_t) 0x2a, + (uint8_t) 0x40, (uint8_t) 0x01, (uint8_t) 0xb4, (uint8_t) 0x00, + (uint8_t) 0x07, (uint8_t) 0x07, (uint8_t) 0x09, (uint8_t) 0x03, + (uint8_t) 0x0b, (uint8_t) 0x0b, (uint8_t) 0x11, (uint8_t) 0x33, + (uint8_t) 0x09, (uint8_t) 0x10, (uint8_t) 0x4b, (uint8_t) 0x00, + (uint8_t) 0x00, (uint8_t) 0x0c, (uint8_t) 0x2c, (uint8_t) 0x09, + (uint8_t) 0xee, (uint8_t) 0x0d, (uint8_t) 0x02, (uint8_t) 0xc9, + (uint8_t) 0x3e, (uint8_t) 0xd7, (uint8_t) 0xb7, (uint8_t) 0x36, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, + (uint8_t) 0x4e, (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, + (uint8_t) 0x70, (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x70, + (uint8_t) 0xf6, (uint8_t) 0x4e, (uint8_t) 0x5c, (uint8_t) 0x00, + (uint8_t) 0xfe, (uint8_t) 0xef, (uint8_t) 0xb9, (uint8_t) 0x00 + }; + + uint16_t keyframe_length = sizeof(vp8_keyframe)/sizeof(vp8_keyframe[0]); + erizo::RtpHeader *header = reinterpret_cast(packet->data); + const uint16_t packet_length = header->getHeaderLength() + keyframe_length; + char packet_buffer[kMaxPacketSize]; + + erizo::RtpHeader *new_header = reinterpret_cast(packet_buffer); + memset(packet_buffer, 0, packet_length); + memcpy(packet_buffer, reinterpret_cast(header), header->getHeaderLength()); + memcpy(packet_buffer + header->getHeaderLength(), reinterpret_cast(vp8_keyframe), keyframe_length); + new_header->setMarker(true); + std::shared_ptr keyframe_packet = + std::make_shared(packet->comp, packet_buffer, packet_length, packet->type); + keyframe_packet->is_keyframe = true; + + return keyframe_packet; +} + } // namespace erizo diff --git a/erizo/src/erizo/rtp/RtpUtils.h b/erizo/src/erizo/rtp/RtpUtils.h index 92f72b1837..e2abd1113b 100644 --- a/erizo/src/erizo/rtp/RtpUtils.h +++ b/erizo/src/erizo/rtp/RtpUtils.h @@ -35,6 +35,7 @@ class RtpUtils { static int getPaddingLength(std::shared_ptr packet); static std::shared_ptr makePaddingPacket(std::shared_ptr packet, uint8_t padding_size); + static std::shared_ptr makeVP8BlackKeyframePacket(std::shared_ptr packet); }; } // namespace erizo diff --git a/erizo/src/test/rtp/FakeKeyframeGeneratorHandlerTest.cpp b/erizo/src/test/rtp/FakeKeyframeGeneratorHandlerTest.cpp new file mode 100644 index 0000000000..d4de0d63cd --- /dev/null +++ b/erizo/src/test/rtp/FakeKeyframeGeneratorHandlerTest.cpp @@ -0,0 +1,128 @@ +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "../utils/Mocks.h" +#include "../utils/Tools.h" +#include "../utils/Matchers.h" + +using ::testing::_; +using ::testing::IsNull; +using ::testing::Args; +using ::testing::Return; +using ::testing::AtLeast; +using ::testing::Not; +using erizo::DataPacket; +using erizo::packetType; +using erizo::AUDIO_PACKET; +using erizo::VIDEO_PACKET; +using erizo::MovingIntervalRateStat; +using erizo::IceConfig; +using erizo::RtpMap; +using erizo::FakeKeyframeGeneratorHandler; +using erizo::WebRtcConnection; +using erizo::Pipeline; +using erizo::InboundHandler; +using erizo::OutboundHandler; +using erizo::Worker; +using std::queue; + + +class FakeKeyframeGeneratorHandlerTest : public erizo::HandlerTest { + public: + FakeKeyframeGeneratorHandlerTest() {} + + protected: + void setHandler() { + keyframe_generator = std::make_shared(); + pipeline->addBack(keyframe_generator); + } + std::shared_ptr keyframe_generator; +}; + +TEST_F(FakeKeyframeGeneratorHandlerTest, basicBehaviourShouldReadPackets) { + auto packet = erizo::PacketTools::createDataPacket(erizo::kArbitrarySeqNumber, AUDIO_PACKET); + + EXPECT_CALL(*reader.get(), read(_, _)). + With(Args<1>(erizo::RtpHasSequenceNumber(erizo::kArbitrarySeqNumber))).Times(1); + pipeline->read(packet); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, basicBehaviourShouldWritePackets) { + auto packet = erizo::PacketTools::createDataPacket(erizo::kArbitrarySeqNumber, AUDIO_PACKET); + + EXPECT_CALL(*writer.get(), write(_, _)). + With(Args<1>(erizo::RtpHasSequenceNumber(erizo::kArbitrarySeqNumber))).Times(1); + pipeline->write(packet); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldNotChangePacketsWhenDisabled) { + keyframe_generator->disable(); + + auto packet = erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, false, false); + int length = packet->length; + EXPECT_CALL(*writer.get(), write(_, _)). + With(Args<1>(erizo::PacketLengthIs(length))).Times(1); + pipeline->write(packet); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldTransformVP8Packet) { + keyframe_generator->enable(); + EXPECT_CALL(*writer.get(), write(_, _)). + With(Args<1>(erizo::PacketIsKeyframe())).Times(1); + pipeline->write(erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, false, false)); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldNotTransformVP8Keyframes) { + auto packet = erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, true, true); + int length = packet->length; + EXPECT_CALL(*writer.get(), write(_, _)). + With(Args<1>(erizo::PacketLengthIs(length))).Times(1); + pipeline->write(packet); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldNotTransformUnsupportedCodecPackets) { + auto packet = erizo::PacketTools::createVP9Packet(erizo::kArbitrarySeqNumber, false, false); + int length = packet->length; + EXPECT_CALL(*writer.get(), write(_, _)). + With(Args<1>(erizo::PacketLengthIs(length))).Times(1); + pipeline->write(packet); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldSendPLIInmediatelyIfNoKeyframeIsReceived) { + auto packet = erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, false, false); + EXPECT_CALL(*reader.get(), read(_, _)).With(Args<1>(erizo::IsPLI())).Times(1); + pipeline->write(packet); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldNotSendPLIIfKeyframeIsFirstPacket) { + auto packet = erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, true, true); + EXPECT_CALL(*reader.get(), read(_, _)).With(Args<1>(erizo::IsPLI())).Times(0); + pipeline->write(packet); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldSendPLIsPeriodically) { + auto packet = erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, false, false); + EXPECT_CALL(*reader.get(), read(_, _)).With(Args<1>(erizo::IsPLI())).Times(2); + pipeline->write(packet); + executeTasksInNextMs(350); +} + +TEST_F(FakeKeyframeGeneratorHandlerTest, shouldNotSendPeriodicPLISIfKeyframeIsReceived) { + auto packet = erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, false, false); + auto keyframe_packet = erizo::PacketTools::createVP8Packet(erizo::kArbitrarySeqNumber, true, true); + EXPECT_CALL(*reader.get(), read(_, _)).With(Args<1>(erizo::IsPLI())).Times(1); + pipeline->write(packet); + pipeline->write(keyframe_packet); + executeTasksInNextMs(350); +} + diff --git a/erizo/src/test/utils/Matchers.h b/erizo/src/test/utils/Matchers.h index 108db4b82d..efb8287d9b 100644 --- a/erizo/src/test/utils/Matchers.h +++ b/erizo/src/test/utils/Matchers.h @@ -61,6 +61,9 @@ MATCHER(IsFIR, "") { return RtpUtils::isFIR(packet); } +MATCHER_P(PacketLengthIs, packet_length, "") { + return (std::get<0>(arg)->length == packet_length); +} MATCHER_P(PacketBelongsToSpatialLayer, spatial_layer_id, "") { return std::get<0>(arg)->belongsToSpatialLayer(spatial_layer_id); } @@ -80,7 +83,6 @@ MATCHER(PacketIsNotKeyframe, "") { return !std::get<0>(arg)->is_keyframe; } - MATCHER(IsKeyframeFirstPacket, "") { erizo::RtpHeader *packet = reinterpret_cast(std::get<0>(arg)); char* data_pointer; diff --git a/erizo/src/test/utils/Tools.h b/erizo/src/test/utils/Tools.h index 04db2e854d..6ada488b42 100644 --- a/erizo/src/test/utils/Tools.h +++ b/erizo/src/test/utils/Tools.h @@ -102,6 +102,7 @@ class PacketTools { *parsing_pointer = is_keyframe? 0x00: 0x01; auto packet = std::make_shared(0, packet_buffer, 200, VIDEO_PACKET); + packet->codec = "VP8"; packet->is_keyframe = is_keyframe; return packet; } @@ -127,6 +128,7 @@ class PacketTools { *parsing_pointer = is_keyframe? 0x00: 0x01; auto packet = std::make_shared(0, packet_buffer, 200, VIDEO_PACKET); + packet->codec = "VP8"; packet->is_keyframe = is_keyframe; return packet; } @@ -149,6 +151,7 @@ class PacketTools { *parsing_pointer = is_keyframe ? 0x5 : 0x1; auto packet = std::make_shared(0, packet_buffer, 200, VIDEO_PACKET); + packet->codec = "H264"; packet->is_keyframe = is_keyframe; return packet; } @@ -185,6 +188,7 @@ class PacketTools { ptr += nal_2_len; auto packet = std::make_shared(0, static_cast(packet_buffer), packet_length, VIDEO_PACKET); + packet->codec = "H264"; return packet; } @@ -213,6 +217,7 @@ class PacketTools { *ptr = change_bit(*ptr, 6, is_end); auto packet = std::make_shared(0, static_cast(packet_buffer), packet_length, VIDEO_PACKET); + packet->codec = "H264"; return packet; } @@ -234,6 +239,7 @@ class PacketTools { *parsing_pointer = is_keyframe? 0x00: 0x40; auto packet = std::make_shared(0, packet_buffer, 200, VIDEO_PACKET); + packet->codec = "VP9"; packet->is_keyframe = is_keyframe; return packet; }