Add simulcast support (#189)

* Add basic simulcast support

* Fix simulcast examples

* Fix media engine missing

* Add h264 keyframe detection

* Fwd big remb packets on simulcast start only

* Remove pli loop

* Remove receivers unused simulcast methods

* Request keyframe on track change

* Improve temporal layer change alg

* Move routers into senders

* Add VP8 temporal switch support

* Fix vp8 temporal scalability

* Partially fix tests

* Fix linting issues

* Simplify keyframe detection

* Add simulcast configuration and move router config

* Add simulcast tests

* Make senders rtp forwarding sync
The rtp packets are already cached in receivers, it's not required to set them in another cache.

* Move nack limiter to receiver

* Set simulcast custom label

* Disable data channel for simulcast changes

* a few lint fixes

* Fix typos

Co-authored-by: tarrencev <tarrence13@gmail.com>

cmd/server/grpc/main.go

@@ -186,7 +186,7 @@ func (s *server) Signal(stream pb.SFU_SignalServer) error {
 				SDP:  string(payload.Join.Offer.Sdp),
 			}
 
-			me := sfu.MediaEngine{}
+			me := webrtc.MediaEngine{}
 			err = me.PopulateFromSDP(offer)
 			if err != nil {
 				log.Errorf("join error: %v", err)

cmd/server/json-rpc/main.go

@@ -161,7 +161,7 @@ func (r *RPC) Handle(ctx context.Context, conn *jsonrpc2.Conn, req *jsonrpc2.Req
 			break
 		}
 
-		me := sfu.MediaEngine{}
+		me := webrtc.MediaEngine{}
 		err = me.PopulateFromSDP(join.Offer)
 		if err != nil {
 			log.Errorf("connect: error creating peer: %v", err)

config.toml

@@ -3,21 +3,23 @@
 subrembfeedback = false
 # Limit the remb bandwidth in kbps
 # zero means no limits
-maxbandwidth = 1000
+maxbandwidth = 400
 # Rate limit nack packets from senders to 1 nack per maxNackTime in seconds
 # zero means no rate limit
 maxnacktime = 1
 
 [router.video]
 # the remb cycle sending to pub, this told the pub it's bandwidth
 rembcycle = 2
-# pli cycle sending to pub, and pub will send a key frame
-plicycle = 1
 # transport-cc cycle in (ms) (experiment feature)
 tcccycle = 0
 # max buffer time by ms
 maxbuffertime = 1000
 
+[router.simulcast]
+# Prefer best quality initially
+bestqualityfirst = true
+
 [webrtc]
 # Range of ports that ion accepts WebRTC traffic on
 # Format: [min, max]   and max - min >= 100

examples/custom-signaling/main.go

@@ -161,7 +161,7 @@ func (r *RPC) Handle(ctx context.Context, conn *jsonrpc2.Conn, req *jsonrpc2.Req
 			break
 		}
 
-		me := sfu.MediaEngine{}
+		me := webrtc.MediaEngine{}
 		err = me.PopulateFromSDP(join.Offer)
 		if err != nil {
 			log.Errorf("connect: error creating peer: %v", err)

examples/simulcast/README.md

@@ -0,0 +1,12 @@
+# pub-sub-in-browser
+Demonstrates the simulcast capabilities of ion-sfu
+
+>IMPORTANT: Work in progress this example may not work yet
+
+## Instructions
+```
+go build cmd/server/json-rpc/main.go
+./main -c config.toml
+```
+### Open pub-sub-in-browser fiddle
+Open publisher fiddle [here](https://jsfiddle.net/orlandoco/k38Lyjvg/). First, click "Publish" you should be prompted to allow media access. Once you accept, you will see your local video. Once it is published, open [subscriber](https://jsfiddle.net/orlandoco/jkdq1uow/) fiddle instance and click join. click the layer buttons, and you will change within temporal layers.

examples/simulcast/config.toml

@@ -0,0 +1,47 @@
+[router]
+# pass bandwidth feeback from subs to pubs
+subrembfeedback = false
+# Limit the remb bandwidth in kbps
+# zero means no limits
+maxbandwidth = 7000
+# Rate limit nack packets from senders to 1 nack per maxNackTime in seconds
+# zero means no rate limit
+maxnacktime = 1
+
+[router.video]
+# the remb cycle sending to pub, this told the pub it's bandwidth
+rembcycle = 2
+# pli cycle sending to pub, and pub will send a key frame
+plicycle = 30
+# transport-cc cycle in (ms) (experiment feature)
+tcccycle = 0
+# max buffer time by ms
+maxbuffertime = 1000
+
+[webrtc]
+# Range of ports that ion accepts WebRTC traffic on
+# Format: [min, max]   and max - min >= 100
+# portrange = [50000, 60000]
+# if sfu behind nat, set iceserver
+[[webrtc.iceserver]]
+urls = ["stun:stun.stunprotocol.org:3478"]
+# [[webrtc.iceserver]]
+# urls = ["turn:turn.awsome.org:3478"]
+# username = "awsome"
+# credential = "awsome"
+
+# In case you're deploying ion-sfu on a server which is configured with
+# a 1:1 NAT (e.g., Amazon EC2), you might want to also specify the public
+# address of the machine using the setting below. This will result in
+# all host candidates (which normally have a private IP address) to
+# be rewritten with the public address provided in the settings. As
+# such, use the option with caution and only if you know what you're doing.
+# Multiple public IP addresses can be specified as a comma separated list
+# if the sfu is deployed in a DMZ between two 1-1 NAT for internal and
+# external users.
+# nat1to1 = ["1.2.3.4"]
+
+[log]
+stats = false
+level = "debug"
+fix = ["proc.go", "asm_amd64.s", "jsonrpc2.go"]

examples/simulcast/jsfiddle/publisher/demo.html

@@ -0,0 +1,11 @@
+Local Video
+<div id="localVideos"></div>
+<button onclick="window.publish()">Publish</button><br />
+
+
+Remote Video<br />
+<div id="remoteVideos"></div>
+<br />
+
+Logs<br />
+<div id="logs"></div>

examples/simulcast/jsfiddle/publisher/demo.js

@@ -0,0 +1,127 @@
+/* eslint-env browser */
+const log = msg =>
+  document.getElementById('logs').innerHTML += msg + '<br>'
+
+const config = {
+  iceServers: [{
+    urls: 'stun:stun.l.google.com:19302'
+  }]
+}
+
+const socket = new WebSocket("ws://localhost:7000/ws");
+const pc = new RTCPeerConnection(config)
+
+pc.ontrack = function ({ track, streams }) {
+  if (track.kind === "video") {
+    log("got track")
+    track.onunmute = () => {
+      let el = document.createElement(track.kind)
+      el.srcObject = streams[0]
+      el.autoplay = true
+
+      document.getElementById('remoteVideos').appendChild(el)
+    }
+  }
+}
+
+pc.oniceconnectionstatechange = e => log(`ICE connection state: ${pc.iceConnectionState}`)
+pc.onicecandidate = event => {
+  if (event.candidate !== null) {
+    socket.send(JSON.stringify({
+      method: "trickle",
+      params: {
+        candidate: event.candidate,
+      }
+    }))
+  }
+}
+
+socket.addEventListener('message', async (event) => {
+  const resp = JSON.parse(event.data)
+
+  // Listen for server renegotiation notifications
+  if (!resp.id && resp.method === "offer") {
+    log(`Got offer notification`)
+    await pc.setRemoteDescription(resp.params)
+    const answer = await pc.createAnswer()
+    await pc.setLocalDescription(answer)
+
+    const id = Math.random().toString()
+    log(`Sending answer`)
+    socket.send(JSON.stringify({
+      method: "answer",
+      params: { desc: answer },
+      id
+    }))
+  } else if (resp.method === "trickle") {
+    pc.addIceCandidate(resp.params).catch(log);
+  }
+})
+
+const join = async () => {
+  const offer = await pc.createOffer()
+  await pc.setLocalDescription(offer)
+  const id = Math.random().toString()
+
+  socket.send(JSON.stringify({
+    method: "join",
+    params: { sid: "test room", offer: pc.localDescription },
+    id
+  }))
+
+
+  socket.addEventListener('message', (event) => {
+    const resp = JSON.parse(event.data)
+    if (resp.id === id) {
+      log(`Got publish answer`)
+
+      // Hook this here so it's not called before joining
+      pc.onnegotiationneeded = async function () {
+        log("Renegotiating")
+        const offer = await pc.createOffer()
+        await pc.setLocalDescription(offer)
+        const id = Math.random().toString()
+        socket.send(JSON.stringify({
+          method: "offer",
+          params: { desc: offer },
+          id
+        }))
+
+        socket.addEventListener('message', (event) => {
+          const resp = JSON.parse(event.data)
+          if (resp.id === id) {
+            log(`Got renegotiation answer`)
+            pc.setRemoteDescription(resp.result)
+          }
+        })
+      }
+
+      pc.setRemoteDescription(resp.result)
+    }
+  })
+}
+
+let localStream
+let pid
+navigator.mediaDevices.getUserMedia({
+  video: true,
+  audio: true
+}).then(stream => {
+  let el = document.createElement("Video")
+  el.srcObject = stream
+  el.autoplay = true
+  el.controls = true
+  el.muted = true
+  document.getElementById('localVideos').appendChild(el)
+
+  localStream = stream
+}).catch(log)
+
+window.publish = () => {
+  log("Publishing stream")
+  localStream.getTracks().forEach((track) => {
+    pc.addTrack(track, localStream);
+  });
+
+  join()
+}

examples/simulcast/jsfiddle/subscriber/demo.html

@@ -0,0 +1,11 @@
+Local Video
+<div id="localVideos"></div>
+<button onclick="window.publish()">Publish</button><br />
+
+
+Remote Video<br />
+<div id="remoteVideos"></div>
+<br />
+
+Logs<br />
+<div id="logs"></div>