Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve setting initial audio and video status when the HPB is used #4181

Merged
merged 9 commits into from
Oct 15, 2020
Merged

Improve setting initial audio and video status when the HPB is used #4181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a10718f
Fix some signaling messages not handled with external signaling server
danxuliu Sep 19, 2020
535ba2b
Send initial state messages for connections also through signaling
danxuliu Sep 19, 2020
e374156
Extract function to send current media state
danxuliu Sep 19, 2020
25f3cd5
Repeat initial state messages when the HPB is used
danxuliu Sep 19, 2020
f43bdaf
Fix sending initial media state to participants without audio or video
danxuliu Sep 24, 2020
b03939e
Remove unneeded code
danxuliu Sep 19, 2020
62dad5c
Remove unneeded condition
danxuliu Sep 25, 2020
cfdcddb
Extract functions to stop checking audio and media state
danxuliu Sep 25, 2020
1440652
Stop checking for peer media if audio/video status is received
danxuliu Sep 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 1 addition & 15 deletions src/utils/webrtc/simplewebrtc/simplewebrtc.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ function SimpleWebRTC(opts) {
}
} else if (peers.length) {
peers.forEach(function(peer) {
if (message.sid) {
if (message.sid && !self.connection.hasFeature('mcu')) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but now it still (always) runs peer.handleMessage(message) or is that what was missing?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now peer.handleMessage(message) is always run; before peer.handleMessage(message) was not run when the HPB was used due to the peer.sid === message.sid check. For the full explanation please refer to the message of the first commit ;-)

if (peer.sid === message.sid) {
peer.handleMessage(message)
}
Expand Down Expand Up @@ -260,7 +260,6 @@ SimpleWebRTC.prototype.disconnect = function() {
}

SimpleWebRTC.prototype.handlePeerStreamAdded = function(peer) {
const self = this
const container = this.getRemoteVideoContainer()
if (container) {
// If there is a video track Chromium does not play audio in a video element
Expand Down Expand Up @@ -291,19 +290,6 @@ SimpleWebRTC.prototype.handlePeerStreamAdded = function(peer) {

this.emit('videoAdded', video, audio, peer)
}

// send our mute status to new peer if we're muted
// currently called with a small delay because it arrives before
// the video element is created otherwise (which happens after
// the async setRemoteDescription-createAnswer)
window.setTimeout(function() {
if (!self.webrtc.isAudioEnabled()) {
peer.send('mute', { name: 'audio' })
}
if (!self.webrtc.isVideoEnabled()) {
peer.send('mute', { name: 'video' })
}
}, 250)
}

SimpleWebRTC.prototype.handlePeerStreamRemoved = function(peer) {
Expand Down
110 changes: 90 additions & 20 deletions src/utils/webrtc/webrtc.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ const delayedConnectionToPeer = []
let callParticipantCollection = null
let localCallParticipantModel = null
let showedTURNWarning = false
let sendCurrentMediaStateWithRepetitionTimeout = null

function arrayDiff(a, b) {
return a.filter(function(i) {
Expand Down Expand Up @@ -141,6 +142,44 @@ function checkStartPublishOwnPeer(signaling) {
localCallParticipantModel.setPeer(ownPeer)
}

function sendCurrentMediaState() {
if (!webrtc.webrtc.isVideoEnabled()) {
webrtc.webrtc.emit('videoOff')
} else {
webrtc.webrtc.emit('videoOn')
}
if (!webrtc.webrtc.isAudioEnabled()) {
webrtc.webrtc.emit('audioOff')
} else {
webrtc.webrtc.emit('audioOn')
}
}

function sendCurrentMediaStateWithRepetition(timeout) {
if (!timeout) {
timeout = 0

clearTimeout(sendCurrentMediaStateWithRepetitionTimeout)
}

sendCurrentMediaStateWithRepetitionTimeout = setTimeout(function() {
sendCurrentMediaState()

if (!timeout) {
timeout = 1000
} else {
timeout *= 2
}

if (timeout > 16000) {
sendCurrentMediaStateWithRepetitionTimeout = null
return
}

sendCurrentMediaStateWithRepetition(timeout)
}, timeout)
}

function userHasStreams(user) {
let flags = user
if (flags.hasOwnProperty('inCall')) {
Expand Down Expand Up @@ -194,6 +233,17 @@ function usersChanged(signaling, newUsers, disconnectedSessionIds) {
if ((signaling.hasFeature('mcu') && user && !userHasStreams(user))
|| (!signaling.hasFeature('mcu') && user && !userHasStreams(user) && !webrtc.webrtc.localStreams.length)) {
callParticipantModel.setPeer(null)

// As there is no Peer for the other participant the current media
// state will not be sent once it is connected, so it needs to be
// sent now.
// When there is no MCU this is not needed; as the local participant
// has no streams it will be automatically marked with audio and
// video not available on the other end, so there is no need to send
// the media state.
if (signaling.hasFeature('mcu')) {
sendCurrentMediaStateWithRepetition()
}
}

const createPeer = function() {
Expand Down Expand Up @@ -485,16 +535,12 @@ export default function initWebRTC(signaling, _callParticipantCollection, _local
function handleIceConnectionStateConnected(peer) {
// Send the current information about the video and microphone
// state.
if (!webrtc.webrtc.isVideoEnabled()) {
webrtc.emit('videoOff')
} else {
webrtc.emit('videoOn')
}
if (!webrtc.webrtc.isAudioEnabled()) {
webrtc.emit('audioOff')
if (!signaling.hasFeature('mcu')) {
sendCurrentMediaState()
} else {
webrtc.emit('audioOn')
sendCurrentMediaStateWithRepetition()
}

if (signaling.settings.userId === null) {
const currentGuestNick = store.getters.getDisplayName()
sendDataChannelToAll('status', 'nickChanged', currentGuestNick)
Expand Down Expand Up @@ -713,15 +759,41 @@ export default function initWebRTC(signaling, _callParticipantCollection, _local
})
}

function stopPeerCheckAudioMedia(peer) {
clearInterval(peer.check_audio_interval)
peer.check_audio_interval = null
}

function stopPeerCheckVideoMedia(peer) {
clearInterval(peer.check_video_interval)
peer.check_video_interval = null
}

function stopPeerIdCheckMediaType(peerId, mediaType) {
// There should be just one video peer with that id, but iterating is
// safer.
const peers = webrtc.getPeers(peerId, 'video')
peers.forEach(function(peer) {
if (mediaType === 'audio') {
stopPeerCheckAudioMedia(peer)
} else if (mediaType === 'video') {
stopPeerCheckVideoMedia(peer)
}
})
}

if (signaling.hasFeature('mcu')) {
webrtc.on('mute', function(data) {
stopPeerIdCheckMediaType(data.id, data.name)
})
webrtc.on('unmute', function(data) {
stopPeerIdCheckMediaType(data.id, data.name)
})
}

function stopPeerCheckMedia(peer) {
if (peer.check_audio_interval) {
clearInterval(peer.check_audio_interval)
peer.check_audio_interval = null
}
if (peer.check_video_interval) {
clearInterval(peer.check_video_interval)
peer.check_video_interval = null
}
stopPeerCheckAudioMedia(peer)
stopPeerCheckVideoMedia(peer)
stopSendingNick(peer)
}

Expand All @@ -730,17 +802,15 @@ export default function initWebRTC(signaling, _callParticipantCollection, _local
peer.check_video_interval = setInterval(function() {
stream.getVideoTracks().forEach(function(video) {
checkPeerMedia(peer, video, 'video').then(function() {
clearInterval(peer.check_video_interval)
peer.check_video_interval = null
stopPeerCheckVideoMedia(peer)
}).catch(() => {
})
})
}, 1000)
peer.check_audio_interval = setInterval(function() {
stream.getAudioTracks().forEach(function(audio) {
checkPeerMedia(peer, audio, 'audio').then(function() {
clearInterval(peer.check_audio_interval)
peer.check_audio_interval = null
stopPeerCheckAudioMedia(peer)
}).catch(() => {
})
})
Expand Down