From 6265caf689cc487cd71e4cf2e3b2ab9dc53e3e02 Mon Sep 17 00:00:00 2001 From: EnricoSchw Date: Fri, 6 Dec 2024 13:57:48 +0100 Subject: [PATCH 1/6] feat: set new screen share constrains --- package.json | 2 +- src/script/calling/CallingRepository.ts | 3 ++ src/script/calling/Participant.ts | 2 +- .../media/MediaConstraintsHandler.test.ts | 52 +------------------ src/script/media/MediaConstraintsHandler.ts | 47 ++++------------- src/script/media/MediaStreamHandler.ts | 2 +- yarn.lock | 10 ++-- 7 files changed, 21 insertions(+), 97 deletions(-) diff --git a/package.json b/package.json index 15af22da9dd..84ad4031a42 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "@lexical/react": "0.20.2", "@mediapipe/tasks-vision": "0.10.18", "@wireapp/avs": "10.0.4", - "@wireapp/avs-debugger": "0.0.5", + "@wireapp/avs-debugger": "0.0.7", "@wireapp/commons": "5.4.0", "@wireapp/core": "46.11.6", "@wireapp/react-ui-kit": "9.28.0", diff --git a/src/script/calling/CallingRepository.ts b/src/script/calling/CallingRepository.ts index 212fcdbaae1..f42c9113fa0 100644 --- a/src/script/calling/CallingRepository.ts +++ b/src/script/calling/CallingRepository.ts @@ -1020,6 +1020,9 @@ export class CallingRepository { try { const mediaStream = await this.getMediaStream({audio: true, screen: true}, call.isGroupOrConference); + if ('contentHint' in mediaStream.getVideoTracks()[0]) { + mediaStream.getVideoTracks()[0].contentHint = 'detail'; + } // If the screen share is stopped by the os system or the browser, an "ended" event is triggered. We listen for // this event to clean up the screen share state in this case. diff --git a/src/script/calling/Participant.ts b/src/script/calling/Participant.ts index 0a6165e703c..3bed0faeabf 100644 --- a/src/script/calling/Participant.ts +++ b/src/script/calling/Participant.ts @@ -78,7 +78,7 @@ export class Participant { if (AvsDebugger.hasTrack(this.user.id)) { AvsDebugger.removeTrack(this.user.id); } - AvsDebugger.addTrack(this.user.id, this.user.name(), stream.getVideoTracks()[0]); + AvsDebugger.addTrack(this.user.id, this.user.name(), stream.getVideoTracks()[0], this.sharesScreen()); } }); } diff --git a/src/script/media/MediaConstraintsHandler.test.ts b/src/script/media/MediaConstraintsHandler.test.ts index b8e9d918b39..c14ff8dc748 100644 --- a/src/script/media/MediaConstraintsHandler.test.ts +++ b/src/script/media/MediaConstraintsHandler.test.ts @@ -77,27 +77,11 @@ describe('MediaConstraintsHandler', () => { }); describe('getScreenStreamConstraints', () => { - it('returns constraints to get the screen stream if browser supports getDisplayMedia in conference call', () => { + it('returns constraints to get the screen stream if browser supports getDisplayMedia', () => { const constraintsHandler = createConstraintsHandler(); const constraints: MediaStreamConstraints | undefined = constraintsHandler.getScreenStreamConstraints( ScreensharingMethods.DISPLAY_MEDIA, - true, - ); - - expect(constraints?.audio).toBe(false); - expect((constraints?.video as MediaTrackConstraints).height).toEqual( - jasmine.objectContaining({ideal: jasmine.any(Number), max: jasmine.any(Number)}), - ); - expect((constraints?.video as MediaTrackConstraints).frameRate).toEqual(jasmine.any(Number)); - }); - - it('returns constraints to get the screen stream if browser supports getDisplayMedia in one to one call', () => { - const constraintsHandler = createConstraintsHandler(); - - const constraints: MediaStreamConstraints | undefined = constraintsHandler.getScreenStreamConstraints( - ScreensharingMethods.DISPLAY_MEDIA, - false, ); expect(constraints?.audio).toBe(false); @@ -105,26 +89,11 @@ describe('MediaConstraintsHandler', () => { expect((constraints?.video as MediaTrackConstraints).frameRate).toEqual(jasmine.any(Number)); }); - it('returns constraints to get the screen stream if browser uses desktopCapturer in conference call', () => { - const constraintsHandler = createConstraintsHandler(); - - const constraints: MediaStreamConstraints | undefined = constraintsHandler.getScreenStreamConstraints( - ScreensharingMethods.DESKTOP_CAPTURER, - true, - ); - - expect(constraints?.audio).toBe(false); - expect((constraints?.video as MediaTrackConstraintsExt).mandatory).toEqual( - jasmine.objectContaining({maxHeight: jasmine.any(Number), minHeight: jasmine.any(Number)}), - ); - }); - it('returns constraints to get the screen stream if browser uses desktopCapturer in one to one call', () => { const constraintsHandler = createConstraintsHandler(); const constraints: MediaStreamConstraints | undefined = constraintsHandler.getScreenStreamConstraints( ScreensharingMethods.DESKTOP_CAPTURER, - false, ); expect(constraints?.audio).toBe(false); @@ -134,30 +103,11 @@ describe('MediaConstraintsHandler', () => { }); }); - it('returns constraints to get the screen stream if browser uses getUserMedia in conference call', () => { - const constraintsHandler = createConstraintsHandler(); - - const constraints: MediaStreamConstraints | undefined = constraintsHandler.getScreenStreamConstraints( - ScreensharingMethods.USER_MEDIA, - true, - ); - - expect(constraints?.audio).toBe(false); - expect(constraints?.video as MediaTrackConstraints).toEqual( - jasmine.objectContaining({ - frameRate: jasmine.any(Number), - height: {exact: jasmine.any(Number)}, - mediaSource: 'screen', - }), - ); - }); - it('returns constraints to get the screen stream if browser uses getUserMedia in one to one call', () => { const constraintsHandler = createConstraintsHandler(); const constraints: MediaStreamConstraints | undefined = constraintsHandler.getScreenStreamConstraints( ScreensharingMethods.USER_MEDIA, - false, ); expect(constraints?.audio).toBe(false); diff --git a/src/script/media/MediaConstraintsHandler.ts b/src/script/media/MediaConstraintsHandler.ts index 5b0fadbc889..8002fe9a440 100644 --- a/src/script/media/MediaConstraintsHandler.ts +++ b/src/script/media/MediaConstraintsHandler.ts @@ -29,15 +29,10 @@ import {UserState} from '../user/UserState'; interface Config { CONSTRAINTS: { SCREEN: { - DESKTOP_CAPTURER_FULL: MediaTrackConstraints & { - mandatory: {chromeMediaSource: string; chromeMediaSourceId?: string; maxHeight?: number; minHeight?: number}; - }; DESKTOP_CAPTURER: MediaTrackConstraints & { - mandatory: {chromeMediaSource: string; chromeMediaSourceId?: string; maxHeight: number; minHeight: number}; + mandatory?: {chromeMediaSource: string; chromeMediaSourceId?: string; maxFrameRate?: number}; }; - DISPLAY_MEDIA_FULL: MediaTrackConstraints; DISPLAY_MEDIA: MediaTrackConstraints; - USER_MEDIA_FULL: MediaTrackConstraints & {mediaSource: string}; USER_MEDIA: MediaTrackConstraints & {mediaSource: string}; }; VIDEO: Record & {PREFERRED_FACING_MODE: string}; @@ -59,35 +54,17 @@ export class MediaConstraintsHandler { return { CONSTRAINTS: { SCREEN: { - DESKTOP_CAPTURER_FULL: { - mandatory: { - chromeMediaSource: 'desktop', - }, - }, DESKTOP_CAPTURER: { mandatory: { chromeMediaSource: 'desktop', - maxHeight: 1080, - minHeight: 1080, + maxFrameRate: 5, }, }, - DISPLAY_MEDIA_FULL: { - frameRate: 12, - }, DISPLAY_MEDIA: { frameRate: 5, - height: { - ideal: 1080, - max: 1080, - }, - }, - USER_MEDIA_FULL: { - frameRate: 12, - mediaSource: 'screen', }, USER_MEDIA: { frameRate: 5, - height: {exact: 720}, mediaSource: 'screen', }, }, @@ -153,14 +130,12 @@ export class MediaConstraintsHandler { }; } - getScreenStreamConstraints(method: ScreensharingMethods, isGroup: boolean): MediaStreamConstraints | undefined { + getScreenStreamConstraints(method: ScreensharingMethods): MediaStreamConstraints | undefined { switch (method) { case ScreensharingMethods.DESKTOP_CAPTURER: - this.logger.info(`Enabling screen sharing from desktopCapturer (with fULL resolution: ${isGroup})`); + this.logger.info(`Enabling screen sharing from desktopCapturer (with fULL resolution)`); - const desktopCapturer = isGroup - ? MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.DESKTOP_CAPTURER - : MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.DESKTOP_CAPTURER_FULL; + const desktopCapturer = MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.DESKTOP_CAPTURER; const streamConstraints = { audio: false, @@ -172,22 +147,18 @@ export class MediaConstraintsHandler { return streamConstraints; case ScreensharingMethods.DISPLAY_MEDIA: - this.logger.info(`Enabling screen sharing from getDisplayMedia (with fULL resolution: ${isGroup})`); + this.logger.info(`Enabling screen sharing from getDisplayMedia (with fULL resolution)`); - const display = isGroup - ? MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.DISPLAY_MEDIA - : MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.DISPLAY_MEDIA_FULL; + const display = MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.DISPLAY_MEDIA; return { audio: false, video: display, }; case ScreensharingMethods.USER_MEDIA: - this.logger.info(`Enabling screen sharing from getUserMedia (with fULL resolution: ${isGroup})`); + this.logger.info(`Enabling screen sharing from getUserMedia (with fULL resolution)`); - const userMedia = isGroup - ? MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.USER_MEDIA - : MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.USER_MEDIA_FULL; + const userMedia = MediaConstraintsHandler.CONFIG.CONSTRAINTS.SCREEN.USER_MEDIA; return { audio: false, diff --git a/src/script/media/MediaStreamHandler.ts b/src/script/media/MediaStreamHandler.ts index 4324a2cabbe..8be15b8d745 100644 --- a/src/script/media/MediaStreamHandler.ts +++ b/src/script/media/MediaStreamHandler.ts @@ -154,7 +154,7 @@ export class MediaStreamHandler { hasPermission: boolean, ): Promise { const mediaConstraints = screen - ? this.constraintsHandler.getScreenStreamConstraints(this.screensharingMethod, isGroup) + ? this.constraintsHandler.getScreenStreamConstraints(this.screensharingMethod) : this.constraintsHandler.getMediaStreamConstraints(audio, video, isGroup); const willPromptForPermission = !hasPermission && !Runtime.isDesktopApp(); diff --git a/yarn.lock b/yarn.lock index 11569e38759..c7e6067224b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5969,14 +5969,14 @@ __metadata: languageName: node linkType: hard -"@wireapp/avs-debugger@npm:0.0.5": - version: 0.0.5 - resolution: "@wireapp/avs-debugger@npm:0.0.5" +"@wireapp/avs-debugger@npm:0.0.7": + version: 0.0.7 + resolution: "@wireapp/avs-debugger@npm:0.0.7" dependencies: debug: "npm:^4.3.5" uuid: "npm:^10.0.0" webrtc-internals: "npm:^0.0.8" - checksum: 10/0ca37f7af760eab50c16ffafb964901190e4a4ed50d8ff3db8f3a1af25265c2534a1683cce2a52076c3b6994ad90e6e46c8b30845ba12d0c0f7a929f5f5d99de + checksum: 10/c666a696c13a52251f7430abb4d703350cc92907a2f93e24ac23932be19191e72125900ecacc76b173f17e29d0ad399557cd35d8cc4d15a5d0baf5549eedfd9c languageName: node linkType: hard @@ -18760,7 +18760,7 @@ __metadata: "@types/webpack-env": "npm:1.18.5" "@types/wicg-file-system-access": "npm:^2023.10.5" "@wireapp/avs": "npm:10.0.4" - "@wireapp/avs-debugger": "npm:0.0.5" + "@wireapp/avs-debugger": "npm:0.0.7" "@wireapp/commons": "npm:5.4.0" "@wireapp/copy-config": "npm:2.2.10" "@wireapp/core": "npm:46.11.6" From 8657873e0c7dd0ff0f940998130e23f3658af016 Mon Sep 17 00:00:00 2001 From: EnricoSchw Date: Fri, 6 Dec 2024 17:54:39 +0100 Subject: [PATCH 2/6] feat: add a zoom for screen share reader --- .../CallParticipantStatusIcons.tsx | 1 + .../components/calling/GroupVideoGridTile.tsx | 31 +++++++++++++++++-- src/style/components/group-video-grid.less | 22 +++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/script/components/calling/CallParticipantsListItem/CallParticipantStatusIcons.tsx b/src/script/components/calling/CallParticipantsListItem/CallParticipantStatusIcons.tsx index b7a113dfb21..0b06fb2daf4 100644 --- a/src/script/components/calling/CallParticipantsListItem/CallParticipantStatusIcons.tsx +++ b/src/script/components/calling/CallParticipantsListItem/CallParticipantStatusIcons.tsx @@ -56,6 +56,7 @@ export const CallParticipantStatusIcons = ({callParticipant}: CallParticipantSta {isMuted ? ( + ) : ( = ({ 'isAudioEstablished', 'isSwitchingVideoResolution', ]); + + const [isZoomedIn, setIsZoomedIn] = useState(false); + const {name} = useKoSubscribableChildren(participant?.user, ['name']); const sharesScreen = videoState === VIDEO_STATE.SCREENSHARE; @@ -103,8 +106,29 @@ const GroupVideoGridTile: React.FC = ({ } }; + const handleZoomClick = () => { + if (isZoomedIn) { + setIsZoomedIn(false); + } else { + setIsZoomedIn(true); + } + }; + const participantNameColor = getParticipantNameColor({isActivelySpeaking, isAudioEstablished}); + const actionItem = !minimized && sharesScreen && ( + + ); + const nameContainer = !minimized && (
= ({ tabIndex={isMaximized ? TabIndex.FOCUSABLE : TabIndex.UNFOCUSABLE} > {hasActiveVideo ? ( -
+
@@ -226,6 +251,8 @@ const GroupVideoGridTile: React.FC = ({
)} + {actionItem} + {nameContainer} {(hasPausedVideo || isSwitchingVideoResolution) && ( diff --git a/src/style/components/group-video-grid.less b/src/style/components/group-video-grid.less index 0a00d1d849c..b2e6a95212e 100644 --- a/src/style/components/group-video-grid.less +++ b/src/style/components/group-video-grid.less @@ -151,9 +151,31 @@ object-fit: cover; } + &__action_icon { + position: absolute; + z-index: 1; + bottom: 30px; + left: 50%; + display: flex; + padding: 4px; + border: 0px; + border-radius: 4px; + margin: 8px; + background-color: var(--gray-100); + transform: translate(-50%); + svg { + width: 24px; + fill: var(--white); + } + } + &__action_icon:hover { + background-color: var(--black); + } + &__label { .label-small-medium; position: absolute; + z-index: 1; bottom: 0; left: 50%; display: flex; From 82093a92a8b687cbf53c70e1f912300fcce3bafb Mon Sep 17 00:00:00 2001 From: EnricoSchw Date: Fri, 6 Dec 2024 18:12:10 +0100 Subject: [PATCH 3/6] feat: fix failing test --- src/script/media/MediaConstraintsHandler.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/script/media/MediaConstraintsHandler.test.ts b/src/script/media/MediaConstraintsHandler.test.ts index c14ff8dc748..f932a9ac8f2 100644 --- a/src/script/media/MediaConstraintsHandler.test.ts +++ b/src/script/media/MediaConstraintsHandler.test.ts @@ -100,6 +100,7 @@ describe('MediaConstraintsHandler', () => { expect((constraints?.video as MediaTrackConstraintsExt).mandatory).toEqual({ chromeMediaSource: 'desktop', chromeMediaSourceId: 'camera', + maxFrameRate: 5, }); }); From 6785b21d1a833bc641e9471e2b30e2684be64bb4 Mon Sep 17 00:00:00 2001 From: EnricoSchw Date: Mon, 9 Dec 2024 12:01:41 +0100 Subject: [PATCH 4/6] feat: fix comment --- src/script/calling/CallingRepository.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/script/calling/CallingRepository.ts b/src/script/calling/CallingRepository.ts index f42c9113fa0..4a216d70349 100644 --- a/src/script/calling/CallingRepository.ts +++ b/src/script/calling/CallingRepository.ts @@ -291,8 +291,8 @@ export class CallingRepository { const maximizedParticipant = call.maximizedParticipant(); if (maximizedParticipant !== null) { maximizedParticipant.isSwitchingVideoResolution(true); - // This is a temporary solution. The SFT does not send a response when a track change has occurred. - // To prevent the wrong video from being briefly displayed, we introduce a timeout here. + // This is a temporary solution. The SFT does not send a response when a track change has occurred. To prevent + // the wrong video from being briefly displayed, we introduce a timeout here. window.setTimeout(() => { maximizedParticipant.isSwitchingVideoResolution(false); }, 1000); From c357946991098f16096573d89562f4260aa3fb65 Mon Sep 17 00:00:00 2001 From: EnricoSchw Date: Mon, 9 Dec 2024 13:35:44 +0100 Subject: [PATCH 5/6] feat: dump config repo --- app-config/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app-config/package.json b/app-config/package.json index cf2c7bbac25..98f50cba563 100644 --- a/app-config/package.json +++ b/app-config/package.json @@ -1,6 +1,6 @@ { "dependencies": { - "wire-web-config-default-master": "https://github.com/wireapp/wire-web-config-wire#v0.31.36", - "wire-web-config-default-staging": "https://github.com/wireapp/wire-web-config-default#v0.31.36" + "wire-web-config-default-master": "https://github.com/wireapp/wire-web-config-wire#v0.32.01", + "wire-web-config-default-staging": "https://github.com/wireapp/wire-web-config-default#v0.32.01" } } From e7db9b8ab319d5b0e702507bed5485a792523844 Mon Sep 17 00:00:00 2001 From: EnricoSchw Date: Tue, 10 Dec 2024 10:52:30 +0100 Subject: [PATCH 6/6] feat: add PR suggestions --- src/script/components/calling/GroupVideoGridTile.tsx | 7 +------ src/script/team/TeamState.ts | 4 +++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/script/components/calling/GroupVideoGridTile.tsx b/src/script/components/calling/GroupVideoGridTile.tsx index fc4d4de972f..a657d46bfc6 100644 --- a/src/script/components/calling/GroupVideoGridTile.tsx +++ b/src/script/components/calling/GroupVideoGridTile.tsx @@ -109,11 +109,7 @@ const GroupVideoGridTile: React.FC = ({ }; const handleZoomClick = () => { - if (isZoomedIn) { - setIsZoomedIn(false); - } else { - setIsZoomedIn(true); - } + setIsZoomedIn(prev => !prev); }; const participantNameColor = getParticipantNameColor({isActivelySpeaking, isAudioEstablished}); @@ -124,7 +120,6 @@ const GroupVideoGridTile: React.FC = ({ data-user-id={participant?.user.id} className="group-video-grid__element__action_icon" onClick={handleZoomClick} - onKeyDown={handleZoomClick} > {!isZoomedIn && } {isZoomedIn && } diff --git a/src/script/team/TeamState.ts b/src/script/team/TeamState.ts index 9e368138208..abbb2b2bf00 100644 --- a/src/script/team/TeamState.ts +++ b/src/script/team/TeamState.ts @@ -117,7 +117,9 @@ export class TeamState { () => this.teamFeatures()?.mls?.config.protocolToggleUsers.includes(this.userState.self().id) ?? false, ); - this.isConferenceCallingEnabled = ko.pureComputed(() => true); + this.isConferenceCallingEnabled = ko.pureComputed( + () => this.teamFeatures()?.conferenceCalling?.status === FeatureStatus.ENABLED, + ); this.isGuestLinkEnabled = ko.pureComputed( () => this.teamFeatures()?.conversationGuestLinks?.status === FeatureStatus.ENABLED,