diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 5e73b38b6..895af23f4 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -8,8 +8,8 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject, Observable } from "rxjs"; import { ComponentType } from "react"; -import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel"; import { LayoutProps } from "./Grid"; +import { TileViewModel } from "../state/TileViewModel"; export interface Bounds { width: number; @@ -42,19 +42,6 @@ export interface CallLayoutInputs { pipAlignment: BehaviorSubject; } -export interface GridTileModel { - type: "grid"; - vm: UserMediaViewModel; -} - -export interface SpotlightTileModel { - type: "spotlight"; - vms: MediaViewModel[]; - maximised: boolean; -} - -export type TileModel = GridTileModel | SpotlightTileModel; - export interface CallLayoutOutputs { /** * Whether the scrolling layer of the layout should appear on top. @@ -63,11 +50,11 @@ export interface CallLayoutOutputs { /** * The visually fixed (non-scrolling) layer of the layout. */ - fixed: ComponentType>; + fixed: ComponentType>; /** * The layer of the layout that can overflow and be scrolled. */ - scrolling: ComponentType>; + scrolling: ComponentType>; } /** diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 51d258e3b..2983357b1 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -24,6 +24,7 @@ import { createContext, forwardRef, memo, + useCallback, useContext, useEffect, useMemo, @@ -33,6 +34,8 @@ import { import useMeasure from "react-use-measure"; import classNames from "classnames"; import { logger } from "matrix-js-sdk/src/logger"; +import { useObservableEagerState } from "observable-hooks"; +import { fromEvent, map, startWith } from "rxjs"; import styles from "./Grid.module.css"; import { useMergedRefs } from "../useMergedRefs"; @@ -51,6 +54,7 @@ interface Tile { id: string; model: Model; onDrag: DragCallback | undefined; + setVisible: (visible: boolean) => void; } type PlacedTile = Tile & Rect; @@ -84,6 +88,7 @@ interface SlotProps extends Omit, "onDrag"> { id: string; model: Model; onDrag?: DragCallback; + onVisibilityChange?: (visible: boolean) => void; style?: CSSProperties; className?: string; } @@ -131,6 +136,11 @@ export function useUpdateLayout(): void { ); } +const windowHeightObservable = fromEvent(window, "resize").pipe( + startWith(null), + map(() => window.innerHeight), +); + export interface LayoutProps { ref: LegacyRef; model: LayoutModel; @@ -232,6 +242,7 @@ export function Grid< const [gridRoot, gridRef2] = useState(null); const gridRef = useMergedRefs(gridRef1, gridRef2); + const windowHeight = useObservableEagerState(windowHeightObservable); const [layoutRoot, setLayoutRoot] = useState(null); const [generation, setGeneration] = useState(null); const tiles = useInitial(() => new Map>()); @@ -239,12 +250,34 @@ export function Grid< const Slot: FC> = useMemo( () => - function Slot({ id, model, onDrag, style, className, ...props }) { + function Slot({ + id, + model, + onDrag, + onVisibilityChange, + style, + className, + ...props + }) { const ref = useRef(null); + const prevVisible = useRef(null); + const setVisible = useCallback( + (visible: boolean) => { + if ( + onVisibilityChange !== undefined && + visible !== prevVisible.current + ) { + onVisibilityChange(visible); + prevVisible.current = visible; + } + }, + [onVisibilityChange], + ); + useEffect(() => { - tiles.set(id, { id, model, onDrag }); + tiles.set(id, { id, model, onDrag, setVisible }); return (): void => void tiles.delete(id); - }, [id, model, onDrag]); + }, [id, model, onDrag, setVisible]); return (
Math.min(gridBounds.bottom, windowHeight) - gridBounds.top, + [gridBounds, windowHeight], + ); + + useEffect(() => { + for (const tile of placedTiles) + tile.setVisible(tile.y + tile.height <= visibleHeight); + }, [placedTiles, visibleHeight]); + // Drag state is stored in a ref rather than component state, because we use // react-spring's imperative API during gestures to improve responsiveness const dragState = useRef(null); diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index ec6937e47..3dc3bef1d 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -12,12 +12,7 @@ import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; import styles from "./GridLayout.module.css"; import { useInitial } from "../useInitial"; -import { - CallLayout, - GridTileModel, - TileModel, - arrangeTiles, -} from "./CallLayout"; +import { CallLayout, arrangeTiles } from "./CallLayout"; import { DragCallback, useUpdateLayout } from "./Grid"; interface GridCSSProperties extends CSSProperties { @@ -49,15 +44,6 @@ export const makeGridLayout: CallLayout = ({ ), ), ); - const tileModel: TileModel | undefined = useMemo( - () => - model.spotlight && { - type: "spotlight", - vms: model.spotlight, - maximised: false, - }, - [model.spotlight], - ); const onDragSpotlight: DragCallback = useCallback( ({ xRatio, yRatio }) => @@ -70,11 +56,11 @@ export const makeGridLayout: CallLayout = ({ return (
- {tileModel && ( + {model.spotlight && ( = ({ [width, minHeight, model.grid.length], ); - const tileModels: GridTileModel[] = useMemo( - () => model.grid.map((vm) => ({ type: "grid", vm })), - [model.grid], - ); - return (
= ({ } as GridCSSProperties } > - {tileModels.map((m) => ( - + {model.grid.map((m) => ( + ))}
); diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx index 71db635d9..03ff5b32e 100644 --- a/src/grid/OneOnOneLayout.tsx +++ b/src/grid/OneOnOneLayout.tsx @@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel"; -import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout"; +import { CallLayout, arrangeTiles } from "./CallLayout"; import styles from "./OneOnOneLayout.module.css"; import { DragCallback, useUpdateLayout } from "./Grid"; @@ -38,15 +38,6 @@ export const makeOneOnOneLayout: CallLayout = ({ [width, height], ); - const remoteTileModel: GridTileModel = useMemo( - () => ({ type: "grid", vm: model.remote }), - [model.remote], - ); - const localTileModel: GridTileModel = useMemo( - () => ({ type: "grid", vm: model.local }), - [model.local], - ); - const onDragLocalTile: DragCallback = useCallback( ({ xRatio, yRatio }) => pipAlignment.next({ @@ -59,16 +50,18 @@ export const makeOneOnOneLayout: CallLayout = ({ return (
diff --git a/src/grid/SpotlightExpandedLayout.tsx b/src/grid/SpotlightExpandedLayout.tsx index 34464bcc7..084950360 100644 --- a/src/grid/SpotlightExpandedLayout.tsx +++ b/src/grid/SpotlightExpandedLayout.tsx @@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { forwardRef, useCallback, useMemo } from "react"; +import { forwardRef, useCallback } from "react"; import { useObservableEagerState } from "observable-hooks"; import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel"; -import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout"; +import { CallLayout } from "./CallLayout"; import { DragCallback, useUpdateLayout } from "./Grid"; import styles from "./SpotlightExpandedLayout.module.css"; @@ -27,17 +27,13 @@ export const makeSpotlightExpandedLayout: CallLayout< ref, ) { useUpdateLayout(); - const spotlightTileModel: SpotlightTileModel = useMemo( - () => ({ type: "spotlight", vms: model.spotlight, maximised: true }), - [model.spotlight], - ); return (
); @@ -50,11 +46,6 @@ export const makeSpotlightExpandedLayout: CallLayout< useUpdateLayout(); const pipAlignmentValue = useObservableEagerState(pipAlignment); - const pipTileModel: GridTileModel | undefined = useMemo( - () => model.pip && { type: "grid", vm: model.pip }, - [model.pip], - ); - const onDragPip: DragCallback = useCallback( ({ xRatio, yRatio }) => pipAlignment.next({ @@ -66,12 +57,13 @@ export const makeSpotlightExpandedLayout: CallLayout< return (
- {pipTileModel && ( + {model.pip && ( diff --git a/src/grid/SpotlightLandscapeLayout.tsx b/src/grid/SpotlightLandscapeLayout.tsx index 4132535a9..b9e6b2891 100644 --- a/src/grid/SpotlightLandscapeLayout.tsx +++ b/src/grid/SpotlightLandscapeLayout.tsx @@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { forwardRef, useMemo } from "react"; +import { forwardRef } from "react"; import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; -import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { CallLayout } from "./CallLayout"; import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel"; import styles from "./SpotlightLandscapeLayout.module.css"; import { useUpdateLayout } from "./Grid"; @@ -30,19 +30,15 @@ export const makeSpotlightLandscapeLayout: CallLayout< ) { useUpdateLayout(); useObservableEagerState(minBounds); - const tileModel: TileModel = useMemo( - () => ({ - type: "spotlight", - vms: model.spotlight, - maximised: false, - }), - [model.spotlight], - ); return (
- +
@@ -55,25 +51,24 @@ export const makeSpotlightLandscapeLayout: CallLayout< ) { useUpdateLayout(); useObservableEagerState(minBounds); - const tileModels: GridTileModel[] = useMemo( - () => model.grid.map((vm) => ({ type: "grid", vm })), - [model.grid], - ); + const withIndicators = + useObservableEagerState(model.spotlight.media).length > 1; return (
1, + [styles.withIndicators]: withIndicators, })} />
- {tileModels.map((m) => ( + {model.grid.map((m) => ( ))}
diff --git a/src/grid/SpotlightPortraitLayout.tsx b/src/grid/SpotlightPortraitLayout.tsx index 56c5e07bf..e617160e9 100644 --- a/src/grid/SpotlightPortraitLayout.tsx +++ b/src/grid/SpotlightPortraitLayout.tsx @@ -5,16 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { CSSProperties, forwardRef, useMemo } from "react"; +import { CSSProperties, forwardRef } from "react"; import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; -import { - CallLayout, - GridTileModel, - TileModel, - arrangeTiles, -} from "./CallLayout"; +import { CallLayout, arrangeTiles } from "./CallLayout"; import { SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel"; import styles from "./SpotlightPortraitLayout.module.css"; import { useUpdateLayout } from "./Grid"; @@ -40,19 +35,15 @@ export const makeSpotlightPortraitLayout: CallLayout< ref, ) { useUpdateLayout(); - const tileModel: TileModel = useMemo( - () => ({ - type: "spotlight", - vms: model.spotlight, - maximised: true, - }), - [model.spotlight], - ); return (
- +
); @@ -71,10 +62,8 @@ export const makeSpotlightPortraitLayout: CallLayout< width, model.grid.length, ); - const tileModels: GridTileModel[] = useMemo( - () => model.grid.map((vm) => ({ type: "grid", vm })), - [model.grid], - ); + const withIndicators = + useObservableEagerState(model.spotlight.media).length > 1; return (
1, + [styles.withIndicators]: withIndicators, })} />
- {tileModels.map((m) => ( + {model.grid.map((m) => ( ))}
diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 9492b2f01..cd980234c 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -28,7 +28,7 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { BehaviorSubject, of } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { logger } from "matrix-js-sdk/src/logger"; @@ -73,7 +73,6 @@ import { E2eeType } from "../e2ee/e2eeType"; import { makeGridLayout } from "../grid/GridLayout"; import { CallLayoutOutputs, - TileModel, defaultPipAlignment, defaultSpotlightAlignment, } from "../grid/CallLayout"; @@ -81,6 +80,7 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout"; import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout"; import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout"; import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout"; +import { GridTileViewModel, TileViewModel } from "../state/TileViewModel"; import { ReactionsProvider, useReactions } from "../useReactions"; import handSoundOgg from "../sound/raise_hand.ogg?url"; import handSoundMp3 from "../sound/raise_hand.mp3?url"; @@ -379,7 +379,7 @@ export const InCallView: FC = ({ () => forwardRef< HTMLDivElement, - PropsWithoutRef> + PropsWithoutRef> >(function Tile( { className, style, targetWidth, targetHeight, model }, ref, @@ -388,13 +388,6 @@ export const InCallView: FC = ({ const onToggleExpanded = useObservableEagerState( vm.toggleSpotlightExpanded, ); - const showVideo = useObservableEagerState( - useMemo( - () => - model.type === "grid" ? vm.showGridVideo(model.vm) : of(true), - [model], - ), - ); const showSpeakingIndicatorsValue = useObservableEagerState( vm.showSpeakingIndicators, ); @@ -402,23 +395,21 @@ export const InCallView: FC = ({ vm.showSpotlightIndicators, ); - return model.type === "grid" ? ( + return model instanceof GridTileViewModel ? ( ) : ( = ({ return ( [p.userId, p])); export interface GridLayoutSummary { type: "grid"; @@ -101,38 +109,71 @@ export type LayoutSummary = | OneOnOneLayoutSummary | PipLayoutSummary; -function summarizeLayout(l: Layout): LayoutSummary { - switch (l.type) { - case "grid": - return { - type: l.type, - spotlight: l.spotlight?.map((vm) => vm.id), - grid: l.grid.map((vm) => vm.id), - }; - case "spotlight-landscape": - case "spotlight-portrait": - return { - type: l.type, - spotlight: l.spotlight.map((vm) => vm.id), - grid: l.grid.map((vm) => vm.id), - }; - case "spotlight-expanded": - return { - type: l.type, - spotlight: l.spotlight.map((vm) => vm.id), - pip: l.pip?.id, - }; - case "one-on-one": - return { type: l.type, local: l.local.id, remote: l.remote.id }; - case "pip": - return { type: l.type, spotlight: l.spotlight.map((vm) => vm.id) }; - } +function summarizeLayout(l: Observable): Observable { + return l.pipe( + switchMap((l) => { + switch (l.type) { + case "grid": + return combineLatest( + [ + l.spotlight?.media ?? of(undefined), + ...l.grid.map((vm) => vm.media), + ], + (spotlight, ...grid) => ({ + type: l.type, + spotlight: spotlight?.map((vm) => vm.id), + grid: grid.map((vm) => vm.id), + }), + ); + case "spotlight-landscape": + case "spotlight-portrait": + return combineLatest( + [l.spotlight.media, ...l.grid.map((vm) => vm.media)], + (spotlight, ...grid) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + grid: grid.map((vm) => vm.id), + }), + ); + case "spotlight-expanded": + return combineLatest( + [l.spotlight.media, l.pip?.media ?? of(undefined)], + (spotlight, pip) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + pip: pip?.id, + }), + ); + case "one-on-one": + return combineLatest( + [l.local.media, l.remote.media], + (local, remote) => ({ + type: l.type, + local: local.id, + remote: remote.id, + }), + ); + case "pip": + return l.spotlight.media.pipe( + map((spotlight) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + })), + ); + } + }), + // Sometimes there can be multiple (synchronous) updates per frame. We only + // care about the most recent value for each time step, so discard these + // extra values. + debounceTime(0), + distinctUntilChanged(isEqual), + ); } function withCallViewModel( - { cold }: OurRunHelpers, remoteParticipants: Observable, connectionState: Observable, + speaking: Map>, continuation: (vm: CallViewModel) => void, ): void { const participantsSpy = vi @@ -141,15 +182,17 @@ function withCallViewModel( const mediaSpy = vi .spyOn(ComponentsCore, "observeParticipantMedia") .mockImplementation((p) => - cold("a", { - a: { participant: p } as Partial< - ComponentsCore.ParticipantMedia - > as ComponentsCore.ParticipantMedia, - }), + of({ participant: p } as Partial< + ComponentsCore.ParticipantMedia + > as ComponentsCore.ParticipantMedia), ); const eventsSpy = vi .spyOn(ComponentsCore, "observeParticipantEvents") - .mockImplementation((p) => cold("a", { a: p })); + .mockImplementation((p) => + (speaking.get(p) ?? of(false)).pipe( + map((s) => ({ ...p, isSpeaking: s }) as Participant), + ), + ); const vm = new CallViewModel( mockMatrixRoom({ @@ -176,107 +219,103 @@ function withCallViewModel( } test("participants are retained during a focus switch", () => { - withTestScheduler((helpers) => { - const { hot, expectObservable } = helpers; + withTestScheduler(({ cold, expectObservable }) => { // Participants disappear on frame 2 and come back on frame 3 - const partMarbles = "a-ba"; + const participantMarbles = "a-ba"; // Start switching focus on frame 1 and reconnect on frame 3 - const connMarbles = "ab-a"; + const connectionMarbles = " cs-c"; // The visible participants should remain the same throughout the switch - const laytMarbles = "aaaa 2997ms a 56998ms a"; + const layoutMarbles = " a"; withCallViewModel( - helpers, - hot(partMarbles, { + cold(participantMarbles, { a: [aliceParticipant, bobParticipant], b: [], }), - hot(connMarbles, { - a: ConnectionState.Connected, - b: ECAddonConnectionState.ECSwitchingFocus, + cold(connectionMarbles, { + c: ConnectionState.Connected, + s: ECAddonConnectionState.ECSwitchingFocus, }), + new Map(), (vm) => { - expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe( - laytMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, - ); + }); }, ); }); }); test("screen sharing activates spotlight layout", () => { - withTestScheduler((helpers) => { - const { hot, schedule, expectObservable } = helpers; + withTestScheduler(({ cold, schedule, expectObservable }) => { // Start with no screen shares, then have Alice and Bob share their screens, // then return to no screen shares, then have just Alice share for a bit - const partMarbles = "abc---d---a-b---a"; + const participantMarbles = " abcda-ba"; // While there are no screen shares, switch to spotlight manually, and then // switch back to grid at the end - const modeMarbles = "-----------a--------b"; + const modeMarbles = " -----s--g"; // We should automatically enter spotlight for the first round of screen // sharing, then return to grid, then manually go into spotlight, and // remain in spotlight until we manually go back to grid - const laytMarbles = "ab(cc)(dd)ae(bb)(ee)a 59979ms a"; - // Speaking indicators should always be shown except for when the active - // speaker is present in the spotlight - const showMarbles = "y----------ny---n---y"; - + const layoutMarbles = " abcdaefeg"; + const showSpeakingMarbles = "y----nyny"; withCallViewModel( - helpers, - hot(partMarbles, { + cold(participantMarbles, { a: [aliceParticipant, bobParticipant], b: [aliceSharingScreen, bobParticipant], c: [aliceSharingScreen, bobSharingScreen], d: [aliceParticipant, bobSharingScreen], }), - hot("a", { a: ConnectionState.Connected }), + of(ConnectionState.Connected), + new Map(), (vm) => { schedule(modeMarbles, { - a: () => vm.setGridMode("spotlight"), - b: () => vm.setGridMode("grid"), + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), }); - expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe( - laytMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [ - `${aliceId}:0:screen-share`, - `${bobId}:0:screen-share`, - ], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0:screen-share`], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - e: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], }, - ); - expectObservable(vm.showSpeakingIndicators).toBe(showMarbles, { + b: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`, `${bobId}:0:screen-share`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0:screen-share`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + e: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${bobId}:0`], + }, + f: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], + }, + g: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], + }, + }); + expectObservable(vm.showSpeakingIndicators).toBe(showSpeakingMarbles, { y: true, n: false, }); @@ -284,3 +323,200 @@ test("screen sharing activates spotlight layout", () => { ); }); }); + +test("participants stay in the same order unless to appear/disappear", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + const modeMarbles = "a"; + // First Bob speaks, then Dave, then Alice + const aSpeakingMarbles = "n- 1998ms - 1999ms y"; + const bSpeakingMarbles = "ny 1998ms n 1999ms "; + const dSpeakingMarbles = "n- 1998ms y 1999ms n"; + // Nothing should change when Bob speaks, because Bob is already on screen. + // When Dave speaks he should switch with Alice because she's the one who + // hasn't spoken at all. Then when Alice speaks, she should return to her + // place at the top. + const layoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a"; + + withCallViewModel( + of([aliceParticipant, bobParticipant, daveParticipant]), + of(ConnectionState.Connected), + new Map([ + [aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })], + [bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })], + [daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })], + ]), + (vm) => { + schedule(modeMarbles, { + a: () => { + // We imagine that only three tiles (the first three) will be visible + // on screen at a time + vm.layout.subscribe((layout) => { + if (layout.type === "grid") { + for (let i = 0; i < layout.grid.length; i++) + layout.grid[i].setVisible(i < 3); + } + }); + }, + }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`], + }, + b: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`, `${bobId}:0`], + }, + }); + }, + ); + }); +}); + +test("spotlight speakers swap places", () => { + withTestScheduler(({ cold, schedule, expectObservable }) => { + // Go immediately into spotlight mode for the test + const modeMarbles = " s"; + // First Bob speaks, then Dave, then Alice + const aSpeakingMarbles = "n--y"; + const bSpeakingMarbles = "nyn"; + const dSpeakingMarbles = "n-yn"; + // Alice should start in the spotlight, then Bob, then Dave, then Alice + // again. However, the positions of Dave and Bob in the grid should be + // reversed by the end because they've been swapped in and out of the + // spotlight. + const layoutMarbles = " abcd"; + + withCallViewModel( + of([aliceParticipant, bobParticipant, daveParticipant]), + of(ConnectionState.Connected), + new Map([ + [aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })], + [bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })], + [daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })], + ]), + (vm) => { + schedule(modeMarbles, { s: () => vm.setGridMode("spotlight") }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${bobId}:0`, `${daveId}:0`], + }, + b: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0`], + grid: ["local:0", `${aliceId}:0`, `${daveId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [`${daveId}:0`], + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${daveId}:0`, `${bobId}:0`], + }, + }); + }, + ); + }); +}); + +test("layout enters picture-in-picture mode when requested", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Enable then disable picture-in-picture + const pipControlMarbles = "-ed"; + // Should go into picture-in-picture layout then back to grid + const layoutMarbles = " aba"; + + withCallViewModel( + of([aliceParticipant, bobParticipant]), + of(ConnectionState.Connected), + new Map(), + (vm) => { + schedule(pipControlMarbles, { + e: () => window.controls.enablePip(), + d: () => window.controls.disablePip(), + }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + b: { + type: "pip", + spotlight: [`${aliceId}:0`], + }, + }); + }, + ); + }); +}); + +test("spotlight remembers whether it's expanded", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Start in spotlight mode, then switch to grid and back to spotlight a + // couple times + const modeMarbles = " s-gs-gs"; + // Expand and collapse the spotlight + const expandMarbles = "-a--a"; + // Spotlight should stay expanded during the first mode switch, and stay + // collapsed during the second mode switch + const layoutMarbles = "abcbada"; + + withCallViewModel( + of([aliceParticipant, bobParticipant]), + of(ConnectionState.Connected), + new Map(), + (vm) => { + schedule(modeMarbles, { + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), + }); + schedule(expandMarbles, { + a: () => { + let toggle: () => void; + vm.toggleSpotlightExpanded.subscribe((val) => (toggle = val!)); + toggle!(); + }, + }); + + expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: ["local:0", `${bobId}:0`], + }, + b: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0`], + pip: "local:0", + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${bobId}:0`, `${aliceId}:0`], + }, + }); + }, + ); + }); +}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 39453c601..37531511b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -46,7 +46,6 @@ import { switchMap, switchScan, take, - throttleTime, timer, withLatestFrom, } from "rxjs"; @@ -70,6 +69,12 @@ import { ObservableScope } from "./ObservableScope"; import { duplicateTiles } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled } from "../controls"; +import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel"; +import { TileStore } from "./TileStore"; +import { gridLikeLayout } from "./GridLikeLayout"; +import { spotlightExpandedLayout } from "./SpotlightExpandedLayout"; +import { oneOnOneLayout } from "./OneOnOneLayout"; +import { pipLayout } from "./PipLayout"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; // How long we wait after a focus switch before showing the real participant @@ -80,39 +85,82 @@ const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; // on mobile. No spotlight tile should be shown below this threshold. const smallMobileCallThreshold = 3; -export interface GridLayout { +export interface GridLayoutMedia { type: "grid"; spotlight?: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface SpotlightLandscapeLayout { +export interface SpotlightLandscapeLayoutMedia { type: "spotlight-landscape"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface SpotlightPortraitLayout { +export interface SpotlightPortraitLayoutMedia { type: "spotlight-portrait"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface SpotlightExpandedLayout { +export interface SpotlightExpandedLayoutMedia { type: "spotlight-expanded"; spotlight: MediaViewModel[]; pip?: UserMediaViewModel; } +export interface OneOnOneLayoutMedia { + type: "one-on-one"; + local: UserMediaViewModel; + remote: UserMediaViewModel; +} + +export interface PipLayoutMedia { + type: "pip"; + spotlight: MediaViewModel[]; +} + +export type LayoutMedia = + | GridLayoutMedia + | SpotlightLandscapeLayoutMedia + | SpotlightPortraitLayoutMedia + | SpotlightExpandedLayoutMedia + | OneOnOneLayoutMedia + | PipLayoutMedia; + +export interface GridLayout { + type: "grid"; + spotlight?: SpotlightTileViewModel; + grid: GridTileViewModel[]; +} + +export interface SpotlightLandscapeLayout { + type: "spotlight-landscape"; + spotlight: SpotlightTileViewModel; + grid: GridTileViewModel[]; +} + +export interface SpotlightPortraitLayout { + type: "spotlight-portrait"; + spotlight: SpotlightTileViewModel; + grid: GridTileViewModel[]; +} + +export interface SpotlightExpandedLayout { + type: "spotlight-expanded"; + spotlight: SpotlightTileViewModel; + pip?: GridTileViewModel; +} + export interface OneOnOneLayout { type: "one-on-one"; - local: LocalUserMediaViewModel; - remote: RemoteUserMediaViewModel; + local: GridTileViewModel; + remote: GridTileViewModel; } export interface PipLayout { type: "pip"; - spotlight: MediaViewModel[]; + spotlight: SpotlightTileViewModel; } /** @@ -161,6 +209,12 @@ enum SortingBin { SelfNotAlwaysShown, } +interface LayoutScanState { + layout: Layout | null; + tiles: TileStore; + visibleTiles: Set; +} + class UserMedia { private readonly scope = new ObservableScope(); public readonly vm: UserMediaViewModel; @@ -426,12 +480,6 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); - private readonly hasRemoteScreenShares: Observable = - this.screenShares.pipe( - map((ms) => ms.some((m) => !m.vm.local)), - distinctUntilChanged(), - ); - private readonly spotlightSpeaker: Observable = this.userMedia.pipe( switchMap((mediaItems) => @@ -466,7 +514,6 @@ export class CallViewModel extends ViewModel { ), map((speaker) => speaker.vm), this.scope.state(), - throttleTime(1600, undefined, { leading: true, trailing: true }), ); private readonly grid: Observable = this.userMedia.pipe( @@ -536,6 +583,14 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); + private readonly hasRemoteScreenShares: Observable = + this.spotlight.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + private readonly pip: Observable = this.spotlightAndPip.pipe(switchMap(([, pip]) => pip)); @@ -616,7 +671,7 @@ export class CallViewModel extends ViewModel { screenShares.length === 0, ); - private readonly gridLayout: Observable = combineLatest( + private readonly gridLayout: Observable = combineLatest( [this.grid, this.spotlight], (grid, spotlight) => ({ type: "grid", @@ -627,38 +682,44 @@ export class CallViewModel extends ViewModel { }), ); - private readonly spotlightLandscapeLayout: Observable = combineLatest( - [this.grid, this.spotlight], - (grid, spotlight) => ({ type: "spotlight-landscape", spotlight, grid }), - ); + private readonly spotlightLandscapeLayout: Observable = + combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); - private readonly spotlightPortraitLayout: Observable = combineLatest( - [this.grid, this.spotlight], - (grid, spotlight) => ({ type: "spotlight-portrait", spotlight, grid }), - ); + private readonly spotlightPortraitLayout: Observable = + combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); - private readonly spotlightExpandedLayout: Observable = combineLatest( - [this.spotlight, this.pip], - (spotlight, pip) => ({ + private readonly spotlightExpandedLayout: Observable = + combineLatest([this.spotlight, this.pip], (spotlight, pip) => ({ type: "spotlight-expanded", spotlight, pip: pip ?? undefined, - }), - ); + })); - private readonly oneOnOneLayout: Observable = this.grid.pipe( - map((grid) => ({ - type: "one-on-one", - local: grid.find((vm) => vm.local) as LocalUserMediaViewModel, - remote: grid.find((vm) => !vm.local) as RemoteUserMediaViewModel, - })), - ); + private readonly oneOnOneLayout: Observable = + this.mediaItems.pipe( + map((grid) => ({ + type: "one-on-one", + local: grid.find((vm) => vm.vm.local)!.vm as LocalUserMediaViewModel, + remote: grid.find((vm) => !vm.vm.local)!.vm as RemoteUserMediaViewModel, + })), + ); - private readonly pipLayout: Observable = this.spotlight.pipe( + private readonly pipLayout: Observable = this.spotlight.pipe( map((spotlight) => ({ type: "pip", spotlight })), ); - public readonly layout: Observable = this.windowMode.pipe( + /** + * The media to be used to produce a layout. + */ + private readonly layoutMedia: Observable = this.windowMode.pipe( switchMap((windowMode) => { switch (windowMode) { case "normal": @@ -719,48 +780,95 @@ export class CallViewModel extends ViewModel { this.scope.state(), ); + /** + * The layout of tiles in the call interface. + */ + public readonly layout: Observable = this.layoutMedia.pipe( + // Each layout will produce a set of tiles, and these tiles have an + // observable indicating whether they're visible. We loop this information + // back into the layout process by using switchScan. + switchScan< + LayoutMedia, + LayoutScanState, + Observable + >( + ({ tiles: prevTiles, visibleTiles }, media) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout( + media, + visibleTiles, + prevTiles, + ); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, visibleTiles, prevTiles); + break; + } + + // Take all of the 'visible' observables and combine them into one big + // observable array + const visibilities = + newTiles.gridTiles.length === 0 + ? of([]) + : combineLatest(newTiles.gridTiles.map((tile) => tile.visible)); + return visibilities.pipe( + map((visibilities) => ({ + layout: layout, + tiles: newTiles, + visibleTiles: new Set( + newTiles.gridTiles.filter((_tile, i) => visibilities[i]), + ), + })), + ); + }, + { + layout: null, + tiles: TileStore.empty(), + visibleTiles: new Set(), + }, + ), + map(({ layout }) => layout), + this.scope.state(), + ); + public showSpotlightIndicators: Observable = this.layout.pipe( map((l) => l.type !== "grid"), this.scope.state(), ); - /** - * Determines whether video should be shown for a certain piece of media - * appearing in the grid. - */ - public showGridVideo(vm: MediaViewModel): Observable { - return this.layout.pipe( - map( - (l) => - !( - (l.type === "spotlight-landscape" || - l.type === "spotlight-portrait") && - // This media is already visible in the spotlight; avoid duplication - l.spotlight.some((spotlightVm) => spotlightVm === vm) - ), - ), - distinctUntilChanged(), - ); - } - public showSpeakingIndicators: Observable = this.layout.pipe( - map((l) => { + switchMap((l) => { switch (l.type) { case "spotlight-landscape": case "spotlight-portrait": // If the spotlight is showing the active speaker, we can do without // speaking indicators as they're a redundant visual cue. But if // screen sharing feeds are in the spotlight we still need them. - return l.spotlight[0] instanceof ScreenShareViewModel; + return l.spotlight.media.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); // In expanded spotlight layout, the active speaker is always shown in // the picture-in-picture tile so there is no need for speaking // indicators. And in one-on-one layout there's no question as to who is // speaking. case "spotlight-expanded": case "one-on-one": - return false; + return of(false); default: - return true; + return of(true); } }), this.scope.state(), diff --git a/src/state/GridLikeLayout.ts b/src/state/GridLikeLayout.ts new file mode 100644 index 000000000..7fcada952 --- /dev/null +++ b/src/state/GridLikeLayout.ts @@ -0,0 +1,43 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { Layout, LayoutMedia } from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +export type GridLikeLayoutType = + | "grid" + | "spotlight-landscape" + | "spotlight-portrait"; + +/** + * Produces a grid-like layout (any layout with a grid and possibly a spotlight) + * with the given media. + */ +export function gridLikeLayout( + media: LayoutMedia & { type: GridLikeLayoutType }, + visibleTiles: Set, + prevTiles: TileStore, +): [Layout & { type: GridLikeLayoutType }, TileStore] { + const update = prevTiles.from(visibleTiles); + if (media.spotlight !== undefined) + update.registerSpotlight( + media.spotlight, + media.type === "spotlight-portrait", + ); + for (const mediaVm of media.grid) update.registerGridTile(mediaVm); + const tiles = update.build(); + + return [ + { + type: media.type, + spotlight: tiles.spotlightTile, + grid: tiles.gridTiles, + } as Layout & { type: GridLikeLayoutType }, + tiles, + ]; +} diff --git a/src/state/OneOnOneLayout.ts b/src/state/OneOnOneLayout.ts new file mode 100644 index 000000000..29ed9fc08 --- /dev/null +++ b/src/state/OneOnOneLayout.ts @@ -0,0 +1,32 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { OneOnOneLayout, OneOnOneLayoutMedia } from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +/** + * Produces a one-on-one layout with the given media. + */ +export function oneOnOneLayout( + media: OneOnOneLayoutMedia, + visibleTiles: Set, + prevTiles: TileStore, +): [OneOnOneLayout, TileStore] { + const update = prevTiles.from(visibleTiles); + update.registerGridTile(media.local); + update.registerGridTile(media.remote); + const tiles = update.build(); + return [ + { + type: media.type, + local: tiles.gridTilesByMedia.get(media.local)!, + remote: tiles.gridTilesByMedia.get(media.remote)!, + }, + tiles, + ]; +} diff --git a/src/state/PipLayout.ts b/src/state/PipLayout.ts new file mode 100644 index 000000000..35edeefe7 --- /dev/null +++ b/src/state/PipLayout.ts @@ -0,0 +1,30 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { PipLayout, PipLayoutMedia } from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +/** + * Produces a picture-in-picture layout with the given media. + */ +export function pipLayout( + media: PipLayoutMedia, + visibleTiles: Set, + prevTiles: TileStore, +): [PipLayout, TileStore] { + const update = prevTiles.from(visibleTiles); + update.registerSpotlight(media.spotlight, true); + const tiles = update.build(); + return [ + { + type: media.type, + spotlight: tiles.spotlightTile!, + }, + tiles, + ]; +} diff --git a/src/state/SpotlightExpandedLayout.ts b/src/state/SpotlightExpandedLayout.ts new file mode 100644 index 000000000..83c5a95e3 --- /dev/null +++ b/src/state/SpotlightExpandedLayout.ts @@ -0,0 +1,36 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { + SpotlightExpandedLayout, + SpotlightExpandedLayoutMedia, +} from "./CallViewModel"; +import { TileStore } from "./TileStore"; +import { GridTileViewModel } from "./TileViewModel"; + +/** + * Produces an expanded spotlight layout with the given media. + */ +export function spotlightExpandedLayout( + media: SpotlightExpandedLayoutMedia, + visibleTiles: Set, + prevTiles: TileStore, +): [SpotlightExpandedLayout, TileStore] { + const update = prevTiles.from(visibleTiles); + update.registerSpotlight(media.spotlight, true); + if (media.pip !== undefined) update.registerGridTile(media.pip); + const tiles = update.build(); + + return [ + { + type: media.type, + spotlight: tiles.spotlightTile!, + pip: tiles.gridTiles[0], + }, + tiles, + ]; +} diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts new file mode 100644 index 000000000..0288830c7 --- /dev/null +++ b/src/state/TileStore.ts @@ -0,0 +1,259 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; + +import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel"; +import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel"; +import { fillGaps } from "../utils/iter"; + +class SpotlightTileData { + private readonly media_: BehaviorSubject; + public get media(): MediaViewModel[] { + return this.media_.value; + } + public set media(value: MediaViewModel[]) { + this.media_.next(value); + } + + private readonly maximised_: BehaviorSubject; + public get maximised(): boolean { + return this.maximised_.value; + } + public set maximised(value: boolean) { + this.maximised_.next(value); + } + + public readonly vm: SpotlightTileViewModel; + + public constructor(media: MediaViewModel[], maximised: boolean) { + this.media_ = new BehaviorSubject(media); + this.maximised_ = new BehaviorSubject(maximised); + this.vm = new SpotlightTileViewModel(this.media_, this.maximised_); + } + + public destroy(): void { + this.vm.destroy(); + } +} + +class GridTileData { + private readonly media_: BehaviorSubject; + public get media(): UserMediaViewModel { + return this.media_.value; + } + public set media(value: UserMediaViewModel) { + this.media_.next(value); + } + + public readonly vm: GridTileViewModel; + + public constructor(media: UserMediaViewModel) { + this.media_ = new BehaviorSubject(media); + this.vm = new GridTileViewModel(this.media_); + } + + public destroy(): void { + this.vm.destroy(); + } +} + +/** + * A collection of tiles to be mapped to a layout. + */ +export class TileStore { + private constructor( + private readonly spotlight: SpotlightTileData | null, + private readonly grid: GridTileData[], + ) {} + + public readonly spotlightTile = this.spotlight?.vm; + public readonly gridTiles = this.grid.map(({ vm }) => vm); + public readonly gridTilesByMedia = new Map( + this.grid.map(({ vm, media }) => [media, vm]), + ); + + /** + * Creates an an empty collection of tiles. + */ + public static empty(): TileStore { + return new TileStore(null, []); + } + + /** + * Creates a builder which can be used to update the collection, passing + * ownership of the tiles to the updated collection. + */ + public from(visibleTiles: Set): TileStoreBuilder { + return new TileStoreBuilder( + this.spotlight, + this.grid, + (spotlight, grid) => new TileStore(spotlight, grid), + visibleTiles, + ); + } +} + +/** + * A builder for a new collection of tiles. Will reuse tiles and destroy unused + * tiles from a previous collection where appropriate. + */ +export class TileStoreBuilder { + private spotlight: SpotlightTileData | null = null; + private readonly prevSpotlightSpeaker = + this.prevSpotlight?.media.length === 1 && + "speaking" in this.prevSpotlight.media[0] && + this.prevSpotlight.media[0]; + + private readonly prevGridByMedia = new Map( + this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const), + ); + + // The total number of grid entries that we have so far + private numGridEntries = 0; + // A sparse array of grid entries which should be kept in the same spots as + // which they appeared in the previous grid + private readonly stationaryGridEntries: GridTileData[] = new Array( + this.prevGrid.length, + ); + // Grid entries which should now enter the visible section of the grid + private readonly visibleGridEntries: GridTileData[] = []; + // Grid entries which should now enter the invisible section of the grid + private readonly invisibleGridEntries: GridTileData[] = []; + + public constructor( + private readonly prevSpotlight: SpotlightTileData | null, + private readonly prevGrid: GridTileData[], + private readonly construct: ( + spotlight: SpotlightTileData | null, + grid: GridTileData[], + ) => TileStore, + private readonly visibleTiles: Set, + ) {} + + /** + * Sets the contents of the spotlight tile. If this is never called, there + * will be no spotlight tile. + */ + public registerSpotlight(media: MediaViewModel[], maximised: boolean): void { + if (this.spotlight !== null) throw new Error("Spotlight already set"); + if (this.numGridEntries > 0) + throw new Error("Spotlight must be registered before grid tiles"); + + // Reuse the previous spotlight tile if it exists + if (this.prevSpotlight === null) { + this.spotlight = new SpotlightTileData(media, maximised); + } else { + this.spotlight = this.prevSpotlight; + this.spotlight.media = media; + this.spotlight.maximised = maximised; + } + } + + /** + * Sets up a grid tile for the given media. If this is never called for some + * media, then that media will have no grid tile. + */ + public registerGridTile(media: UserMediaViewModel): void { + if (this.spotlight !== null) { + // We actually *don't* want spotlight speakers to appear in both the + // spotlight and the grid, so they're filtered out here + if (!media.local && this.spotlight.media.includes(media)) return; + // When the spotlight speaker changes, we would see one grid tile appear + // and another grid tile disappear. This would be an undesirable layout + // shift, so instead what we do is take the speaker's grid tile and swap + // the media out, so it can remain where it is in the layout. + if ( + media === this.prevSpotlightSpeaker && + this.spotlight.media.length === 1 && + "speaking" in this.spotlight.media[0] && + this.prevSpotlightSpeaker !== this.spotlight.media[0] + ) { + const prev = this.prevGridByMedia.get(this.spotlight.media[0]); + if (prev !== undefined) { + const [entry, prevIndex] = prev; + const previouslyVisible = this.visibleTiles.has(entry.vm); + const nowVisible = this.visibleTiles.has( + this.prevGrid[this.numGridEntries]?.vm, + ); + + // If it doesn't need to move between the visible/invisible sections of + // the grid, then we can keep it where it was and swap the media + if (previouslyVisible === nowVisible) { + this.stationaryGridEntries[prevIndex] = entry; + // Do the media swap + entry.media = media; + this.prevGridByMedia.delete(this.spotlight.media[0]); + this.prevGridByMedia.set(media, prev); + } else { + // Create a new tile; this will cause a layout shift but I'm not + // sure there's any other straightforward option in this case + (nowVisible + ? this.visibleGridEntries + : this.invisibleGridEntries + ).push(new GridTileData(media)); + } + + this.numGridEntries++; + return; + } + } + } + + // Was there previously a tile with this same media? + const prev = this.prevGridByMedia.get(media); + if (prev === undefined) { + // Create a new tile + (this.visibleTiles.has(this.prevGrid[this.numGridEntries]?.vm) + ? this.visibleGridEntries + : this.invisibleGridEntries + ).push(new GridTileData(media)); + } else { + // Reuse the existing tile + const [entry, prevIndex] = prev; + const previouslyVisible = this.visibleTiles.has(entry.vm); + const nowVisible = this.visibleTiles.has( + this.prevGrid[this.numGridEntries]?.vm, + ); + // If it doesn't need to move between the visible/invisible sections of + // the grid, then we can keep it exactly where it was previously + if (previouslyVisible === nowVisible) + this.stationaryGridEntries[prevIndex] = entry; + // Otherwise, queue this tile to be moved + else + (nowVisible ? this.visibleGridEntries : this.invisibleGridEntries).push( + entry, + ); + } + + this.numGridEntries++; + } + + /** + * Constructs a new collection of all registered tiles, transferring ownership + * of the tiles to the new collection. Any tiles present in the previous + * collection but not the new collection will be destroyed. + */ + public build(): TileStore { + // Piece together the grid + const grid = [ + ...fillGaps(this.stationaryGridEntries, [ + ...this.visibleGridEntries, + ...this.invisibleGridEntries, + ]), + ]; + + // Destroy unused tiles + if (this.spotlight === null && this.prevSpotlight !== null) + this.prevSpotlight.destroy(); + const gridEntries = new Set(grid); + for (const entry of this.prevGrid) + if (!gridEntries.has(entry)) entry.destroy(); + + return this.construct(this.spotlight, grid); + } +} diff --git a/src/state/TileViewModel.ts b/src/state/TileViewModel.ts new file mode 100644 index 000000000..3c25907ea --- /dev/null +++ b/src/state/TileViewModel.ts @@ -0,0 +1,43 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; + +import { ViewModel } from "./ViewModel"; +import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel"; + +let nextId = 0; +function createId(): string { + return (nextId++).toString(); +} + +export class GridTileViewModel extends ViewModel { + public readonly id = createId(); + + private readonly visible_ = new BehaviorSubject(false); + /** + * Whether the tile is visible within the current viewport. + */ + public readonly visible: Observable = this.visible_; + + public setVisible = (value: boolean): void => this.visible_.next(value); + + public constructor(public readonly media: Observable) { + super(); + } +} + +export class SpotlightTileViewModel extends ViewModel { + public constructor( + public readonly media: Observable, + public readonly maximised: Observable, + ) { + super(); + } +} + +export type TileViewModel = GridTileViewModel | SpotlightTileViewModel; diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index 0bf6cab82..81e501102 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -9,12 +9,20 @@ import { RemoteTrackPublication } from "livekit-client"; import { test, expect } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; +import { of } from "rxjs"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { GridTile } from "./GridTile"; import { withRemoteMedia } from "../utils/test"; +import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsProvider } from "../useReactions"; +global.IntersectionObserver = class MockIntersectionObserver { + public observe(): void {} + public unobserve(): void {} + public disconnect(): void {} +} as unknown as typeof IntersectionObserver; + test("GridTile is accessible", async () => { await withRemoteMedia( { @@ -42,11 +50,10 @@ test("GridTile is accessible", async () => { const { container } = render( {}} targetWidth={300} targetHeight={200} - showVideo showSpeakingIndicators /> , diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 3675e9a7c..8252d1086 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -10,6 +10,7 @@ import { ReactNode, forwardRef, useCallback, + useRef, useState, } from "react"; import { animated } from "@react-spring/web"; @@ -44,6 +45,8 @@ import { import { Slider } from "../Slider"; import { MediaView } from "./MediaView"; import { useLatest } from "../useLatest"; +import { GridTileViewModel } from "../state/TileViewModel"; +import { useMergedRefs } from "../useMergedRefs"; import { useReactions } from "../useReactions"; interface TileProps { @@ -52,7 +55,6 @@ interface TileProps { targetWidth: number; targetHeight: number; displayName: string; - showVideo: boolean; showSpeakingIndicators: boolean; } @@ -67,7 +69,6 @@ const UserMediaTile = forwardRef( ( { vm, - showVideo, showSpeakingIndicators, menuStart, menuEnd, @@ -119,7 +120,7 @@ const UserMediaTile = forwardRef( video={video} member={vm.member} unencryptedWarning={unencryptedWarning} - videoEnabled={videoEnabled && showVideo} + videoEnabled={videoEnabled} videoFit={cropVideo ? "cover" : "contain"} className={classNames(className, styles.tile, { [styles.speaking]: showSpeaking, @@ -277,25 +278,27 @@ const RemoteUserMediaTile = forwardRef< RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; interface GridTileProps { - vm: UserMediaViewModel; + vm: GridTileViewModel; onOpenProfile: (() => void) | null; targetWidth: number; targetHeight: number; className?: string; style?: ComponentProps["style"]; - showVideo: boolean; showSpeakingIndicators: boolean; } export const GridTile = forwardRef( - ({ vm, onOpenProfile, ...props }, ref) => { - const displayName = useDisplayName(vm); + ({ vm, onOpenProfile, ...props }, theirRef) => { + const ourRef = useRef(null); + const ref = useMergedRefs(ourRef, theirRef); + const media = useObservableEagerState(vm.media); + const displayName = useDisplayName(media); - if (vm instanceof LocalUserMediaViewModel) { + if (media instanceof LocalUserMediaViewModel) { return ( ( return ( diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index a0fbed45a..cedeea626 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -9,9 +9,11 @@ import { test, expect, vi } from "vitest"; import { isInaccessible, render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import userEvent from "@testing-library/user-event"; +import { of } from "rxjs"; import { SpotlightTile } from "./SpotlightTile"; import { withLocalMedia, withRemoteMedia } from "../utils/test"; +import { SpotlightTileViewModel } from "../state/TileViewModel"; global.IntersectionObserver = class MockIntersectionObserver { public observe(): void {} @@ -36,10 +38,9 @@ test("SpotlightTile is accessible", async () => { const toggleExpanded = vi.fn(); const { container } = render( = { + ref, videoEnabled, videoFit: cropVideo ? "cover" : "contain", ...props, }; return vm instanceof LocalUserMediaViewModel ? ( - + ) : ( ); @@ -175,8 +178,7 @@ const SpotlightItem = forwardRef( SpotlightItem.displayName = "SpotlightItem"; interface Props { - vms: MediaViewModel[]; - maximised: boolean; + vm: SpotlightTileViewModel; expanded: boolean; onToggleExpanded: (() => void) | null; targetWidth: number; @@ -189,8 +191,7 @@ interface Props { export const SpotlightTile = forwardRef( ( { - vms, - maximised, + vm, expanded, onToggleExpanded, targetWidth, @@ -204,12 +205,14 @@ export const SpotlightTile = forwardRef( const { t } = useTranslation(); const [root, ourRef] = useObservableRef(null); const ref = useMergedRefs(ourRef, theirRef); - const [visibleId, setVisibleId] = useState(vms[0].id); - const latestVms = useLatest(vms); + const maximised = useObservableEagerState(vm.maximised); + const media = useObservableEagerState(vm.media); + const [visibleId, setVisibleId] = useState(media[0].id); + const latestMedia = useLatest(media); const latestVisibleId = useLatest(visibleId); - const visibleIndex = vms.findIndex((vm) => vm.id === visibleId); + const visibleIndex = media.findIndex((vm) => vm.id === visibleId); const canGoBack = visibleIndex > 0; - const canGoToNext = visibleIndex !== -1 && visibleIndex < vms.length - 1; + const canGoToNext = visibleIndex !== -1 && visibleIndex < media.length - 1; // To keep track of which item is visible, we need an intersection observer // hooked up to the root element and the items. Because the items will run @@ -234,28 +237,30 @@ export const SpotlightTile = forwardRef( const [scrollToId, setScrollToId] = useReactiveState( (prev) => - prev == null || prev === visibleId || vms.every((vm) => vm.id !== prev) + prev == null || + prev === visibleId || + media.every((vm) => vm.id !== prev) ? null : prev, [visibleId], ); const onBackClick = useCallback(() => { - const vms = latestVms.current; - const visibleIndex = vms.findIndex( + const media = latestMedia.current; + const visibleIndex = media.findIndex( (vm) => vm.id === latestVisibleId.current, ); - if (visibleIndex > 0) setScrollToId(vms[visibleIndex - 1].id); - }, [latestVisibleId, latestVms, setScrollToId]); + if (visibleIndex > 0) setScrollToId(media[visibleIndex - 1].id); + }, [latestVisibleId, latestMedia, setScrollToId]); const onNextClick = useCallback(() => { - const vms = latestVms.current; - const visibleIndex = vms.findIndex( + const media = latestMedia.current; + const visibleIndex = media.findIndex( (vm) => vm.id === latestVisibleId.current, ); - if (visibleIndex !== -1 && visibleIndex !== vms.length - 1) - setScrollToId(vms[visibleIndex + 1].id); - }, [latestVisibleId, latestVms, setScrollToId]); + if (visibleIndex !== -1 && visibleIndex !== media.length - 1) + setScrollToId(media[visibleIndex + 1].id); + }, [latestVisibleId, latestMedia, setScrollToId]); const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon; @@ -277,7 +282,7 @@ export const SpotlightTile = forwardRef( )}
- {vms.map((vm) => ( + {media.map((vm) => ( ( {!expanded && (
1, + [styles.show]: showIndicators && media.length > 1, })} > - {vms.map((vm) => ( + {media.map((vm) => (
{ + expect([ + ...fillGaps([1, undefined, undefined, undefined, 3], [2]), + ]).toStrictEqual([1, 2, 3]); +}); + +test("fillGaps adds extra filler elements to the end", () => { + expect([ + ...fillGaps([1, undefined, 3, undefined], [2, 4, 5, 6]), + ]).toStrictEqual([1, 2, 3, 4, 5, 6]); +}); diff --git a/src/utils/iter.ts b/src/utils/iter.ts new file mode 100644 index 000000000..cf40ae8fe --- /dev/null +++ b/src/utils/iter.ts @@ -0,0 +1,36 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +/** + * Fills in the 'undefined' gaps in a collection by drawing items from a second + * collection, or simply filtering out the gap if no items are left. If filler + * items remain at the end, they will be appended to the resulting collection. + */ +export function fillGaps( + gappy: Iterable, + filler: Iterable, +): Iterable { + return { + [Symbol.iterator](): Iterator { + const gappyIter = gappy[Symbol.iterator](); + const fillerIter = filler[Symbol.iterator](); + return { + next(): IteratorResult { + let gappyItem: IteratorResult; + do { + gappyItem = gappyIter.next(); + if (!gappyItem.done && gappyItem.value !== undefined) + return gappyItem as IteratorYieldResult; + const fillerItem = fillerIter.next(); + if (!fillerItem.done) return fillerItem; + } while (!gappyItem.done); + return gappyItem; + }, + }; + }, + }; +}