Show tiles for members we're trying to connect to

This should help give more context on what's going wrong in splitbrain scenarios. If users leave calls uncleanly, their tile will remain in until their member event times out, which will be an hour from when they joined the call. See #639. Part of #616
element-hq · Oct 21, 2022 · 1ea9432 · 1ea9432
1 parent 54fe2aa
commit 1ea9432
Show file tree

Hide file tree

Showing 11 changed files with 86 additions and 62 deletions.
diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx
@@ -76,7 +76,6 @@ export function GroupCallView({
     toggleScreensharing,
     requestingScreenshare,
     isScreensharing,
-    localScreenshareFeed,
     screenshareFeeds,
     participants,
     unencryptedEventsFromUsers,
@@ -221,6 +220,7 @@ export function GroupCallView({
           client={client}
           roomName={groupCall.room.name}
           avatarUrl={avatarUrl}
+          participants={participants}
           microphoneMuted={microphoneMuted}
           localVideoMuted={localVideoMuted}
           toggleLocalVideoMuted={toggleLocalVideoMuted}
@@ -230,7 +230,6 @@ export function GroupCallView({
           onLeave={onLeave}
           toggleScreensharing={toggleScreensharing}
           isScreensharing={isScreensharing}
-          localScreenshareFeed={localScreenshareFeed}
           screenshareFeeds={screenshareFeeds}
           roomIdOrAlias={roomIdOrAlias}
           unencryptedEventsFromUsers={unencryptedEventsFromUsers}

diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx
@@ -70,6 +70,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 interface Props {
   client: MatrixClient;
   groupCall: GroupCall;
+  participants: RoomMember[];
   roomName: string;
   avatarUrl: string;
   microphoneMuted: boolean;
@@ -82,14 +83,16 @@ interface Props {
   onLeave: () => void;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   roomIdOrAlias: string;
   unencryptedEventsFromUsers: Set<string>;
   hideHeader: boolean;
 }
 
-export interface Participant {
+// Represents something that should get a tile on the layout,
+// ie. a user's video feed or a screen share feed.
+export interface TileDescriptor {
   id: string;
+  member: RoomMember;
   focused: boolean;
   presenter: boolean;
   callFeed?: CallFeed;
@@ -99,6 +102,7 @@ export interface Participant {
 export function InCallView({
   client,
   groupCall,
+  participants,
   roomName,
   avatarUrl,
   microphoneMuted,
@@ -111,7 +115,6 @@ export function InCallView({
   toggleScreensharing,
   isScreensharing,
   screenshareFeeds,
-  localScreenshareFeed,
   roomIdOrAlias,
   unencryptedEventsFromUsers,
   hideHeader,
@@ -185,39 +188,48 @@ export function InCallView({
   }, [setLayout]);
 
   const items = useMemo(() => {
-    const participants: Participant[] = [];
-
-    for (const callFeed of userMediaFeeds) {
-      participants.push({
-        id: callFeed.stream.id,
-        callFeed,
-        focused:
-          screenshareFeeds.length === 0 && callFeed.userId === activeSpeaker,
-        isLocal: callFeed.isLocal(),
+    const tileDescriptors: TileDescriptor[] = [];
+
+    // one tile for each participants, to start with (we want a tile for everyone we
+    // think should be in the call, even if we don't have a media feed for them yet)
+    for (const p of participants) {
+      const userMediaFeed = userMediaFeeds.find((f) => f.userId === p.userId);
+
+      // NB. this assumes that the same user can't join more than once from multiple
+      // devices, but the participants are just RoomMembers, so this assumption is baked
+      // into GroupCall itself.
+      tileDescriptors.push({
+        id: p.userId,
+        member: p,
+        callFeed: userMediaFeed,
+        focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker,
+        isLocal: p.userId === client.getUserId(),
         presenter: false,
       });
     }
 
-    for (const callFeed of screenshareFeeds) {
-      const userMediaItem = participants.find(
-        (item) => item.callFeed.userId === callFeed.userId
+    // add the screenshares too
+    for (const screenshareFeed of screenshareFeeds) {
+      const userMediaItem = tileDescriptors.find(
+        (item) => item.member.userId === screenshareFeed.userId
       );
 
       if (userMediaItem) {
         userMediaItem.presenter = true;
       }
 
-      participants.push({
-        id: callFeed.stream.id,
-        callFeed,
+      tileDescriptors.push({
+        id: screenshareFeed.stream.id,
+        member: userMediaItem?.member,
+        callFeed: screenshareFeed,
         focused: true,
-        isLocal: callFeed.isLocal(),
+        isLocal: screenshareFeed.isLocal(),
         presenter: false,
       });
     }
 
-    return participants;
-  }, [userMediaFeeds, activeSpeaker, screenshareFeeds]);
+    return tileDescriptors;
+  }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]);
 
   // The maximised participant: either the participant that the user has
   // manually put in fullscreen, or the focused (active) participant if the
@@ -281,7 +293,13 @@ export function InCallView({
 
     return (
       <VideoGrid items={items} layout={layout} disableAnimations={isSafari}>
-        {({ item, ...rest }: { item: Participant; [x: string]: unknown }) => (
+        {({
+          item,
+          ...rest
+        }: {
+          item: TileDescriptor;
+          [x: string]: unknown;
+        }) => (
           <VideoTileContainer
             key={item.id}
             item={item}

diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts
@@ -49,7 +49,6 @@ export interface UseGroupCallReturnType {
   requestingScreenshare: boolean;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   participants: RoomMember[];
   hasLocalParticipant: boolean;
@@ -66,7 +65,6 @@ interface State {
   microphoneMuted: boolean;
   localVideoMuted: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   isScreensharing: boolean;
   requestingScreenshare: boolean;
@@ -87,7 +85,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       localVideoMuted,
       isScreensharing,
       screenshareFeeds,
-      localScreenshareFeed,
       localDesktopCapturerSourceId,
       participants,
       hasLocalParticipant,
@@ -105,7 +102,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     localVideoMuted: false,
     isScreensharing: false,
     screenshareFeeds: [],
-    localScreenshareFeed: null,
     localDesktopCapturerSourceId: null,
     requestingScreenshare: false,
     participants: [],
@@ -133,7 +129,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
         microphoneMuted: groupCall.isMicrophoneMuted(),
         localVideoMuted: groupCall.isLocalVideoMuted(),
         isScreensharing: groupCall.isScreensharing(),
-        localScreenshareFeed: groupCall.localScreenshareFeed,
         localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
         screenshareFeeds: [...groupCall.screenshareFeeds],
         participants: [...groupCall.participants],
@@ -170,12 +165,11 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
 
     function onLocalScreenshareStateChanged(
       isScreensharing: boolean,
-      localScreenshareFeed: CallFeed,
+      _localScreenshareFeed: CallFeed,
       localDesktopCapturerSourceId: string
     ): void {
       updateState({
         isScreensharing,
-        localScreenshareFeed,
         localDesktopCapturerSourceId,
       });
     }
@@ -226,7 +220,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       microphoneMuted: groupCall.isMicrophoneMuted(),
       localVideoMuted: groupCall.isLocalVideoMuted(),
       isScreensharing: groupCall.isScreensharing(),
-      localScreenshareFeed: groupCall.localScreenshareFeed,
       localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
       screenshareFeeds: [...groupCall.screenshareFeeds],
       participants: [...groupCall.participants],
@@ -342,7 +335,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,
-    localScreenshareFeed,
     localDesktopCapturerSourceId,
     participants,
     hasLocalParticipant,

diff --git a/src/video-grid/AudioContainer.tsx b/src/video-grid/AudioContainer.tsx
@@ -16,15 +16,15 @@ limitations under the License.
 
 import React, { FC, useEffect, useRef } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useCallFeed } from "./useCallFeed";
 import { useMediaStreamTrackCount } from "./useMediaStream";
 
 // XXX: These in fact do not render anything but to my knowledge this is the
 // only way to a hook on an array
 
 interface AudioForParticipantProps {
-  item: Participant;
+  item: TileDescriptor;
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }
@@ -78,7 +78,7 @@ export const AudioForParticipant: FC<AudioForParticipantProps> = ({
 };
 
 interface AudioContainerProps {
-  items: Participant[];
+  items: TileDescriptor[];
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }

diff --git a/src/video-grid/VideoGrid.stories.tsx b/src/video-grid/VideoGrid.stories.tsx
@@ -16,11 +16,12 @@ limitations under the License.
 
 import React, { useState } from "react";
 import { useMemo } from "react";
+import { RoomMember } from "matrix-js-sdk";
 
 import { VideoGrid, useVideoGridLayout } from "./VideoGrid";
 import { VideoTile } from "./VideoTile";
 import { Button } from "../button";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 export default {
   title: "VideoGrid",
@@ -33,10 +34,11 @@ export const ParticipantsTest = () => {
   const { layout, setLayout } = useVideoGridLayout(false);
   const [participantCount, setParticipantCount] = useState(1);
 
-  const items: Participant[] = useMemo(
+  const items: TileDescriptor[] = useMemo(
     () =>
       new Array(participantCount).fill(undefined).map((_, i) => ({
         id: (i + 1).toString(),
+        member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`),
         focused: false,
         presenter: false,
       })),

diff --git a/src/video-grid/VideoGrid.tsx b/src/video-grid/VideoGrid.tsx
@@ -23,7 +23,7 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ
 
 import styles from "./VideoGrid.module.css";
 import { Layout } from "../room/GridLayoutMenu";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface TilePosition {
   x: number;
@@ -36,7 +36,7 @@ interface TilePosition {
 interface Tile {
   key: Key;
   order: number;
-  item: Participant;
+  item: TileDescriptor;
   remove: boolean;
   focused: boolean;
   presenter: boolean;
@@ -693,12 +693,12 @@ interface ChildrenProperties extends ReactDOMAttributes {
   };
   width: number;
   height: number;
-  item: Participant;
+  item: TileDescriptor;
   [index: string]: unknown;
 }
 
 interface VideoGridProps {
-  items: Participant[];
+  items: TileDescriptor[];
   layout: Layout;
   disableAnimations?: boolean;
   children: (props: ChildrenProperties) => React.ReactNode;

diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx
@@ -26,6 +26,7 @@ import { AudioButton, FullscreenButton } from "../button/Button";
 
 interface Props {
   name: string;
+  hasFeed: Boolean;
   speaking?: boolean;
   audioMuted?: boolean;
   videoMuted?: boolean;
@@ -47,6 +48,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
   (
     {
       name,
+      hasFeed,
       speaking,
       audioMuted,
       videoMuted,
@@ -90,6 +92,8 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       }
     }
 
+    const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name });
+
     return (
       <animated.div
         className={classNames(styles.videoTile, className, {
@@ -120,7 +124,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             <div className={classNames(styles.infoBubble, styles.memberName)}>
               {audioMuted && !videoMuted && <MicMutedIcon />}
               {videoMuted && <VideoMutedIcon />}
-              <span title={name}>{name}</span>
+              <span title={caption}>{caption}</span>
             </div>
           ))}
         <video ref={mediaRef} playsInline disablePictureInPicture />

diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx
@@ -25,10 +25,10 @@ import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
 import { VideoTileSettingsModal } from "./VideoTileSettingsModal";
 import { useModalTriggerState } from "../Modal";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface Props {
-  item: Participant;
+  item: TileDescriptor;
   width?: number;
   height?: number;
   getAvatar: (
@@ -41,7 +41,7 @@ interface Props {
   disableSpeakingIndicator: boolean;
   maximised: boolean;
   fullscreen: boolean;
-  onFullscreen: (item: Participant) => void;
+  onFullscreen: (item: TileDescriptor) => void;
 }
 
 export function VideoTileContainer({
@@ -65,9 +65,8 @@ export function VideoTileContainer({
     speaking,
     stream,
     purpose,
-    member,
   } = useCallFeed(item.callFeed);
-  const { rawDisplayName } = useRoomMemberName(member);
+  const { rawDisplayName } = useRoomMemberName(item.member);
   const [tileRef, mediaRef] = useSpatialMediaStream(
     stream,
     audioContext,
@@ -99,9 +98,10 @@ export function VideoTileContainer({
         videoMuted={videoMuted}
         screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
         name={rawDisplayName}
+        hasFeed={Boolean(item.callFeed)}
         ref={tileRef}
         mediaRef={mediaRef}
-        avatar={getAvatar && getAvatar(member, width, height)}
+        avatar={getAvatar && getAvatar(item.member, width, height)}
         onOptionsPress={onOptionsPress}
         localVolume={localVolume}
         maximised={maximised}