Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for playing a sound when the user exits a call. #2860

Open
wants to merge 28 commits into
base: livekit
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/room/CallEventAudioRenderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import { useLatest } from "../useLatest";
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;

const sounds = prefetchSounds({
export const callEventAudioSounds = prefetchSounds({
join: {
mp3: joinCallSoundMp3,
ogg: joinCallSoundOgg,
Expand All @@ -46,7 +46,7 @@ export function CallEventAudioRenderer({
vm: CallViewModel;
}): ReactNode {
const audioEngineCtx = useAudioContext({
sounds,
sounds: callEventAudioSounds,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
Expand All @@ -60,7 +60,7 @@ export function CallEventAudioRenderer({

useEffect(() => {
if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
audioEngineRef.current.playSound("raiseHand");
void audioEngineRef.current.playSound("raiseHand");
}
}, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);

Expand All @@ -74,7 +74,7 @@ export function CallEventAudioRenderer({
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
audioEngineRef.current?.playSound("join");
void audioEngineRef.current?.playSound("join");
});

const leftSub = vm.memberChanges
Expand All @@ -86,7 +86,7 @@ export function CallEventAudioRenderer({
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
audioEngineRef.current?.playSound("left");
void audioEngineRef.current?.playSound("left");
});

return (): void => {
Expand Down
151 changes: 151 additions & 0 deletions src/room/GroupCallView.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
/*
Copyright 2024 New Vector Ltd.

SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/

import { beforeEach, expect, MockedFunction, test, vitest } from "vitest";
import { render } from "@testing-library/react";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { of } from "rxjs";
import { JoinRule, RoomState } from "matrix-js-sdk/src/matrix";
import { Router } from "react-router-dom";
import { createBrowserHistory } from "history";
import userEvent from "@testing-library/user-event";

import { MuteStates } from "./MuteStates";
import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
import { ActiveCall } from "./InCallView";
import {
mockMatrixRoom,
mockMatrixRoomMember,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import { GroupCallView } from "./GroupCallView";
import { leaveRTCSession } from "../rtcSessionHelpers";
import { WidgetHelpers } from "../widget";
import { LazyEventEmitter } from "../LazyEventEmitter";

vitest.mock("../soundUtils");
vitest.mock("../useAudioContext");
vitest.mock("./InCallView");

vitest.mock("../rtcSessionHelpers", async (importOriginal) => {
const orig = await importOriginal<typeof import("../rtcSessionHelpers")>();
vitest.spyOn(orig, "leaveRTCSession");
return orig;
});

let playSound: MockedFunction<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;

const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const carol = mockMatrixRoomMember(localRtcMember);
const roomMembers = new Map([carol].map((p) => [p.userId, p]));

const roomId = "!foo:bar";
const soundPromise = Promise.resolve(true);

beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
sound: new ArrayBuffer(0),
});
playSound = vitest.fn().mockReturnValue(soundPromise);
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
});
// A trivial implementation of Active call to ensure we are testing GroupCallView exclusively here.
(ActiveCall as MockedFunction<typeof ActiveCall>).mockImplementation(
({ onLeave }) => {
return (
<div>
<button onClick={() => onLeave()}>Leave</button>
</div>
);
},
);
});

function createGroupCallView(widget: WidgetHelpers | null): {
rtcSession: MockRTCSession;
getByText: ReturnType<typeof render>["getByText"];
} {
const history = createBrowserHistory();
const client = {
getUser: () => null,
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
getRoom: (rId) => (rId === roomId ? room : null),
} as Partial<MatrixClient> as MatrixClient;
const room = mockMatrixRoom({
client,
roomId,
getMember: (userId) => roomMembers.get(userId) ?? null,
getMxcAvatarUrl: () => null,
getCanonicalAlias: () => null,
currentState: {
getJoinRule: () => JoinRule.Invite,
} as Partial<RoomState> as RoomState,
});
const rtcSession = new MockRTCSession(
room,
localRtcMember,
[],
).withMemberships(of([]));
const muteState = {
audio: { enabled: false },
video: { enabled: false },
} as MuteStates;
const { getByText } = render(
<Router history={history}>
<GroupCallView
client={client}
isPasswordlessUser={false}
confineToRoom={false}
preload={false}
skipLobby={false}
hideHeader={true}
rtcSession={rtcSession as unknown as MatrixRTCSession}
muteStates={muteState}
widget={widget}
/>
</Router>,
);
return {
getByText,
rtcSession,
};
}

test("will play a leave sound asynchronously in SPA mode", async () => {
const user = userEvent.setup();
const { getByText, rtcSession } = createGroupCallView(null);
const leaveButton = getByText("Leave");
await user.click(leaveButton);
expect(playSound).toHaveBeenCalledWith("left");
expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, undefined);
expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce();
});

test("will play a leave sound synchronously in widget mode", async () => {
const user = userEvent.setup();
const widget = {
api: {
setAlwaysOnScreen: async () => Promise.resolve(true),
} as Partial<WidgetHelpers["api"]>,
lazyActions: new LazyEventEmitter(),
};
const { getByText, rtcSession } = createGroupCallView(
widget as WidgetHelpers,
);
const leaveButton = getByText("Leave");
await user.click(leaveButton);
expect(playSound).toHaveBeenCalledWith("left");
expect(leaveRTCSession).toHaveBeenCalledWith(rtcSession, soundPromise);
expect(rtcSession.leaveRoomSession).toHaveBeenCalledOnce();
});
70 changes: 47 additions & 23 deletions src/room/GroupCallView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { Heading, Text } from "@vector-im/compound-web";
import { useTranslation } from "react-i18next";

import type { IWidgetApiRequest } from "matrix-widget-api";
import { widget, ElementWidgetActions, JoinCallData } from "../widget";
import { ElementWidgetActions, JoinCallData, WidgetHelpers } from "../widget";
import { FullScreenView } from "../FullScreenView";
import { LobbyView } from "./LobbyView";
import { MatrixInfo } from "./VideoPreview";
Expand All @@ -41,6 +41,9 @@ import { InviteModal } from "./InviteModal";
import { useUrlParams } from "../UrlParams";
import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useAudioContext } from "../useAudioContext";
import { callEventAudioSounds } from "./CallEventAudioRenderer";
import { useLatest } from "../useLatest";

declare global {
interface Window {
Expand All @@ -57,6 +60,7 @@ interface Props {
hideHeader: boolean;
rtcSession: MatrixRTCSession;
muteStates: MuteStates;
widget: WidgetHelpers | null;
}

export const GroupCallView: FC<Props> = ({
Expand All @@ -68,10 +72,16 @@ export const GroupCallView: FC<Props> = ({
hideHeader,
rtcSession,
muteStates,
widget,
}) => {
const memberships = useMatrixRTCSessionMemberships(rtcSession);
const isJoined = useMatrixRTCSessionJoinState(rtcSession);

const leaveSoundContext = useLatest(
useAudioContext({
sounds: callEventAudioSounds,
latencyHint: "interactive",
}),
);
// This should use `useEffectEvent` (only available in experimental versions)
useEffect(() => {
if (memberships.length >= MUTE_PARTICIPANT_COUNT)
Expand Down Expand Up @@ -185,14 +195,14 @@ export const GroupCallView: FC<Props> = ({
ev.detail.data as unknown as JoinCallData,
);
await enterRTCSession(rtcSession, perParticipantE2EE);
widget!.api.transport.reply(ev.detail, {});
widget.api.transport.reply(ev.detail, {});
})().catch((e) => {
logger.error("Error joining RTC session", e);
});
};
widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
widget.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
};
} else {
// No lobby and no preload: we enter the rtc session right away
Expand All @@ -206,29 +216,33 @@ export const GroupCallView: FC<Props> = ({
void enterRTCSession(rtcSession, perParticipantE2EE);
}
}
}, [rtcSession, preload, skipLobby, perParticipantE2EE]);
}, [widget, rtcSession, preload, skipLobby, perParticipantE2EE]);

const [left, setLeft] = useState(false);
const [leaveError, setLeaveError] = useState<Error | undefined>(undefined);
const history = useHistory();

const onLeave = useCallback(
(leaveError?: Error): void => {
setLeaveError(leaveError);
setLeft(true);

const audioPromise = leaveSoundContext.current?.playSound("left");
// In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent,
// therefore we want the event to be sent instantly without getting queued/batched.
const sendInstantly = !!widget;
setLeaveError(leaveError);
setLeft(true);
PosthogAnalytics.instance.eventCallEnded.track(
rtcSession.room.roomId,
rtcSession.memberships.length,
sendInstantly,
rtcSession,
);

// Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
leaveRTCSession(rtcSession)
leaveRTCSession(
rtcSession,
// Wait for the sound in widget mode (it's not long)
sendInstantly && audioPromise ? audioPromise : undefined,
)
// Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
.then(() => {
if (
!isPasswordlessUser &&
Expand All @@ -242,29 +256,36 @@ export const GroupCallView: FC<Props> = ({
logger.error("Error leaving RTC session", e);
});
},
[rtcSession, isPasswordlessUser, confineToRoom, history],
[
widget,
rtcSession,
isPasswordlessUser,
confineToRoom,
leaveSoundContext,
history,
],
);

useEffect(() => {
if (widget && isJoined) {
// set widget to sticky once joined.
widget!.api.setAlwaysOnScreen(true).catch((e) => {
widget.api.setAlwaysOnScreen(true).catch((e) => {
logger.error("Error calling setAlwaysOnScreen(true)", e);
});

const onHangup = (ev: CustomEvent<IWidgetApiRequest>): void => {
widget!.api.transport.reply(ev.detail, {});
widget.api.transport.reply(ev.detail, {});
// Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts.
leaveRTCSession(rtcSession).catch((e) => {
logger.error("Failed to leave RTC session", e);
});
};
widget.lazyActions.once(ElementWidgetActions.HangupCall, onHangup);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
};
}
}, [isJoined, rtcSession]);
}, [widget, isJoined, rtcSession]);

const onReconnect = useCallback(() => {
setLeft(false);
Expand Down Expand Up @@ -357,14 +378,17 @@ export const GroupCallView: FC<Props> = ({
leaveError
) {
return (
<CallEndedView
endedCallId={rtcSession.room.roomId}
client={client}
isPasswordlessUser={isPasswordlessUser}
confineToRoom={confineToRoom}
leaveError={leaveError}
reconnect={onReconnect}
/>
<>
<CallEndedView
endedCallId={rtcSession.room.roomId}
client={client}
isPasswordlessUser={isPasswordlessUser}
confineToRoom={confineToRoom}
leaveError={leaveError}
reconnect={onReconnect}
/>
;
</>
);
} else {
// If the user is a regular user, we'll have sent them back to the homepage,
Expand Down
4 changes: 2 additions & 2 deletions src/room/ReactionAudioRenderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@ export function ReactionsAudioRenderer(): ReactNode {
return;
}
if (soundMap[reactionName]) {
audioEngineRef.current.playSound(reactionName);
void audioEngineRef.current.playSound(reactionName);
} else {
// Fallback sounds.
audioEngineRef.current.playSound("generic");
void audioEngineRef.current.playSound("generic");
}
}
}, [audioEngineRef, shouldPlay, oldReactions, reactions]);
Expand Down
1 change: 1 addition & 0 deletions src/room/RoomPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ export const RoomPage: FC = () => {
case "loaded":
return (
<GroupCallView
widget={widget}
client={client!}
rtcSession={groupCallState.rtcSession}
isPasswordlessUser={passwordlessUser}
Expand Down
Loading
Loading