Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add background blur feature for supported devices #2812

Draft
wants to merge 17 commits into
base: livekit
Choose a base branch
from
Draft
4 changes: 4 additions & 0 deletions knip.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ export default {
// then Knip will flag it as a false positive
// https://github.com/webpro-nl/knip/issues/766
"@vector-im/compound-web",
// We need this so the eslint is happy with @livekit/track-processors.
// This might be a bug in the livekit repo but for now we fix it on the
// element call side.
"@types/dom-mediacapture-transform",
"matrix-widget-api",
],
ignoreExportsUsedInFile: true,
Expand Down
3 changes: 3 additions & 0 deletions locales/en-GB/app.json
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
"effect_volume_label": "Sound effect volume"
},
"background_blur_header": "Background",
"background_blur_label": "Blur the background of the video",
"blur_not_supported_by_browser": "(Background blur is not supported by this browser)",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"blur_not_supported_by_browser": "(Background blur is not supported by this browser)",
"blur_not_supported_by_browser": "(Background blur is not supported by this device)",

...because this message also appears when EC is embedded as widget and the user isn't aware that it is a browser.

"developer_settings_label": "Developer Settings",
"developer_settings_label_description": "Expose developer settings in the settings window.",
"developer_tab_title": "Developer",
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"@formatjs/intl-segmenter": "^11.7.3",
"@livekit/components-core": "^0.11.0",
"@livekit/components-react": "^2.0.0",
"@livekit/track-processors": "^0.3.2",
"@opentelemetry/api": "^1.4.0",
"@opentelemetry/core": "^1.25.1",
"@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
Expand All @@ -50,6 +51,7 @@
"@testing-library/react-hooks": "^8.0.1",
"@testing-library/user-event": "^14.5.1",
"@types/content-type": "^1.1.5",
"@types/dom-mediacapture-transform": "^0.1.10",
"@types/grecaptcha": "^3.0.9",
"@types/jsdom": "^21.1.7",
"@types/lodash-es": "^4.17.12",
Expand Down
54 changes: 33 additions & 21 deletions src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import { Initializer } from "./initializer";
import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
import { widget } from "./widget";
import { useTheme } from "./useTheme";
import { ProcessorProvider } from "./livekit/TrackProcessorContext";

const SentryRoute = Sentry.withSentryRouting(Route);

Expand Down Expand Up @@ -82,27 +83,25 @@ export const App: FC<AppProps> = ({ history }) => {
<TooltipProvider>
{loaded ? (
<Suspense fallback={null}>
<ClientProvider>
<MediaDevicesProvider>
<Sentry.ErrorBoundary fallback={errorPage}>
<DisconnectedBanner />
<Switch>
<SentryRoute exact path="/">
<HomePage />
</SentryRoute>
<SentryRoute exact path="/login">
<LoginPage />
</SentryRoute>
<SentryRoute exact path="/register">
<RegisterPage />
</SentryRoute>
<SentryRoute path="*">
<RoomPage />
</SentryRoute>
</Switch>
</Sentry.ErrorBoundary>
</MediaDevicesProvider>
</ClientProvider>
<Providers>
<Sentry.ErrorBoundary fallback={errorPage}>
<DisconnectedBanner />
<Switch>
<SentryRoute exact path="/">
<HomePage />
</SentryRoute>
<SentryRoute exact path="/login">
<LoginPage />
</SentryRoute>
<SentryRoute exact path="/register">
<RegisterPage />
</SentryRoute>
<SentryRoute path="*">
<RoomPage />
</SentryRoute>
</Switch>
</Sentry.ErrorBoundary>
</Providers>
</Suspense>
) : (
<LoadingView />
Expand All @@ -113,3 +112,16 @@ export const App: FC<AppProps> = ({ history }) => {
</Router>
);
};

const Providers: FC<{
children: JSX.Element;
}> = ({ children }) => {
// We use this to stack all used providers to not make the App component to verbose
return (
<ClientProvider>
<MediaDevicesProvider>
<ProcessorProvider>{children}</ProcessorProvider>
</MediaDevicesProvider>
</ClientProvider>
);
};
111 changes: 111 additions & 0 deletions src/livekit/TrackProcessorContext.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
Copyright 2024 New Vector Ltd.

SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/

import {
BackgroundBlur as backgroundBlur,
BackgroundOptions,
ProcessorWrapper,
} from "@livekit/track-processors";
import {
createContext,
FC,
useCallback,
useContext,
useEffect,
useRef,
useState,
} from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { LocalVideoTrack } from "livekit-client";

import {
backgroundBlur as backgroundBlurSettings,
useSetting,
} from "../settings/settings";

type ProcessorState = {
supported: boolean | undefined;
processor: undefined | ProcessorWrapper<BackgroundOptions>;
/**
* Call this method to try to initialize a processor.
* This only needs to happen if supported is undefined.
* If the backgroundBlur setting is set to true this does not need to be called
* and the processorState.supported will update automatically to the correct value.
*/
checkSupported: () => void;
};
const ProcessorContext = createContext<ProcessorState | undefined>(undefined);

export const useTrackProcessor = (): ProcessorState | undefined =>
useContext(ProcessorContext);

export const useTrackProcessorSync = (
videoTrack: LocalVideoTrack | null,
): void => {
const { processor } = useTrackProcessor() || {};
useEffect(() => {
if (processor && !videoTrack?.getProcessor()) {
void videoTrack?.setProcessor(processor);
}
if (!processor && videoTrack?.getProcessor()) {
void videoTrack?.stopProcessor();
}
}, [processor, videoTrack]);
};

interface Props {
children: JSX.Element;
}
export const ProcessorProvider: FC<Props> = ({ children }) => {
// The setting the user wants to have
const [blurActivated] = useSetting(backgroundBlurSettings);

// If `ProcessorState.supported` is undefined the user can activate that we want
// to have it at least checked (this is useful to show the settings menu properly)
// We dont want to try initializing the blur if the user is not even looking at the setting
const [shouldCheckSupport, setShouldCheckSupport] = useState(blurActivated);

// Cache the processor so we only need to initialize it once.
const blur = useRef<ProcessorWrapper<BackgroundOptions> | undefined>(
undefined,
);

const checkSupported = useCallback(() => {
setShouldCheckSupport(true);
}, []);
// This is the actual state exposed through the context
const [processorState, setProcessorState] = useState<ProcessorState>(() => ({
supported: false,
processor: undefined,
checkSupported,
}));

useEffect(() => {
if (!shouldCheckSupport) return;
try {
if (!blur.current) blur.current = backgroundBlur(15, { delegate: "GPU" });
setProcessorState({
checkSupported,
supported: true,
processor: blurActivated ? blur.current : undefined,
});
} catch (e) {
setProcessorState({
checkSupported,
supported: false,
processor: undefined,
});
logger.error("disable background blur", e);
}
}, [blurActivated, checkSupported, shouldCheckSupport]);

return (
<ProcessorContext.Provider value={processorState}>
{children}
</ProcessorContext.Provider>
);
};
22 changes: 21 additions & 1 deletion src/livekit/useLiveKit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
ConnectionState,
E2EEOptions,
ExternalE2EEKeyProvider,
LocalVideoTrack,
Room,
RoomOptions,
Track,
Expand All @@ -33,6 +34,11 @@ import {
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { E2eeType } from "../e2ee/e2eeType";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
useTrackProcessor,
useTrackProcessorSync,
} from "./TrackProcessorContext";
import { useInitial } from "../useInitial";

interface UseLivekitResult {
livekitRoom?: Room;
Expand Down Expand Up @@ -79,12 +85,15 @@ export function useLiveKit(
const devices = useMediaDevices();
const initialDevices = useRef<MediaDevices>(devices);

const { processor } = useTrackProcessor() || {};
const initialProcessor = useInitial(() => processor);
const roomOptions = useMemo(
(): RoomOptions => ({
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: initialDevices.current.videoInput.selectedId,
processor: initialProcessor,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
Expand All @@ -95,7 +104,7 @@ export function useLiveKit(
},
e2ee: e2eeOptions,
}),
[e2eeOptions],
[e2eeOptions, initialProcessor],
);

// Store if audio/video are currently updating. If to prohibit unnecessary calls
Expand All @@ -120,6 +129,15 @@ export function useLiveKit(
return r;
}, [roomOptions, e2eeSystem]);

const videoTrack = useMemo(
() =>
Array.from(room.localParticipant.videoTrackPublications.values()).find(
(v) => v.source === Track.Source.Camera,
)?.track as LocalVideoTrack | null,
[room.localParticipant.videoTrackPublications],
);
useTrackProcessorSync(videoTrack);

const connectionState = useECConnectionState(
{
deviceId: initialDevices.current.audioInput.selectedId,
Expand Down Expand Up @@ -195,13 +213,15 @@ export function useLiveKit(
audioMuteUpdating.current = true;
trackPublication = await participant.setMicrophoneEnabled(
buttonEnabled.current.audio,
room.options.audioCaptureDefaults,
);
audioMuteUpdating.current = false;
break;
case MuteDevice.Camera:
videoMuteUpdating.current = true;
trackPublication = await participant.setCameraEnabled(
buttonEnabled.current.video,
room.options.videoCaptureDefaults,
);
videoMuteUpdating.current = false;
break;
Expand Down
41 changes: 27 additions & 14 deletions src/room/LobbyView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/

import { FC, useCallback, useMemo, useState } from "react";
import { FC, useCallback, useEffect, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Button } from "@vector-im/compound-web";
import classNames from "classnames";
import { useHistory } from "react-router-dom";
import { logger } from "matrix-js-sdk/src/logger";
import { usePreviewTracks } from "@livekit/components-react";
import { LocalVideoTrack, Track } from "livekit-client";
import {
CreateLocalTracksOptions,
LocalVideoTrack,
Track,
} from "livekit-client";
import { useObservable } from "observable-hooks";
import { map } from "rxjs";

Expand All @@ -37,7 +41,11 @@ import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { useInitial } from "../useInitial";
import { useSwitchCamera } from "./useSwitchCamera";
import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
import {
useTrackProcessor,
useTrackProcessorSync,
} from "../livekit/TrackProcessorContext";

interface Props {
client: MatrixClient;
Expand Down Expand Up @@ -108,7 +116,10 @@ export const LobbyView: FC<Props> = ({
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);

const localTrackOptions = useMemo(
const { processor } = useTrackProcessor() || {};

const initialProcessor = useInitial(() => processor);
const localTrackOptions = useMemo<CreateLocalTracksOptions>(
() => ({
// The only reason we request audio here is to get the audio permission
// request over with at the same time. But changing the audio settings
Expand All @@ -119,12 +130,14 @@ export const LobbyView: FC<Props> = ({
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
processor: initialProcessor,
},
}),
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
devices.videoInput.selectedId,
initialProcessor,
],
);

Expand All @@ -139,14 +152,12 @@ export const LobbyView: FC<Props> = ({

const tracks = usePreviewTracks(localTrackOptions, onError);

const videoTrack = useMemo(
() =>
(tracks?.find((t) => t.kind === Track.Kind.Video) ??
null) as LocalVideoTrack | null,
[tracks],
);

const switchCamera = useSwitchCamera(
const videoTrack = useMemo(() => {
const track = tracks?.find((t) => t.kind === Track.Kind.Video);
return track as LocalVideoTrack | null;
}, [tracks]);
useTrackProcessorSync(videoTrack);
const showSwitchCamera = useShowSwitchCamera(
useObservable(
(inputs) => inputs.pipe(map(([video]) => video)),
[videoTrack],
Expand Down Expand Up @@ -208,7 +219,9 @@ export const LobbyView: FC<Props> = ({
onClick={onVideoPress}
disabled={muteStates.video.setEnabled === null}
/>
{switchCamera && <SwitchCameraButton onClick={switchCamera} />}
{showSwitchCamera && (
<SwitchCameraButton onClick={showSwitchCamera} />
)}
<SettingsButton onClick={openSettings} />
{!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
</div>
Expand Down
Loading