Add background blur feature for supported devices

knip.ts

@@ -24,6 +24,10 @@ export default {
     // then Knip will flag it as a false positive
     // https://github.com/webpro-nl/knip/issues/766
     "@vector-im/compound-web",
+    // We need this so the eslint is happy with @livekit/track-processors.
+    // This might be a bug in the livekit repo but for now we fix it on the
+    // element call side.
+    "@types/dom-mediacapture-transform",
     "matrix-widget-api",
   ],
   ignoreExportsUsedInFile: true,

locales/en-GB/app.json

@@ -150,6 +150,9 @@
       "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
       "effect_volume_label": "Sound effect volume"
     },
+    "background_blur_header": "Background",
+    "background_blur_label": "Blur the background of the video",
+    "blur_not_supported_by_browser": "(Background blur is not supported by this browser)",
     "developer_settings_label": "Developer Settings",
     "developer_settings_label_description": "Expose developer settings in the settings window.",
     "developer_tab_title": "Developer",

package.json

@@ -31,6 +31,7 @@
     "@formatjs/intl-segmenter": "^11.7.3",
     "@livekit/components-core": "^0.11.0",
     "@livekit/components-react": "^2.0.0",
+    "@livekit/track-processors": "^0.3.2",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/core": "^1.25.1",
     "@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
@@ -50,6 +51,7 @@
     "@testing-library/react-hooks": "^8.0.1",
     "@testing-library/user-event": "^14.5.1",
     "@types/content-type": "^1.1.5",
+    "@types/dom-mediacapture-transform": "^0.1.10",
     "@types/grecaptcha": "^3.0.9",
     "@types/jsdom": "^21.1.7",
     "@types/lodash-es": "^4.17.12",

src/App.tsx

@@ -28,6 +28,7 @@ import { Initializer } from "./initializer";
 import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
 import { widget } from "./widget";
 import { useTheme } from "./useTheme";
+import { ProcessorProvider } from "./livekit/TrackProcessorContext";
 
 const SentryRoute = Sentry.withSentryRouting(Route);
 
@@ -82,27 +83,25 @@ export const App: FC<AppProps> = ({ history }) => {
           <TooltipProvider>
             {loaded ? (
               <Suspense fallback={null}>
-                <ClientProvider>
-                  <MediaDevicesProvider>
-                    <Sentry.ErrorBoundary fallback={errorPage}>
-                      <DisconnectedBanner />
-                      <Switch>
-                        <SentryRoute exact path="/">
-                          <HomePage />
-                        </SentryRoute>
-                        <SentryRoute exact path="/login">
-                          <LoginPage />
-                        </SentryRoute>
-                        <SentryRoute exact path="/register">
-                          <RegisterPage />
-                        </SentryRoute>
-                        <SentryRoute path="*">
-                          <RoomPage />
-                        </SentryRoute>
-                      </Switch>
-                    </Sentry.ErrorBoundary>
-                  </MediaDevicesProvider>
-                </ClientProvider>
+                <Providers>
+                  <Sentry.ErrorBoundary fallback={errorPage}>
+                    <DisconnectedBanner />
+                    <Switch>
+                      <SentryRoute exact path="/">
+                        <HomePage />
+                      </SentryRoute>
+                      <SentryRoute exact path="/login">
+                        <LoginPage />
+                      </SentryRoute>
+                      <SentryRoute exact path="/register">
+                        <RegisterPage />
+                      </SentryRoute>
+                      <SentryRoute path="*">
+                        <RoomPage />
+                      </SentryRoute>
+                    </Switch>
+                  </Sentry.ErrorBoundary>
+                </Providers>
               </Suspense>
             ) : (
               <LoadingView />
@@ -113,3 +112,16 @@ export const App: FC<AppProps> = ({ history }) => {
     </Router>
   );
 };
+
+const Providers: FC<{
+  children: JSX.Element;
+}> = ({ children }) => {
+  // We use this to stack all used providers to not make the App component to verbose
+  return (
+    <ClientProvider>
+      <MediaDevicesProvider>
+        <ProcessorProvider>{children}</ProcessorProvider>
+      </MediaDevicesProvider>
+    </ClientProvider>
+  );
+};

src/livekit/TrackProcessorContext.tsx

@@ -0,0 +1,111 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  BackgroundBlur as backgroundBlur,
+  BackgroundOptions,
+  ProcessorWrapper,
+} from "@livekit/track-processors";
+import {
+  createContext,
+  FC,
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
+import { logger } from "matrix-js-sdk/src/logger";
+import { LocalVideoTrack } from "livekit-client";
+
+import {
+  backgroundBlur as backgroundBlurSettings,
+  useSetting,
+} from "../settings/settings";
+
+type ProcessorState = {
+  supported: boolean | undefined;
+  processor: undefined | ProcessorWrapper<BackgroundOptions>;
+  /**
+   * Call this method to try to initialize a processor.
+   * This only needs to happen if supported is undefined.
+   * If the backgroundBlur setting is set to true this does not need to be called
+   * and the processorState.supported will update automatically to the correct value.
+   */
+  checkSupported: () => void;
+};
+const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
+
+export const useTrackProcessor = (): ProcessorState | undefined =>
+  useContext(ProcessorContext);
+
+export const useTrackProcessorSync = (
+  videoTrack: LocalVideoTrack | null,
+): void => {
+  const { processor } = useTrackProcessor() || {};
+  useEffect(() => {
+    if (processor && !videoTrack?.getProcessor()) {
+      void videoTrack?.setProcessor(processor);
+    }
+    if (!processor && videoTrack?.getProcessor()) {
+      void videoTrack?.stopProcessor();
+    }
+  }, [processor, videoTrack]);
+};
+
+interface Props {
+  children: JSX.Element;
+}
+export const ProcessorProvider: FC<Props> = ({ children }) => {
+  // The setting the user wants to have
+  const [blurActivated] = useSetting(backgroundBlurSettings);
+
+  // If `ProcessorState.supported` is undefined the user can activate that we want
+  // to have it at least checked (this is useful to show the settings menu properly)
+  // We dont want to try initializing the blur if the user is not even looking at the setting
+  const [shouldCheckSupport, setShouldCheckSupport] = useState(blurActivated);
+
+  // Cache the processor so we only need to initialize it once.
+  const blur = useRef<ProcessorWrapper<BackgroundOptions> | undefined>(
+    undefined,
+  );
+
+  const checkSupported = useCallback(() => {
+    setShouldCheckSupport(true);
+  }, []);
+  // This is the actual state exposed through the context
+  const [processorState, setProcessorState] = useState<ProcessorState>(() => ({
+    supported: false,
+    processor: undefined,
+    checkSupported,
+  }));
+
+  useEffect(() => {
+    if (!shouldCheckSupport) return;
+    try {
+      if (!blur.current) blur.current = backgroundBlur(15, { delegate: "GPU" });
+      setProcessorState({
+        checkSupported,
+        supported: true,
+        processor: blurActivated ? blur.current : undefined,
+      });
+    } catch (e) {
+      setProcessorState({
+        checkSupported,
+        supported: false,
+        processor: undefined,
+      });
+      logger.error("disable background blur", e);
+    }
+  }, [blurActivated, checkSupported, shouldCheckSupport]);
+
+  return (
+    <ProcessorContext.Provider value={processorState}>
+      {children}
+    </ProcessorContext.Provider>
+  );
+};

src/livekit/useLiveKit.ts

@@ -9,6 +9,7 @@ import {
   ConnectionState,
   E2EEOptions,
   ExternalE2EEKeyProvider,
+  LocalVideoTrack,
   Room,
   RoomOptions,
   Track,
@@ -33,6 +34,11 @@ import {
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
 import { E2eeType } from "../e2ee/e2eeType";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "./TrackProcessorContext";
+import { useInitial } from "../useInitial";
 
 interface UseLivekitResult {
   livekitRoom?: Room;
@@ -79,12 +85,15 @@ export function useLiveKit(
   const devices = useMediaDevices();
   const initialDevices = useRef<MediaDevices>(devices);
 
+  const { processor } = useTrackProcessor() || {};
+  const initialProcessor = useInitial(() => processor);
   const roomOptions = useMemo(
     (): RoomOptions => ({
       ...defaultLiveKitOptions,
       videoCaptureDefaults: {
         ...defaultLiveKitOptions.videoCaptureDefaults,
         deviceId: initialDevices.current.videoInput.selectedId,
+        processor: initialProcessor,
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
@@ -95,7 +104,7 @@ export function useLiveKit(
       },
       e2ee: e2eeOptions,
     }),
-    [e2eeOptions],
+    [e2eeOptions, initialProcessor],
   );
 
   // Store if audio/video are currently updating. If to prohibit unnecessary calls
@@ -120,6 +129,15 @@ export function useLiveKit(
     return r;
   }, [roomOptions, e2eeSystem]);
 
+  const videoTrack = useMemo(
+    () =>
+      Array.from(room.localParticipant.videoTrackPublications.values()).find(
+        (v) => v.source === Track.Source.Camera,
+      )?.track as LocalVideoTrack | null,
+    [room.localParticipant.videoTrackPublications],
+  );
+  useTrackProcessorSync(videoTrack);
+
   const connectionState = useECConnectionState(
     {
       deviceId: initialDevices.current.audioInput.selectedId,
@@ -195,13 +213,15 @@ export function useLiveKit(
                 audioMuteUpdating.current = true;
                 trackPublication = await participant.setMicrophoneEnabled(
                   buttonEnabled.current.audio,
+                  room.options.audioCaptureDefaults,
                 );
                 audioMuteUpdating.current = false;
                 break;
               case MuteDevice.Camera:
                 videoMuteUpdating.current = true;
                 trackPublication = await participant.setCameraEnabled(
                   buttonEnabled.current.video,
+                  room.options.videoCaptureDefaults,
                 );
                 videoMuteUpdating.current = false;
                 break;

src/room/LobbyView.tsx

@@ -5,15 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { FC, useCallback, useMemo, useState } from "react";
+import { FC, useCallback, useEffect, useMemo, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { MatrixClient } from "matrix-js-sdk/src/matrix";
 import { Button } from "@vector-im/compound-web";
 import classNames from "classnames";
 import { useHistory } from "react-router-dom";
 import { logger } from "matrix-js-sdk/src/logger";
 import { usePreviewTracks } from "@livekit/components-react";
-import { LocalVideoTrack, Track } from "livekit-client";
+import {
+  CreateLocalTracksOptions,
+  LocalVideoTrack,
+  Track,
+} from "livekit-client";
 import { useObservable } from "observable-hooks";
 import { map } from "rxjs";
 
@@ -37,7 +41,11 @@ import { E2eeType } from "../e2ee/e2eeType";
 import { Link } from "../button/Link";
 import { useMediaDevices } from "../livekit/MediaDevicesContext";
 import { useInitial } from "../useInitial";
-import { useSwitchCamera } from "./useSwitchCamera";
+import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "../livekit/TrackProcessorContext";
 
 interface Props {
   client: MatrixClient;
@@ -108,7 +116,10 @@ export const LobbyView: FC<Props> = ({
       muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
   );
 
-  const localTrackOptions = useMemo(
+  const { processor } = useTrackProcessor() || {};
+
+  const initialProcessor = useInitial(() => processor);
+  const localTrackOptions = useMemo<CreateLocalTracksOptions>(
     () => ({
       // The only reason we request audio here is to get the audio permission
       // request over with at the same time. But changing the audio settings
@@ -119,12 +130,14 @@ export const LobbyView: FC<Props> = ({
       audio: Object.assign({}, initialAudioOptions),
       video: muteStates.video.enabled && {
         deviceId: devices.videoInput.selectedId,
+        processor: initialProcessor,
       },
     }),
     [
       initialAudioOptions,
-      devices.videoInput.selectedId,
       muteStates.video.enabled,
+      devices.videoInput.selectedId,
+      initialProcessor,
     ],
   );
 
@@ -139,14 +152,12 @@ export const LobbyView: FC<Props> = ({
 
   const tracks = usePreviewTracks(localTrackOptions, onError);
 
-  const videoTrack = useMemo(
-    () =>
-      (tracks?.find((t) => t.kind === Track.Kind.Video) ??
-        null) as LocalVideoTrack | null,
-    [tracks],
-  );
-
-  const switchCamera = useSwitchCamera(
+  const videoTrack = useMemo(() => {
+    const track = tracks?.find((t) => t.kind === Track.Kind.Video);
+    return track as LocalVideoTrack | null;
+  }, [tracks]);
+  useTrackProcessorSync(videoTrack);
+  const showSwitchCamera = useShowSwitchCamera(
     useObservable(
       (inputs) => inputs.pipe(map(([video]) => video)),
       [videoTrack],
@@ -208,7 +219,9 @@ export const LobbyView: FC<Props> = ({
               onClick={onVideoPress}
               disabled={muteStates.video.setEnabled === null}
             />
-            {switchCamera && <SwitchCameraButton onClick={switchCamera} />}
+            {showSwitchCamera && (
+              <SwitchCameraButton onClick={showSwitchCamera} />
+            )}
             <SettingsButton onClick={openSettings} />
             {!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
           </div>