Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add background blur feature for supported devices #2812

Draft
wants to merge 16 commits into
base: livekit
Choose a base branch
from
4 changes: 4 additions & 0 deletions knip.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ export default {
// then Knip will flag it as a false positive
// https://github.com/webpro-nl/knip/issues/766
"@vector-im/compound-web",
// We need this so the eslint is happy with @livekit/track-processors.
// This might be a bug in the livekit repo but for now we fix it on the
// element call side.
"@types/dom-mediacapture-transform",
"matrix-widget-api",
],
ignoreExportsUsedInFile: true,
Expand Down
3 changes: 3 additions & 0 deletions locales/en-GB/app.json
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
"effect_volume_label": "Sound effect volume"
},
"background_blur_header": "Background",
"background_blur_label": "Blur the background of the video",
"blur_not_supported_by_browser": "(Background blur is not supported by this browser)",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"blur_not_supported_by_browser": "(Background blur is not supported by this browser)",
"blur_not_supported_by_browser": "(Background blur is not supported by this device)",

...because this message also appears when EC is embedded as widget and the user isn't aware that it is a browser.

"developer_settings_label": "Developer Settings",
"developer_settings_label_description": "Expose developer settings in the settings window.",
"developer_tab_title": "Developer",
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"@formatjs/intl-segmenter": "^11.7.3",
"@livekit/components-core": "^0.11.0",
"@livekit/components-react": "^2.0.0",
"@livekit/track-processors": "^0.3.2",
"@opentelemetry/api": "^1.4.0",
"@opentelemetry/core": "^1.25.1",
"@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
Expand All @@ -50,6 +51,7 @@
"@testing-library/react-hooks": "^8.0.1",
"@testing-library/user-event": "^14.5.1",
"@types/content-type": "^1.1.5",
"@types/dom-mediacapture-transform": "^0.1.10",
"@types/grecaptcha": "^3.0.9",
"@types/jsdom": "^21.1.7",
"@types/lodash-es": "^4.17.12",
Expand Down
70 changes: 68 additions & 2 deletions src/livekit/useLiveKit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,17 @@ import {
ConnectionState,
E2EEOptions,
ExternalE2EEKeyProvider,
LocalTrackPublication,
Room,
RoomEvent,
RoomOptions,
Track,
} from "livekit-client";
import { useEffect, useMemo, useRef } from "react";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import { logger } from "matrix-js-sdk/src/logger";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors";

import { defaultLiveKitOptions } from "./options";
import { SFUConfig } from "./openIDSFU";
Expand All @@ -26,13 +29,15 @@ import {
MediaDevices,
useMediaDevices,
} from "./MediaDevicesContext";
import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
import {
ECConnectionState,
useECConnectionState,
} from "./useECConnectionState";
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { E2eeType } from "../e2ee/e2eeType";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { useSetting } from "../settings/settings";

interface UseLivekitResult {
livekitRoom?: Room;
Expand Down Expand Up @@ -78,13 +83,22 @@ export function useLiveKit(
const initialMuteStates = useRef<MuteStates>(muteStates);
const devices = useMediaDevices();
const initialDevices = useRef<MediaDevices>(devices);

const blur = useMemo(() => {
let b = undefined;
try {
b = backgroundBlur(15, { delegate: "GPU" });
} catch (e) {
logger.error("disable background blur", e);
}
return b;
}, []);
const roomOptions = useMemo(
(): RoomOptions => ({
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: initialDevices.current.videoInput.selectedId,
processor: blur,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
Expand All @@ -95,7 +109,7 @@ export function useLiveKit(
},
e2ee: e2eeOptions,
}),
[e2eeOptions],
[blur, e2eeOptions],
);

// Store if audio/video are currently updating. If to prohibit unnecessary calls
Expand Down Expand Up @@ -129,6 +143,58 @@ export function useLiveKit(
sfuConfig,
);

const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
const videoTrackPromise = useRef<
undefined | Promise<LocalTrackPublication | undefined>
>(undefined);

useEffect(() => {
// Don't even try if we cannot blur on this platform
if (!blur) return;
if (!room || videoTrackPromise.current) return;
const update = async (): Promise<void> => {
let publishCallback: undefined | ((track: LocalTrackPublication) => void);
videoTrackPromise.current = new Promise<
LocalTrackPublication | undefined
>((resolve) => {
const videoTrack = Array.from(
room.localParticipant.videoTrackPublications.values(),
).find((v) => v.source === Track.Source.Camera);
if (videoTrack) {
resolve(videoTrack);
}
publishCallback = (videoTrack: LocalTrackPublication): void => {
if (videoTrack.source === Track.Source.Camera) {
resolve(videoTrack);
}
};
room.on(RoomEvent.LocalTrackPublished, publishCallback);
});

const videoTrack = await videoTrackPromise.current;

if (publishCallback)
room.off(RoomEvent.LocalTrackPublished, publishCallback);

if (videoTrack !== undefined) {
if (
showBackgroundBlur &&
videoTrack.track?.getProcessor()?.name !== "background-blur"
) {
logger.info("Blur: set blur");

void videoTrack.track?.setProcessor(blur);
} else if (
videoTrack.track?.getProcessor()?.name === "background-blur"
) {
void videoTrack.track?.stopProcessor();
}
}
videoTrackPromise.current = undefined;
};
void update();
}, [blur, room, showBackgroundBlur]);

useEffect(() => {
// Sync the requested mute states with LiveKit's mute states. We do it this
// way around rather than using LiveKit as the source of truth, so that the
Expand Down
45 changes: 40 additions & 5 deletions src/room/LobbyView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/

import { FC, useCallback, useMemo, useState } from "react";
import { FC, useCallback, useEffect, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Button } from "@vector-im/compound-web";
Expand All @@ -16,6 +16,7 @@ import { usePreviewTracks } from "@livekit/components-react";
import { LocalVideoTrack, Track } from "livekit-client";
import { useObservable } from "observable-hooks";
import { map } from "rxjs";
import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors";

import inCallStyles from "./InCallView.module.css";
import styles from "./LobbyView.module.css";
Expand All @@ -32,12 +33,14 @@ import {
VideoButton,
} from "../button/Button";
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
import { useMediaQuery } from "../useMediaQuery";
import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { useInitial } from "../useInitial";
import { useSwitchCamera } from "./useSwitchCamera";
import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
import { useSetting } from "../settings/settings";

interface Props {
client: MatrixClient;
Expand Down Expand Up @@ -108,6 +111,19 @@ export const LobbyView: FC<Props> = ({
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);

const blur = useMemo(() => {
let b = undefined;
try {
b = backgroundBlur(15, { delegate: "GPU" });
} catch (e) {
logger.error(
"disable background blur because its not supported by the platform.",
e,
);
}
return b;
}, []);

const localTrackOptions = useMemo(
() => ({
// The only reason we request audio here is to get the audio permission
Expand All @@ -119,12 +135,14 @@ export const LobbyView: FC<Props> = ({
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
// It should be possible to set a processor here:
// processor: blur,
},
}),
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
devices.videoInput.selectedId,
],
);

Expand All @@ -146,7 +164,22 @@ export const LobbyView: FC<Props> = ({
[tracks],
);

const switchCamera = useSwitchCamera(
const [showBackgroundBlur] = useSetting(backgroundBlurSettings);

useEffect(() => {
// Fon't even try if we cannot blur on this platform
if (!blur) return;
const updateBlur = async (showBlur: boolean): Promise<void> => {
if (showBlur && !videoTrack?.getProcessor()) {
await videoTrack?.setProcessor(blur);
} else {
await videoTrack?.stopProcessor();
}
};
if (videoTrack) void updateBlur(showBackgroundBlur);
}, [videoTrack, showBackgroundBlur, blur]);

const showSwitchCamera = useShowSwitchCamera(
useObservable(
(inputs) => inputs.pipe(map(([video]) => video)),
[videoTrack],
Expand Down Expand Up @@ -208,7 +241,9 @@ export const LobbyView: FC<Props> = ({
onClick={onVideoPress}
disabled={muteStates.video.setEnabled === null}
/>
{switchCamera && <SwitchCameraButton onClick={switchCamera} />}
{showSwitchCamera && (
<SwitchCameraButton onClick={showSwitchCamera} />
)}
<SettingsButton onClick={openSettings} />
{!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
</div>
Expand Down
58 changes: 50 additions & 8 deletions src/settings/SettingsModal.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/

import { ChangeEvent, FC, useCallback } from "react";
import { ChangeEvent, FC, ReactNode, useCallback } from "react";
import { Trans, useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Root as Form, Text } from "@vector-im/compound-web";
import { Root as Form, Separator, Text } from "@vector-im/compound-web";
import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors";
import { logger } from "matrix-js-sdk/src/logger";

import { Modal } from "../Modal";
import styles from "./SettingsModal.module.css";
Expand All @@ -26,6 +28,7 @@ import {
useSetting,
developerSettingsTab as developerSettingsTabSetting,
duplicateTiles as duplicateTilesSetting,
backgroundBlur as backgroundBlurSetting,
useOptInAnalytics,
soundEffectVolumeSetting,
} from "./settings";
Expand Down Expand Up @@ -70,6 +73,41 @@ export const SettingsModal: FC<Props> = ({
);
const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting);

// Generate a `Checkbox` input to turn blur on or off.
const BlurCheckbox: React.FC = (): ReactNode => {
const [blur, setBlur] = useSetting(backgroundBlurSetting);
let canBlur = true;
try {
backgroundBlur(15);
} catch (e) {
logger.debug(
"Cannot blur, so we do not show the option in settings. error: ",
e,
);
canBlur = false;
setBlur(false);
}
return (
<>
<h4>{t("settings.background_blur_header")}</h4>

<FieldRow>
<InputField
id="activateBackgroundBlur"
label={t("settings.background_blur_label")}
description={
canBlur ? "" : t("settings.blur_not_supported_by_browser")
}
type="checkbox"
checked={blur}
onChange={(b): void => setBlur(b.target.checked)}
disabled={!canBlur}
/>
</FieldRow>
</>
);
};

const optInDescription = (
<Text size="sm">
<Trans i18nKey="settings.opt_in_description">
Expand Down Expand Up @@ -123,12 +161,16 @@ export const SettingsModal: FC<Props> = ({
key: "video",
name: t("common.video"),
content: (
<Form>
<DeviceSelection
devices={devices.videoInput}
caption={t("common.camera")}
/>
</Form>
<>
<Form>
<DeviceSelection
devices={devices.videoInput}
caption={t("common.camera")}
/>
</Form>
<Separator />
<BlurCheckbox />
</>
),
};

Expand Down
2 changes: 2 additions & 0 deletions src/settings/settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ export const videoInput = new Setting<string | undefined>(
undefined,
);

export const backgroundBlur = new Setting<boolean>("background-blur", false);

export const showHandRaisedTimer = new Setting<boolean>(
"hand-raised-show-timer",
false,
Expand Down
Loading
Loading