Skip to content

Commit

Permalink
backgroundBlur squashed
Browse files Browse the repository at this point in the history
  • Loading branch information
@toger5
toger5 committed Dec 2, 2024
1 parent a37c3bf commit ffd2cff
Show file tree
Hide file tree
Showing 8 changed files with 202 additions and 15 deletions.
4 changes: 4 additions & 0 deletions knip.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ export default {
// then Knip will flag it as a false positive
// https://github.com/webpro-nl/knip/issues/766
"@vector-im/compound-web",
// We need this so the eslint is happy with @livekit/track-processors.
// This might be a bug in the livekit repo but for now we fix it on the
// element call side.
"@types/dom-mediacapture-transform",
"matrix-widget-api",
],
ignoreExportsUsedInFile: true,
Expand Down
3 changes: 3 additions & 0 deletions locales/en-GB/app.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
"effect_volume_label": "Sound effect volume"
},
"background_blur_header": "Background",
"background_blur_label": "Enable background blurring",
"blur_not_supported_by_browser": "(Background blur is not supported by this browser.)",
"developer_settings_label": "Developer Settings",
"developer_settings_label_description": "Expose developer settings in the settings window.",
"developer_tab_title": "Developer",
Expand Down
2 changes: 2 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"@formatjs/intl-segmenter": "^11.7.3",
"@livekit/components-core": "^0.11.0",
"@livekit/components-react": "^2.0.0",
"@livekit/track-processors": "^0.3.2",
"@opentelemetry/api": "^1.4.0",
"@opentelemetry/core": "^1.25.1",
"@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
Expand All @@ -50,6 +51,7 @@
"@testing-library/react-hooks": "^8.0.1",
"@testing-library/user-event": "^14.5.1",
"@types/content-type": "^1.1.5",
"@types/dom-mediacapture-transform": "^0.1.10",
"@types/grecaptcha": "^3.0.9",
"@types/jsdom": "^21.1.7",
"@types/lodash-es": "^4.17.12",
Expand Down
70 changes: 68 additions & 2 deletions src/livekit/useLiveKit.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,17 @@ import {
ConnectionState,
E2EEOptions,
ExternalE2EEKeyProvider,
LocalTrackPublication,
Room,
RoomEvent,
RoomOptions,
Track,
} from "livekit-client";
import { useEffect, useMemo, useRef } from "react";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import { logger } from "matrix-js-sdk/src/logger";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors";

import { defaultLiveKitOptions } from "./options";
import { SFUConfig } from "./openIDSFU";
Expand All @@ -26,13 +29,15 @@ import {
MediaDevices,
useMediaDevices,
} from "./MediaDevicesContext";
import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
import {
ECConnectionState,
useECConnectionState,
} from "./useECConnectionState";
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { E2eeType } from "../e2ee/e2eeType";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { useSetting } from "../settings/settings";

interface UseLivekitResult {
livekitRoom?: Room;
Expand Down Expand Up @@ -78,13 +83,22 @@ export function useLiveKit(
const initialMuteStates = useRef<MuteStates>(muteStates);
const devices = useMediaDevices();
const initialDevices = useRef<MediaDevices>(devices);

const blur = useMemo(() => {
let b = undefined;
try {
b = backgroundBlur(15, { delegate: "GPU" });
} catch (e) {
logger.error("disable background blur", e);
}
return b;
}, []);
const roomOptions = useMemo(
(): RoomOptions => ({
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: initialDevices.current.videoInput.selectedId,
processor: blur,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
Expand All @@ -95,7 +109,7 @@ export function useLiveKit(
},
e2ee: e2eeOptions,
}),
[e2eeOptions],
[blur, e2eeOptions],
);

// Store if audio/video are currently updating. If to prohibit unnecessary calls
Expand Down Expand Up @@ -129,6 +143,58 @@ export function useLiveKit(
sfuConfig,
);

const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
const videoTrackPromise = useRef<
undefined | Promise<LocalTrackPublication | undefined>
>(undefined);

useEffect(() => {
// Don't even try if we cannot blur on this platform
if (!blur) return;
if (!room || videoTrackPromise.current) return;
const update = async (): Promise<void> => {
let publishCallback: undefined | ((track: LocalTrackPublication) => void);
videoTrackPromise.current = new Promise<
LocalTrackPublication | undefined
>((resolve) => {
const videoTrack = Array.from(
room.localParticipant.videoTrackPublications.values(),
).find((v) => v.source === Track.Source.Camera);
if (videoTrack) {
resolve(videoTrack);
}
publishCallback = (videoTrack: LocalTrackPublication): void => {
if (videoTrack.source === Track.Source.Camera) {
resolve(videoTrack);
}
};
room.on(RoomEvent.LocalTrackPublished, publishCallback);
});

const videoTrack = await videoTrackPromise.current;

if (publishCallback)
room.off(RoomEvent.LocalTrackPublished, publishCallback);

if (videoTrack !== undefined) {
if (
showBackgroundBlur &&
videoTrack.track?.getProcessor()?.name !== "background-blur"
) {
logger.info("Blur: set blur");

void videoTrack.track?.setProcessor(blur);
} else if (
videoTrack.track?.getProcessor()?.name === "background-blur"
) {
void videoTrack.track?.stopProcessor();
}
}
videoTrackPromise.current = undefined;
};
void update();
}, [blur, room, showBackgroundBlur]);

useEffect(() => {
// Sync the requested mute states with LiveKit's mute states. We do it this
// way around rather than using LiveKit as the source of truth, so that the
Expand Down
47 changes: 42 additions & 5 deletions src/room/LobbyView.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/

import { FC, useCallback, useMemo, useState } from "react";
import { FC, useCallback, useEffect, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Button } from "@vector-im/compound-web";
Expand All @@ -16,6 +16,7 @@ import { usePreviewTracks } from "@livekit/components-react";
import { LocalVideoTrack, Track } from "livekit-client";
import { useObservable } from "observable-hooks";
import { map } from "rxjs";
import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors";

import inCallStyles from "./InCallView.module.css";
import styles from "./LobbyView.module.css";
Expand All @@ -32,12 +33,14 @@ import {
VideoButton,
} from "../button/Button";
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
import { useMediaQuery } from "../useMediaQuery";
import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { useInitial } from "../useInitial";
import { useSwitchCamera } from "./useSwitchCamera";
import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
import { useSetting } from "../settings/settings";

interface Props {
client: MatrixClient;
Expand Down Expand Up @@ -108,6 +111,20 @@ export const LobbyView: FC<Props> = ({
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);

// eslint-disable-next-line new-cap
const blur = useMemo(() => {
let b = undefined;
try {
b = backgroundBlur(15, { delegate: "GPU" });
} catch (e) {
logger.error(
"disable background blur because its not supported by the platform.",
e,
);
}
return b;
}, []);

const localTrackOptions = useMemo(
() => ({
// The only reason we request audio here is to get the audio permission
Expand All @@ -119,12 +136,15 @@ export const LobbyView: FC<Props> = ({
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
// It should be possible to set a processor here:
processor: blur,
},
}),
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
devices.videoInput.selectedId,
blur,
],
);

Expand All @@ -146,7 +166,22 @@ export const LobbyView: FC<Props> = ({
[tracks],
);

const switchCamera = useSwitchCamera(
const [showBackgroundBlur] = useSetting(backgroundBlurSettings);

useEffect(() => {
// Fon't even try if we cannot blur on this platform
if (!blur) return;
const updateBlur = async (showBlur: boolean): Promise<void> => {
if (showBlur && !videoTrack?.getProcessor()) {
await videoTrack?.setProcessor(blur);
} else {
await videoTrack?.stopProcessor();
}
};
if (videoTrack) void updateBlur(showBackgroundBlur);
}, [videoTrack, showBackgroundBlur, blur]);

const showSwitchCamera = useShowSwitchCamera(
useObservable(
(inputs) => inputs.pipe(map(([video]) => video)),
[videoTrack],
Expand Down Expand Up @@ -208,7 +243,9 @@ export const LobbyView: FC<Props> = ({
onClick={onVideoPress}
disabled={muteStates.video.setEnabled === null}
/>
{switchCamera && <SwitchCameraButton onClick={switchCamera} />}
{showSwitchCamera && (
<SwitchCameraButton onClick={showSwitchCamera} />
)}
<SettingsButton onClick={openSettings} />
{!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
</div>
Expand Down
59 changes: 51 additions & 8 deletions src/settings/SettingsModal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/

import { ChangeEvent, FC, useCallback } from "react";
import { ChangeEvent, FC, ReactNode, useCallback } from "react";
import { Trans, useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Root as Form, Text } from "@vector-im/compound-web";
import { Root as Form, Separator, Text } from "@vector-im/compound-web";
import { BackgroundBlur } from "@livekit/track-processors";
import { logger } from "matrix-js-sdk/src/logger";

import { Modal } from "../Modal";
import styles from "./SettingsModal.module.css";
Expand All @@ -26,6 +28,7 @@ import {
useSetting,
developerSettingsTab as developerSettingsTabSetting,
duplicateTiles as duplicateTilesSetting,
backgroundBlur as backgroundBlurSetting,
useOptInAnalytics,
soundEffectVolumeSetting,
} from "./settings";
Expand Down Expand Up @@ -70,6 +73,42 @@ export const SettingsModal: FC<Props> = ({
);
const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting);

// Generate a `Checkbox` input to turn blur on or off.
const BlurCheckbox: React.FC = (): ReactNode => {
const [blur, setBlur] = useSetting(backgroundBlurSetting);
let canBlur = true;
try {
// eslint-disable-next-line new-cap
BackgroundBlur(15);
} catch (e) {
logger.debug(
"Cannot blur, so we do not show the option in settings. error: ",
e,
);
canBlur = false;
setBlur(false);
}
return (
<>
<h4>{t("settings.background_blur_header")}</h4>

<FieldRow>
<InputField
id="activateBackgroundBlur"
label={t("settings.background_blur_label")}
description={
canBlur ? "" : t("settings.blur_not_supported_by_browser")
}
type="checkbox"
checked={blur}
onChange={(b): void => setBlur(b.target.checked)}
disabled={!canBlur}
/>
</FieldRow>
</>
);
};

const optInDescription = (
<Text size="sm">
<Trans i18nKey="settings.opt_in_description">
Expand Down Expand Up @@ -123,12 +162,16 @@ export const SettingsModal: FC<Props> = ({
key: "video",
name: t("common.video"),
content: (
<Form>
<DeviceSelection
devices={devices.videoInput}
caption={t("common.camera")}
/>
</Form>
<>
<Form>
<DeviceSelection
devices={devices.videoInput}
caption={t("common.camera")}
/>
</Form>
<Separator />
<BlurCheckbox />
</>
),
};

Expand Down
2 changes: 2 additions & 0 deletions src/settings/settings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ export const videoInput = new Setting<string | undefined>(
undefined,
);

export const backgroundBlur = new Setting<boolean>("background-blur", false);

export const showHandRaisedTimer = new Setting<boolean>(
"hand-raised-show-timer",
false,
Expand Down
Loading

0 comments on commit ffd2cff

Please sign in to comment.