Add audio analyser util function (livekit#515)

* add audio analyser

* convert to util function

* add comment

* cleanup

* more comments

* update import

* add vu meter in example, give option to clone track when creating analyser

* revert room options

* changeset

.changeset/lazy-elephants-work.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Add createAudioAnalyser util function

example/sample.ts

@@ -1,8 +1,10 @@
 import {
   ConnectionQuality,
   ConnectionState,
+  createAudioAnalyser,
   DataPacket_Kind,
   DisconnectReason,
+  LocalAudioTrack,
   LocalParticipant,
   LogLevel,
   MediaDeviceFailure,
@@ -112,7 +114,16 @@ const appActions = {
       .on(RoomEvent.Reconnected, () => {
         appendLog('Successfully reconnected. server', room.engine.connectedServerAddress);
       })
-      .on(RoomEvent.LocalTrackPublished, () => {
+      .on(RoomEvent.LocalTrackPublished, (pub) => {
+        const track = pub.track as LocalAudioTrack;
+
+        if (track instanceof LocalAudioTrack) {
+          const { calculateVolume } = createAudioAnalyser(track);
+
+          setInterval(() => {
+            $('local-volume')?.setAttribute('value', calculateVolume().toFixed(4));
+          }, 200);
+        }
         renderParticipant(room.localParticipant);
         updateButtonsForPublishState();
         renderScreenShare(room);
@@ -143,7 +154,7 @@ const appActions = {
           appendLog('connection quality changed', participant?.identity, quality);
         },
       )
-      .on(RoomEvent.TrackSubscribed, (_1, pub, participant) => {
+      .on(RoomEvent.TrackSubscribed, (track, pub, participant) => {
         appendLog('subscribed to track', pub.trackSid, participant.identity);
         renderParticipant(participant);
         renderScreenShare(room);
@@ -464,10 +475,11 @@ function renderParticipant(participant: Participant, remove: boolean = false) {
         </div>
       </div>
       ${
-        participant instanceof RemoteParticipant &&
-        `<div class="volume-control">
+        participant instanceof RemoteParticipant
+          ? `<div class="volume-control">
         <input id="volume-${identity}" type="range" min="0" max="1" step="0.1" value="1" orient="vertical" />
       </div>`
+          : `<progress id="local-volume" max="1" value="0" />`
       }
 
     `;

example/styles.css

@@ -158,3 +158,9 @@
   height: 40%;
   -webkit-appearance: slider-vertical; /* Chromium */
 }
+
+
+.participant .volume-meter {
+  position: absolute;
+  z-index: 4;
+}

src/index.ts

@@ -23,8 +23,11 @@ import {
   supportsAdaptiveStream,
   supportsAV1,
   supportsDynacast,
+  createAudioAnalyser,
 } from './room/utils';
 
+import type { AudioAnalyserOptions } from './room/utils';
+
 export * from './options';
 export * from './room/errors';
 export * from './room/events';
@@ -43,6 +46,8 @@ export {
   supportsAdaptiveStream,
   supportsDynacast,
   supportsAV1,
+  createAudioAnalyser,
+  AudioAnalyserOptions,
   LogLevel,
   Room,
   ConnectionState,

src/room/utils.ts

@@ -1,6 +1,9 @@
 import UAParser from 'ua-parser-js';
 import { ClientInfo, ClientInfo_SDK } from '../proto/livekit_models';
 import { protocolVersion, version } from '../version';
+import type LocalAudioTrack from './track/LocalAudioTrack';
+import type RemoteAudioTrack from './track/RemoteAudioTrack';
+import { getNewAudioContext } from './track/utils';
 
 const separator = '|';
 
@@ -236,3 +239,83 @@ export class Future<T> {
     }).finally(() => this.onFinally?.());
   }
 }
+
+export type AudioAnalyserOptions = {
+  /**
+   * If set to true, the analyser will use a cloned version of the underlying mediastreamtrack, which won't be impacted by muting the track.
+   * Useful for local tracks when implementing things like "seems like you're muted, but trying to speak".
+   * Defaults to false
+   */
+  cloneTrack?: boolean;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/fftSize
+   */
+  fftSize?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/smoothingTimeConstant
+   */
+  smoothingTimeConstant?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/minDecibels
+   */
+  minDecibels?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/maxDecibels
+   */
+  maxDecibels?: number;
+};
+
+/**
+ * Creates and returns an analyser web audio node that is attached to the provided track.
+ * Additionally returns a convenience method `calculateVolume` to perform instant volume readings on that track.
+ * Call the returned `cleanup` function to close the audioContext that has been created for the instance of this helper
+ */
+export function createAudioAnalyser(
+  track: LocalAudioTrack | RemoteAudioTrack,
+  options?: AudioAnalyserOptions,
+) {
+  const opts = {
+    cloneTrack: false,
+    fftSize: 2048,
+    smoothingTimeConstant: 0.8,
+    minDecibels: -100,
+    maxDecibels: -80,
+    ...options,
+  };
+  const audioContext = getNewAudioContext();
+
+  if (!audioContext) {
+    throw new Error('Audio Context not supported on this browser');
+  }
+  const streamTrack = opts.cloneTrack ? track.mediaStreamTrack.clone() : track.mediaStreamTrack;
+  const mediaStreamSource = audioContext.createMediaStreamSource(new MediaStream([streamTrack]));
+  const analyser = audioContext.createAnalyser();
+  analyser.minDecibels = opts.minDecibels;
+  analyser.fftSize = opts.fftSize;
+  analyser.smoothingTimeConstant = opts.smoothingTimeConstant;
+
+  mediaStreamSource.connect(analyser);
+  const dataArray = new Uint8Array(analyser.frequencyBinCount);
+
+  /**
+   * Calculates the current volume of the track in the range from 0 to 1
+   */
+  const calculateVolume = () => {
+    analyser.getByteFrequencyData(dataArray);
+    let sum = 0;
+    for (const amplitude of dataArray) {
+      sum += Math.pow(amplitude / 255, 2);
+    }
+    const volume = Math.sqrt(sum / dataArray.length);
+    return volume;
+  };
+
+  const cleanup = () => {
+    audioContext.close();
+    if (opts.cloneTrack) {
+      streamTrack.stop();
+    }
+  };
+
+  return { calculateVolume, analyser, cleanup };
+}