From 22e3c7774d54ce73306d4a6b1cc5c01e812f61cc Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 26 Aug 2022 11:50:25 +0100 Subject: [PATCH 01/18] Quell a bunch of React props errors --- src/components/views/context_menus/MessageContextMenu.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/components/views/context_menus/MessageContextMenu.tsx b/src/components/views/context_menus/MessageContextMenu.tsx index 9452fa28268..c2298955cc0 100644 --- a/src/components/views/context_menus/MessageContextMenu.tsx +++ b/src/components/views/context_menus/MessageContextMenu.tsx @@ -382,7 +382,8 @@ export default class MessageContextMenu extends React.Component public render(): JSX.Element { const cli = MatrixClientPeg.get(); const me = cli.getUserId(); - const { mxEvent, rightClick, link, eventTileOps, reactions, collapseReplyChain } = this.props; + const { mxEvent, rightClick, link, eventTileOps, reactions, collapseReplyChain, + permalinkCreator, getRelationsForEvent, ...otherProps } = this.props; const eventStatus = mxEvent.status; const unsentReactionsCount = this.getUnsentReactions().length; const contentActionable = isContentActionable(mxEvent); @@ -747,7 +748,7 @@ export default class MessageContextMenu extends React.Component return ( Date: Fri, 26 Aug 2022 11:54:44 +0100 Subject: [PATCH 02/18] Extract WorkerManager from BlurhashEncoder --- src/BlurhashEncoder.ts | 34 +++--------------------- src/WorkerManager.ts | 47 ++++++++++++++++++++++++++++++++++ src/workers/blurhash.worker.ts | 11 +++++--- src/workers/worker.ts | 19 ++++++++++++++ 4 files changed, 78 insertions(+), 33 deletions(-) create mode 100644 src/WorkerManager.ts create mode 100644 src/workers/worker.ts diff --git a/src/BlurhashEncoder.ts b/src/BlurhashEncoder.ts index 2aee370fe90..12b1a180b98 100644 --- a/src/BlurhashEncoder.ts +++ b/src/BlurhashEncoder.ts @@ -14,15 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { defer, IDeferred } from "matrix-js-sdk/src/utils"; - // @ts-ignore - `.ts` is needed here to make TS happy -import BlurhashWorker from "./workers/blurhash.worker.ts"; - -interface IBlurhashWorkerResponse { - seq: number; - blurhash: string; -} +import BlurhashWorker, { Request, Response } from "./workers/blurhash.worker.ts"; +import { WorkerManager } from "./WorkerManager"; export class BlurhashEncoder { private static internalInstance = new BlurhashEncoder(); @@ -31,30 +25,10 @@ export class BlurhashEncoder { return BlurhashEncoder.internalInstance; } - private readonly worker: Worker; - private seq = 0; - private pendingDeferredMap = new Map>(); - - constructor() { - this.worker = new BlurhashWorker(); - this.worker.onmessage = this.onMessage; - } - - private onMessage = (ev: MessageEvent) => { - const { seq, blurhash } = ev.data; - const deferred = this.pendingDeferredMap.get(seq); - if (deferred) { - this.pendingDeferredMap.delete(seq); - deferred.resolve(blurhash); - } - }; + private readonly worker = new WorkerManager(BlurhashWorker); public getBlurhash(imageData: ImageData): Promise { - const seq = this.seq++; - const deferred = defer(); - this.pendingDeferredMap.set(seq, deferred); - this.worker.postMessage({ seq, imageData }); - return deferred.promise; + return this.worker.call({ imageData }).then(resp => resp.blurhash); } } diff --git a/src/WorkerManager.ts b/src/WorkerManager.ts new file mode 100644 index 00000000000..4373a293040 --- /dev/null +++ b/src/WorkerManager.ts @@ -0,0 +1,47 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { defer, IDeferred } from "matrix-js-sdk/src/utils"; + +import { WorkerPayload } from "./workers/worker"; + +export class WorkerManager { + private readonly worker: Worker; + private seq = 0; + private pendingDeferredMap = new Map>(); + + constructor(WorkerConstructor: { new (): Worker }) { + this.worker = new WorkerConstructor(); + this.worker.onmessage = this.onMessage; + } + + private onMessage = (ev: MessageEvent) => { + const deferred = this.pendingDeferredMap.get(ev.data.seq); + if (deferred) { + this.pendingDeferredMap.delete(ev.data.seq); + deferred.resolve(ev.data); + } + }; + + public call(request: Request): Promise { + const seq = this.seq++; + const deferred = defer(); + this.pendingDeferredMap.set(seq, deferred); + this.worker.postMessage({ seq, ...request }); + return deferred.promise; + } +} + diff --git a/src/workers/blurhash.worker.ts b/src/workers/blurhash.worker.ts index 031cc67c905..183ffefc43e 100644 --- a/src/workers/blurhash.worker.ts +++ b/src/workers/blurhash.worker.ts @@ -16,14 +16,19 @@ limitations under the License. import { encode } from "blurhash"; +import { WorkerPayload } from "./worker"; + const ctx: Worker = self as any; -interface IBlurhashWorkerRequest { - seq: number; +export interface Request { imageData: ImageData; } -ctx.addEventListener("message", (event: MessageEvent): void => { +export interface Response { + blurhash: string; +} + +ctx.addEventListener("message", (event: MessageEvent): void => { const { seq, imageData } = event.data; const blurhash = encode( imageData.data, diff --git a/src/workers/worker.ts b/src/workers/worker.ts new file mode 100644 index 00000000000..b62fb887269 --- /dev/null +++ b/src/workers/worker.ts @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface WorkerPayload { + seq: number; +} From e7e41bd0dc7ee3d5933b668c0280a505c92ded25 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 26 Aug 2022 12:28:38 +0100 Subject: [PATCH 03/18] Stash playback worker --- src/audio/Playback.ts | 23 +++++++++---------- src/workers/playback.worker.ts | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 src/workers/playback.worker.ts diff --git a/src/audio/Playback.ts b/src/audio/Playback.ts index a25794b59ed..1cf5c582111 100644 --- a/src/audio/Playback.ts +++ b/src/audio/Playback.ts @@ -18,12 +18,15 @@ import EventEmitter from "events"; import { SimpleObservable } from "matrix-widget-api"; import { logger } from "matrix-js-sdk/src/logger"; +// @ts-ignore - `.ts` is needed here to make TS happy +import PlaybackWorker, { Request, Response } from "../workers/playback.worker.ts"; import { UPDATE_EVENT } from "../stores/AsyncStore"; -import { arrayFastResample, arrayRescale, arraySeed, arraySmoothingResample } from "../utils/arrays"; +import { arrayFastResample, arraySeed } from "../utils/arrays"; import { IDestroyable } from "../utils/IDestroyable"; import { PlaybackClock } from "./PlaybackClock"; import { createAudioContext, decodeOgg } from "./compat"; import { clamp } from "../utils/numbers"; +import { WorkerManager } from "../WorkerManager"; export enum PlaybackState { Decoding = "decoding", @@ -36,15 +39,6 @@ export const PLAYBACK_WAVEFORM_SAMPLES = 39; const THUMBNAIL_WAVEFORM_SAMPLES = 100; // arbitrary: [30,120] export const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES); -function makePlaybackWaveform(input: number[]): number[] { - // First, convert negative amplitudes to positive so we don't detect zero as "noisy". - const noiseWaveform = input.map(v => Math.abs(v)); - - // Then, we'll resample the waveform using a smoothing approach so we can keep the same rough shape. - // We also rescale the waveform to be 0-1 so we end up with a clamped waveform to rely upon. - return arrayRescale(arraySmoothingResample(noiseWaveform, PLAYBACK_WAVEFORM_SAMPLES), 0, 1); -} - export class Playback extends EventEmitter implements IDestroyable { /** * Stable waveform for representing a thumbnail of the media. Values are @@ -61,6 +55,7 @@ export class Playback extends EventEmitter implements IDestroyable { private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; private readonly fileSize: number; + private readonly worker = new WorkerManager(PlaybackWorker); /** * Creates a new playback instance from a buffer. @@ -182,8 +177,7 @@ export class Playback extends EventEmitter implements IDestroyable { // Update the waveform to the real waveform once we have channel data to use. We don't // exactly trust the user-provided waveform to be accurate... - const waveform = Array.from(this.audioBuf.getChannelData(0)); - this.resampledWaveform = makePlaybackWaveform(waveform); + this.resampledWaveform = await this.makePlaybackWaveform(this.audioBuf.getChannelData(0)); } this.waveformObservable.update(this.resampledWaveform); @@ -196,6 +190,11 @@ export class Playback extends EventEmitter implements IDestroyable { this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore } + private makePlaybackWaveform(input: Float32Array): Promise { + // const waveform = Array.from(); + return this.worker.call({ data: Array.from(input) }).then(resp => resp.waveform); + } + private onPlaybackEnd = async () => { await this.context.suspend(); this.emit(PlaybackState.Stopped); diff --git a/src/workers/playback.worker.ts b/src/workers/playback.worker.ts new file mode 100644 index 00000000000..e59242be168 --- /dev/null +++ b/src/workers/playback.worker.ts @@ -0,0 +1,42 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { WorkerPayload } from "./worker"; +import { arrayRescale, arraySmoothingResample } from "../utils/arrays"; +import { PLAYBACK_WAVEFORM_SAMPLES } from "../audio/Playback"; + +const ctx: Worker = self as any; + +export interface Request { + data: number[]; +} + +export interface Response { + waveform: number[]; +} + +ctx.addEventListener("message", async (event: MessageEvent): Promise => { + const { seq, data } = event.data; + + // First, convert negative amplitudes to positive so we don't detect zero as "noisy". + const noiseWaveform = data.map(v => Math.abs(v)); + + // Then, we'll resample the waveform using a smoothing approach so we can keep the same rough shape. + // We also rescale the waveform to be 0-1 so we end up with a clamped waveform to rely upon. + const waveform = arrayRescale(arraySmoothingResample(noiseWaveform, PLAYBACK_WAVEFORM_SAMPLES), 0, 1); + + ctx.postMessage({ seq, waveform }); +}); From 76d3c26530beb9b4ef47cd6255a71610c50f0777 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 26 Aug 2022 12:28:48 +0100 Subject: [PATCH 04/18] Fix yet more React errors --- src/components/structures/ContextMenu.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/components/structures/ContextMenu.tsx b/src/components/structures/ContextMenu.tsx index 2445e0b38aa..d4d4d0aa95d 100644 --- a/src/components/structures/ContextMenu.tsx +++ b/src/components/structures/ContextMenu.tsx @@ -246,6 +246,8 @@ export default class ContextMenu extends React.PureComponent { wrapperClassName, chevronFace: propsChevronFace, chevronOffset: propsChevronOffset, + hasBackground: _, + onFinished: __, ...props } = this.props; From 1eb3870208a0751278bde3f549b1129f2ac64892 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 25 Apr 2023 11:12:12 +0100 Subject: [PATCH 05/18] Iterate and avoid import cycles in worker-loader --- src/WorkerManager.ts | 5 ++--- src/audio/ManagedPlayback.ts | 3 ++- src/audio/Playback.ts | 13 ++++++------- src/audio/PlaybackManager.ts | 3 ++- src/audio/consts.ts | 5 +++++ .../views/audio_messages/PlaybackWaveform.tsx | 3 ++- src/workers/playback.worker.ts | 4 ++-- 7 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/WorkerManager.ts b/src/WorkerManager.ts index 4373a293040..9ad8355d9b8 100644 --- a/src/WorkerManager.ts +++ b/src/WorkerManager.ts @@ -23,12 +23,12 @@ export class WorkerManager { private seq = 0; private pendingDeferredMap = new Map>(); - constructor(WorkerConstructor: { new (): Worker }) { + public constructor(WorkerConstructor: { new (): Worker }) { this.worker = new WorkerConstructor(); this.worker.onmessage = this.onMessage; } - private onMessage = (ev: MessageEvent) => { + private onMessage = (ev: MessageEvent): void => { const deferred = this.pendingDeferredMap.get(ev.data.seq); if (deferred) { this.pendingDeferredMap.delete(ev.data.seq); @@ -44,4 +44,3 @@ export class WorkerManager { return deferred.promise; } } - diff --git a/src/audio/ManagedPlayback.ts b/src/audio/ManagedPlayback.ts index c33d032b688..6ecedd6766a 100644 --- a/src/audio/ManagedPlayback.ts +++ b/src/audio/ManagedPlayback.ts @@ -14,8 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { DEFAULT_WAVEFORM, Playback } from "./Playback"; +import { Playback } from "./Playback"; import { PlaybackManager } from "./PlaybackManager"; +import { DEFAULT_WAVEFORM } from "./consts"; /** * A managed playback is a Playback instance that is guided by a PlaybackManager. diff --git a/src/audio/Playback.ts b/src/audio/Playback.ts index 5e649e03b92..362fc16bb6b 100644 --- a/src/audio/Playback.ts +++ b/src/audio/Playback.ts @@ -21,12 +21,13 @@ import { logger } from "matrix-js-sdk/src/logger"; // @ts-ignore - `.ts` is needed here to make TS happy import PlaybackWorker, { Request, Response } from "../workers/playback.worker.ts"; import { UPDATE_EVENT } from "../stores/AsyncStore"; -import { arrayFastResample, arraySeed } from "../utils/arrays"; +import { arrayFastResample } from "../utils/arrays"; import { IDestroyable } from "../utils/IDestroyable"; import { PlaybackClock } from "./PlaybackClock"; import { createAudioContext, decodeOgg } from "./compat"; import { clamp } from "../utils/numbers"; import { WorkerManager } from "../WorkerManager"; +import { DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES } from "./consts"; export enum PlaybackState { Decoding = "decoding", @@ -42,9 +43,7 @@ export interface PlaybackInterface { skipTo(timeSeconds: number): Promise; } -export const PLAYBACK_WAVEFORM_SAMPLES = 39; const THUMBNAIL_WAVEFORM_SAMPLES = 100; // arbitrary: [30,120] -export const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES); export class Playback extends EventEmitter implements IDestroyable { /** @@ -54,10 +53,10 @@ export class Playback extends EventEmitter implements IDestroyable { public readonly thumbnailWaveform: number[]; private readonly context: AudioContext; - private source: AudioBufferSourceNode | MediaElementAudioSourceNode; + private source?: AudioBufferSourceNode | MediaElementAudioSourceNode; private state = PlaybackState.Decoding; - private audioBuf: AudioBuffer; - private element: HTMLAudioElement; + private audioBuf?: AudioBuffer; + private element?: HTMLAudioElement; private resampledWaveform: number[]; private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; @@ -220,7 +219,7 @@ export class Playback extends EventEmitter implements IDestroyable { private makePlaybackWaveform(input: Float32Array): Promise { // const waveform = Array.from(); - return this.worker.call({ data: Array.from(input) }).then(resp => resp.waveform); + return this.worker.call({ data: Array.from(input) }).then((resp) => resp.waveform); } private onPlaybackEnd = async (): Promise => { diff --git a/src/audio/PlaybackManager.ts b/src/audio/PlaybackManager.ts index 0cc52e7f0e7..30bc3876766 100644 --- a/src/audio/PlaybackManager.ts +++ b/src/audio/PlaybackManager.ts @@ -14,8 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { DEFAULT_WAVEFORM, Playback, PlaybackState } from "./Playback"; +import { Playback, PlaybackState } from "./Playback"; import { ManagedPlayback } from "./ManagedPlayback"; +import { DEFAULT_WAVEFORM } from "./consts"; /** * Handles management of playback instances to ensure certain functionality, like diff --git a/src/audio/consts.ts b/src/audio/consts.ts index 39e9b309042..4c8fb4602b3 100644 --- a/src/audio/consts.ts +++ b/src/audio/consts.ts @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { arraySeed } from "../utils/arrays"; + export const WORKLET_NAME = "mx-voice-worklet"; export enum PayloadEvent { @@ -35,3 +37,6 @@ export interface IAmplitudePayload extends IPayload { forIndex: number; amplitude: number; } + +export const PLAYBACK_WAVEFORM_SAMPLES = 39; +export const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES); diff --git a/src/components/views/audio_messages/PlaybackWaveform.tsx b/src/components/views/audio_messages/PlaybackWaveform.tsx index 19fd5de7970..c0752371f18 100644 --- a/src/components/views/audio_messages/PlaybackWaveform.tsx +++ b/src/components/views/audio_messages/PlaybackWaveform.tsx @@ -18,8 +18,9 @@ import React from "react"; import { arraySeed, arrayTrimFill } from "../../../utils/arrays"; import Waveform from "./Waveform"; -import { Playback, PLAYBACK_WAVEFORM_SAMPLES } from "../../../audio/Playback"; +import { Playback } from "../../../audio/Playback"; import { percentageOf } from "../../../utils/numbers"; +import { PLAYBACK_WAVEFORM_SAMPLES } from "../../../audio/consts"; interface IProps { playback: Playback; diff --git a/src/workers/playback.worker.ts b/src/workers/playback.worker.ts index e59242be168..6288013f676 100644 --- a/src/workers/playback.worker.ts +++ b/src/workers/playback.worker.ts @@ -16,7 +16,7 @@ limitations under the License. import { WorkerPayload } from "./worker"; import { arrayRescale, arraySmoothingResample } from "../utils/arrays"; -import { PLAYBACK_WAVEFORM_SAMPLES } from "../audio/Playback"; +import { PLAYBACK_WAVEFORM_SAMPLES } from "../audio/consts"; const ctx: Worker = self as any; @@ -32,7 +32,7 @@ ctx.addEventListener("message", async (event: MessageEvent Math.abs(v)); + const noiseWaveform = data.map((v) => Math.abs(v)); // Then, we'll resample the waveform using a smoothing approach so we can keep the same rough shape. // We also rescale the waveform to be 0-1 so we end up with a clamped waveform to rely upon. From a9d4ff81f6c161856f436f7ebd48c8aa29754dc8 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 25 Apr 2023 11:16:29 +0100 Subject: [PATCH 06/18] Iterate --- src/utils/arrays.ts | 57 --------------------------------- src/workers/playback.worker.ts | 58 +++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 58 deletions(-) diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts index d9c7c3735ea..c9a6005a1b7 100644 --- a/src/utils/arrays.ts +++ b/src/utils/arrays.ts @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { percentageOf, percentageWithin } from "./numbers"; - /** * Quickly resample an array to have less/more data points. If an input which is larger * than the desired size is provided, it will be downsampled. Similarly, if the input @@ -50,61 +48,6 @@ export function arrayFastResample(input: number[], points: number): number[] { return arrayTrimFill(samples, points, arraySeed(input[input.length - 1], points)); } -/** - * Attempts a smooth resample of the given array. This is functionally similar to arrayFastResample - * though can take longer due to the smoothing of data. - * @param {number[]} input The input array to resample. - * @param {number} points The number of samples to end up with. - * @returns {number[]} The resampled array. - */ -export function arraySmoothingResample(input: number[], points: number): number[] { - if (input.length === points) return input; // short-circuit a complicated call - - let samples: number[] = []; - if (input.length > points) { - // We're downsampling. To preserve the curve we'll actually reduce our sample - // selection and average some points between them. - - // All we're doing here is repeatedly averaging the waveform down to near our - // target value. We don't average down to exactly our target as the loop might - // never end, and we can over-average the data. Instead, we'll get as far as - // we can and do a followup fast resample (the neighbouring points will be close - // to the actual waveform, so we can get away with this safely). - while (samples.length > points * 2 || samples.length === 0) { - samples = []; - for (let i = 1; i < input.length - 1; i += 2) { - const prevPoint = input[i - 1]; - const nextPoint = input[i + 1]; - const currPoint = input[i]; - const average = (prevPoint + nextPoint + currPoint) / 3; - samples.push(average); - } - input = samples; - } - - return arrayFastResample(samples, points); - } else { - // In practice there's not much purpose in burning CPU for short arrays only to - // end up with a result that can't possibly look much different than the fast - // resample, so just skip ahead to the fast resample. - return arrayFastResample(input, points); - } -} - -/** - * Rescales the input array to have values that are inclusively within the provided - * minimum and maximum. - * @param {number[]} input The array to rescale. - * @param {number} newMin The minimum value to scale to. - * @param {number} newMax The maximum value to scale to. - * @returns {number[]} The rescaled array. - */ -export function arrayRescale(input: number[], newMin: number, newMax: number): number[] { - const min: number = Math.min(...input); - const max: number = Math.max(...input); - return input.map((v) => percentageWithin(percentageOf(v, min, max), newMin, newMax)); -} - /** * Creates an array of the given length, seeded with the given value. * @param {T} val The value to seed the array with. diff --git a/src/workers/playback.worker.ts b/src/workers/playback.worker.ts index 6288013f676..a0b25f220bf 100644 --- a/src/workers/playback.worker.ts +++ b/src/workers/playback.worker.ts @@ -15,11 +15,67 @@ limitations under the License. */ import { WorkerPayload } from "./worker"; -import { arrayRescale, arraySmoothingResample } from "../utils/arrays"; +import { arrayFastResample } from "../utils/arrays"; import { PLAYBACK_WAVEFORM_SAMPLES } from "../audio/consts"; +import { percentageOf, percentageWithin } from "../utils/numbers"; const ctx: Worker = self as any; +/** + * Attempts a smooth resample of the given array. This is functionally similar to arrayFastResample + * though can take longer due to the smoothing of data. + * @param {number[]} input The input array to resample. + * @param {number} points The number of samples to end up with. + * @returns {number[]} The resampled array. + */ +export function arraySmoothingResample(input: number[], points: number): number[] { + if (input.length === points) return input; // short-circuit a complicated call + + let samples: number[] = []; + if (input.length > points) { + // We're downsampling. To preserve the curve we'll actually reduce our sample + // selection and average some points between them. + + // All we're doing here is repeatedly averaging the waveform down to near our + // target value. We don't average down to exactly our target as the loop might + // never end, and we can over-average the data. Instead, we'll get as far as + // we can and do a followup fast resample (the neighbouring points will be close + // to the actual waveform, so we can get away with this safely). + while (samples.length > points * 2 || samples.length === 0) { + samples = []; + for (let i = 1; i < input.length - 1; i += 2) { + const prevPoint = input[i - 1]; + const nextPoint = input[i + 1]; + const currPoint = input[i]; + const average = (prevPoint + nextPoint + currPoint) / 3; + samples.push(average); + } + input = samples; + } + + return arrayFastResample(samples, points); + } else { + // In practice there's not much purpose in burning CPU for short arrays only to + // end up with a result that can't possibly look much different than the fast + // resample, so just skip ahead to the fast resample. + return arrayFastResample(input, points); + } +} + +/** + * Rescales the input array to have values that are inclusively within the provided + * minimum and maximum. + * @param {number[]} input The array to rescale. + * @param {number} newMin The minimum value to scale to. + * @param {number} newMax The maximum value to scale to. + * @returns {number[]} The rescaled array. + */ +export function arrayRescale(input: number[], newMin: number, newMax: number): number[] { + const min: number = Math.min(...input); + const max: number = Math.max(...input); + return input.map((v) => percentageWithin(percentageOf(v, min, max), newMin, newMax)); +} + export interface Request { data: number[]; } From f530c7eb2bca8cff97d7268264497a1a5d0920f4 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 25 Apr 2023 11:35:14 +0100 Subject: [PATCH 07/18] Iterate --- src/BlurhashEncoder.ts | 2 +- src/utils/arrays.ts | 59 ++++++++++++++++++++++++++++++++++ src/workers/playback.worker.ts | 58 +-------------------------------- 3 files changed, 61 insertions(+), 58 deletions(-) diff --git a/src/BlurhashEncoder.ts b/src/BlurhashEncoder.ts index bf4cfa00fbb..89ed2b56e54 100644 --- a/src/BlurhashEncoder.ts +++ b/src/BlurhashEncoder.ts @@ -28,6 +28,6 @@ export class BlurhashEncoder { private readonly worker = new WorkerManager(BlurhashWorker); public getBlurhash(imageData: ImageData): Promise { - return this.worker.call({ imageData }).then(resp => resp.blurhash); + return this.worker.call({ imageData }).then((resp) => resp.blurhash); } } diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts index c9a6005a1b7..63d2e26739e 100644 --- a/src/utils/arrays.ts +++ b/src/utils/arrays.ts @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { percentageOf, percentageWithin } from "./numbers"; + /** * Quickly resample an array to have less/more data points. If an input which is larger * than the desired size is provided, it will be downsampled. Similarly, if the input @@ -48,6 +50,63 @@ export function arrayFastResample(input: number[], points: number): number[] { return arrayTrimFill(samples, points, arraySeed(input[input.length - 1], points)); } +/** + * Attempts a smooth resample of the given array. This is functionally similar to arrayFastResample + * though can take longer due to the smoothing of data. + * @param {number[]} input The input array to resample. + * @param {number} points The number of samples to end up with. + * @returns {number[]} The resampled array. + */ +// ts-prune-ignore-next +export function arraySmoothingResample(input: number[], points: number): number[] { + if (input.length === points) return input; // short-circuit a complicated call + + let samples: number[] = []; + if (input.length > points) { + // We're downsampling. To preserve the curve we'll actually reduce our sample + // selection and average some points between them. + + // All we're doing here is repeatedly averaging the waveform down to near our + // target value. We don't average down to exactly our target as the loop might + // never end, and we can over-average the data. Instead, we'll get as far as + // we can and do a followup fast resample (the neighbouring points will be close + // to the actual waveform, so we can get away with this safely). + while (samples.length > points * 2 || samples.length === 0) { + samples = []; + for (let i = 1; i < input.length - 1; i += 2) { + const prevPoint = input[i - 1]; + const nextPoint = input[i + 1]; + const currPoint = input[i]; + const average = (prevPoint + nextPoint + currPoint) / 3; + samples.push(average); + } + input = samples; + } + + return arrayFastResample(samples, points); + } else { + // In practice there's not much purpose in burning CPU for short arrays only to + // end up with a result that can't possibly look much different than the fast + // resample, so just skip ahead to the fast resample. + return arrayFastResample(input, points); + } +} + +/** + * Rescales the input array to have values that are inclusively within the provided + * minimum and maximum. + * @param {number[]} input The array to rescale. + * @param {number} newMin The minimum value to scale to. + * @param {number} newMax The maximum value to scale to. + * @returns {number[]} The rescaled array. + */ +// ts-prune-ignore-next +export function arrayRescale(input: number[], newMin: number, newMax: number): number[] { + const min: number = Math.min(...input); + const max: number = Math.max(...input); + return input.map((v) => percentageWithin(percentageOf(v, min, max), newMin, newMax)); +} + /** * Creates an array of the given length, seeded with the given value. * @param {T} val The value to seed the array with. diff --git a/src/workers/playback.worker.ts b/src/workers/playback.worker.ts index a0b25f220bf..6288013f676 100644 --- a/src/workers/playback.worker.ts +++ b/src/workers/playback.worker.ts @@ -15,67 +15,11 @@ limitations under the License. */ import { WorkerPayload } from "./worker"; -import { arrayFastResample } from "../utils/arrays"; +import { arrayRescale, arraySmoothingResample } from "../utils/arrays"; import { PLAYBACK_WAVEFORM_SAMPLES } from "../audio/consts"; -import { percentageOf, percentageWithin } from "../utils/numbers"; const ctx: Worker = self as any; -/** - * Attempts a smooth resample of the given array. This is functionally similar to arrayFastResample - * though can take longer due to the smoothing of data. - * @param {number[]} input The input array to resample. - * @param {number} points The number of samples to end up with. - * @returns {number[]} The resampled array. - */ -export function arraySmoothingResample(input: number[], points: number): number[] { - if (input.length === points) return input; // short-circuit a complicated call - - let samples: number[] = []; - if (input.length > points) { - // We're downsampling. To preserve the curve we'll actually reduce our sample - // selection and average some points between them. - - // All we're doing here is repeatedly averaging the waveform down to near our - // target value. We don't average down to exactly our target as the loop might - // never end, and we can over-average the data. Instead, we'll get as far as - // we can and do a followup fast resample (the neighbouring points will be close - // to the actual waveform, so we can get away with this safely). - while (samples.length > points * 2 || samples.length === 0) { - samples = []; - for (let i = 1; i < input.length - 1; i += 2) { - const prevPoint = input[i - 1]; - const nextPoint = input[i + 1]; - const currPoint = input[i]; - const average = (prevPoint + nextPoint + currPoint) / 3; - samples.push(average); - } - input = samples; - } - - return arrayFastResample(samples, points); - } else { - // In practice there's not much purpose in burning CPU for short arrays only to - // end up with a result that can't possibly look much different than the fast - // resample, so just skip ahead to the fast resample. - return arrayFastResample(input, points); - } -} - -/** - * Rescales the input array to have values that are inclusively within the provided - * minimum and maximum. - * @param {number[]} input The array to rescale. - * @param {number} newMin The minimum value to scale to. - * @param {number} newMax The maximum value to scale to. - * @returns {number[]} The rescaled array. - */ -export function arrayRescale(input: number[], newMin: number, newMax: number): number[] { - const min: number = Math.min(...input); - const max: number = Math.max(...input); - return input.map((v) => percentageWithin(percentageOf(v, min, max), newMin, newMax)); -} - export interface Request { data: number[]; } From aa472154d57d3408b68684d4a29a822cf31ab033 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 25 Apr 2023 11:44:06 +0100 Subject: [PATCH 08/18] Iterate types --- src/audio/Playback.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/audio/Playback.ts b/src/audio/Playback.ts index 362fc16bb6b..a438c300a4e 100644 --- a/src/audio/Playback.ts +++ b/src/audio/Playback.ts @@ -17,6 +17,7 @@ limitations under the License. import EventEmitter from "events"; import { SimpleObservable } from "matrix-widget-api"; import { logger } from "matrix-js-sdk/src/logger"; +import { defer } from "matrix-js-sdk/src/utils"; // @ts-ignore - `.ts` is needed here to make TS happy import PlaybackWorker, { Request, Response } from "../workers/playback.worker.ts"; @@ -164,12 +165,11 @@ export class Playback extends EventEmitter implements IDestroyable { // 5mb logger.log("Audio file too large: processing through