From cdbf2a1187976e70bb4b58370361dbb5fea8b918 Mon Sep 17 00:00:00 2001 From: ComfyFluffy <24245520+ComfyFluffy@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:02:53 +0800 Subject: [PATCH 1/5] smoothed waveform --- .../src/components/editor/sound/wavesurfer.ts | 162 +++++++++++++----- spx-gui/src/utils/wavesurfer-record.ts | 21 +-- 2 files changed, 128 insertions(+), 55 deletions(-) diff --git a/spx-gui/src/components/editor/sound/wavesurfer.ts b/spx-gui/src/components/editor/sound/wavesurfer.ts index cc7d9810b..00e37df2d 100644 --- a/spx-gui/src/components/editor/sound/wavesurfer.ts +++ b/spx-gui/src/components/editor/sound/wavesurfer.ts @@ -7,7 +7,11 @@ import WaveSurfer from 'wavesurfer.js' import { useUIVariables } from '@/components/ui' import { getAudioContext } from '@/utils/audio' -export function useWavesurfer(container: () => HTMLElement | undefined, gain: () => number) { +export function useWavesurfer( + container: () => HTMLElement | undefined, + gain: () => number, + recording?: boolean +) { const uiVariables = useUIVariables() const wavesurfer = ref(null) @@ -35,6 +39,17 @@ export function useWavesurfer(container: () => HTMLElement | undefined, gain: () gainNode_.gain.value = gain() gainNode = gainNode_ + /** + * Cache for averaged data. Only used for recording. + * The recorder will set the cache to null and enable it when recording starts, + * and will disable it when recording stops. + */ + const cachedAveragedData: { + cache: { averagedData: number[]; originalLength: number; blockSize: number } | null + } = { + cache: null + } + wavesurfer.value = new WaveSurfer({ interact: false, container: newContainer, @@ -46,54 +61,123 @@ export function useWavesurfer(container: () => HTMLElement | undefined, gain: () normalize: true, media: audioElement, renderFunction: (peaks: (Float32Array | number[])[], ctx: CanvasRenderingContext2D): void => { - // TODO: Better drawing algorithm to reduce flashing? - const smoothAndDrawChannel = (channel: Float32Array, vScale: number) => { - const { width, height } = ctx.canvas - const halfHeight = height / 2 - const numPoints = Math.floor(width / 5) - const blockSize = Math.floor(channel.length / numPoints) - const smoothedData = new Float32Array(numPoints) - - // Smooth the data by averaging blocks - for (let i = 0; i < numPoints; i++) { - let sum = 0 - for (let j = 0; j < blockSize; j++) { - sum += Math.abs(channel[i * blockSize + j]) + try { + const halfHeight = ctx.canvas.height / 2 + + const averageBlock = (data: number[] | Float32Array, blockSize: number): number[] => { + const { cache } = cachedAveragedData + + // Check if we can use the cached data + if ( + recording && + cache && + cache.blockSize === blockSize && + cache.originalLength <= data.length + ) { + const newBlocks = + Math.floor(data.length / blockSize) - Math.floor(cache.originalLength / blockSize) + + // If there are new blocks to process + if (newBlocks > 0) { + const newAveragedData = cache.averagedData.slice() + const startIndex = cache.originalLength + for (let i = 0; i < newBlocks; i++) { + let sum = 0 + for (let j = 0; j < blockSize; j++) { + const index = startIndex + i * blockSize + j + sum += Math.max(0, data[index]) + } + newAveragedData.push(sum / blockSize) + } + + cachedAveragedData.cache = { + averagedData: newAveragedData, + originalLength: data.length, + blockSize + } + return newAveragedData + } + + // If no new blocks to process, return cached data + return cache.averagedData + } + + // Calculate new averaged data if cache is not valid or not present + const averagedData: number[] = new Array(Math.floor(data.length / blockSize)) + for (let i = 0; i < averagedData.length; i++) { + let sum = 0 + for (let j = 0; j < blockSize; j++) { + const index = i * blockSize + j + sum += Math.max(0, data[index]) + } + averagedData[i] = sum / blockSize + } + + // Update cache with new averaged data + cachedAveragedData.cache = { + averagedData, + originalLength: data.length, + blockSize } - smoothedData[i] = sum / blockSize + + return averagedData } - // Draw with bezier curves - ctx.beginPath() - ctx.moveTo(0, halfHeight) + const drawSmoothCurve = ( + ctx: CanvasRenderingContext2D, + points: number[], + getPoint: (index: number) => number + ) => { + const segmentLength = ctx.canvas.width / (points.length - 1) + + ctx.beginPath() + ctx.moveTo(0, halfHeight) - for (let i = 1; i < smoothedData.length; i++) { - const prevX = (i - 1) * (width / numPoints) - const currX = i * (width / numPoints) - const midX = (prevX + currX) / 2 - const prevY = halfHeight + smoothedData[i - 1] * halfHeight * vScale - const currY = halfHeight + smoothedData[i] * halfHeight * vScale + for (let i = 0; i < points.length - 2; i++) { + const xc = (i * segmentLength + (i + 1) * segmentLength) / 2 + const yc = (getPoint(i) + getPoint(i + 1)) / 2 + ctx.quadraticCurveTo(i * segmentLength, getPoint(i), xc, yc) + } - // Use a quadratic bezier curve to the middle of the interval for a smoother line - ctx.quadraticCurveTo(prevX, prevY, midX, (prevY + currY) / 2) - ctx.quadraticCurveTo(midX, (prevY + currY) / 2, currX, currY) + ctx.quadraticCurveTo( + (points.length - 2) * segmentLength, + getPoint(points.length - 2), + ctx.canvas.width, + getPoint(points.length - 1) + ) + + ctx.lineTo(ctx.canvas.width, halfHeight) + + ctx.strokeStyle = uiVariables.color.sound[400] + ctx.lineWidth = 2 + ctx.stroke() + ctx.closePath() + ctx.fillStyle = uiVariables.color.sound[400] + ctx.fill() } - ctx.lineTo(width, halfHeight) - ctx.strokeStyle = uiVariables.color.sound[400] - ctx.stroke() - ctx.closePath() - ctx.fillStyle = uiVariables.color.sound[400] - ctx.fill() - } + const channel = peaks[0] + + const scale = gain() * 3000 - const channel = Array.isArray(peaks[0]) ? new Float32Array(peaks[0] as number[]) : peaks[0] + const blockSize = channel.length > 200000 ? 2000 : channel.length > 100000 ? 1000 : 500 - const scale = gain() * 5 + const smoothedChannel = averageBlock(channel, blockSize) - // Only one channel is assumed, render it twice (mirrored) - smoothAndDrawChannel(channel, scale) // Upper part - smoothAndDrawChannel(channel, -scale) // Lower part (mirrored) + drawSmoothCurve( + ctx, + smoothedChannel, + (index: number) => smoothedChannel[index] * scale + halfHeight + ) + drawSmoothCurve( + ctx, + smoothedChannel, + (index: number) => -smoothedChannel[index] * scale + halfHeight + ) + } catch (e) { + // wavesurfer does not log errors so we do it ourselves + console.error(e) + } } }) diff --git a/spx-gui/src/utils/wavesurfer-record.ts b/spx-gui/src/utils/wavesurfer-record.ts index 5766cfe5c..e57b7600d 100644 --- a/spx-gui/src/utils/wavesurfer-record.ts +++ b/spx-gui/src/utils/wavesurfer-record.ts @@ -46,7 +46,6 @@ const findSupportedMimeType = () => export class RecordPlugin extends BasePlugin { private stream: MediaStream | null = null private mediaRecorder: MediaRecorder | null = null - private dataWindow: Float32Array | null = null private isWaveformPaused = false private originalOptions: { cursorWidth: number; interact: boolean } | undefined private timer: Timer @@ -82,10 +81,11 @@ export class RecordPlugin extends BasePlugin Date: Tue, 11 Jun 2024 13:10:34 +0800 Subject: [PATCH 2/5] update comments --- spx-gui/src/components/editor/sound/wavesurfer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spx-gui/src/components/editor/sound/wavesurfer.ts b/spx-gui/src/components/editor/sound/wavesurfer.ts index 00e37df2d..d03c6c2f3 100644 --- a/spx-gui/src/components/editor/sound/wavesurfer.ts +++ b/spx-gui/src/components/editor/sound/wavesurfer.ts @@ -41,8 +41,8 @@ export function useWavesurfer( /** * Cache for averaged data. Only used for recording. - * The recorder will set the cache to null and enable it when recording starts, - * and will disable it when recording stops. + * As we expect the `WaveSurfer` to be destroyed and recreated on every recording, + * we don't need to reset the cache when the recording stops. */ const cachedAveragedData: { cache: { averagedData: number[]; originalLength: number; blockSize: number } | null From 3d08de099ca226756ec8662aa5e4963d88f3b55b Mon Sep 17 00:00:00 2001 From: ComfyFluffy <24245520+ComfyFluffy@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:35:05 +0800 Subject: [PATCH 3/5] do not reload audio on record end --- .../components/editor/sound/SoundRecorder.vue | 11 ++++++++--- .../editor/sound/WavesurferWithRange.vue | 4 +++- .../src/components/editor/sound/wavesurfer.ts | 16 +++++++--------- spx-gui/src/utils/wavesurfer-record.ts | 1 - 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/spx-gui/src/components/editor/sound/SoundRecorder.vue b/spx-gui/src/components/editor/sound/SoundRecorder.vue index 0574f551a..a49c8479c 100644 --- a/spx-gui/src/components/editor/sound/SoundRecorder.vue +++ b/spx-gui/src/components/editor/sound/SoundRecorder.vue @@ -6,6 +6,7 @@ v-if="recording || audioBlob" ref="wavesurferRef" v-model:range="audioRange" + recording :gain="gain" @init="handleWaveSurferInit" /> @@ -99,6 +100,7 @@ import { RecordPlugin } from '@/utils/wavesurfer-record' import VolumeSlider from './VolumeSlider.vue' import WavesurferWithRange from './WavesurferWithRange.vue' import type WaveSurfer from 'wavesurfer.js' +import { toWav } from '@/utils/audio' const emit = defineEmits<{ saved: [Sound] @@ -162,10 +164,13 @@ const stopRecording = () => { } const saveRecording = async () => { - if (!wavesurferRef.value) return - const wav = await wavesurferRef.value.exportWav() + if (!wavesurferRef.value || !audioBlob.value) return + const wav = await toWav(await audioBlob.value.arrayBuffer()) - const file = fromBlob(`Recording_${dayjs().format('YYYY-MM-DD_HH:mm:ss')}.webm`, wav) + const file = fromBlob( + `Recording_${dayjs().format('YYYY-MM-DD_HH:mm:ss')}.wav`, + new Blob([wav], { type: 'audio/wav' }) + ) const sound = await Sound.create('recording', file) const action = { name: { en: 'Add recording', zh: '添加录音' } } await editorCtx.project.history.doAction(action, () => editorCtx.project.addSound(sound)) diff --git a/spx-gui/src/components/editor/sound/WavesurferWithRange.vue b/spx-gui/src/components/editor/sound/WavesurferWithRange.vue index fc86e2212..360e105e9 100644 --- a/spx-gui/src/components/editor/sound/WavesurferWithRange.vue +++ b/spx-gui/src/components/editor/sound/WavesurferWithRange.vue @@ -16,6 +16,7 @@ const props = defineProps<{ audioUrl?: string | null range: { left: number; right: number } gain: number + recording?: boolean }>() const emit = defineEmits<{ @@ -75,7 +76,8 @@ const wavesurferDiv = ref() const wavesurferRef = useWavesurfer( () => wavesurferDiv.value, - () => props.gain + () => props.gain, + props.recording ) // we assume that wavesurferDiv.value will not change diff --git a/spx-gui/src/components/editor/sound/wavesurfer.ts b/spx-gui/src/components/editor/sound/wavesurfer.ts index d03c6c2f3..57b9201fd 100644 --- a/spx-gui/src/components/editor/sound/wavesurfer.ts +++ b/spx-gui/src/components/editor/sound/wavesurfer.ts @@ -10,7 +10,7 @@ import { getAudioContext } from '@/utils/audio' export function useWavesurfer( container: () => HTMLElement | undefined, gain: () => number, - recording?: boolean + recording: boolean ) { const uiVariables = useUIVariables() @@ -44,10 +44,10 @@ export function useWavesurfer( * As we expect the `WaveSurfer` to be destroyed and recreated on every recording, * we don't need to reset the cache when the recording stops. */ - const cachedAveragedData: { - cache: { averagedData: number[]; originalLength: number; blockSize: number } | null - } = { - cache: null + let cache: { + averagedData: number[] + originalLength: number + blockSize: number } wavesurfer.value = new WaveSurfer({ @@ -65,8 +65,6 @@ export function useWavesurfer( const halfHeight = ctx.canvas.height / 2 const averageBlock = (data: number[] | Float32Array, blockSize: number): number[] => { - const { cache } = cachedAveragedData - // Check if we can use the cached data if ( recording && @@ -90,7 +88,7 @@ export function useWavesurfer( newAveragedData.push(sum / blockSize) } - cachedAveragedData.cache = { + cache = { averagedData: newAveragedData, originalLength: data.length, blockSize @@ -114,7 +112,7 @@ export function useWavesurfer( } // Update cache with new averaged data - cachedAveragedData.cache = { + cache = { averagedData, originalLength: data.length, blockSize diff --git a/spx-gui/src/utils/wavesurfer-record.ts b/spx-gui/src/utils/wavesurfer-record.ts index e57b7600d..517bb99a8 100644 --- a/spx-gui/src/utils/wavesurfer-record.ts +++ b/spx-gui/src/utils/wavesurfer-record.ts @@ -179,7 +179,6 @@ export class RecordPlugin extends BasePlugin Date: Wed, 12 Jun 2024 11:37:57 +0800 Subject: [PATCH 4/5] adjust waveform scale to match scratch --- spx-gui/src/components/editor/sound/wavesurfer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spx-gui/src/components/editor/sound/wavesurfer.ts b/spx-gui/src/components/editor/sound/wavesurfer.ts index 57b9201fd..44b619924 100644 --- a/spx-gui/src/components/editor/sound/wavesurfer.ts +++ b/spx-gui/src/components/editor/sound/wavesurfer.ts @@ -156,7 +156,7 @@ export function useWavesurfer( const channel = peaks[0] - const scale = gain() * 3000 + const scale = gain() * 1200 const blockSize = channel.length > 200000 ? 2000 : channel.length > 100000 ? 1000 : 500 From 653aabf62b3562fb087745cfeb5dbb24de4c7e80 Mon Sep 17 00:00:00 2001 From: ComfyFluffy <24245520+ComfyFluffy@users.noreply.github.com> Date: Fri, 14 Jun 2024 13:34:41 +0800 Subject: [PATCH 5/5] add comments --- spx-gui/src/components/editor/sound/WavesurferWithRange.vue | 5 +++++ spx-gui/src/components/editor/sound/wavesurfer.ts | 3 +++ 2 files changed, 8 insertions(+) diff --git a/spx-gui/src/components/editor/sound/WavesurferWithRange.vue b/spx-gui/src/components/editor/sound/WavesurferWithRange.vue index 360e105e9..cfae925d2 100644 --- a/spx-gui/src/components/editor/sound/WavesurferWithRange.vue +++ b/spx-gui/src/components/editor/sound/WavesurferWithRange.vue @@ -16,6 +16,11 @@ const props = defineProps<{ audioUrl?: string | null range: { left: number; right: number } gain: number + // When true, the component will render the waveform differently. + // It enables the cache and updates the waveform in real-time. + // TODO: Currently in the recording mode, the waveform will be rendered + // in different samples than in the normal mode, resulting different smoothness. + // We may fix this in the future. recording?: boolean }>() diff --git a/spx-gui/src/components/editor/sound/wavesurfer.ts b/spx-gui/src/components/editor/sound/wavesurfer.ts index 44b619924..e7744bd31 100644 --- a/spx-gui/src/components/editor/sound/wavesurfer.ts +++ b/spx-gui/src/components/editor/sound/wavesurfer.ts @@ -158,6 +158,9 @@ export function useWavesurfer( const scale = gain() * 1200 + // TODO: For now, blockSize is fixed to 2000 for longer recordings + // to address the issue of slow update. This should be optimized + // in the future. const blockSize = channel.length > 200000 ? 2000 : channel.length > 100000 ? 1000 : 500 const smoothedChannel = averageBlock(channel, blockSize)