diff --git a/spx-gui/src/components/editor/sound/SoundRecorder.vue b/spx-gui/src/components/editor/sound/SoundRecorder.vue index 0574f551a..a49c8479c 100644 --- a/spx-gui/src/components/editor/sound/SoundRecorder.vue +++ b/spx-gui/src/components/editor/sound/SoundRecorder.vue @@ -6,6 +6,7 @@ v-if="recording || audioBlob" ref="wavesurferRef" v-model:range="audioRange" + recording :gain="gain" @init="handleWaveSurferInit" /> @@ -99,6 +100,7 @@ import { RecordPlugin } from '@/utils/wavesurfer-record' import VolumeSlider from './VolumeSlider.vue' import WavesurferWithRange from './WavesurferWithRange.vue' import type WaveSurfer from 'wavesurfer.js' +import { toWav } from '@/utils/audio' const emit = defineEmits<{ saved: [Sound] @@ -162,10 +164,13 @@ const stopRecording = () => { } const saveRecording = async () => { - if (!wavesurferRef.value) return - const wav = await wavesurferRef.value.exportWav() + if (!wavesurferRef.value || !audioBlob.value) return + const wav = await toWav(await audioBlob.value.arrayBuffer()) - const file = fromBlob(`Recording_${dayjs().format('YYYY-MM-DD_HH:mm:ss')}.webm`, wav) + const file = fromBlob( + `Recording_${dayjs().format('YYYY-MM-DD_HH:mm:ss')}.wav`, + new Blob([wav], { type: 'audio/wav' }) + ) const sound = await Sound.create('recording', file) const action = { name: { en: 'Add recording', zh: '添加录音' } } await editorCtx.project.history.doAction(action, () => editorCtx.project.addSound(sound)) diff --git a/spx-gui/src/components/editor/sound/WavesurferWithRange.vue b/spx-gui/src/components/editor/sound/WavesurferWithRange.vue index fc86e2212..cfae925d2 100644 --- a/spx-gui/src/components/editor/sound/WavesurferWithRange.vue +++ b/spx-gui/src/components/editor/sound/WavesurferWithRange.vue @@ -16,6 +16,12 @@ const props = defineProps<{ audioUrl?: string | null range: { left: number; right: number } gain: number + // When true, the component will render the waveform differently. + // It enables the cache and updates the waveform in real-time. + // TODO: Currently in the recording mode, the waveform will be rendered + // in different samples than in the normal mode, resulting different smoothness. + // We may fix this in the future. + recording?: boolean }>() const emit = defineEmits<{ @@ -75,7 +81,8 @@ const wavesurferDiv = ref() const wavesurferRef = useWavesurfer( () => wavesurferDiv.value, - () => props.gain + () => props.gain, + props.recording ) // we assume that wavesurferDiv.value will not change diff --git a/spx-gui/src/components/editor/sound/wavesurfer.ts b/spx-gui/src/components/editor/sound/wavesurfer.ts index cc7d9810b..e7744bd31 100644 --- a/spx-gui/src/components/editor/sound/wavesurfer.ts +++ b/spx-gui/src/components/editor/sound/wavesurfer.ts @@ -7,7 +7,11 @@ import WaveSurfer from 'wavesurfer.js' import { useUIVariables } from '@/components/ui' import { getAudioContext } from '@/utils/audio' -export function useWavesurfer(container: () => HTMLElement | undefined, gain: () => number) { +export function useWavesurfer( + container: () => HTMLElement | undefined, + gain: () => number, + recording: boolean +) { const uiVariables = useUIVariables() const wavesurfer = ref(null) @@ -35,6 +39,17 @@ export function useWavesurfer(container: () => HTMLElement | undefined, gain: () gainNode_.gain.value = gain() gainNode = gainNode_ + /** + * Cache for averaged data. Only used for recording. + * As we expect the `WaveSurfer` to be destroyed and recreated on every recording, + * we don't need to reset the cache when the recording stops. + */ + let cache: { + averagedData: number[] + originalLength: number + blockSize: number + } + wavesurfer.value = new WaveSurfer({ interact: false, container: newContainer, @@ -46,54 +61,124 @@ export function useWavesurfer(container: () => HTMLElement | undefined, gain: () normalize: true, media: audioElement, renderFunction: (peaks: (Float32Array | number[])[], ctx: CanvasRenderingContext2D): void => { - // TODO: Better drawing algorithm to reduce flashing? - const smoothAndDrawChannel = (channel: Float32Array, vScale: number) => { - const { width, height } = ctx.canvas - const halfHeight = height / 2 - const numPoints = Math.floor(width / 5) - const blockSize = Math.floor(channel.length / numPoints) - const smoothedData = new Float32Array(numPoints) - - // Smooth the data by averaging blocks - for (let i = 0; i < numPoints; i++) { - let sum = 0 - for (let j = 0; j < blockSize; j++) { - sum += Math.abs(channel[i * blockSize + j]) + try { + const halfHeight = ctx.canvas.height / 2 + + const averageBlock = (data: number[] | Float32Array, blockSize: number): number[] => { + // Check if we can use the cached data + if ( + recording && + cache && + cache.blockSize === blockSize && + cache.originalLength <= data.length + ) { + const newBlocks = + Math.floor(data.length / blockSize) - Math.floor(cache.originalLength / blockSize) + + // If there are new blocks to process + if (newBlocks > 0) { + const newAveragedData = cache.averagedData.slice() + const startIndex = cache.originalLength + for (let i = 0; i < newBlocks; i++) { + let sum = 0 + for (let j = 0; j < blockSize; j++) { + const index = startIndex + i * blockSize + j + sum += Math.max(0, data[index]) + } + newAveragedData.push(sum / blockSize) + } + + cache = { + averagedData: newAveragedData, + originalLength: data.length, + blockSize + } + return newAveragedData + } + + // If no new blocks to process, return cached data + return cache.averagedData } - smoothedData[i] = sum / blockSize - } - // Draw with bezier curves - ctx.beginPath() - ctx.moveTo(0, halfHeight) + // Calculate new averaged data if cache is not valid or not present + const averagedData: number[] = new Array(Math.floor(data.length / blockSize)) + for (let i = 0; i < averagedData.length; i++) { + let sum = 0 + for (let j = 0; j < blockSize; j++) { + const index = i * blockSize + j + sum += Math.max(0, data[index]) + } + averagedData[i] = sum / blockSize + } - for (let i = 1; i < smoothedData.length; i++) { - const prevX = (i - 1) * (width / numPoints) - const currX = i * (width / numPoints) - const midX = (prevX + currX) / 2 - const prevY = halfHeight + smoothedData[i - 1] * halfHeight * vScale - const currY = halfHeight + smoothedData[i] * halfHeight * vScale + // Update cache with new averaged data + cache = { + averagedData, + originalLength: data.length, + blockSize + } - // Use a quadratic bezier curve to the middle of the interval for a smoother line - ctx.quadraticCurveTo(prevX, prevY, midX, (prevY + currY) / 2) - ctx.quadraticCurveTo(midX, (prevY + currY) / 2, currX, currY) + return averagedData } - ctx.lineTo(width, halfHeight) - ctx.strokeStyle = uiVariables.color.sound[400] - ctx.stroke() - ctx.closePath() - ctx.fillStyle = uiVariables.color.sound[400] - ctx.fill() - } + const drawSmoothCurve = ( + ctx: CanvasRenderingContext2D, + points: number[], + getPoint: (index: number) => number + ) => { + const segmentLength = ctx.canvas.width / (points.length - 1) - const channel = Array.isArray(peaks[0]) ? new Float32Array(peaks[0] as number[]) : peaks[0] + ctx.beginPath() + ctx.moveTo(0, halfHeight) - const scale = gain() * 5 + for (let i = 0; i < points.length - 2; i++) { + const xc = (i * segmentLength + (i + 1) * segmentLength) / 2 + const yc = (getPoint(i) + getPoint(i + 1)) / 2 + ctx.quadraticCurveTo(i * segmentLength, getPoint(i), xc, yc) + } - // Only one channel is assumed, render it twice (mirrored) - smoothAndDrawChannel(channel, scale) // Upper part - smoothAndDrawChannel(channel, -scale) // Lower part (mirrored) + ctx.quadraticCurveTo( + (points.length - 2) * segmentLength, + getPoint(points.length - 2), + ctx.canvas.width, + getPoint(points.length - 1) + ) + + ctx.lineTo(ctx.canvas.width, halfHeight) + + ctx.strokeStyle = uiVariables.color.sound[400] + ctx.lineWidth = 2 + ctx.stroke() + ctx.closePath() + ctx.fillStyle = uiVariables.color.sound[400] + ctx.fill() + } + + const channel = peaks[0] + + const scale = gain() * 1200 + + // TODO: For now, blockSize is fixed to 2000 for longer recordings + // to address the issue of slow update. This should be optimized + // in the future. + const blockSize = channel.length > 200000 ? 2000 : channel.length > 100000 ? 1000 : 500 + + const smoothedChannel = averageBlock(channel, blockSize) + + drawSmoothCurve( + ctx, + smoothedChannel, + (index: number) => smoothedChannel[index] * scale + halfHeight + ) + drawSmoothCurve( + ctx, + smoothedChannel, + (index: number) => -smoothedChannel[index] * scale + halfHeight + ) + } catch (e) { + // wavesurfer does not log errors so we do it ourselves + console.error(e) + } } }) diff --git a/spx-gui/src/utils/wavesurfer-record.ts b/spx-gui/src/utils/wavesurfer-record.ts index 5766cfe5c..517bb99a8 100644 --- a/spx-gui/src/utils/wavesurfer-record.ts +++ b/spx-gui/src/utils/wavesurfer-record.ts @@ -46,7 +46,6 @@ const findSupportedMimeType = () => export class RecordPlugin extends BasePlugin { private stream: MediaStream | null = null private mediaRecorder: MediaRecorder | null = null - private dataWindow: Float32Array | null = null private isWaveformPaused = false private originalOptions: { cursorWidth: number; interact: boolean } | undefined private timer: Timer @@ -82,10 +81,11 @@ export class RecordPlugin extends BasePlugin