From 97bade877750dbe682aa7e91a7e11dcd5d3a577d Mon Sep 17 00:00:00 2001 From: WofWca Date: Thu, 25 Feb 2021 16:53:14 +0800 Subject: [PATCH] WIP improvement(content): improve sound quality on stretched parts (marginBefore) Set `element.preservesPitch = false` and do pitch correction in the extension code only so we don't have two pitch correctors shifting pitch in opposite directions --- src/content/Controller.ts | 13 +- src/content/PitchPreservingStretcherNode.ts | 215 ++++++++++---------- src/popup/Chart.svelte | 12 ++ 3 files changed, 130 insertions(+), 110 deletions(-) diff --git a/src/content/Controller.ts b/src/content/Controller.ts index a6f4a875..5873cba8 100644 --- a/src/content/Controller.ts +++ b/src/content/Controller.ts @@ -107,6 +107,12 @@ export default class Controller { element.playbackRate = elementPlaybackRateBeforeInitialization; }); + (element as any).preservesPitch = false; + const elementPreservesPitchBeforeInitialization = (element as any).preservesPitch; + this._onDestroyCallbacks.push(() => { + (element as any).preservesPitch = elementPreservesPitchBeforeInitialization; + }); + this._elementVolumeCache = element.volume; const onElementVolumeChange = () => this._elementVolumeCache = element.volume; element.addEventListener('volumechange', onElementVolumeChange); @@ -156,10 +162,10 @@ export default class Controller { const volumeFilter = new AudioWorkletNode(ctx, 'VolumeFilter', { outputChannelCount: [1], processorOptions: { - maxSmoothingWindowLength: 0.03, + maxSmoothingWindowLength: 0.3, }, parameterData: { - smoothingWindowLength: 0.03, // TODO make a setting out of it. + smoothingWindowLength: 0.3, // TODO make a setting out of it. }, }); audioWorklets.push(volumeFilter); @@ -409,12 +415,13 @@ export default class Controller { lastActualPlaybackRateChange: this._lastActualPlaybackRateChange, elementVolume: this._elementVolumeCache, totalOutputDelay: this._lookahead && this._stretcher - ? getTotalDelay(this._lookahead.delayTime.value, this._stretcher.delayNode.delayTime.value) + ? getTotalDelay(this._lookahead.delayTime.value, this._stretcher.stretcherNode.delayTime.value) : 0, // TODO also log `interruptLastScheduledStretch` calls. // lastScheduledStretch: this._stretcher.lastScheduledStretch, lastScheduledStretchInputTime: this._stretcher?.lastScheduledStretch && stretchToInputTime(this._stretcher.lastScheduledStretch), + pitch: this._stretcher?.pitchCorrector.pitch, // TODO remove }; } } diff --git a/src/content/PitchPreservingStretcherNode.ts b/src/content/PitchPreservingStretcherNode.ts index 358fe849..a8c43f4a 100644 --- a/src/content/PitchPreservingStretcherNode.ts +++ b/src/content/PitchPreservingStretcherNode.ts @@ -16,21 +16,20 @@ import type { Time, StretchInfo } from '@/helpers'; import { assert } from '@/helpers'; -// TODO make it into a setting? -const CROSS_FADE_DURATION = 0.01; +function speedChangeMultiplierToSemitones(m: number) { + return -12 * Math.log2(m); +} + +// // TODO make it into a setting? +// const CROSS_FADE_DURATION = 0.01; type PitchSetting = 'slowdown' | 'speedup' | 'normal'; export default class PitchPreservingStretcherNode { // 2 pitch shifts and 3 gains because `.pitch` of `PitchShift` is not an AudioParam, therefore doesn't support // scheduling. - speedUpGain: GainNode; - slowDownGain: GainNode; - normalSpeedGain: GainNode; - speedUpPitchShift: PitchShift; - slowDownPitchShift: PitchShift; - originalPitchCompensationDelay: DelayNode; - delayNode: DelayNode; + pitchCorrector: PitchShift; + stretcherNode: DelayNode; lastScheduledStretch?: StretchInfo & { speedupOrSlowdown: 'speedup' | 'slowdown' }; lastElementSpeedChangeAtInputTime?: Time; @@ -46,51 +45,38 @@ export default class PitchPreservingStretcherNode { }), private getLookaheadDelay: () => number, ) { - this.speedUpGain = context.createGain(); - this.slowDownGain = context.createGain(); - this.normalSpeedGain = context.createGain(); - this.speedUpGain.gain.value = 0; - this.slowDownGain.gain.value = 0 - this.normalSpeedGain.gain.value = 1; - toneSetContext(context); - this.speedUpPitchShift = new PitchShift(); - this.slowDownPitchShift = new PitchShift(); + this.pitchCorrector = new PitchShift(); // Why this value? // 1. Withing the range recommended by Tone.js documentation: // https://tonejs.github.io/docs/14.7.39/PitchShift#windowSize // 2. I played around with it a bit and this sounded best for me. // TODO make it into a setting? - const windowSize = 0.1; - this.speedUpPitchShift.windowSize = windowSize; - this.slowDownPitchShift.windowSize = windowSize; - - // `PitchShift` nodes introduce a delay: - // https://github.com/Tonejs/Tone.js/blob/ed0d3b08be2b95220fffe7cce7eac32a5b77580e/Tone/effect/PitchShift.ts#L97-L117 - // This is so their outputs and original pitch outputs are in sync. - const averagePitchShiftDelay = windowSize / 2; - this.originalPitchCompensationDelay = context.createDelay(averagePitchShiftDelay); - this.originalPitchCompensationDelay.delayTime.value = averagePitchShiftDelay; - this.delayNode = context.createDelay(maxDelay); - this.delayNode.delayTime.value = initialDelay; - - this.delayNode.connect(this.speedUpGain); - this.delayNode.connect(this.slowDownGain); - this.delayNode.connect(this.normalSpeedGain); - - ToneConnect(this.speedUpGain, this.speedUpPitchShift); - ToneConnect(this.slowDownGain, this.slowDownPitchShift); - this.normalSpeedGain.connect(this.originalPitchCompensationDelay); + const windowSize = 0.05; + this.pitchCorrector.windowSize = windowSize; + + this.stretcherNode = context.createDelay(maxDelay); + this.stretcherNode.delayTime.value = initialDelay; + + // Why in that order and not the other way? + // Because currently (and, perhaps in general) PitchShift nodes add delay and the smaller the amount of things that + // introduce delay the easier it is to calculate stuff. + // + // But maybe it makes sense that the delay node may only reduce the amount of information the signal carries + // (e.i. when downsampling) and maybe the pitchCorrector node will produce better results on a signal with more + // information. + // Also I think we'd only be required to change the `getTotalDelay` function. + // But at the end of the day it's better to just check out how real programmers implement pitch-preserving time + // stretching, the task is pretty trivial. TODO. + ToneConnect(this.stretcherNode, this.pitchCorrector); } connectInputFrom(sourceNode: AudioNode): void { - sourceNode.connect(this.delayNode); + sourceNode.connect(this.stretcherNode); } connectOutputTo(destinationNode: AudioNode): void { - this.speedUpPitchShift.connect(destinationNode) - this.slowDownPitchShift.connect(destinationNode) - this.originalPitchCompensationDelay.connect(destinationNode) + this.pitchCorrector.connect(destinationNode) } onSilenceEnd(eventTime: Time): void { @@ -184,6 +170,12 @@ export default class PitchPreservingStretcherNode { // A.k.a. `marginBeforePartAtSilenceSpeedStartOutputTime + silenceSpeedPartStretchedDuration` const endTime = eventTime + getTotalDelay(lookaheadDelay, finalStretcherDelay); this.stretch(startValue, endValue, startTime, endTime); + const speedChangeMultiplier = settings.soundedSpeed; + // console.warn('silenceEnd, change pitch in (ms)', (startTime - this.context.currentTime) * 1000, startTime - eventTime); + // TODO replace `setTimeout` with `setValueAtTime`. + setTimeout(() => { + this.pitchCorrector.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier); + }, (startTime - this.context.currentTime) * 1000); // if (isLogging(this)) { // this._log({ type: 'stretch', lastScheduledStretch: this.lastScheduledStretch }); // } @@ -212,75 +204,80 @@ export default class PitchPreservingStretcherNode { startTime, endTime, ); + const speedChangeMultiplier = settings.silenceSpeed; + // console.warn('silenceStart, change pitch in (ms)', (startTime - this.context.currentTime) * 1000, startTime - eventTime); + setTimeout(() => { + this.pitchCorrector.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier); + }, (startTime - this.context.currentTime) * 1000); // if (isLogging(this)) { // this._log({ type: 'reset', lastScheduledStretch: this.lastScheduledStretch }); // } } - private setOutputPitchAt(pitchSetting: PitchSetting, time: Time, oldPitchSetting: PitchSetting) { - if (process.env.NODE_ENV !== 'production') { - if (!['slowdown', 'speedup', 'normal'].includes(pitchSetting)) { - // TODO replace with TypeScript? - throw new Error(`Invalid pitchSetting "${pitchSetting}"`); - } - if (pitchSetting === oldPitchSetting) { - console.warn(`New pitchSetting is the same as oldPitchSetting: ${pitchSetting}`); - } - if ( - pitchSetting === 'speedup' && oldPitchSetting === 'slowdown' - || pitchSetting === 'slowdown' && oldPitchSetting === 'speedup' - ) { - console.warn(`Switching from ${oldPitchSetting} to ${pitchSetting} immediately. It hasn't been happening` - + 'at the time of writing, so not sure if it works as intended.'); - } - } - - // Cross-fade to avoid glitches. - // TODO make sure the cross-fade behaves well in cases when `interruptLastScheduledStretch()` is called. - const crossFadeHalfDuration = CROSS_FADE_DURATION / 2; - const crossFadeStart = time - crossFadeHalfDuration; - const crossFadeEnd = time + crossFadeHalfDuration; - const pitchSettingToItsGainNode = { - 'normal': this.normalSpeedGain, - 'speedup': this.speedUpGain, - 'slowdown': this.slowDownGain, - }; - const fromNode = pitchSettingToItsGainNode[oldPitchSetting]; - const toNode = pitchSettingToItsGainNode[pitchSetting]; - fromNode.gain.setValueAtTime(1, crossFadeStart); - toNode.gain.setValueAtTime(0, crossFadeStart); - fromNode.gain.linearRampToValueAtTime(0, crossFadeEnd); - toNode.gain.linearRampToValueAtTime(1, crossFadeEnd); - } + // private setOutputPitchAt(pitchSetting: PitchSetting, time: Time, oldPitchSetting: PitchSetting) { + // if (process.env.NODE_ENV !== 'production') { + // if (!['slowdown', 'speedup', 'normal'].includes(pitchSetting)) { + // // TODO replace with TypeScript? + // throw new Error(`Invalid pitchSetting "${pitchSetting}"`); + // } + // if (pitchSetting === oldPitchSetting) { + // console.warn(`New pitchSetting is the same as oldPitchSetting: ${pitchSetting}`); + // } + // if ( + // pitchSetting === 'speedup' && oldPitchSetting === 'slowdown' + // || pitchSetting === 'slowdown' && oldPitchSetting === 'speedup' + // ) { + // console.warn(`Switching from ${oldPitchSetting} to ${pitchSetting} immediately. It hasn't been happening` + // + 'at the time of writing, so not sure if it works as intended.'); + // } + // } + + // // Cross-fade to avoid glitches. + // // TODO make sure the cross-fade behaves well in cases when `interruptLastScheduledStretch()` is called. + // const crossFadeHalfDuration = CROSS_FADE_DURATION / 2; + // const crossFadeStart = time - crossFadeHalfDuration; + // const crossFadeEnd = time + crossFadeHalfDuration; + // const pitchSettingToItsGainNode = { + // 'normal': this.normalSpeedGain, + // 'speedup': this.speedUpGain, + // 'slowdown': this.slowDownGain, + // }; + // const fromNode = pitchSettingToItsGainNode[oldPitchSetting]; + // const toNode = pitchSettingToItsGainNode[pitchSetting]; + // fromNode.gain.setValueAtTime(1, crossFadeStart); + // toNode.gain.setValueAtTime(0, crossFadeStart); + // fromNode.gain.linearRampToValueAtTime(0, crossFadeEnd); + // toNode.gain.linearRampToValueAtTime(1, crossFadeEnd); + // } stretch(startValue: Time, endValue: Time, startTime: Time, endTime: Time): void { if (startValue === endValue) { return; } - this.delayNode.delayTime + this.stretcherNode.delayTime .setValueAtTime(startValue, startTime) .linearRampToValueAtTime(endValue, endTime); const speedupOrSlowdown = endValue > startValue ? 'slowdown' : 'speedup'; - this.setOutputPitchAt( - speedupOrSlowdown, - startTime, - 'normal' - ); - this.setOutputPitchAt('normal', endTime, speedupOrSlowdown); + // this.setOutputPitchAt( + // speedupOrSlowdown, + // startTime, + // 'normal' + // ); + // this.setOutputPitchAt('normal', endTime, speedupOrSlowdown); - const speedChangeMultiplier = getStretchSpeedChangeMultiplier({ startValue, endValue, startTime, endTime }); - // Acutally we only need to do this when the user changes settings. - setTimeout(() => { - function speedChangeMultiplierToSemitones(m: number) { - return -12 * Math.log2(m); - } - const node = speedupOrSlowdown === 'speedup' - ? this.speedUpPitchShift - : this.slowDownPitchShift; - node.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier); - }, startTime - this.context.currentTime); + // const speedChangeMultiplier = getStretchSpeedChangeMultiplier({ startValue, endValue, startTime, endTime }); + // // Acutally we only need to do this when the user changes settings. + // setTimeout(() => { + // function speedChangeMultiplierToSemitones(m: number) { + // return -12 * Math.log2(m); + // } + // const node = speedupOrSlowdown === 'speedup' + // ? this.speedUpPitchShift + // : this.slowDownPitchShift; + // node.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier); + // }, startTime - this.context.currentTime); this.lastScheduledStretch = { startValue, @@ -299,19 +296,19 @@ export default class PitchPreservingStretcherNode { assert(this.lastScheduledStretch, 'Called `interruptLastScheduledStretch`, but no stretch has been scheduled ' + 'yet'); // We don't need to specify the start time since it has been scheduled before in the `stretch` method - this.delayNode.delayTime + this.stretcherNode.delayTime .cancelAndHoldAtTime(interruptAtTime) .linearRampToValueAtTime(interruptAtTimeValue, interruptAtTime); - const allGainNodes = [ - this.speedUpGain, - this.slowDownGain, - this.normalSpeedGain, - ]; - for (const node of allGainNodes) { - node.gain.cancelAndHoldAtTime(interruptAtTime); - } - this.setOutputPitchAt('normal', interruptAtTime, this.lastScheduledStretch.speedupOrSlowdown); + // const allGainNodes = [ + // this.speedUpGain, + // this.slowDownGain, + // this.normalSpeedGain, + // ]; + // for (const node of allGainNodes) { + // node.gain.cancelAndHoldAtTime(interruptAtTime); + // } + // this.setOutputPitchAt('normal', interruptAtTime, this.lastScheduledStretch.speedupOrSlowdown); } // setDelay(value: Time): void { @@ -319,15 +316,19 @@ export default class PitchPreservingStretcherNode { // } onSettingsUpdate(): void { const newSettings = this.getSettings(); - this.delayNode.delayTime.value = getStretcherSoundedDelay( + this.stretcherNode.delayTime.value = getStretcherSoundedDelay( newSettings.marginBefore, newSettings.soundedSpeed, newSettings.silenceSpeed ); + // Just a dumb temporary workaround so pitch is updated when we change soundedSpeed. TODO + setTimeout(() => { + this.pitchCorrector.pitch = speedChangeMultiplierToSemitones(newSettings.soundedSpeed); + }, this.getLookaheadDelay() * 1000); } destroy(): void { - const toneAudioNodes = [this.speedUpPitchShift, this.slowDownPitchShift]; + const toneAudioNodes = [this.pitchCorrector]; for (const node of toneAudioNodes) { node.dispose(); } diff --git a/src/popup/Chart.svelte b/src/popup/Chart.svelte index 6f6e848f..cbb06055 100644 --- a/src/popup/Chart.svelte +++ b/src/popup/Chart.svelte @@ -30,6 +30,8 @@ let stretchSeries: TimeSeries; let shrinkSeries: TimeSeries; + let pitchSeries: TimeSeries; + const bestYAxisRelativeVolumeThreshold = 1/6; let chartMaxValue: number; function setMaxChartValueToBest() { @@ -100,6 +102,7 @@ volumeThresholdSeries = new TimeSeries(); stretchSeries = new TimeSeries(); shrinkSeries = new TimeSeries(); + pitchSeries = new TimeSeries(); // Order determines z-index const soundedSpeedColor = 'rgba(0, 255, 0, 0.3)'; const silenceSpeedColor = 'rgba(255, 0, 0, 0.3)'; @@ -133,6 +136,11 @@ strokeStyle: '#f44', fillStyle: 'transparent', }); + smoothie.addTimeSeries(pitchSeries, { + lineWidth: 1, + strokeStyle: '#808', + fillStyle: 'transparent', + }); const canvasContext = canvasEl.getContext('2d')!; (function drawAndScheduleAnother() { @@ -232,6 +240,10 @@ volumeSeries.append(sToMs(r.unixTime), r.inputVolume) })(); + const pitchChartVal = -1 * (r.pitch ?? 0) / 500; + // pitchSeries.append(sToMs(r.unixTime), pitchChartVal); + pitchSeries.append(sToMs(r.unixTime - r.totalOutputDelay), pitchChartVal); + function arePlaybackRateChangeObjectsEqual( a: TelemetryRecord['lastActualPlaybackRateChange'] | undefined, b: TelemetryRecord['lastActualPlaybackRateChange'] | undefined,