Skip to content

Commit

Permalink
WIP improvement(content): improve sound quality on stretched parts (m…
Browse files Browse the repository at this point in the history 
…arginBefore)

Set `element.preservesPitch = false` and do pitch correction in the extension code only
so we don't have two pitch correctors shifting pitch in opposite directions
  • Loading branch information
@WofWca
WofWca committed Feb 26, 2021
1 parent d9e9fd5 commit 97bade8
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 110 deletions.
13 changes: 10 additions & 3 deletions src/content/Controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,12 @@ export default class Controller {
element.playbackRate = elementPlaybackRateBeforeInitialization;
});

(element as any).preservesPitch = false;
const elementPreservesPitchBeforeInitialization = (element as any).preservesPitch;
this._onDestroyCallbacks.push(() => {
(element as any).preservesPitch = elementPreservesPitchBeforeInitialization;
});

this._elementVolumeCache = element.volume;
const onElementVolumeChange = () => this._elementVolumeCache = element.volume;
element.addEventListener('volumechange', onElementVolumeChange);
Expand Down Expand Up @@ -156,10 +162,10 @@ export default class Controller {
const volumeFilter = new AudioWorkletNode(ctx, 'VolumeFilter', {
outputChannelCount: [1],
processorOptions: {
maxSmoothingWindowLength: 0.03,
maxSmoothingWindowLength: 0.3,
},
parameterData: {
smoothingWindowLength: 0.03, // TODO make a setting out of it.
smoothingWindowLength: 0.3, // TODO make a setting out of it.
},
});
audioWorklets.push(volumeFilter);
Expand Down Expand Up @@ -409,12 +415,13 @@ export default class Controller {
lastActualPlaybackRateChange: this._lastActualPlaybackRateChange,
elementVolume: this._elementVolumeCache,
totalOutputDelay: this._lookahead && this._stretcher
? getTotalDelay(this._lookahead.delayTime.value, this._stretcher.delayNode.delayTime.value)
? getTotalDelay(this._lookahead.delayTime.value, this._stretcher.stretcherNode.delayTime.value)
: 0,
// TODO also log `interruptLastScheduledStretch` calls.
// lastScheduledStretch: this._stretcher.lastScheduledStretch,
lastScheduledStretchInputTime:
this._stretcher?.lastScheduledStretch && stretchToInputTime(this._stretcher.lastScheduledStretch),
pitch: this._stretcher?.pitchCorrector.pitch, // TODO remove
};
}
}
215 changes: 108 additions & 107 deletions src/content/PitchPreservingStretcherNode.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,20 @@ import type { Time, StretchInfo } from '@/helpers';
import { assert } from '@/helpers';


// TODO make it into a setting?
const CROSS_FADE_DURATION = 0.01;
function speedChangeMultiplierToSemitones(m: number) {
return -12 * Math.log2(m);
}

// // TODO make it into a setting?
// const CROSS_FADE_DURATION = 0.01;

type PitchSetting = 'slowdown' | 'speedup' | 'normal';

export default class PitchPreservingStretcherNode {
// 2 pitch shifts and 3 gains because `.pitch` of `PitchShift` is not an AudioParam, therefore doesn't support
// scheduling.
speedUpGain: GainNode;
slowDownGain: GainNode;
normalSpeedGain: GainNode;
speedUpPitchShift: PitchShift;
slowDownPitchShift: PitchShift;
originalPitchCompensationDelay: DelayNode;
delayNode: DelayNode;
pitchCorrector: PitchShift;
stretcherNode: DelayNode;
lastScheduledStretch?: StretchInfo & { speedupOrSlowdown: 'speedup' | 'slowdown' };
lastElementSpeedChangeAtInputTime?: Time;

Expand All @@ -46,51 +45,38 @@ export default class PitchPreservingStretcherNode {
}),
private getLookaheadDelay: () => number,
) {
this.speedUpGain = context.createGain();
this.slowDownGain = context.createGain();
this.normalSpeedGain = context.createGain();
this.speedUpGain.gain.value = 0;
this.slowDownGain.gain.value = 0
this.normalSpeedGain.gain.value = 1;

toneSetContext(context);
this.speedUpPitchShift = new PitchShift();
this.slowDownPitchShift = new PitchShift();
this.pitchCorrector = new PitchShift();

// Why this value?
// 1. Withing the range recommended by Tone.js documentation:
// https://tonejs.github.io/docs/14.7.39/PitchShift#windowSize
// 2. I played around with it a bit and this sounded best for me.
// TODO make it into a setting?
const windowSize = 0.1;
this.speedUpPitchShift.windowSize = windowSize;
this.slowDownPitchShift.windowSize = windowSize;

// `PitchShift` nodes introduce a delay:
// https://github.com/Tonejs/Tone.js/blob/ed0d3b08be2b95220fffe7cce7eac32a5b77580e/Tone/effect/PitchShift.ts#L97-L117
// This is so their outputs and original pitch outputs are in sync.
const averagePitchShiftDelay = windowSize / 2;
this.originalPitchCompensationDelay = context.createDelay(averagePitchShiftDelay);
this.originalPitchCompensationDelay.delayTime.value = averagePitchShiftDelay;
this.delayNode = context.createDelay(maxDelay);
this.delayNode.delayTime.value = initialDelay;

this.delayNode.connect(this.speedUpGain);
this.delayNode.connect(this.slowDownGain);
this.delayNode.connect(this.normalSpeedGain);

ToneConnect(this.speedUpGain, this.speedUpPitchShift);
ToneConnect(this.slowDownGain, this.slowDownPitchShift);
this.normalSpeedGain.connect(this.originalPitchCompensationDelay);
const windowSize = 0.05;
this.pitchCorrector.windowSize = windowSize;

this.stretcherNode = context.createDelay(maxDelay);
this.stretcherNode.delayTime.value = initialDelay;

// Why in that order and not the other way?
// Because currently (and, perhaps in general) PitchShift nodes add delay and the smaller the amount of things that
// introduce delay the easier it is to calculate stuff.
//
// But maybe it makes sense that the delay node may only reduce the amount of information the signal carries
// (e.i. when downsampling) and maybe the pitchCorrector node will produce better results on a signal with more
// information.
// Also I think we'd only be required to change the `getTotalDelay` function.
// But at the end of the day it's better to just check out how real programmers implement pitch-preserving time
// stretching, the task is pretty trivial. TODO.
ToneConnect(this.stretcherNode, this.pitchCorrector);
}

connectInputFrom(sourceNode: AudioNode): void {
sourceNode.connect(this.delayNode);
sourceNode.connect(this.stretcherNode);
}
connectOutputTo(destinationNode: AudioNode): void {
this.speedUpPitchShift.connect(destinationNode)
this.slowDownPitchShift.connect(destinationNode)
this.originalPitchCompensationDelay.connect(destinationNode)
this.pitchCorrector.connect(destinationNode)
}

onSilenceEnd(eventTime: Time): void {
Expand Down Expand Up @@ -184,6 +170,12 @@ export default class PitchPreservingStretcherNode {
// A.k.a. `marginBeforePartAtSilenceSpeedStartOutputTime + silenceSpeedPartStretchedDuration`
const endTime = eventTime + getTotalDelay(lookaheadDelay, finalStretcherDelay);
this.stretch(startValue, endValue, startTime, endTime);
const speedChangeMultiplier = settings.soundedSpeed;
// console.warn('silenceEnd, change pitch in (ms)', (startTime - this.context.currentTime) * 1000, startTime - eventTime);
// TODO replace `setTimeout` with `setValueAtTime`.
setTimeout(() => {
this.pitchCorrector.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier);
}, (startTime - this.context.currentTime) * 1000);
// if (isLogging(this)) {
// this._log({ type: 'stretch', lastScheduledStretch: this.lastScheduledStretch });
// }
Expand Down Expand Up @@ -212,75 +204,80 @@ export default class PitchPreservingStretcherNode {
startTime,
endTime,
);
const speedChangeMultiplier = settings.silenceSpeed;
// console.warn('silenceStart, change pitch in (ms)', (startTime - this.context.currentTime) * 1000, startTime - eventTime);
setTimeout(() => {
this.pitchCorrector.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier);
}, (startTime - this.context.currentTime) * 1000);

// if (isLogging(this)) {
// this._log({ type: 'reset', lastScheduledStretch: this.lastScheduledStretch });
// }
}

private setOutputPitchAt(pitchSetting: PitchSetting, time: Time, oldPitchSetting: PitchSetting) {
if (process.env.NODE_ENV !== 'production') {
if (!['slowdown', 'speedup', 'normal'].includes(pitchSetting)) {
// TODO replace with TypeScript?
throw new Error(`Invalid pitchSetting "${pitchSetting}"`);
}
if (pitchSetting === oldPitchSetting) {
console.warn(`New pitchSetting is the same as oldPitchSetting: ${pitchSetting}`);
}
if (
pitchSetting === 'speedup' && oldPitchSetting === 'slowdown'
|| pitchSetting === 'slowdown' && oldPitchSetting === 'speedup'
) {
console.warn(`Switching from ${oldPitchSetting} to ${pitchSetting} immediately. It hasn't been happening`
+ 'at the time of writing, so not sure if it works as intended.');
}
}

// Cross-fade to avoid glitches.
// TODO make sure the cross-fade behaves well in cases when `interruptLastScheduledStretch()` is called.
const crossFadeHalfDuration = CROSS_FADE_DURATION / 2;
const crossFadeStart = time - crossFadeHalfDuration;
const crossFadeEnd = time + crossFadeHalfDuration;
const pitchSettingToItsGainNode = {
'normal': this.normalSpeedGain,
'speedup': this.speedUpGain,
'slowdown': this.slowDownGain,
};
const fromNode = pitchSettingToItsGainNode[oldPitchSetting];
const toNode = pitchSettingToItsGainNode[pitchSetting];
fromNode.gain.setValueAtTime(1, crossFadeStart);
toNode.gain.setValueAtTime(0, crossFadeStart);
fromNode.gain.linearRampToValueAtTime(0, crossFadeEnd);
toNode.gain.linearRampToValueAtTime(1, crossFadeEnd);
}
// private setOutputPitchAt(pitchSetting: PitchSetting, time: Time, oldPitchSetting: PitchSetting) {
// if (process.env.NODE_ENV !== 'production') {
// if (!['slowdown', 'speedup', 'normal'].includes(pitchSetting)) {
// // TODO replace with TypeScript?
// throw new Error(`Invalid pitchSetting "${pitchSetting}"`);
// }
// if (pitchSetting === oldPitchSetting) {
// console.warn(`New pitchSetting is the same as oldPitchSetting: ${pitchSetting}`);
// }
// if (
// pitchSetting === 'speedup' && oldPitchSetting === 'slowdown'
// || pitchSetting === 'slowdown' && oldPitchSetting === 'speedup'
// ) {
// console.warn(`Switching from ${oldPitchSetting} to ${pitchSetting} immediately. It hasn't been happening`
// + 'at the time of writing, so not sure if it works as intended.');
// }
// }

// // Cross-fade to avoid glitches.
// // TODO make sure the cross-fade behaves well in cases when `interruptLastScheduledStretch()` is called.
// const crossFadeHalfDuration = CROSS_FADE_DURATION / 2;
// const crossFadeStart = time - crossFadeHalfDuration;
// const crossFadeEnd = time + crossFadeHalfDuration;
// const pitchSettingToItsGainNode = {
// 'normal': this.normalSpeedGain,
// 'speedup': this.speedUpGain,
// 'slowdown': this.slowDownGain,
// };
// const fromNode = pitchSettingToItsGainNode[oldPitchSetting];
// const toNode = pitchSettingToItsGainNode[pitchSetting];
// fromNode.gain.setValueAtTime(1, crossFadeStart);
// toNode.gain.setValueAtTime(0, crossFadeStart);
// fromNode.gain.linearRampToValueAtTime(0, crossFadeEnd);
// toNode.gain.linearRampToValueAtTime(1, crossFadeEnd);
// }

stretch(startValue: Time, endValue: Time, startTime: Time, endTime: Time): void {
if (startValue === endValue) {
return;
}

this.delayNode.delayTime
this.stretcherNode.delayTime
.setValueAtTime(startValue, startTime)
.linearRampToValueAtTime(endValue, endTime);
const speedupOrSlowdown = endValue > startValue ? 'slowdown' : 'speedup';
this.setOutputPitchAt(
speedupOrSlowdown,
startTime,
'normal'
);
this.setOutputPitchAt('normal', endTime, speedupOrSlowdown);
// this.setOutputPitchAt(
// speedupOrSlowdown,
// startTime,
// 'normal'
// );
// this.setOutputPitchAt('normal', endTime, speedupOrSlowdown);

const speedChangeMultiplier = getStretchSpeedChangeMultiplier({ startValue, endValue, startTime, endTime });
// Acutally we only need to do this when the user changes settings.
setTimeout(() => {
function speedChangeMultiplierToSemitones(m: number) {
return -12 * Math.log2(m);
}
const node = speedupOrSlowdown === 'speedup'
? this.speedUpPitchShift
: this.slowDownPitchShift;
node.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier);
}, startTime - this.context.currentTime);
// const speedChangeMultiplier = getStretchSpeedChangeMultiplier({ startValue, endValue, startTime, endTime });
// // Acutally we only need to do this when the user changes settings.
// setTimeout(() => {
// function speedChangeMultiplierToSemitones(m: number) {
// return -12 * Math.log2(m);
// }
// const node = speedupOrSlowdown === 'speedup'
// ? this.speedUpPitchShift
// : this.slowDownPitchShift;
// node.pitch = speedChangeMultiplierToSemitones(speedChangeMultiplier);
// }, startTime - this.context.currentTime);

this.lastScheduledStretch = {
startValue,
Expand All @@ -299,35 +296,39 @@ export default class PitchPreservingStretcherNode {
assert(this.lastScheduledStretch, 'Called `interruptLastScheduledStretch`, but no stretch has been scheduled '
+ 'yet');
// We don't need to specify the start time since it has been scheduled before in the `stretch` method
this.delayNode.delayTime
this.stretcherNode.delayTime
.cancelAndHoldAtTime(interruptAtTime)
.linearRampToValueAtTime(interruptAtTimeValue, interruptAtTime);

const allGainNodes = [
this.speedUpGain,
this.slowDownGain,
this.normalSpeedGain,
];
for (const node of allGainNodes) {
node.gain.cancelAndHoldAtTime(interruptAtTime);
}
this.setOutputPitchAt('normal', interruptAtTime, this.lastScheduledStretch.speedupOrSlowdown);
// const allGainNodes = [
// this.speedUpGain,
// this.slowDownGain,
// this.normalSpeedGain,
// ];
// for (const node of allGainNodes) {
// node.gain.cancelAndHoldAtTime(interruptAtTime);
// }
// this.setOutputPitchAt('normal', interruptAtTime, this.lastScheduledStretch.speedupOrSlowdown);
}

// setDelay(value: Time): void {
// this.delayNode.delayTime.value = value;
// }
onSettingsUpdate(): void {
const newSettings = this.getSettings();
this.delayNode.delayTime.value = getStretcherSoundedDelay(
this.stretcherNode.delayTime.value = getStretcherSoundedDelay(
newSettings.marginBefore,
newSettings.soundedSpeed,
newSettings.silenceSpeed
);
// Just a dumb temporary workaround so pitch is updated when we change soundedSpeed. TODO
setTimeout(() => {
this.pitchCorrector.pitch = speedChangeMultiplierToSemitones(newSettings.soundedSpeed);
}, this.getLookaheadDelay() * 1000);
}

destroy(): void {
const toneAudioNodes = [this.speedUpPitchShift, this.slowDownPitchShift];
const toneAudioNodes = [this.pitchCorrector];
for (const node of toneAudioNodes) {
node.dispose();
}
Expand Down
12 changes: 12 additions & 0 deletions src/popup/Chart.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
let stretchSeries: TimeSeries;
let shrinkSeries: TimeSeries;
let pitchSeries: TimeSeries;
const bestYAxisRelativeVolumeThreshold = 1/6;
let chartMaxValue: number;
function setMaxChartValueToBest() {
Expand Down Expand Up @@ -100,6 +102,7 @@
volumeThresholdSeries = new TimeSeries();
stretchSeries = new TimeSeries();
shrinkSeries = new TimeSeries();
pitchSeries = new TimeSeries();
// Order determines z-index
const soundedSpeedColor = 'rgba(0, 255, 0, 0.3)';
const silenceSpeedColor = 'rgba(255, 0, 0, 0.3)';
Expand Down Expand Up @@ -133,6 +136,11 @@
strokeStyle: '#f44',
fillStyle: 'transparent',
});
smoothie.addTimeSeries(pitchSeries, {
lineWidth: 1,
strokeStyle: '#808',
fillStyle: 'transparent',
});
const canvasContext = canvasEl.getContext('2d')!;
(function drawAndScheduleAnother() {
Expand Down Expand Up @@ -232,6 +240,10 @@
volumeSeries.append(sToMs(r.unixTime), r.inputVolume)
})();
const pitchChartVal = -1 * (r.pitch ?? 0) / 500;
// pitchSeries.append(sToMs(r.unixTime), pitchChartVal);
pitchSeries.append(sToMs(r.unixTime - r.totalOutputDelay), pitchChartVal);
function arePlaybackRateChangeObjectsEqual(
a: TelemetryRecord['lastActualPlaybackRateChange'] | undefined,
b: TelemetryRecord['lastActualPlaybackRateChange'] | undefined,
Expand Down

0 comments on commit 97bade8

Please sign in to comment.