Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: [Spectrogram] fix cropping and scaling issues #3796

Merged
merged 2 commits into from
Dec 23, 2024
Merged

Fix: [Spectrogram] fix cropping and scaling issues #3796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2b9bb14
Fix(Spectrogram): fix cropping and scaling issues
jonom Dec 18, 2024
ba25bfe
Fix(Spectrogram): fix mel scale labels
jonom Dec 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion examples/spectrogram.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const ws = WaveSurfer.create({
waveColor: 'rgb(200, 0, 200)',
progressColor: 'rgb(100, 0, 100)',
url: '/examples/audio/audio.wav',
sampleRate: 22050,
sampleRate: 44100,
})

// Initialize the Spectrogram plugin
Expand All @@ -18,6 +18,11 @@ ws.registerPlugin(
labels: true,
height: 200,
splitChannels: true,
scale: 'mel', // or 'linear'
frequencyMax: 8000,
frequencyMin: 0,
fftSamples: 1024,
labelsBackground: 'rgba(0, 0, 0, 0.1)',
}),
)

Expand Down
78 changes: 37 additions & 41 deletions src/plugins/spectrogram.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ export type SpectrogramPluginOptions = {
alpha?: number
/** Min frequency to scale spectrogram. */
frequencyMin?: number
/** Max frequency to scale spectrogram. Set this to samplerate/2 to draw whole range of spectrogram. */
/** Max frequency to scale spectrogram. Set this to samplerate/4 to draw whole range of spectrogram. */
frequencyMax?: number
/**
* Based on: https://manual.audacityteam.org/man/spectrogram_settings.html
Expand Down Expand Up @@ -336,7 +336,8 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
}
}
this.fftSamples = options.fftSamples || 512
this.noverlap = options.noverlap || this.fftSamples * 0.75
this.height = options.height || 200
this.noverlap = options.noverlap || null // Will be calculated later based on canvas size
this.windowFunc = options.windowFunc || 'hann'
this.alpha = options.alpha

Expand All @@ -348,14 +349,7 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
this.gainDB = options.gainDB || 20
this.rangeDB = options.rangeDB || 80
this.scale = options.scale || 'mel'
this.numMelFilters = options.numMelFilters || this.fftSamples / 8
if (this.scale == 'mel') {
this.height = options.height || this.numMelFilters
this.height = Math.min(this.height, this.numMelFilters)
} else {
this.height = options.height || this.fftSamples / 2
this.height = Math.min(this.height, this.fftSamples / 2)
}
this.numMelFilters = this.fftSamples / 4

this.createWrapper()
this.createCanvas()
Expand Down Expand Up @@ -466,14 +460,13 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
// Set the height to fit all channels
this.wrapper.style.height = this.height * frequenciesData.length + 'px'

this.width = this.wavesurfer.getWrapper().offsetWidth
this.canvas.width = this.width
this.canvas.width = this.getWidth()
this.canvas.height = this.height * frequenciesData.length

const spectrCc = this.spectrCc
const height = this.height
const width = this.width
const freqFrom = this.buffer.sampleRate / 2
const width = this.getWidth()
const freqFrom = this.buffer.sampleRate / 4
const freqMin = this.frequencyMin
const freqMax = this.frequencyMax

Expand All @@ -484,12 +477,13 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
for (let c = 0; c < frequenciesData.length; c++) {
// for each channel
const pixels = this.resample(frequenciesData[c])
const imageData = new ImageData(width, height)
const bitmapHeight = pixels[0].length
const imageData = new ImageData(width, bitmapHeight)
Comment on lines +480 to +481
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Check potential out-of-bounds index.
Accessing “pixels[0]” might fail if “pixels” is empty. Ensure “frequenciesData[c]” is never empty or handle errors gracefully.


for (let i = 0; i < pixels.length; i++) {
for (let j = 0; j < pixels[i].length; j++) {
const colorMap = this.colorMap[pixels[i][j]]
const redIndex = ((height - j) * width + i) * 4
const redIndex = ((bitmapHeight / 2 - j) * width + i) * 4
imageData.data[redIndex] = colorMap[0] * 255
imageData.data[redIndex + 1] = colorMap[1] * 255
imageData.data[redIndex + 2] = colorMap[2] * 255
Expand All @@ -498,18 +492,14 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
}

// scale and stack spectrograms
createImageBitmap(imageData).then((renderer) => {
spectrCc.drawImage(
renderer,
0,
height * (1 - freqMax / freqFrom), // source x, y
width,
(height * (freqMax - freqMin)) / freqFrom, // source width, height
0,
height * c, // destination x, y
width,
height, // destination width, height
)
createImageBitmap(
imageData,
0,
Math.round(bitmapHeight - bitmapHeight * (freqMax / freqFrom)) / 2,
width,
Math.round(bitmapHeight * ((freqMax - freqMin) / freqFrom)),
).then((bitmap) => {
jonom marked this conversation as resolved.
Show resolved Hide resolved
spectrCc.drawImage(bitmap, 0, height * c, width, height * 2)
})
}

Expand Down Expand Up @@ -562,11 +552,15 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
return melSpectrum
}

private getWidth() {
return this.wavesurfer.getWrapper().offsetWidth
}

private getFrequencies(buffer: AudioBuffer): number[] {
const fftSamples = this.fftSamples
const channels = this.options.splitChannels ?? this.wavesurfer?.options.splitChannels ? buffer.numberOfChannels : 1

this.frequencyMax = this.frequencyMax || buffer.sampleRate / 2
this.frequencyMax = this.frequencyMax || buffer.sampleRate / 4

if (!buffer) return

Expand Down Expand Up @@ -680,21 +674,23 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
ctx.textAlign = textAlign
ctx.textBaseline = 'middle'

const freq = freqStart + step * i
let freq
if (this.scale == 'mel') {
const melMin = this.hzToMel(0)
const melMax = this.hzToMel(this.frequencyMax)
freq = this.melToHz(melMin + (i / labelIndex) * (melMax - melMin))
} else {
freq = freqStart + step * i
}

const label = this.freqType(freq)
const units = this.unitType(freq)
const yLabelOffset = 2
const x = 16
let y
let y = (1 + c) * getMaxY - (i / labelIndex) * getMaxY

// Make sure label remains in view
y = Math.min(Math.max(y, c * getMaxY + 10), (1 + c) * getMaxY - 10)

if (i == 0) {
y = (1 + c) * getMaxY + i - 10
} else {
y = (1 + c) * getMaxY - i * 50 + yLabelOffset
}
if (this.scale == 'mel' && freq != 0) {
y = y * this.hzToMel(freq) / freq
}
// unit label
ctx.fillStyle = textColorUnit
ctx.font = fontSizeUnit + ' ' + fontType
Expand All @@ -708,7 +704,7 @@ class SpectrogramPlugin extends BasePlugin<SpectrogramPluginEvents, SpectrogramP
}

private resample(oldMatrix) {
const columnsNumber = this.width
const columnsNumber = this.getWidth()
const newMatrix = []

const oldPiece = 1 / oldMatrix.length
Expand Down
Loading