From f7739477cae647c5fc9c9ae9572472a541a834e8 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sat, 29 Jan 2022 16:16:39 -0500 Subject: [PATCH 01/22] Add first pass at a GSM compression plugin. --- .gitmodules | 3 + pedalboard/plugins/GSMCompressor.h | 158 +++++++++++++++++++++++++++++ pedalboard/python_bindings.cpp | 2 + setup.py | 5 + vendors/libgsm | 1 + 5 files changed, 169 insertions(+) create mode 100644 pedalboard/plugins/GSMCompressor.h create mode 160000 vendors/libgsm diff --git a/.gitmodules b/.gitmodules index dc118c22..0f442f8a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "vendors/lame"] path = vendors/lame url = https://github.com/lameproject/lame.git +[submodule "vendors/libgsm"] + path = vendors/libgsm + url = git@github.com:timothytylee/libgsm.git diff --git a/pedalboard/plugins/GSMCompressor.h b/pedalboard/plugins/GSMCompressor.h new file mode 100644 index 00000000..13dbdf28 --- /dev/null +++ b/pedalboard/plugins/GSMCompressor.h @@ -0,0 +1,158 @@ +/* + * pedalboard + * Copyright 2022 Spotify AB + * + * Licensed under the GNU Public License, Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.gnu.org/licenses/gpl-3.0.html + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../Plugin.h" +extern "C" { +#include +} + +namespace Pedalboard { + +/* + * A small C++ wrapper around the C-based libgsm object. + * Used mostly to avoid leaking memory. + */ +class GSMWrapper { +public: + GSMWrapper() {} + ~GSMWrapper() { reset(); } + + operator bool() const { return _gsm != nullptr; } + + void reset() { + gsm_destroy(_gsm); + _gsm = nullptr; + } + + gsm getContext() { + if (!_gsm) + _gsm = gsm_create(); + return _gsm; + } + +private: + gsm _gsm = nullptr; +}; + +class GSMCompressor : public Plugin { +public: + virtual ~GSMCompressor(){}; + + virtual void prepare(const juce::dsp::ProcessSpec &spec) override { + bool specChanged = lastSpec.sampleRate != spec.sampleRate || + lastSpec.maximumBlockSize < spec.maximumBlockSize || + lastSpec.numChannels != spec.numChannels; + if (!encoder || specChanged) { + reset(); + + if (spec.sampleRate != 8000) { + throw std::domain_error( + "GSM compression currently only works at 8kHz."); + } + + if (spec.numChannels != 1) { + throw std::domain_error( + "GSM compression currently only works on mono signals."); + } + + if (spec.maximumBlockSize % GSM_FRAME_SIZE_SAMPLES != 0) { + throw std::domain_error("GSM compression currently requires a buffer " + "size of a multiple of " + + std::to_string(GSM_FRAME_SIZE_SAMPLES) + "."); + } + + if (!encoder.getContext()) { + throw std::runtime_error("Failed to initialize GSM encoder."); + } + if (!decoder.getContext()) { + throw std::runtime_error("Failed to initialize GSM decoder."); + } + + lastSpec = spec; + } + } + + int process( + const juce::dsp::ProcessContextReplacing &context) override final { + auto ioBlock = context.getOutputBlock(); + + for (int blockStart = 0; blockStart < ioBlock.getNumSamples(); + blockStart += GSM_FRAME_SIZE_SAMPLES) { + int blockEnd = blockStart + GSM_FRAME_SIZE_SAMPLES; + if (blockEnd > ioBlock.getNumSamples()) + blockEnd = ioBlock.getNumSamples(); + + int blockSize = blockEnd - blockStart; + + // Convert samples to signed 16-bit integer first, + // then pass to the GSM Encoder, then immediately back + // around to the GSM decoder. + short frame[GSM_FRAME_SIZE_SAMPLES]; + + juce::AudioDataConverters::convertFloatToInt16LE( + ioBlock.getChannelPointer(0) + blockStart, frame, + GSM_FRAME_SIZE_SAMPLES); + + gsm_frame encodedFrame; + + gsm_encode(encoder.getContext(), frame, encodedFrame); + if (gsm_decode(decoder.getContext(), encodedFrame, frame) < 0) { + throw std::runtime_error("GSM decoder could not decode frame!"); + } + + juce::AudioDataConverters::convertInt16LEToFloat( + frame, ioBlock.getChannelPointer(0) + blockStart, + GSM_FRAME_SIZE_SAMPLES); + } + + return ioBlock.getNumSamples(); + } + + void reset() override final { + encoder.reset(); + decoder.reset(); + + samplesProduced = 0; + encoderInStreamLatency = 0; + } + +protected: + virtual int getLatencyHint() override { return 0; } + +private: + GSMWrapper encoder; + GSMWrapper decoder; + + static constexpr size_t GSM_FRAME_SIZE_SAMPLES = 160; +}; + +inline void init_gsm_compressor(py::module &m) { + py::class_( + m, "GSMCompressor", + "Apply an GSM compressor to emulate the sound of a GSM (\"2G\") cellular " + "phone connection.") + .def(py::init([]() { return new GSMCompressor(); })) + .def("__repr__", [](const GSMCompressor &plugin) { + std::ostringstream ss; + ss << ""; + return ss.str(); + }); +} + +}; // namespace Pedalboard \ No newline at end of file diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index 170809e4..9df44d97 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -39,6 +39,7 @@ namespace py = pybind11; #include "plugins/Delay.h" #include "plugins/Distortion.h" #include "plugins/Gain.h" +#include "plugins/GSMCompressor.h" #include "plugins/HighpassFilter.h" #include "plugins/Invert.h" #include "plugins/LadderFilter.h" @@ -144,6 +145,7 @@ PYBIND11_MODULE(pedalboard_native, m) { init_delay(m); init_distortion(m); init_gain(m); + init_gsm_compressor(m); init_highpass(m); init_invert(m); init_ladderfilter(m); diff --git a/setup.py b/setup.py index 364616c0..4360afae 100644 --- a/setup.py +++ b/setup.py @@ -107,6 +107,11 @@ ] +# libgsm +ALL_SOURCE_PATHS += list(Path("vendors/libgsm/src").glob("*.c")) +ALL_INCLUDES += ['vendors/libgsm/inc'] + + # Add platform-specific flags: if platform.system() == "Darwin": ALL_CPPFLAGS.append("-DMACOS=1") diff --git a/vendors/libgsm b/vendors/libgsm new file mode 160000 index 00000000..98f1708f --- /dev/null +++ b/vendors/libgsm @@ -0,0 +1 @@ +Subproject commit 98f1708fb5e06a0dfebd58a3b40d610823db9715 From dd828bf31f960944f0440e94ea3a39cbc504a046 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 30 Jan 2022 11:07:07 -0500 Subject: [PATCH 02/22] First test of GSM compressor with built-in resampler. --- pedalboard/plugins/GSMCompressor.h | 60 ++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/pedalboard/plugins/GSMCompressor.h b/pedalboard/plugins/GSMCompressor.h index 13dbdf28..1faf55cf 100644 --- a/pedalboard/plugins/GSMCompressor.h +++ b/pedalboard/plugins/GSMCompressor.h @@ -59,21 +59,25 @@ class GSMCompressor : public Plugin { if (!encoder || specChanged) { reset(); - if (spec.sampleRate != 8000) { - throw std::domain_error( - "GSM compression currently only works at 8kHz."); - } + resamplerRatio = spec.sampleRate / GSM_SAMPLE_RATE; + resampledBuffer.setSize( + spec.numChannels, + (spec.maximumBlockSize * GSM_SAMPLE_RATE / spec.sampleRate)); if (spec.numChannels != 1) { throw std::domain_error( "GSM compression currently only works on mono signals."); } - if (spec.maximumBlockSize % GSM_FRAME_SIZE_SAMPLES != 0) { - throw std::domain_error("GSM compression currently requires a buffer " - "size of a multiple of " + - std::to_string(GSM_FRAME_SIZE_SAMPLES) + "."); + inputBlockSizeForGSM = (GSM_FRAME_SIZE_SAMPLES * resamplerRatio); + if (spec.maximumBlockSize % (inputBlockSizeForGSM) != 0) { + throw std::domain_error( + "GSM compression currently requires a buffer " + "size of a multiple of " + + std::to_string(inputBlockSizeForGSM) + + ", but got " + std::to_string(inputBlockSizeForGSM) + "."); } + printf("Maximum block size is %d, setting inputBlockSizeForGSM to %d samples at %f Hz.\n", spec.maximumBlockSize, inputBlockSizeForGSM, spec.sampleRate); if (!encoder.getContext()) { throw std::runtime_error("Failed to initialize GSM encoder."); @@ -91,21 +95,26 @@ class GSMCompressor : public Plugin { auto ioBlock = context.getOutputBlock(); for (int blockStart = 0; blockStart < ioBlock.getNumSamples(); - blockStart += GSM_FRAME_SIZE_SAMPLES) { - int blockEnd = blockStart + GSM_FRAME_SIZE_SAMPLES; + blockStart += inputBlockSizeForGSM) { + int blockEnd = blockStart + inputBlockSizeForGSM; if (blockEnd > ioBlock.getNumSamples()) blockEnd = ioBlock.getNumSamples(); int blockSize = blockEnd - blockStart; + // Resample the input audio down to 8kHz. + float *inputSamples = ioBlock.getChannelPointer(0) + blockStart; + float *tempSamples = resampledBuffer.getWritePointer(0); + int samplesUsed = nativeToGSMResampler.process(resamplerRatio, inputSamples, tempSamples, + GSM_FRAME_SIZE_SAMPLES); + // Convert samples to signed 16-bit integer first, // then pass to the GSM Encoder, then immediately back // around to the GSM decoder. short frame[GSM_FRAME_SIZE_SAMPLES]; - juce::AudioDataConverters::convertFloatToInt16LE( - ioBlock.getChannelPointer(0) + blockStart, frame, - GSM_FRAME_SIZE_SAMPLES); + juce::AudioDataConverters::convertFloatToInt16LE(tempSamples, frame, + GSM_FRAME_SIZE_SAMPLES); gsm_frame encodedFrame; @@ -114,9 +123,15 @@ class GSMCompressor : public Plugin { throw std::runtime_error("GSM decoder could not decode frame!"); } - juce::AudioDataConverters::convertInt16LEToFloat( - frame, ioBlock.getChannelPointer(0) + blockStart, - GSM_FRAME_SIZE_SAMPLES); + juce::AudioDataConverters::convertInt16LEToFloat(frame, tempSamples, + GSM_FRAME_SIZE_SAMPLES); + + // Resample back up to the native sample rate: + samplesUsed = gsmToNativeResampler.process(1.0 / resamplerRatio, tempSamples, + inputSamples, blockSize); + // if (samplesUsed != blockSize) { + // throw std::runtime_error("An incorrect number of samples were returned after resampling for GSM. This is an internal Pedalboard error and should be reported."); + // } } return ioBlock.getNumSamples(); @@ -125,9 +140,9 @@ class GSMCompressor : public Plugin { void reset() override final { encoder.reset(); decoder.reset(); - - samplesProduced = 0; - encoderInStreamLatency = 0; + nativeToGSMResampler.reset(); + gsmToNativeResampler.reset(); + resampledBuffer.clear(); } protected: @@ -137,7 +152,14 @@ class GSMCompressor : public Plugin { GSMWrapper encoder; GSMWrapper decoder; + int inputBlockSizeForGSM; + double resamplerRatio = 1.0; + juce::Interpolators::ZeroOrderHold nativeToGSMResampler; + juce::Interpolators::WindowedSinc gsmToNativeResampler; + juce::AudioBuffer resampledBuffer; + static constexpr size_t GSM_FRAME_SIZE_SAMPLES = 160; + static constexpr float GSM_SAMPLE_RATE = 8000; }; inline void init_gsm_compressor(py::module &m) { From d29179c1efcb18f500d338eaeca42255dfd28772 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 30 Jan 2022 16:48:20 -0500 Subject: [PATCH 03/22] Working, click-free, properly aligned GSM compression except at 11.025kHz. --- pedalboard/plugins/GSMCompressor.h | 273 ++++++++++++++++++++++++----- 1 file changed, 229 insertions(+), 44 deletions(-) diff --git a/pedalboard/plugins/GSMCompressor.h b/pedalboard/plugins/GSMCompressor.h index 1faf55cf..f1d0009d 100644 --- a/pedalboard/plugins/GSMCompressor.h +++ b/pedalboard/plugins/GSMCompressor.h @@ -60,24 +60,16 @@ class GSMCompressor : public Plugin { reset(); resamplerRatio = spec.sampleRate / GSM_SAMPLE_RATE; - resampledBuffer.setSize( - spec.numChannels, - (spec.maximumBlockSize * GSM_SAMPLE_RATE / spec.sampleRate)); + inverseResamplerRatio = GSM_SAMPLE_RATE / spec.sampleRate; - if (spec.numChannels != 1) { - throw std::domain_error( - "GSM compression currently only works on mono signals."); - } + gsmFrameSizeInNativeSampleRate = GSM_FRAME_SIZE_SAMPLES * resamplerRatio; + int maximumBlockSizeInGSMSampleRate = + spec.maximumBlockSize / resamplerRatio; - inputBlockSizeForGSM = (GSM_FRAME_SIZE_SAMPLES * resamplerRatio); - if (spec.maximumBlockSize % (inputBlockSizeForGSM) != 0) { - throw std::domain_error( - "GSM compression currently requires a buffer " - "size of a multiple of " + - std::to_string(inputBlockSizeForGSM) + - ", but got " + std::to_string(inputBlockSizeForGSM) + "."); - } - printf("Maximum block size is %d, setting inputBlockSizeForGSM to %d samples at %f Hz.\n", spec.maximumBlockSize, inputBlockSizeForGSM, spec.sampleRate); + // Store the remainder of the input: any samples that weren't consumed in + // one pushSamples() call but would be consumable in the next one. + inputReservoir.setSize(1, (int)std::ceil(resamplerRatio) + + spec.maximumBlockSize); if (!encoder.getContext()) { throw std::runtime_error("Failed to initialize GSM encoder."); @@ -86,6 +78,25 @@ class GSMCompressor : public Plugin { throw std::runtime_error("Failed to initialize GSM decoder."); } + inStreamLatency = 0; + + // Add the resamplers' latencies so the output is properly aligned: + inStreamLatency += nativeToGSMResampler.getBaseLatency() * resamplerRatio; + inStreamLatency += gsmToNativeResampler.getBaseLatency() * resamplerRatio; + + resampledBuffer.setSize(1, maximumBlockSizeInGSMSampleRate + + GSM_FRAME_SIZE_SAMPLES + + (inStreamLatency / resamplerRatio)); + outputBuffer.setSize(1, spec.maximumBlockSize + + gsmFrameSizeInNativeSampleRate + + inStreamLatency); + + // Feed one GSM frame's worth of silence at the start so that we + // can tolerate different buffer sizes without underrunning any buffers. + std::vector silence(gsmFrameSizeInNativeSampleRate); + inStreamLatency += silence.size(); + pushSamples(silence.data(), silence.size()); + lastSpec = spec; } } @@ -94,28 +105,158 @@ class GSMCompressor : public Plugin { const juce::dsp::ProcessContextReplacing &context) override final { auto ioBlock = context.getOutputBlock(); - for (int blockStart = 0; blockStart < ioBlock.getNumSamples(); - blockStart += inputBlockSizeForGSM) { - int blockEnd = blockStart + inputBlockSizeForGSM; - if (blockEnd > ioBlock.getNumSamples()) - blockEnd = ioBlock.getNumSamples(); + // Mix all channels to mono first, if necessary; GSM (in reality) is + // mono-only. + if (ioBlock.getNumChannels() > 1) { + float channelVolume = 1.0f / ioBlock.getNumChannels(); + for (int i = 0; i < ioBlock.getNumChannels(); i++) { + ioBlock.getSingleChannelBlock(i) *= channelVolume; + } + + // Copy all of the latter channels into the first channel, + // which will be used for processing: + auto firstChannel = ioBlock.getSingleChannelBlock(0); + for (int i = 1; i < ioBlock.getNumChannels(); i++) { + firstChannel += ioBlock.getSingleChannelBlock(i); + } + } + + // Actually do the GSM processing! + pushSamples(ioBlock.getChannelPointer(0), ioBlock.getNumSamples()); + int samplesOutput = + pullSamples(ioBlock.getChannelPointer(0), ioBlock.getNumSamples()); + + // Copy the mono signal back out to all other channels: + if (ioBlock.getNumChannels() > 1) { + auto firstChannel = ioBlock.getSingleChannelBlock(0); + for (int i = 1; i < ioBlock.getNumChannels(); i++) { + ioBlock.getSingleChannelBlock(i).copyFrom(firstChannel); + } + } + + if (samplesProduced > 0 && samplesOutput < ioBlock.getNumSamples()) { + throw std::runtime_error("Buffer underrun on the output!"); + } - int blockSize = blockEnd - blockStart; + samplesProduced += samplesOutput; + int samplesToReturn = std::min((long)(samplesProduced - inStreamLatency), + (long)ioBlock.getNumSamples()); + if (samplesToReturn < 0) + samplesToReturn = 0; - // Resample the input audio down to 8kHz. - float *inputSamples = ioBlock.getChannelPointer(0) + blockStart; - float *tempSamples = resampledBuffer.getWritePointer(0); - int samplesUsed = nativeToGSMResampler.process(resamplerRatio, inputSamples, tempSamples, - GSM_FRAME_SIZE_SAMPLES); + return samplesToReturn; + } + + /* + * Return the number of samples needed for this + * plugin to return a single GSM frame's worth of audio. + */ + int spaceAvailableInResampledBuffer() const { + return resampledBuffer.getNumSamples() - samplesInResampledBuffer; + } + + int spaceAvailableInOutputBuffer() const { + return outputBuffer.getNumSamples() - samplesInOutputBuffer; + } + + /* + * Push a certain number of input samples into the internal buffer(s) + * of this plugin, as GSM coding processes audio 160 samples at a time. + */ + void pushSamples(float *inputSamples, int numInputSamples) { + float expectedOutputSamples = numInputSamples / resamplerRatio; + + if (spaceAvailableInResampledBuffer() < expectedOutputSamples) { + throw std::runtime_error( + "More samples were provided than can be buffered! This is an " + "internal Pedalboard error and should be reported. Buffer had " + + std::to_string(samplesInResampledBuffer) + "/" + + std::to_string(resampledBuffer.getNumSamples()) + + " samples at 8kHz, but was provided " + + std::to_string(expectedOutputSamples) + "."); + } + + float *resampledBufferPointer = + resampledBuffer.getWritePointer(0) + samplesInResampledBuffer; + + int samplesUsed = 0; + if (samplesInInputReservoir) { + // Copy the input samples into the input reservoir and use that as the + // resampler's input: + expectedOutputSamples += (float)samplesInInputReservoir / resamplerRatio; + inputReservoir.copyFrom(0, samplesInInputReservoir, inputSamples, + numInputSamples); + samplesUsed = nativeToGSMResampler.process( + resamplerRatio, inputReservoir.getReadPointer(0), + resampledBufferPointer, expectedOutputSamples); + + if (samplesUsed < numInputSamples + samplesInInputReservoir) { + // Take the missing samples and put them at the start of the input + // reservoir for next time: + int unusedInputSampleCount = + (numInputSamples + samplesInInputReservoir) - samplesUsed; + inputReservoir.copyFrom(0, 0, + inputReservoir.getReadPointer(0) + samplesUsed, + unusedInputSampleCount); + samplesInInputReservoir = unusedInputSampleCount; + } else { + samplesInInputReservoir = 0; + } + } else { + samplesUsed = nativeToGSMResampler.process(resamplerRatio, inputSamples, + resampledBufferPointer, + expectedOutputSamples); + + if (samplesUsed < numInputSamples) { + // Take the missing samples and put them at the start of the input + // reservoir for next time: + int unusedInputSampleCount = numInputSamples - samplesUsed; + inputReservoir.copyFrom(0, 0, inputSamples + samplesUsed, + unusedInputSampleCount); + samplesInInputReservoir = unusedInputSampleCount; + } + } + + samplesInResampledBuffer += expectedOutputSamples; + + performEncodeAndDecode(); + } + + int pullSamples(float *outputSamples, int maxOutputSamples) { + performEncodeAndDecode(); + + // Copy the data out of outputBuffer and into the provided pointer, at + // the right side of the buffer: + int samplesToCopy = std::min(samplesInOutputBuffer, maxOutputSamples); + int offsetInOutput = maxOutputSamples - samplesToCopy; + juce::FloatVectorOperations::copy(outputSamples + offsetInOutput, + outputBuffer.getWritePointer(0), + samplesToCopy); + samplesInOutputBuffer -= samplesToCopy; + + // Move remaining samples to the left side of the output buffer: + std::memmove((char *)outputBuffer.getWritePointer(0), + (char *)(outputBuffer.getWritePointer(0) + samplesToCopy), + samplesInOutputBuffer * sizeof(float)); + + performEncodeAndDecode(); + + return samplesToCopy; + } + + void performEncodeAndDecode() { + while (samplesInResampledBuffer >= GSM_FRAME_SIZE_SAMPLES) { + float *encodeBuffer = resampledBuffer.getWritePointer(0); // Convert samples to signed 16-bit integer first, // then pass to the GSM Encoder, then immediately back // around to the GSM decoder. short frame[GSM_FRAME_SIZE_SAMPLES]; - juce::AudioDataConverters::convertFloatToInt16LE(tempSamples, frame, + juce::AudioDataConverters::convertFloatToInt16LE(encodeBuffer, frame, GSM_FRAME_SIZE_SAMPLES); + // Actually do the GSM encoding/decoding: gsm_frame encodedFrame; gsm_encode(encoder.getContext(), frame, encodedFrame); @@ -123,18 +264,40 @@ class GSMCompressor : public Plugin { throw std::runtime_error("GSM decoder could not decode frame!"); } - juce::AudioDataConverters::convertInt16LEToFloat(frame, tempSamples, + if (spaceAvailableInOutputBuffer() < gsmFrameSizeInNativeSampleRate) { + throw std::runtime_error( + "Not enough space in output buffer to store a GSM frame! Needed " + + std::to_string(gsmFrameSizeInNativeSampleRate) + + " samples but only had " + + std::to_string(spaceAvailableInOutputBuffer()) + + " samples available. This is " + "an internal Pedalboard error and should be reported."); + } + float *outputBufferPointer = + outputBuffer.getWritePointer(0) + samplesInOutputBuffer; + + float floatFrame[GSM_FRAME_SIZE_SAMPLES]; + juce::AudioDataConverters::convertInt16LEToFloat(frame, floatFrame, GSM_FRAME_SIZE_SAMPLES); - // Resample back up to the native sample rate: - samplesUsed = gsmToNativeResampler.process(1.0 / resamplerRatio, tempSamples, - inputSamples, blockSize); - // if (samplesUsed != blockSize) { - // throw std::runtime_error("An incorrect number of samples were returned after resampling for GSM. This is an internal Pedalboard error and should be reported."); - // } + // Resample back up to the native sample rate and store in outputBuffer: + int expectedOutputSamples = gsmFrameSizeInNativeSampleRate; + int samplesConsumed = gsmToNativeResampler.process( + inverseResamplerRatio, floatFrame, outputBufferPointer, + expectedOutputSamples); + + samplesInOutputBuffer += expectedOutputSamples; + + // Now that we're done with this chunk of resampledBuffer, move its + // contents to the left: + int samplesRemainingInResampledBuffer = + samplesInResampledBuffer - samplesConsumed; + std::memmove( + (char *)resampledBuffer.getWritePointer(0), + (char *)(resampledBuffer.getWritePointer(0) + samplesConsumed), + samplesRemainingInResampledBuffer * sizeof(float)); + samplesInResampledBuffer -= samplesConsumed; } - - return ioBlock.getNumSamples(); } void reset() override final { @@ -142,21 +305,43 @@ class GSMCompressor : public Plugin { decoder.reset(); nativeToGSMResampler.reset(); gsmToNativeResampler.reset(); + resampledBuffer.clear(); + outputBuffer.clear(); + inputReservoir.clear(); + + samplesInResampledBuffer = 0; + samplesInOutputBuffer = 0; + samplesInInputReservoir = 0; + + samplesProduced = 0; + inStreamLatency = 0; } protected: - virtual int getLatencyHint() override { return 0; } + virtual int getLatencyHint() override { return inStreamLatency; } private: + double resamplerRatio = 1.0; + double inverseResamplerRatio = 1.0; + float gsmFrameSizeInNativeSampleRate; + + juce::AudioBuffer inputReservoir; + int samplesInInputReservoir = 0; + + juce::Interpolators::Lagrange nativeToGSMResampler; + juce::AudioBuffer resampledBuffer; + int samplesInResampledBuffer = 0; + GSMWrapper encoder; GSMWrapper decoder; - int inputBlockSizeForGSM; - double resamplerRatio = 1.0; - juce::Interpolators::ZeroOrderHold nativeToGSMResampler; - juce::Interpolators::WindowedSinc gsmToNativeResampler; - juce::AudioBuffer resampledBuffer; + juce::Interpolators::Lagrange gsmToNativeResampler; + juce::AudioBuffer outputBuffer; + int samplesInOutputBuffer = 0; + + int samplesProduced = 0; + int inStreamLatency = 0; static constexpr size_t GSM_FRAME_SIZE_SAMPLES = 160; static constexpr float GSM_SAMPLE_RATE = 8000; @@ -166,7 +351,7 @@ inline void init_gsm_compressor(py::module &m) { py::class_( m, "GSMCompressor", "Apply an GSM compressor to emulate the sound of a GSM (\"2G\") cellular " - "phone connection.") + "phone connection. This plugin internally resamples the input audio to 8kHz.") .def(py::init([]() { return new GSMCompressor(); })) .def("__repr__", [](const GSMCompressor &plugin) { std::ostringstream ss; From 812ed286aeed43b1b818e952d44fdc31e22f0eee Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 30 Jan 2022 17:25:05 -0500 Subject: [PATCH 04/22] Working, click-free, correct GSM compressor at all sample rates. --- pedalboard/plugins/GSMCompressor.h | 49 +++++++++++++---------- tests/test_gsm_compressor.py | 62 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 20 deletions(-) create mode 100644 tests/test_gsm_compressor.py diff --git a/pedalboard/plugins/GSMCompressor.h b/pedalboard/plugins/GSMCompressor.h index f1d0009d..fe7038f8 100644 --- a/pedalboard/plugins/GSMCompressor.h +++ b/pedalboard/plugins/GSMCompressor.h @@ -87,9 +87,10 @@ class GSMCompressor : public Plugin { resampledBuffer.setSize(1, maximumBlockSizeInGSMSampleRate + GSM_FRAME_SIZE_SAMPLES + (inStreamLatency / resamplerRatio)); - outputBuffer.setSize(1, spec.maximumBlockSize + - gsmFrameSizeInNativeSampleRate + - inStreamLatency); + outputBuffer.setSize(1, + spec.maximumBlockSize + + gsmFrameSizeInNativeSampleRate + + inStreamLatency); // Feed one GSM frame's worth of silence at the start so that we // can tolerate different buffer sizes without underrunning any buffers. @@ -134,10 +135,6 @@ class GSMCompressor : public Plugin { } } - if (samplesProduced > 0 && samplesOutput < ioBlock.getNumSamples()) { - throw std::runtime_error("Buffer underrun on the output!"); - } - samplesProduced += samplesOutput; int samplesToReturn = std::min((long)(samplesProduced - inStreamLatency), (long)ioBlock.getNumSamples()); @@ -276,27 +273,34 @@ class GSMCompressor : public Plugin { float *outputBufferPointer = outputBuffer.getWritePointer(0) + samplesInOutputBuffer; - float floatFrame[GSM_FRAME_SIZE_SAMPLES]; - juce::AudioDataConverters::convertInt16LEToFloat(frame, floatFrame, - GSM_FRAME_SIZE_SAMPLES); + juce::AudioDataConverters::convertInt16LEToFloat( + frame, gsmOutputFrame + samplesInGsmOutputFrame, + GSM_FRAME_SIZE_SAMPLES); + samplesInGsmOutputFrame += GSM_FRAME_SIZE_SAMPLES; - // Resample back up to the native sample rate and store in outputBuffer: - int expectedOutputSamples = gsmFrameSizeInNativeSampleRate; + // Resample back up to the native sample rate and store in outputBuffer, + // using gsmOutputFrame as a temporary buffer to store up to 1 extra + // sample to compensate for rounding errors: + int expectedOutputSamples = samplesInGsmOutputFrame * resamplerRatio; int samplesConsumed = gsmToNativeResampler.process( - inverseResamplerRatio, floatFrame, outputBufferPointer, + inverseResamplerRatio, gsmOutputFrame, outputBufferPointer, expectedOutputSamples); + std::memmove((char *)gsmOutputFrame, + (char *)(gsmOutputFrame + samplesConsumed), + (samplesInGsmOutputFrame - samplesConsumed) * sizeof(float)); + samplesInGsmOutputFrame -= samplesConsumed; samplesInOutputBuffer += expectedOutputSamples; // Now that we're done with this chunk of resampledBuffer, move its // contents to the left: int samplesRemainingInResampledBuffer = - samplesInResampledBuffer - samplesConsumed; + samplesInResampledBuffer - GSM_FRAME_SIZE_SAMPLES; std::memmove( (char *)resampledBuffer.getWritePointer(0), - (char *)(resampledBuffer.getWritePointer(0) + samplesConsumed), + (char *)(resampledBuffer.getWritePointer(0) + GSM_FRAME_SIZE_SAMPLES), samplesRemainingInResampledBuffer * sizeof(float)); - samplesInResampledBuffer -= samplesConsumed; + samplesInResampledBuffer -= GSM_FRAME_SIZE_SAMPLES; } } @@ -313,6 +317,7 @@ class GSMCompressor : public Plugin { samplesInResampledBuffer = 0; samplesInOutputBuffer = 0; samplesInInputReservoir = 0; + samplesInGsmOutputFrame = 0; samplesProduced = 0; inStreamLatency = 0; @@ -322,6 +327,9 @@ class GSMCompressor : public Plugin { virtual int getLatencyHint() override { return inStreamLatency; } private: + static constexpr size_t GSM_FRAME_SIZE_SAMPLES = 160; + static constexpr float GSM_SAMPLE_RATE = 8000; + double resamplerRatio = 1.0; double inverseResamplerRatio = 1.0; float gsmFrameSizeInNativeSampleRate; @@ -337,21 +345,22 @@ class GSMCompressor : public Plugin { GSMWrapper decoder; juce::Interpolators::Lagrange gsmToNativeResampler; + float gsmOutputFrame[GSM_FRAME_SIZE_SAMPLES + 1]; + int samplesInGsmOutputFrame = 0; + juce::AudioBuffer outputBuffer; int samplesInOutputBuffer = 0; int samplesProduced = 0; int inStreamLatency = 0; - - static constexpr size_t GSM_FRAME_SIZE_SAMPLES = 160; - static constexpr float GSM_SAMPLE_RATE = 8000; }; inline void init_gsm_compressor(py::module &m) { py::class_( m, "GSMCompressor", "Apply an GSM compressor to emulate the sound of a GSM (\"2G\") cellular " - "phone connection. This plugin internally resamples the input audio to 8kHz.") + "phone connection. This plugin internally resamples the input audio to " + "8kHz.") .def(py::init([]() { return new GSMCompressor(); })) .def("__repr__", [](const GSMCompressor &plugin) { std::ostringstream ss; diff --git a/tests/test_gsm_compressor.py b/tests/test_gsm_compressor.py new file mode 100644 index 00000000..b241fe72 --- /dev/null +++ b/tests/test_gsm_compressor.py @@ -0,0 +1,62 @@ +#! /usr/bin/env python +# +# Copyright 2021 Spotify AB +# +# Licensed under the GNU Public License, Version 3.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/gpl-3.0.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import numpy as np +from pedalboard import GSMCompressor + +# GSM is a _very_ lossy codec: +GSM_ABSOLUTE_TOLERANCE = 0.75 + +# Passing in a full-scale sine wave seems to often make GSM clip: +SINE_WAVE_VOLUME = 0.9 + + +def generate_sine_at( + sample_rate: float, + fundamental_hz: float = 440.0, + num_seconds: float = 3.0, + num_channels: int = 1, +) -> np.ndarray: + samples = np.arange(num_seconds * sample_rate) + sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + if num_channels == 2: + return np.stack([sine_wave, sine_wave]) + return sine_wave + + +@pytest.mark.parametrize("fundamental_hz", [440.0]) +@pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 32000, 32001, 44100, 48000]) +@pytest.mark.parametrize("buffer_size", [1, 32, 160, 8192]) +@pytest.mark.parametrize("duration", [1.0, 3.0, 30.0]) +@pytest.mark.parametrize("num_channels", [1, 2]) +def test_gsm_compressor(fundamental_hz: float, sample_rate: float, buffer_size: int, duration: float, num_channels: int): + signal = generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) * SINE_WAVE_VOLUME + compressed = GSMCompressor()(signal, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(signal, compressed, atol=GSM_ABSOLUTE_TOLERANCE) + + +@pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 32000, 32001, 44100, 48000]) +@pytest.mark.parametrize("num_channels", [1])#, 2]) +def test_gsm_compressor_invariant_to_buffer_size(sample_rate: float, num_channels: int): + fundamental_hz = 400.0 + duration = 3.0 + signal = generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) + + compressed = [GSMCompressor()(signal, sample_rate, buffer_size=buffer_size) for buffer_size in (1, 32, 8192)] + for a, b in zip(compressed, compressed[1:]): + np.testing.assert_allclose(a, b) From fbee204c706b1b321c114fcbb9ff14ce4039d4e4 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 30 Jan 2022 17:25:49 -0500 Subject: [PATCH 05/22] Re-ran black. --- tests/test_gsm_compressor.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_gsm_compressor.py b/tests/test_gsm_compressor.py index b241fe72..9408cc2f 100644 --- a/tests/test_gsm_compressor.py +++ b/tests/test_gsm_compressor.py @@ -44,19 +44,26 @@ def generate_sine_at( @pytest.mark.parametrize("buffer_size", [1, 32, 160, 8192]) @pytest.mark.parametrize("duration", [1.0, 3.0, 30.0]) @pytest.mark.parametrize("num_channels", [1, 2]) -def test_gsm_compressor(fundamental_hz: float, sample_rate: float, buffer_size: int, duration: float, num_channels: int): - signal = generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) * SINE_WAVE_VOLUME +def test_gsm_compressor( + fundamental_hz: float, sample_rate: float, buffer_size: int, duration: float, num_channels: int +): + signal = ( + generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) * SINE_WAVE_VOLUME + ) compressed = GSMCompressor()(signal, sample_rate, buffer_size=buffer_size) np.testing.assert_allclose(signal, compressed, atol=GSM_ABSOLUTE_TOLERANCE) @pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 32000, 32001, 44100, 48000]) -@pytest.mark.parametrize("num_channels", [1])#, 2]) +@pytest.mark.parametrize("num_channels", [1]) # , 2]) def test_gsm_compressor_invariant_to_buffer_size(sample_rate: float, num_channels: int): fundamental_hz = 400.0 duration = 3.0 signal = generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) - compressed = [GSMCompressor()(signal, sample_rate, buffer_size=buffer_size) for buffer_size in (1, 32, 8192)] + compressed = [ + GSMCompressor()(signal, sample_rate, buffer_size=buffer_size) + for buffer_size in (1, 32, 8192) + ] for a, b in zip(compressed, compressed[1:]): np.testing.assert_allclose(a, b) From ed9f839eec8db15653309dbc10b481593dd1a8d1 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 30 Jan 2022 17:33:49 -0500 Subject: [PATCH 06/22] Added GSM compressor to the README. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index d3fee27a..81160251 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Internally at Spotify, `pedalboard` is used for [data augmentation](https://en.w - `Phaser` - `PitchShift` (provided by Chris Cannam's [Rubber Band Library](https://github.com/breakfastquay/rubberband)) - `Reverb` + - Built-in lossy compression algorithms: + - `GSMCompressor`, an implementation of the GSM ("2G") lossy voice compression codec - Supports VST3® plugins on macOS, Windows, and Linux (`pedalboard.load_plugin`) - Supports Audio Units on macOS - Strong thread-safety, memory usage, and speed guarantees @@ -242,5 +244,6 @@ Not yet, either - although the underlying framework (JUCE) supports passing MIDI - The [VST3 SDK](https://github.com/steinbergmedia/vst3sdk), bundled with JUCE, is owned by [Steinberg® Media Technologies GmbH](https://www.steinberg.net/en/home.html) and licensed under the GPLv3. - The `PitchShift` plugin uses [the Rubber Band Library](https://github.com/breakfastquay/rubberband), which is [dual-licensed under a commercial license](https://breakfastquay.com/technology/license.html) and the GPLv2 (or newer). - The `MP3Compressor` plugin uses [`libmp3lame` from the LAME project](https://lame.sourceforge.io/), which is [licensed under the LGPLv2](https://github.com/lameproject/lame/blob/master/README) and [upgraded to the GPLv3 for inclusion in this project (as permitted by the LGPLv2)](https://www.gnu.org/licenses/gpl-faq.html#AllCompatibility). + - The `GSMCompressor` plugin uses [`libgsm`](http://quut.com/gsm/), which is [licensed under the ISC license](https://github.com/timothytylee/libgsm/blob/master/COPYRIGHT) and [compatible with the GPLv3](https://www.gnu.org/licenses/license-list.en.html#ISC). _VST is a registered trademark of Steinberg Media Technologies GmbH._ From f38aaaaf9505dd2715ecdb5b8035ec5a42fcddfe Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 30 Jan 2022 21:09:33 -0500 Subject: [PATCH 07/22] Clang-format. --- pedalboard/plugins/GSMCompressor.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pedalboard/plugins/GSMCompressor.h b/pedalboard/plugins/GSMCompressor.h index fe7038f8..0ad25069 100644 --- a/pedalboard/plugins/GSMCompressor.h +++ b/pedalboard/plugins/GSMCompressor.h @@ -87,10 +87,9 @@ class GSMCompressor : public Plugin { resampledBuffer.setSize(1, maximumBlockSizeInGSMSampleRate + GSM_FRAME_SIZE_SAMPLES + (inStreamLatency / resamplerRatio)); - outputBuffer.setSize(1, - spec.maximumBlockSize + - gsmFrameSizeInNativeSampleRate + - inStreamLatency); + outputBuffer.setSize(1, spec.maximumBlockSize + + gsmFrameSizeInNativeSampleRate + + inStreamLatency); // Feed one GSM frame's worth of silence at the start so that we // can tolerate different buffer sizes without underrunning any buffers. From 75ed90c34ae6c111c04e4815efbaa4aa57a0919d Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Fri, 4 Feb 2022 15:40:55 -0500 Subject: [PATCH 08/22] Temp: working resampler, but broken delay --- .../plugin_templates/ResamplingPlugin.h | 348 ++++++++++++++++++ pedalboard/plugins/AddLatency.h | 2 + pedalboard/python_bindings.cpp | 3 + tests/test_resample.py | 58 +++ 4 files changed, 411 insertions(+) create mode 100644 pedalboard/plugin_templates/ResamplingPlugin.h create mode 100644 tests/test_resample.py diff --git a/pedalboard/plugin_templates/ResamplingPlugin.h b/pedalboard/plugin_templates/ResamplingPlugin.h new file mode 100644 index 00000000..e70f3eac --- /dev/null +++ b/pedalboard/plugin_templates/ResamplingPlugin.h @@ -0,0 +1,348 @@ +/* + * pedalboard + * Copyright 2022 Spotify AB + * + * Licensed under the GNU Public License, Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.gnu.org/licenses/gpl-3.0.html + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../JuceHeader.h" +#include "../Plugin.h" +#include "../plugins/AddLatency.h" +#include + +namespace Pedalboard { + +/** + * A test plugin used to verify the behaviour of the ResamplingPlugin wrapper. + */ +template +class Passthrough : public Plugin { +public: + virtual ~Passthrough(){}; + + virtual void prepare(const juce::dsp::ProcessSpec &spec) {} + + virtual int + process(const juce::dsp::ProcessContextReplacing &context) { + return context.getInputBlock().getNumSamples(); + } + + virtual void reset() {} +}; + +/** + * A template class that wraps a Pedalboard plugin and resamples + * the audio to the provided sample rate. The wrapped plugin receives + * resampled audio and its sampleRate and maximumBlockSize parameters + * are adjusted accordingly. + */ +template , + typename SampleType = float> class Resample : public Plugin { +public: + virtual ~Resample(){}; + + virtual void prepare(const juce::dsp::ProcessSpec &spec) { + bool specChanged = lastSpec.sampleRate != spec.sampleRate || + lastSpec.maximumBlockSize < spec.maximumBlockSize || + lastSpec.numChannels != spec.numChannels; + if (specChanged) { + reset(); + + nativeToTargetResamplers.resize(spec.numChannels); + targetToNativeResamplers.resize(spec.numChannels); + + resamplerRatio = spec.sampleRate / targetSampleRate; + inverseResamplerRatio = targetSampleRate / spec.sampleRate; + + const juce::dsp::ProcessSpec subSpec = { + .numChannels = spec.numChannels, + .sampleRate = targetSampleRate, + .maximumBlockSize = static_cast(spec.maximumBlockSize * resamplerRatio) + }; + plugin.prepare(subSpec); + + int maximumBlockSizeInSampleRate = + spec.maximumBlockSize / resamplerRatio; + + // Store the remainder of the input: any samples that weren't consumed in + // one pushSamples() call but would be consumable in the next one. + inputReservoir.setSize(spec.numChannels, (int)std::ceil(resamplerRatio) + + spec.maximumBlockSize); + + inStreamLatency = 0; + + // Add the resamplers' latencies so the output is properly aligned: + printf("nativeToTarget latency: %f, * %f = %f\n", nativeToTargetResamplers[0].getBaseLatency(), resamplerRatio, nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio); + printf("targetToNative latency: %f, * %f = %f\n", targetToNativeResamplers[0].getBaseLatency(), inverseResamplerRatio, targetToNativeResamplers[0].getBaseLatency() * inverseResamplerRatio); + inStreamLatency += std::round(nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio + targetToNativeResamplers[0].getBaseLatency()); + printf("total in-stream latency: %d\n", inStreamLatency); + + resampledBuffer.setSize(spec.numChannels, 30 * maximumBlockSizeInSampleRate + (inStreamLatency / resamplerRatio)); + outputBuffer.setSize(spec.numChannels, spec.maximumBlockSize * 3 + inStreamLatency); + lastSpec = spec; + } + } + + int process( + const juce::dsp::ProcessContextReplacing &context) override final { + auto ioBlock = context.getOutputBlock(); + + printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + + float expectedResampledSamples = ioBlock.getNumSamples() / resamplerRatio; + + if (spaceAvailableInResampledBuffer() < expectedResampledSamples) { + throw std::runtime_error( + "More samples were provided than can be buffered! This is an " + "internal Pedalboard error and should be reported. Buffer had " + + std::to_string(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer) + "/" + + std::to_string(resampledBuffer.getNumSamples()) + + " samples at target sample rate, but was provided " + + std::to_string(expectedResampledSamples) + "."); + } + + int samplesUsed = 0; + if (samplesInInputReservoir) { + // Copy the input samples into the input reservoir and use that as the + // resampler's input: + expectedResampledSamples += (float)samplesInInputReservoir / resamplerRatio; + + printf( + "Copying ioBlock[%d:%d] into inputReservoir[%d:%d]\n", + 0, ioBlock.getNumSamples(), samplesInInputReservoir, samplesInInputReservoir + ioBlock.getNumSamples() + ); + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + inputReservoir.copyFrom(c, samplesInInputReservoir, ioBlock.getChannelPointer(c), + ioBlock.getNumSamples()); + SampleType *resampledBufferPointer = resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer; + samplesUsed = nativeToTargetResamplers[c].process( + resamplerRatio, inputReservoir.getReadPointer(c), + resampledBufferPointer, expectedResampledSamples); + } + + printf("Ran nativeToTargetResamplers on inputReservoir[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); + + if (samplesUsed < ioBlock.getNumSamples() + samplesInInputReservoir) { + // Take the missing samples and put them at the start of the input + // reservoir for next time: + int unusedInputSampleCount = + (ioBlock.getNumSamples() + samplesInInputReservoir) - samplesUsed; + + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + inputReservoir.copyFrom(c, 0, + inputReservoir.getReadPointer(c) + samplesUsed, + unusedInputSampleCount); + } + + samplesInInputReservoir = unusedInputSampleCount; + printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); + } else { + samplesInInputReservoir = 0; + printf("Clearing input reservoir.\n"); + } + } else { + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + SampleType *resampledBufferPointer = resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer; + samplesUsed = nativeToTargetResamplers[c].process( + resamplerRatio, ioBlock.getChannelPointer(c), + resampledBufferPointer, (int)expectedResampledSamples); + } + + printf("Ran nativeToTargetResamplers on input[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); + + if (samplesUsed < ioBlock.getNumSamples()) { + // Take the missing samples and put them at the start of the input + // reservoir for next time: + int unusedInputSampleCount = ioBlock.getNumSamples() - samplesUsed; + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + inputReservoir.copyFrom(c, 0, ioBlock.getChannelPointer(c) + samplesUsed, + unusedInputSampleCount); + } + printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); + samplesInInputReservoir = unusedInputSampleCount; + } + } + + cleanSamplesInResampledBuffer += (int)expectedResampledSamples; + + printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + + // Pass resampledBuffer to the plugin: + juce::dsp::AudioBlock resampledBlock(resampledBuffer); + printf("Processing resampledBuffer[%d:%d] (%d samples) to plugin\n", processedSamplesInResampledBuffer, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, cleanSamplesInResampledBuffer); + juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock(processedSamplesInResampledBuffer, cleanSamplesInResampledBuffer); + // TODO: Check that samplesInResampledBuffer is not greater than the plugin's maximumBlockSize! + juce::dsp::ProcessContextReplacing subContext(subBlock); + int resampledSamplesOutput = plugin.process(subContext); + + cleanSamplesInResampledBuffer -= resampledSamplesOutput; + processedSamplesInResampledBuffer += resampledSamplesOutput; + + printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + + // Resample back to the intended sample rate: + int expectedOutputSamples = processedSamplesInResampledBuffer * resamplerRatio; + + int samplesConsumed = 0; + + if (spaceAvailableInOutputBuffer() < expectedOutputSamples) { + throw std::runtime_error( + "More samples were provided than can be buffered! This is an " + "internal Pedalboard error and should be reported. Buffer had " + + std::to_string(samplesInOutputBuffer) + "/" + + std::to_string(outputBuffer.getNumSamples()) + + " samples at native sample rate, but was provided " + + std::to_string(expectedOutputSamples) + "."); + } + + + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + float *outputBufferPointer = + outputBuffer.getWritePointer(c) + samplesInOutputBuffer; + samplesConsumed = targetToNativeResamplers[c].process( + inverseResamplerRatio, resampledBuffer.getReadPointer(c), + outputBufferPointer, expectedOutputSamples); + } + printf("Ran targetToNativeResampler on resampledBuffer[0:%d] -> outputBuffer[%d:%d]\n", samplesConsumed, samplesInOutputBuffer, samplesInOutputBuffer + expectedOutputSamples); + + samplesInOutputBuffer += expectedOutputSamples; + + printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + + int samplesRemainingInResampledBuffer = processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer - samplesConsumed; + if (samplesRemainingInResampledBuffer > 0) { + printf("Moving %d samples left by %d\n", samplesRemainingInResampledBuffer, samplesConsumed); + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + // Move the contents of the resampled block to the left: + std::memmove((char *)resampledBuffer.getWritePointer(c), + (char *)(resampledBuffer.getWritePointer(c) + samplesConsumed), + (samplesRemainingInResampledBuffer) * sizeof(SampleType)); + } + } + + processedSamplesInResampledBuffer -= samplesConsumed; + + // Copy from output buffer to output block: + int samplesToOutput = std::min(ioBlock.getNumSamples(), (unsigned long) samplesInOutputBuffer); + printf("Copying %d samples from output buffer to ioBlock at %d (%d samples large)\n", samplesToOutput, ioBlock.getNumSamples() - samplesToOutput, ioBlock.getNumSamples()); + ioBlock.copyFrom(outputBuffer, 0, ioBlock.getNumSamples() - samplesToOutput, samplesToOutput); + + int samplesRemainingInOutputBuffer = samplesInOutputBuffer - samplesToOutput; + if (samplesRemainingInOutputBuffer > 0) { + printf("Moving %d samples left in output buffer by %d\n", samplesRemainingInOutputBuffer, samplesToOutput); + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + // Move the contents of the resampled block to the left: + std::memmove((char *)outputBuffer.getWritePointer(c), + (char *)(outputBuffer.getWritePointer(c) + samplesToOutput), + (samplesRemainingInOutputBuffer) * sizeof(SampleType)); + } + } + samplesInOutputBuffer -= samplesToOutput; + + printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + + samplesProduced += samplesToOutput; + int samplesToReturn = std::min((long)(samplesProduced - inStreamLatency), + (long)samplesToOutput); + if (samplesToReturn < 0) + samplesToReturn = 0; + + printf("Returning %d samples\n", samplesToReturn); + return samplesToReturn; + } + + void setTargetSampleRate(float newSampleRate) { + targetSampleRate = newSampleRate; + } + + float getTargetSampleRate() const { + return targetSampleRate; + } + + T &getNestedPlugin() { return plugin; } + + virtual void reset() override final { + for (int c = 0; c < nativeToTargetResamplers.size(); c++) { + nativeToTargetResamplers[c].reset(); + targetToNativeResamplers[c].reset(); + } + + + resampledBuffer.clear(); + outputBuffer.clear(); + inputReservoir.clear(); + + cleanSamplesInResampledBuffer = 0; + processedSamplesInResampledBuffer = 0; + samplesInOutputBuffer = 0; + samplesInInputReservoir = 0; + + samplesProduced = 0; + inStreamLatency = 0; + } + + virtual int getLatencyHint() override { return inStreamLatency; } + +private: + T plugin; + float targetSampleRate = 44100.0f; + + double resamplerRatio = 1.0; + double inverseResamplerRatio = 1.0; + + juce::AudioBuffer inputReservoir; + int samplesInInputReservoir = 0; + + std::vector nativeToTargetResamplers; + juce::AudioBuffer resampledBuffer; + int cleanSamplesInResampledBuffer = 0; + int processedSamplesInResampledBuffer = 0; + std::vector targetToNativeResamplers; + + juce::AudioBuffer outputBuffer; + int samplesInOutputBuffer = 0; + + int samplesProduced = 0; + int inStreamLatency = 0; + + int spaceAvailableInResampledBuffer() const { + return resampledBuffer.getNumSamples() - std::max(cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer); + } + + int spaceAvailableInOutputBuffer() const { + return outputBuffer.getNumSamples() - samplesInOutputBuffer; + } +}; + +inline void init_resampling_test_plugin(py::module &m) { + py::class_, Plugin>(m, "Resample") + .def(py::init([](float targetSampleRate) { + auto plugin = std::make_unique>(); + plugin->setTargetSampleRate(targetSampleRate); + plugin->getNestedPlugin().getDSP().setMaximumDelayInSamples(1024); + plugin->getNestedPlugin().getDSP().setDelay(1024); + return plugin; + }), + py::arg("target_sample_rate") = 8000.0) + .def("__repr__", [](const Resample &plugin) { + std::ostringstream ss; + ss << ""; + return ss.str(); + }); +} + +} // namespace Pedalboard \ No newline at end of file diff --git a/pedalboard/plugins/AddLatency.h b/pedalboard/plugins/AddLatency.h index fa993c80..041f8696 100644 --- a/pedalboard/plugins/AddLatency.h +++ b/pedalboard/plugins/AddLatency.h @@ -16,6 +16,8 @@ */ #pragma once +#pragma once + #include "../JucePlugin.h" namespace Pedalboard { diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index 9df44d97..8d2cd44a 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -32,6 +32,8 @@ namespace py = pybind11; #include "Plugin.h" #include "process.h" +#include "plugin_templates/ResamplingPlugin.h" + #include "plugins/AddLatency.h" #include "plugins/Chorus.h" #include "plugins/Compressor.h" @@ -155,6 +157,7 @@ PYBIND11_MODULE(pedalboard_native, m) { init_noisegate(m); init_phaser(m); init_pitch_shift(m); + init_resampling_test_plugin(m); init_reverb(m); init_external_plugins(m); diff --git a/tests/test_resample.py b/tests/test_resample.py new file mode 100644 index 00000000..e900aa61 --- /dev/null +++ b/tests/test_resample.py @@ -0,0 +1,58 @@ +#! /usr/bin/env python +# +# Copyright 2021 Spotify AB +# +# Licensed under the GNU Public License, Version 3.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/gpl-3.0.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import numpy as np +from pedalboard import Pedalboard, Resample + + +@pytest.mark.parametrize("fundamental_hz", [440, 880]) +@pytest.mark.parametrize("sample_rate", [22050, 44100, 48000]) +@pytest.mark.parametrize("target_sample_rate", [22050, 44100, 48000, 8000]) +@pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) +@pytest.mark.parametrize("duration", [0.5, 2.0, 3.14159]) +@pytest.mark.parametrize("num_channels", [1, 2]) +def test_resample(fundamental_hz, sample_rate, target_sample_rate, buffer_size, duration, num_channels): + samples = np.arange(duration * sample_rate) + sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + if num_channels == 2: + sine_wave = np.stack([sine_wave, sine_wave]) + + plugin = Resample(target_sample_rate) + output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + + try: + np.testing.assert_allclose(sine_wave, output, atol=0.15) + except AssertionError: + import matplotlib.pyplot as plt + + for cut in (buffer_size * 2, len(sine_wave) // 200, len(sine_wave)): + fig, ax = plt.subplots(3) + ax[0].plot(sine_wave[:cut]) + ax[0].set_title("Input") + ax[1].plot(output[:cut]) + ax[1].set_title("Output") + ax[2].plot(np.abs(sine_wave - output)[:cut]) + ax[2].set_title("Diff") + ax[2].set_ylim(0, 1) + fig.suptitle(f"fundamental_hz={fundamental_hz}, sample_rate={sample_rate}, target_sample_rate={target_sample_rate}") + plt.savefig(f"{fundamental_hz}-{sample_rate}-{target_sample_rate}-{buffer_size}-{cut}.png", dpi=300) + plt.clf() + + import soundfile as sf + sf.write(f"{fundamental_hz}-{sample_rate}-{target_sample_rate}-{buffer_size}.wav", output, sample_rate) + raise From b4e4db2dc4705948b6dbe77aaffd9179253b68bf Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Fri, 4 Feb 2022 16:23:47 -0500 Subject: [PATCH 09/22] TEMP: Working except for slight boundary condition at end --- .../plugin_templates/ResamplingPlugin.h | 73 +++++++++++-------- tests/test_resample.py | 8 +- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/pedalboard/plugin_templates/ResamplingPlugin.h b/pedalboard/plugin_templates/ResamplingPlugin.h index e70f3eac..81dc7b75 100644 --- a/pedalboard/plugin_templates/ResamplingPlugin.h +++ b/pedalboard/plugin_templates/ResamplingPlugin.h @@ -77,16 +77,16 @@ template , // Store the remainder of the input: any samples that weren't consumed in // one pushSamples() call but would be consumable in the next one. - inputReservoir.setSize(spec.numChannels, (int)std::ceil(resamplerRatio) + + inputReservoir.setSize(spec.numChannels, (int)std::ceil(resamplerRatio) + (int)std::ceil(inverseResamplerRatio) + spec.maximumBlockSize); inStreamLatency = 0; // Add the resamplers' latencies so the output is properly aligned: - printf("nativeToTarget latency: %f, * %f = %f\n", nativeToTargetResamplers[0].getBaseLatency(), resamplerRatio, nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio); - printf("targetToNative latency: %f, * %f = %f\n", targetToNativeResamplers[0].getBaseLatency(), inverseResamplerRatio, targetToNativeResamplers[0].getBaseLatency() * inverseResamplerRatio); + // printf("nativeToTarget latency: %f, * %f = %f\n", nativeToTargetResamplers[0].getBaseLatency(), resamplerRatio, nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio); + // printf("targetToNative latency: %f, * %f = %f\n", targetToNativeResamplers[0].getBaseLatency(), inverseResamplerRatio, targetToNativeResamplers[0].getBaseLatency() * inverseResamplerRatio); inStreamLatency += std::round(nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio + targetToNativeResamplers[0].getBaseLatency()); - printf("total in-stream latency: %d\n", inStreamLatency); + // printf("total in-stream latency: %d\n", inStreamLatency); resampledBuffer.setSize(spec.numChannels, 30 * maximumBlockSizeInSampleRate + (inStreamLatency / resamplerRatio)); outputBuffer.setSize(spec.numChannels, spec.maximumBlockSize * 3 + inStreamLatency); @@ -98,7 +98,7 @@ template , const juce::dsp::ProcessContextReplacing &context) override final { auto ioBlock = context.getOutputBlock(); - printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); float expectedResampledSamples = ioBlock.getNumSamples() / resamplerRatio; @@ -118,10 +118,10 @@ template , // resampler's input: expectedResampledSamples += (float)samplesInInputReservoir / resamplerRatio; - printf( - "Copying ioBlock[%d:%d] into inputReservoir[%d:%d]\n", - 0, ioBlock.getNumSamples(), samplesInInputReservoir, samplesInInputReservoir + ioBlock.getNumSamples() - ); + // printf( + // "Copying ioBlock[%d:%d] into inputReservoir[%d:%d]\n", + // 0, ioBlock.getNumSamples(), samplesInInputReservoir, samplesInInputReservoir + ioBlock.getNumSamples() + // ); for (int c = 0; c < ioBlock.getNumChannels(); c++) { inputReservoir.copyFrom(c, samplesInInputReservoir, ioBlock.getChannelPointer(c), ioBlock.getNumSamples()); @@ -131,7 +131,7 @@ template , resampledBufferPointer, expectedResampledSamples); } - printf("Ran nativeToTargetResamplers on inputReservoir[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); + // printf("Ran nativeToTargetResamplers on inputReservoir[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); if (samplesUsed < ioBlock.getNumSamples() + samplesInInputReservoir) { // Take the missing samples and put them at the start of the input @@ -146,10 +146,10 @@ template , } samplesInInputReservoir = unusedInputSampleCount; - printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); + // printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); } else { samplesInInputReservoir = 0; - printf("Clearing input reservoir.\n"); + // printf("Clearing input reservoir.\n"); } } else { for (int c = 0; c < ioBlock.getNumChannels(); c++) { @@ -159,7 +159,7 @@ template , resampledBufferPointer, (int)expectedResampledSamples); } - printf("Ran nativeToTargetResamplers on input[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); + // printf("Ran nativeToTargetResamplers on input[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); if (samplesUsed < ioBlock.getNumSamples()) { // Take the missing samples and put them at the start of the input @@ -169,27 +169,40 @@ template , inputReservoir.copyFrom(c, 0, ioBlock.getChannelPointer(c) + samplesUsed, unusedInputSampleCount); } - printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); + // printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); samplesInInputReservoir = unusedInputSampleCount; } } cleanSamplesInResampledBuffer += (int)expectedResampledSamples; - printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); // Pass resampledBuffer to the plugin: juce::dsp::AudioBlock resampledBlock(resampledBuffer); - printf("Processing resampledBuffer[%d:%d] (%d samples) to plugin\n", processedSamplesInResampledBuffer, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, cleanSamplesInResampledBuffer); + // printf("Processing resampledBuffer[%d:%d] (%d samples) to plugin\n", processedSamplesInResampledBuffer, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, cleanSamplesInResampledBuffer); juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock(processedSamplesInResampledBuffer, cleanSamplesInResampledBuffer); - // TODO: Check that samplesInResampledBuffer is not greater than the plugin's maximumBlockSize! - juce::dsp::ProcessContextReplacing subContext(subBlock); - int resampledSamplesOutput = plugin.process(subContext); - - cleanSamplesInResampledBuffer -= resampledSamplesOutput; - processedSamplesInResampledBuffer += resampledSamplesOutput; + if (cleanSamplesInResampledBuffer) { + // TODO: Check that samplesInResampledBuffer is not greater than the plugin's maximumBlockSize! + juce::dsp::ProcessContextReplacing subContext(subBlock); + int resampledSamplesOutput = plugin.process(subContext); + + if (resampledSamplesOutput < cleanSamplesInResampledBuffer) { + // Move processed samples to the left of the buffer: + int offset = cleanSamplesInResampledBuffer - resampledSamplesOutput; + // printf("Moving %d processed samples left by %d\n", resampledSamplesOutput, offset); + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + // Move the contents of the resampled block to the left: + std::memmove((char *)resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer, + (char *)(resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer + offset), + (resampledSamplesOutput) * sizeof(SampleType)); + } + } + cleanSamplesInResampledBuffer = 0; + processedSamplesInResampledBuffer += resampledSamplesOutput; + } - printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); // Resample back to the intended sample rate: int expectedOutputSamples = processedSamplesInResampledBuffer * resamplerRatio; @@ -214,15 +227,15 @@ template , inverseResamplerRatio, resampledBuffer.getReadPointer(c), outputBufferPointer, expectedOutputSamples); } - printf("Ran targetToNativeResampler on resampledBuffer[0:%d] -> outputBuffer[%d:%d]\n", samplesConsumed, samplesInOutputBuffer, samplesInOutputBuffer + expectedOutputSamples); + // printf("Ran targetToNativeResampler on resampledBuffer[0:%d] -> outputBuffer[%d:%d]\n", samplesConsumed, samplesInOutputBuffer, samplesInOutputBuffer + expectedOutputSamples); samplesInOutputBuffer += expectedOutputSamples; - printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); int samplesRemainingInResampledBuffer = processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer - samplesConsumed; if (samplesRemainingInResampledBuffer > 0) { - printf("Moving %d samples left by %d\n", samplesRemainingInResampledBuffer, samplesConsumed); + // printf("Moving %d samples left by %d\n", samplesRemainingInResampledBuffer, samplesConsumed); for (int c = 0; c < ioBlock.getNumChannels(); c++) { // Move the contents of the resampled block to the left: std::memmove((char *)resampledBuffer.getWritePointer(c), @@ -235,12 +248,12 @@ template , // Copy from output buffer to output block: int samplesToOutput = std::min(ioBlock.getNumSamples(), (unsigned long) samplesInOutputBuffer); - printf("Copying %d samples from output buffer to ioBlock at %d (%d samples large)\n", samplesToOutput, ioBlock.getNumSamples() - samplesToOutput, ioBlock.getNumSamples()); + // printf("Copying %d samples from output buffer to ioBlock at %d (%d samples large)\n", samplesToOutput, ioBlock.getNumSamples() - samplesToOutput, ioBlock.getNumSamples()); ioBlock.copyFrom(outputBuffer, 0, ioBlock.getNumSamples() - samplesToOutput, samplesToOutput); int samplesRemainingInOutputBuffer = samplesInOutputBuffer - samplesToOutput; if (samplesRemainingInOutputBuffer > 0) { - printf("Moving %d samples left in output buffer by %d\n", samplesRemainingInOutputBuffer, samplesToOutput); + // printf("Moving %d samples left in output buffer by %d\n", samplesRemainingInOutputBuffer, samplesToOutput); for (int c = 0; c < ioBlock.getNumChannels(); c++) { // Move the contents of the resampled block to the left: std::memmove((char *)outputBuffer.getWritePointer(c), @@ -250,7 +263,7 @@ template , } samplesInOutputBuffer -= samplesToOutput; - printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); + // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); samplesProduced += samplesToOutput; int samplesToReturn = std::min((long)(samplesProduced - inStreamLatency), @@ -258,7 +271,7 @@ template , if (samplesToReturn < 0) samplesToReturn = 0; - printf("Returning %d samples\n", samplesToReturn); + // printf("Returning %d samples\n", samplesToReturn); return samplesToReturn; } diff --git a/tests/test_resample.py b/tests/test_resample.py index e900aa61..2bc602ec 100644 --- a/tests/test_resample.py +++ b/tests/test_resample.py @@ -20,11 +20,11 @@ from pedalboard import Pedalboard, Resample -@pytest.mark.parametrize("fundamental_hz", [440, 880]) -@pytest.mark.parametrize("sample_rate", [22050, 44100, 48000]) -@pytest.mark.parametrize("target_sample_rate", [22050, 44100, 48000, 8000]) +@pytest.mark.parametrize("fundamental_hz", [440]) +@pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) +@pytest.mark.parametrize("target_sample_rate", [8000, 22050, 44100, 48000]) @pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) -@pytest.mark.parametrize("duration", [0.5, 2.0, 3.14159]) +@pytest.mark.parametrize("duration", [0.5, 1.2345]) @pytest.mark.parametrize("num_channels", [1, 2]) def test_resample(fundamental_hz, sample_rate, target_sample_rate, buffer_size, duration, num_channels): samples = np.arange(duration * sample_rate) From 34635636f1c4ddcb16fbb98017b9b8af2219fe0f Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sat, 5 Feb 2022 10:01:48 -0500 Subject: [PATCH 10/22] Got resampler working at any sample rate. --- .../{ResamplingPlugin.h => Resample.h} | 182 +++++++++--------- pedalboard/python_bindings.cpp | 4 +- tests/test_resample.py | 37 ++-- 3 files changed, 102 insertions(+), 121 deletions(-) rename pedalboard/plugin_templates/{ResamplingPlugin.h => Resample.h} (55%) diff --git a/pedalboard/plugin_templates/ResamplingPlugin.h b/pedalboard/plugin_templates/Resample.h similarity index 55% rename from pedalboard/plugin_templates/ResamplingPlugin.h rename to pedalboard/plugin_templates/Resample.h index 81dc7b75..d9cedca1 100644 --- a/pedalboard/plugin_templates/ResamplingPlugin.h +++ b/pedalboard/plugin_templates/Resample.h @@ -26,8 +26,7 @@ namespace Pedalboard { /** * A test plugin used to verify the behaviour of the ResamplingPlugin wrapper. */ -template -class Passthrough : public Plugin { +template class Passthrough : public Plugin { public: virtual ~Passthrough(){}; @@ -47,8 +46,8 @@ class Passthrough : public Plugin { * resampled audio and its sampleRate and maximumBlockSize parameters * are adjusted accordingly. */ -template , - typename SampleType = float> class Resample : public Plugin { +template , typename SampleType = float> +class Resample : public Plugin { public: virtual ~Resample(){}; @@ -66,48 +65,51 @@ template , inverseResamplerRatio = targetSampleRate / spec.sampleRate; const juce::dsp::ProcessSpec subSpec = { - .numChannels = spec.numChannels, - .sampleRate = targetSampleRate, - .maximumBlockSize = static_cast(spec.maximumBlockSize * resamplerRatio) - }; + .numChannels = spec.numChannels, + .sampleRate = targetSampleRate, + .maximumBlockSize = static_cast(spec.maximumBlockSize * + resamplerRatio)}; plugin.prepare(subSpec); - int maximumBlockSizeInSampleRate = - spec.maximumBlockSize / resamplerRatio; + int maximumBlockSizeInSampleRate = spec.maximumBlockSize / resamplerRatio; // Store the remainder of the input: any samples that weren't consumed in // one pushSamples() call but would be consumable in the next one. - inputReservoir.setSize(spec.numChannels, (int)std::ceil(resamplerRatio) + (int)std::ceil(inverseResamplerRatio) + - spec.maximumBlockSize); + inputReservoir.setSize(spec.numChannels, + 2 * ((int)std::ceil(resamplerRatio) + + (int)std::ceil(inverseResamplerRatio)) + + spec.maximumBlockSize); inStreamLatency = 0; // Add the resamplers' latencies so the output is properly aligned: - // printf("nativeToTarget latency: %f, * %f = %f\n", nativeToTargetResamplers[0].getBaseLatency(), resamplerRatio, nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio); - // printf("targetToNative latency: %f, * %f = %f\n", targetToNativeResamplers[0].getBaseLatency(), inverseResamplerRatio, targetToNativeResamplers[0].getBaseLatency() * inverseResamplerRatio); - inStreamLatency += std::round(nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio + targetToNativeResamplers[0].getBaseLatency()); - // printf("total in-stream latency: %d\n", inStreamLatency); + inStreamLatency += std::round( + nativeToTargetResamplers[0].getBaseLatency() * resamplerRatio + + targetToNativeResamplers[0].getBaseLatency()); + + resampledBuffer.setSize(spec.numChannels, + maximumBlockSizeInSampleRate + + (inStreamLatency / resamplerRatio)); + outputBuffer.setSize(spec.numChannels, + spec.maximumBlockSize * 3 + inStreamLatency); - resampledBuffer.setSize(spec.numChannels, 30 * maximumBlockSizeInSampleRate + (inStreamLatency / resamplerRatio)); - outputBuffer.setSize(spec.numChannels, spec.maximumBlockSize * 3 + inStreamLatency); lastSpec = spec; } } - int process( - const juce::dsp::ProcessContextReplacing &context) override final { + int process(const juce::dsp::ProcessContextReplacing &context) + override final { auto ioBlock = context.getOutputBlock(); - // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); - float expectedResampledSamples = ioBlock.getNumSamples() / resamplerRatio; if (spaceAvailableInResampledBuffer() < expectedResampledSamples) { throw std::runtime_error( "More samples were provided than can be buffered! This is an " "internal Pedalboard error and should be reported. Buffer had " + - std::to_string(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer) + "/" + - std::to_string(resampledBuffer.getNumSamples()) + + std::to_string(processedSamplesInResampledBuffer + + cleanSamplesInResampledBuffer) + + "/" + std::to_string(resampledBuffer.getNumSamples()) + " samples at target sample rate, but was provided " + std::to_string(expectedResampledSamples) + "."); } @@ -116,23 +118,21 @@ template , if (samplesInInputReservoir) { // Copy the input samples into the input reservoir and use that as the // resampler's input: - expectedResampledSamples += (float)samplesInInputReservoir / resamplerRatio; - - // printf( - // "Copying ioBlock[%d:%d] into inputReservoir[%d:%d]\n", - // 0, ioBlock.getNumSamples(), samplesInInputReservoir, samplesInInputReservoir + ioBlock.getNumSamples() - // ); - for (int c = 0; c < ioBlock.getNumChannels(); c++) { - inputReservoir.copyFrom(c, samplesInInputReservoir, ioBlock.getChannelPointer(c), - ioBlock.getNumSamples()); - SampleType *resampledBufferPointer = resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer; + expectedResampledSamples += + (float)samplesInInputReservoir / resamplerRatio; + + for (int c = 0; c < ioBlock.getNumChannels(); c++) { + inputReservoir.copyFrom(c, samplesInInputReservoir, + ioBlock.getChannelPointer(c), + ioBlock.getNumSamples()); + SampleType *resampledBufferPointer = + resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer; samplesUsed = nativeToTargetResamplers[c].process( - resamplerRatio, inputReservoir.getReadPointer(c), - resampledBufferPointer, expectedResampledSamples); + resamplerRatio, inputReservoir.getReadPointer(c), + resampledBufferPointer, expectedResampledSamples); } - // printf("Ran nativeToTargetResamplers on inputReservoir[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); - if (samplesUsed < ioBlock.getNumSamples() + samplesInInputReservoir) { // Take the missing samples and put them at the start of the input // reservoir for next time: @@ -140,72 +140,70 @@ template , (ioBlock.getNumSamples() + samplesInInputReservoir) - samplesUsed; for (int c = 0; c < ioBlock.getNumChannels(); c++) { - inputReservoir.copyFrom(c, 0, - inputReservoir.getReadPointer(c) + samplesUsed, - unusedInputSampleCount); + inputReservoir.copyFrom( + c, 0, inputReservoir.getReadPointer(c) + samplesUsed, + unusedInputSampleCount); } samplesInInputReservoir = unusedInputSampleCount; - // printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); } else { samplesInInputReservoir = 0; - // printf("Clearing input reservoir.\n"); } } else { for (int c = 0; c < ioBlock.getNumChannels(); c++) { - SampleType *resampledBufferPointer = resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer; + SampleType *resampledBufferPointer = + resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer; samplesUsed = nativeToTargetResamplers[c].process( - resamplerRatio, ioBlock.getChannelPointer(c), - resampledBufferPointer, (int)expectedResampledSamples); + resamplerRatio, ioBlock.getChannelPointer(c), + resampledBufferPointer, (int)expectedResampledSamples); } - // printf("Ran nativeToTargetResamplers on input[0:%d] -> resampledBuffer[%d:%d]\n", samplesUsed, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, (int)(processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer + expectedResampledSamples)); - if (samplesUsed < ioBlock.getNumSamples()) { // Take the missing samples and put them at the start of the input // reservoir for next time: int unusedInputSampleCount = ioBlock.getNumSamples() - samplesUsed; for (int c = 0; c < ioBlock.getNumChannels(); c++) { - inputReservoir.copyFrom(c, 0, ioBlock.getChannelPointer(c) + samplesUsed, + inputReservoir.copyFrom(c, 0, + ioBlock.getChannelPointer(c) + samplesUsed, unusedInputSampleCount); } - // printf("Copied remaining %d samples into input reservoir\n", unusedInputSampleCount); samplesInInputReservoir = unusedInputSampleCount; } } cleanSamplesInResampledBuffer += (int)expectedResampledSamples; - // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); - // Pass resampledBuffer to the plugin: juce::dsp::AudioBlock resampledBlock(resampledBuffer); - // printf("Processing resampledBuffer[%d:%d] (%d samples) to plugin\n", processedSamplesInResampledBuffer, processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer, cleanSamplesInResampledBuffer); - juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock(processedSamplesInResampledBuffer, cleanSamplesInResampledBuffer); + juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock( + processedSamplesInResampledBuffer, cleanSamplesInResampledBuffer); if (cleanSamplesInResampledBuffer) { - // TODO: Check that samplesInResampledBuffer is not greater than the plugin's maximumBlockSize! + // TODO: Check that samplesInResampledBuffer is not greater than the + // plugin's maximumBlockSize! juce::dsp::ProcessContextReplacing subContext(subBlock); int resampledSamplesOutput = plugin.process(subContext); if (resampledSamplesOutput < cleanSamplesInResampledBuffer) { // Move processed samples to the left of the buffer: int offset = cleanSamplesInResampledBuffer - resampledSamplesOutput; - // printf("Moving %d processed samples left by %d\n", resampledSamplesOutput, offset); + for (int c = 0; c < ioBlock.getNumChannels(); c++) { // Move the contents of the resampled block to the left: - std::memmove((char *)resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer, - (char *)(resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer + offset), - (resampledSamplesOutput) * sizeof(SampleType)); + std::memmove((char *)resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer, + (char *)(resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer + offset), + (resampledSamplesOutput) * sizeof(SampleType)); } } cleanSamplesInResampledBuffer = 0; processedSamplesInResampledBuffer += resampledSamplesOutput; } - // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); - // Resample back to the intended sample rate: - int expectedOutputSamples = processedSamplesInResampledBuffer * resamplerRatio; + int expectedOutputSamples = + processedSamplesInResampledBuffer * resamplerRatio; int samplesConsumed = 0; @@ -217,61 +215,58 @@ template , std::to_string(outputBuffer.getNumSamples()) + " samples at native sample rate, but was provided " + std::to_string(expectedOutputSamples) + "."); - } + } - for (int c = 0; c < ioBlock.getNumChannels(); c++) { float *outputBufferPointer = outputBuffer.getWritePointer(c) + samplesInOutputBuffer; samplesConsumed = targetToNativeResamplers[c].process( - inverseResamplerRatio, resampledBuffer.getReadPointer(c), - outputBufferPointer, expectedOutputSamples); + inverseResamplerRatio, resampledBuffer.getReadPointer(c), + outputBufferPointer, expectedOutputSamples); } - // printf("Ran targetToNativeResampler on resampledBuffer[0:%d] -> outputBuffer[%d:%d]\n", samplesConsumed, samplesInOutputBuffer, samplesInOutputBuffer + expectedOutputSamples); samplesInOutputBuffer += expectedOutputSamples; - // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); - - int samplesRemainingInResampledBuffer = processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer - samplesConsumed; + int samplesRemainingInResampledBuffer = processedSamplesInResampledBuffer + + cleanSamplesInResampledBuffer - + samplesConsumed; if (samplesRemainingInResampledBuffer > 0) { - // printf("Moving %d samples left by %d\n", samplesRemainingInResampledBuffer, samplesConsumed); for (int c = 0; c < ioBlock.getNumChannels(); c++) { // Move the contents of the resampled block to the left: - std::memmove((char *)resampledBuffer.getWritePointer(c), - (char *)(resampledBuffer.getWritePointer(c) + samplesConsumed), - (samplesRemainingInResampledBuffer) * sizeof(SampleType)); + std::memmove( + (char *)resampledBuffer.getWritePointer(c), + (char *)(resampledBuffer.getWritePointer(c) + samplesConsumed), + (samplesRemainingInResampledBuffer) * sizeof(SampleType)); } } processedSamplesInResampledBuffer -= samplesConsumed; // Copy from output buffer to output block: - int samplesToOutput = std::min(ioBlock.getNumSamples(), (unsigned long) samplesInOutputBuffer); - // printf("Copying %d samples from output buffer to ioBlock at %d (%d samples large)\n", samplesToOutput, ioBlock.getNumSamples() - samplesToOutput, ioBlock.getNumSamples()); - ioBlock.copyFrom(outputBuffer, 0, ioBlock.getNumSamples() - samplesToOutput, samplesToOutput); + int samplesToOutput = + std::min(ioBlock.getNumSamples(), (unsigned long)samplesInOutputBuffer); + ioBlock.copyFrom(outputBuffer, 0, ioBlock.getNumSamples() - samplesToOutput, + samplesToOutput); - int samplesRemainingInOutputBuffer = samplesInOutputBuffer - samplesToOutput; + int samplesRemainingInOutputBuffer = + samplesInOutputBuffer - samplesToOutput; if (samplesRemainingInOutputBuffer > 0) { - // printf("Moving %d samples left in output buffer by %d\n", samplesRemainingInOutputBuffer, samplesToOutput); for (int c = 0; c < ioBlock.getNumChannels(); c++) { // Move the contents of the resampled block to the left: - std::memmove((char *)outputBuffer.getWritePointer(c), - (char *)(outputBuffer.getWritePointer(c) + samplesToOutput), - (samplesRemainingInOutputBuffer) * sizeof(SampleType)); + std::memmove( + (char *)outputBuffer.getWritePointer(c), + (char *)(outputBuffer.getWritePointer(c) + samplesToOutput), + (samplesRemainingInOutputBuffer) * sizeof(SampleType)); } } samplesInOutputBuffer -= samplesToOutput; - // printf("[BUFFERS] [I] %d [R] C:%d P:%d [O] %d\n", ioBlock.getNumSamples(), cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer, samplesInOutputBuffer); - samplesProduced += samplesToOutput; int samplesToReturn = std::min((long)(samplesProduced - inStreamLatency), (long)samplesToOutput); if (samplesToReturn < 0) samplesToReturn = 0; - // printf("Returning %d samples\n", samplesToReturn); return samplesToReturn; } @@ -279,18 +274,13 @@ template , targetSampleRate = newSampleRate; } - float getTargetSampleRate() const { - return targetSampleRate; - } + float getTargetSampleRate() const { return targetSampleRate; } T &getNestedPlugin() { return plugin; } virtual void reset() override final { - for (int c = 0; c < nativeToTargetResamplers.size(); c++) { - nativeToTargetResamplers[c].reset(); - targetToNativeResamplers[c].reset(); - } - + nativeToTargetResamplers.clear(); + targetToNativeResamplers.clear(); resampledBuffer.clear(); outputBuffer.clear(); @@ -330,7 +320,9 @@ template , int inStreamLatency = 0; int spaceAvailableInResampledBuffer() const { - return resampledBuffer.getNumSamples() - std::max(cleanSamplesInResampledBuffer, processedSamplesInResampledBuffer); + return resampledBuffer.getNumSamples() - + std::max(cleanSamplesInResampledBuffer, + processedSamplesInResampledBuffer); } int spaceAvailableInOutputBuffer() const { @@ -343,6 +335,8 @@ inline void init_resampling_test_plugin(py::module &m) { .def(py::init([](float targetSampleRate) { auto plugin = std::make_unique>(); plugin->setTargetSampleRate(targetSampleRate); + + // Set a delay on this test plugin: plugin->getNestedPlugin().getDSP().setMaximumDelayInSamples(1024); plugin->getNestedPlugin().getDSP().setDelay(1024); return plugin; diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index 8d2cd44a..9be1653d 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -32,7 +32,7 @@ namespace py = pybind11; #include "Plugin.h" #include "process.h" -#include "plugin_templates/ResamplingPlugin.h" +#include "plugin_templates/Resample.h" #include "plugins/AddLatency.h" #include "plugins/Chorus.h" @@ -40,8 +40,8 @@ namespace py = pybind11; #include "plugins/Convolution.h" #include "plugins/Delay.h" #include "plugins/Distortion.h" -#include "plugins/Gain.h" #include "plugins/GSMCompressor.h" +#include "plugins/Gain.h" #include "plugins/HighpassFilter.h" #include "plugins/Invert.h" #include "plugins/LadderFilter.h" diff --git a/tests/test_resample.py b/tests/test_resample.py index 2bc602ec..e465a7ae 100644 --- a/tests/test_resample.py +++ b/tests/test_resample.py @@ -22,37 +22,24 @@ @pytest.mark.parametrize("fundamental_hz", [440]) @pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) -@pytest.mark.parametrize("target_sample_rate", [8000, 22050, 44100, 48000]) +@pytest.mark.parametrize("target_sample_rate", [8000, 22050, 44100, 48000, 1234.56]) @pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) -@pytest.mark.parametrize("duration", [0.5, 1.2345]) +@pytest.mark.parametrize("duration", [0.5, 1.23456]) @pytest.mark.parametrize("num_channels", [1, 2]) -def test_resample(fundamental_hz, sample_rate, target_sample_rate, buffer_size, duration, num_channels): +def test_resample( + fundamental_hz, sample_rate, target_sample_rate, buffer_size, duration, num_channels +): samples = np.arange(duration * sample_rate) sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + # Fade the sine wave in at the start and out at the end to remove any transients: + fade_duration = int(sample_rate * 0.1) + sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) + sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) if num_channels == 2: sine_wave = np.stack([sine_wave, sine_wave]) + if num_channels == 2: + np.testing.assert_allclose(sine_wave[0], sine_wave[1]) + plugin = Resample(target_sample_rate) output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) - - try: - np.testing.assert_allclose(sine_wave, output, atol=0.15) - except AssertionError: - import matplotlib.pyplot as plt - - for cut in (buffer_size * 2, len(sine_wave) // 200, len(sine_wave)): - fig, ax = plt.subplots(3) - ax[0].plot(sine_wave[:cut]) - ax[0].set_title("Input") - ax[1].plot(output[:cut]) - ax[1].set_title("Output") - ax[2].plot(np.abs(sine_wave - output)[:cut]) - ax[2].set_title("Diff") - ax[2].set_ylim(0, 1) - fig.suptitle(f"fundamental_hz={fundamental_hz}, sample_rate={sample_rate}, target_sample_rate={target_sample_rate}") - plt.savefig(f"{fundamental_hz}-{sample_rate}-{target_sample_rate}-{buffer_size}-{cut}.png", dpi=300) - plt.clf() - - import soundfile as sf - sf.write(f"{fundamental_hz}-{sample_rate}-{target_sample_rate}-{buffer_size}.wav", output, sample_rate) - raise From 844fcdd47e6e97c4e953cc1f66a937ec3e15bbaf Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sat, 5 Feb 2022 10:52:01 -0500 Subject: [PATCH 11/22] Add proper Resampler plugin interface. --- pedalboard/plugin_templates/Resample.h | 121 ++++++++++++++++--------- pedalboard/python_bindings.cpp | 3 +- tests/test_resample.py | 13 ++- 3 files changed, 91 insertions(+), 46 deletions(-) diff --git a/pedalboard/plugin_templates/Resample.h b/pedalboard/plugin_templates/Resample.h index d9cedca1..be1e0513 100644 --- a/pedalboard/plugin_templates/Resample.h +++ b/pedalboard/plugin_templates/Resample.h @@ -63,16 +63,15 @@ class Resample : public Plugin { resamplerRatio = spec.sampleRate / targetSampleRate; inverseResamplerRatio = targetSampleRate / spec.sampleRate; + maximumBlockSizeInTargetSampleRate = + std::ceil(spec.maximumBlockSize / resamplerRatio); const juce::dsp::ProcessSpec subSpec = { .numChannels = spec.numChannels, .sampleRate = targetSampleRate, - .maximumBlockSize = static_cast(spec.maximumBlockSize * - resamplerRatio)}; + .maximumBlockSize = maximumBlockSizeInTargetSampleRate}; plugin.prepare(subSpec); - int maximumBlockSizeInSampleRate = spec.maximumBlockSize / resamplerRatio; - // Store the remainder of the input: any samples that weren't consumed in // one pushSamples() call but would be consumable in the next one. inputReservoir.setSize(spec.numChannels, @@ -88,7 +87,7 @@ class Resample : public Plugin { targetToNativeResamplers[0].getBaseLatency()); resampledBuffer.setSize(spec.numChannels, - maximumBlockSizeInSampleRate + + maximumBlockSizeInTargetSampleRate + (inStreamLatency / resamplerRatio)); outputBuffer.setSize(spec.numChannels, spec.maximumBlockSize * 3 + inStreamLatency); @@ -114,14 +113,14 @@ class Resample : public Plugin { std::to_string(expectedResampledSamples) + "."); } - int samplesUsed = 0; + unsigned long samplesUsed = 0; if (samplesInInputReservoir) { // Copy the input samples into the input reservoir and use that as the // resampler's input: expectedResampledSamples += (float)samplesInInputReservoir / resamplerRatio; - for (int c = 0; c < ioBlock.getNumChannels(); c++) { + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { inputReservoir.copyFrom(c, samplesInInputReservoir, ioBlock.getChannelPointer(c), ioBlock.getNumSamples()); @@ -139,7 +138,7 @@ class Resample : public Plugin { int unusedInputSampleCount = (ioBlock.getNumSamples() + samplesInInputReservoir) - samplesUsed; - for (int c = 0; c < ioBlock.getNumChannels(); c++) { + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { inputReservoir.copyFrom( c, 0, inputReservoir.getReadPointer(c) + samplesUsed, unusedInputSampleCount); @@ -150,7 +149,7 @@ class Resample : public Plugin { samplesInInputReservoir = 0; } } else { - for (int c = 0; c < ioBlock.getNumChannels(); c++) { + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { SampleType *resampledBufferPointer = resampledBuffer.getWritePointer(c) + processedSamplesInResampledBuffer + cleanSamplesInResampledBuffer; @@ -163,7 +162,7 @@ class Resample : public Plugin { // Take the missing samples and put them at the start of the input // reservoir for next time: int unusedInputSampleCount = ioBlock.getNumSamples() - samplesUsed; - for (int c = 0; c < ioBlock.getNumChannels(); c++) { + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { inputReservoir.copyFrom(c, 0, ioBlock.getChannelPointer(c) + samplesUsed, unusedInputSampleCount); @@ -174,31 +173,40 @@ class Resample : public Plugin { cleanSamplesInResampledBuffer += (int)expectedResampledSamples; - // Pass resampledBuffer to the plugin: + // Pass resampledBuffer to the plugin, in chunks: juce::dsp::AudioBlock resampledBlock(resampledBuffer); - juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock( - processedSamplesInResampledBuffer, cleanSamplesInResampledBuffer); + if (cleanSamplesInResampledBuffer) { - // TODO: Check that samplesInResampledBuffer is not greater than the - // plugin's maximumBlockSize! - juce::dsp::ProcessContextReplacing subContext(subBlock); - int resampledSamplesOutput = plugin.process(subContext); - - if (resampledSamplesOutput < cleanSamplesInResampledBuffer) { - // Move processed samples to the left of the buffer: - int offset = cleanSamplesInResampledBuffer - resampledSamplesOutput; - - for (int c = 0; c < ioBlock.getNumChannels(); c++) { - // Move the contents of the resampled block to the left: - std::memmove((char *)resampledBuffer.getWritePointer(c) + - processedSamplesInResampledBuffer, - (char *)(resampledBuffer.getWritePointer(c) + - processedSamplesInResampledBuffer + offset), - (resampledSamplesOutput) * sizeof(SampleType)); + // Only pass in the maximumBlockSize (in target sample rate) that the + // sub-plugin expects: + while (cleanSamplesInResampledBuffer > 0) { + int cleanSamplesThisBlock = + std::min((int)maximumBlockSizeInTargetSampleRate, + cleanSamplesInResampledBuffer); + + juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock( + processedSamplesInResampledBuffer, cleanSamplesThisBlock); + juce::dsp::ProcessContextReplacing subContext(subBlock); + + int resampledSamplesOutput = plugin.process(subContext); + + if (resampledSamplesOutput < cleanSamplesThisBlock) { + // Move processed samples to the left of the buffer: + int offset = cleanSamplesThisBlock - resampledSamplesOutput; + + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { + // Move the contents of the resampled block to the left: + std::memmove((char *)resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer, + (char *)(resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer + offset), + (resampledSamplesOutput) * sizeof(SampleType)); + } } + + processedSamplesInResampledBuffer += resampledSamplesOutput; + cleanSamplesInResampledBuffer -= resampledSamplesOutput; } - cleanSamplesInResampledBuffer = 0; - processedSamplesInResampledBuffer += resampledSamplesOutput; } // Resample back to the intended sample rate: @@ -217,7 +225,7 @@ class Resample : public Plugin { std::to_string(expectedOutputSamples) + "."); } - for (int c = 0; c < ioBlock.getNumChannels(); c++) { + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { float *outputBufferPointer = outputBuffer.getWritePointer(c) + samplesInOutputBuffer; samplesConsumed = targetToNativeResamplers[c].process( @@ -231,7 +239,7 @@ class Resample : public Plugin { cleanSamplesInResampledBuffer - samplesConsumed; if (samplesRemainingInResampledBuffer > 0) { - for (int c = 0; c < ioBlock.getNumChannels(); c++) { + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { // Move the contents of the resampled block to the left: std::memmove( (char *)resampledBuffer.getWritePointer(c), @@ -251,7 +259,7 @@ class Resample : public Plugin { int samplesRemainingInOutputBuffer = samplesInOutputBuffer - samplesToOutput; if (samplesRemainingInOutputBuffer > 0) { - for (int c = 0; c < ioBlock.getNumChannels(); c++) { + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { // Move the contents of the resampled block to the left: std::memmove( (char *)outputBuffer.getWritePointer(c), @@ -293,9 +301,12 @@ class Resample : public Plugin { samplesProduced = 0; inStreamLatency = 0; + maximumBlockSizeInTargetSampleRate = 0; } - virtual int getLatencyHint() override { return inStreamLatency; } + virtual int getLatencyHint() override { + return inStreamLatency + (plugin.getLatencyHint() * resamplerRatio); + } private: T plugin; @@ -318,6 +329,7 @@ class Resample : public Plugin { int samplesProduced = 0; int inStreamLatency = 0; + unsigned int maximumBlockSizeInTargetSampleRate = 0; int spaceAvailableInResampledBuffer() const { return resampledBuffer.getNumSamples() - @@ -330,19 +342,16 @@ class Resample : public Plugin { } }; -inline void init_resampling_test_plugin(py::module &m) { - py::class_, Plugin>(m, "Resample") + +inline void init_resample(py::module &m) { + py::class_, float>, Plugin>(m, "Resample") .def(py::init([](float targetSampleRate) { - auto plugin = std::make_unique>(); + auto plugin = std::make_unique, float>>(); plugin->setTargetSampleRate(targetSampleRate); - - // Set a delay on this test plugin: - plugin->getNestedPlugin().getDSP().setMaximumDelayInSamples(1024); - plugin->getNestedPlugin().getDSP().setDelay(1024); return plugin; }), py::arg("target_sample_rate") = 8000.0) - .def("__repr__", [](const Resample &plugin) { + .def("__repr__", [](const Resample, float> &plugin) { std::ostringstream ss; ss << ">(); + plugin->setTargetSampleRate(targetSampleRate); + plugin->getNestedPlugin().getDSP().setMaximumDelayInSamples(internalLatency); + plugin->getNestedPlugin().getDSP().setDelay(internalLatency); + return plugin; + }), + py::arg("target_sample_rate") = 8000.0, py::arg("internal_latency") = 1024) + .def("__repr__", [](Resample &plugin) { + std::ostringstream ss; + ss << ""; + return ss.str(); + }); +} + } // namespace Pedalboard \ No newline at end of file diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index 9be1653d..eff491f8 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -157,7 +157,7 @@ PYBIND11_MODULE(pedalboard_native, m) { init_noisegate(m); init_phaser(m); init_pitch_shift(m); - init_resampling_test_plugin(m); + init_resample(m); init_reverb(m); init_external_plugins(m); @@ -166,4 +166,5 @@ PYBIND11_MODULE(pedalboard_native, m) { py::module internal = m.def_submodule("_internal"); init_add_latency(internal); init_prime_with_silence_test_plugin(internal); + init_resample_with_latency(internal); }; diff --git a/tests/test_resample.py b/tests/test_resample.py index e465a7ae..7980aafa 100644 --- a/tests/test_resample.py +++ b/tests/test_resample.py @@ -18,6 +18,8 @@ import pytest import numpy as np from pedalboard import Pedalboard, Resample +from pedalboard_native._internal import ResampleWithLatency + @pytest.mark.parametrize("fundamental_hz", [440]) @@ -26,8 +28,15 @@ @pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) @pytest.mark.parametrize("duration", [0.5, 1.23456]) @pytest.mark.parametrize("num_channels", [1, 2]) +@pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) def test_resample( - fundamental_hz, sample_rate, target_sample_rate, buffer_size, duration, num_channels + fundamental_hz: float, + sample_rate: float, + target_sample_rate: float, + buffer_size: int, + duration: float, + num_channels: int, + plugin_class, ): samples = np.arange(duration * sample_rate) sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) @@ -41,5 +50,5 @@ def test_resample( if num_channels == 2: np.testing.assert_allclose(sine_wave[0], sine_wave[1]) - plugin = Resample(target_sample_rate) + plugin = plugin_class(target_sample_rate) output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) From 07e6d6fd23a0b7bbf42d892eeb3621c7b1a382d2 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sat, 5 Feb 2022 14:25:28 -0500 Subject: [PATCH 12/22] Add different quality levels. --- pedalboard/plugin_templates/Resample.h | 291 +++++++++++++++++++------ tests/test_resample.py | 117 +++++++++- 2 files changed, 345 insertions(+), 63 deletions(-) diff --git a/pedalboard/plugin_templates/Resample.h b/pedalboard/plugin_templates/Resample.h index be1e0513..a95019bb 100644 --- a/pedalboard/plugin_templates/Resample.h +++ b/pedalboard/plugin_templates/Resample.h @@ -23,6 +23,80 @@ namespace Pedalboard { +#include + +/** + * The various levels of resampler quality available from JUCE. + * More could be added here, but these should cover the vast + * majority of use cases. + */ +enum class ResamplingQuality { + ZeroOrderHold = 0, + Linear = 1, + CatmullRom = 2, + Lagrange = 3, + WindowedSinc = 4, +}; + +/** + * A wrapper class that allows changing the quality of a resampler, + * as the JUCE GenericInterpolator implementations are each separate classes. + */ +class VariableQualityResampler { +public: + void setQuality(const ResamplingQuality newQuality) { + switch (newQuality) { + case ResamplingQuality::ZeroOrderHold: + interpolator = juce::Interpolators::ZeroOrderHold(); + break; + case ResamplingQuality::Linear: + interpolator = juce::Interpolators::Linear(); + break; + case ResamplingQuality::CatmullRom: + interpolator = juce::Interpolators::CatmullRom(); + break; + case ResamplingQuality::Lagrange: + interpolator = juce::Interpolators::Lagrange(); + break; + case ResamplingQuality::WindowedSinc: + interpolator = juce::Interpolators::WindowedSinc(); + break; + default: + throw std::domain_error("Unknown resampler quality received!"); + } + } + + ResamplingQuality getQuality() const { + return (ResamplingQuality)interpolator.index(); + } + + float getBaseLatency() const { + return std::visit([](auto &&i) -> float { return i.getBaseLatency(); }, + interpolator); + } + + void reset() noexcept { + std::visit([](auto &&i) { return i.reset(); }, interpolator); + } + + int process(double speedRatio, const float *inputSamples, + float *outputSamples, int numOutputSamplesToProduce) noexcept { + return std::visit( + [speedRatio, inputSamples, outputSamples, + numOutputSamplesToProduce](auto &&i) -> float { + return i.process(speedRatio, inputSamples, outputSamples, + numOutputSamplesToProduce); + }, + interpolator); + } + +private: + std::variant + interpolator; +}; + /** * A test plugin used to verify the behaviour of the ResamplingPlugin wrapper. */ @@ -61,6 +135,12 @@ class Resample : public Plugin { nativeToTargetResamplers.resize(spec.numChannels); targetToNativeResamplers.resize(spec.numChannels); + // Set the quality on each resampler: + for (int i = 0; i < spec.numChannels; i++) { + nativeToTargetResamplers[i].setQuality(quality); + targetToNativeResamplers[i].setQuality(quality); + } + resamplerRatio = spec.sampleRate / targetSampleRate; inverseResamplerRatio = targetSampleRate / spec.sampleRate; maximumBlockSizeInTargetSampleRate = @@ -87,10 +167,12 @@ class Resample : public Plugin { targetToNativeResamplers[0].getBaseLatency()); resampledBuffer.setSize(spec.numChannels, - maximumBlockSizeInTargetSampleRate + + ((maximumBlockSizeInTargetSampleRate + 1) * 3) + (inStreamLatency / resamplerRatio)); - outputBuffer.setSize(spec.numChannels, - spec.maximumBlockSize * 3 + inStreamLatency); + outputBuffer.setSize( + spec.numChannels, + (int)std::ceil(resampledBuffer.getNumSamples() * resamplerRatio) + + spec.maximumBlockSize); lastSpec = spec; } @@ -176,37 +258,36 @@ class Resample : public Plugin { // Pass resampledBuffer to the plugin, in chunks: juce::dsp::AudioBlock resampledBlock(resampledBuffer); - if (cleanSamplesInResampledBuffer) { - // Only pass in the maximumBlockSize (in target sample rate) that the - // sub-plugin expects: - while (cleanSamplesInResampledBuffer > 0) { - int cleanSamplesThisBlock = - std::min((int)maximumBlockSizeInTargetSampleRate, - cleanSamplesInResampledBuffer); - - juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock( - processedSamplesInResampledBuffer, cleanSamplesThisBlock); - juce::dsp::ProcessContextReplacing subContext(subBlock); - - int resampledSamplesOutput = plugin.process(subContext); - - if (resampledSamplesOutput < cleanSamplesThisBlock) { - // Move processed samples to the left of the buffer: - int offset = cleanSamplesThisBlock - resampledSamplesOutput; - - for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { - // Move the contents of the resampled block to the left: - std::memmove((char *)resampledBuffer.getWritePointer(c) + - processedSamplesInResampledBuffer, - (char *)(resampledBuffer.getWritePointer(c) + - processedSamplesInResampledBuffer + offset), - (resampledSamplesOutput) * sizeof(SampleType)); - } - } + // Only pass in the maximumBlockSize (in target sample rate) that the + // sub-plugin expects: + while (cleanSamplesInResampledBuffer > 0) { + int cleanSamplesToProcess = + std::min((int)maximumBlockSizeInTargetSampleRate, + cleanSamplesInResampledBuffer); + + juce::dsp::AudioBlock subBlock = resampledBlock.getSubBlock( + processedSamplesInResampledBuffer, cleanSamplesToProcess); + juce::dsp::ProcessContextReplacing subContext(subBlock); + + int resampledSamplesOutput = plugin.process(subContext); + + if (resampledSamplesOutput < cleanSamplesToProcess) { + // Move all remaining samples to the left of the buffer: + int offset = cleanSamplesToProcess - resampledSamplesOutput; - processedSamplesInResampledBuffer += resampledSamplesOutput; - cleanSamplesInResampledBuffer -= resampledSamplesOutput; + for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { + // Move the contents of the resampled block to the left: + std::memmove( + (char *)resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer, + (char *)(resampledBuffer.getWritePointer(c) + + processedSamplesInResampledBuffer + offset), + (resampledSamplesOutput + cleanSamplesInResampledBuffer) * + sizeof(SampleType)); + } } + processedSamplesInResampledBuffer += resampledSamplesOutput; + cleanSamplesInResampledBuffer -= cleanSamplesToProcess; } // Resample back to the intended sample rate: @@ -278,11 +359,19 @@ class Resample : public Plugin { return samplesToReturn; } - void setTargetSampleRate(float newSampleRate) { - targetSampleRate = newSampleRate; - } + SampleType getTargetSampleRate() const { return targetSampleRate; } + void setTargetSampleRate(const SampleType value) { + if (value <= 0.0) { + throw std::range_error("Target sample rate must be greater than 0Hz."); + } + targetSampleRate = value; + }; - float getTargetSampleRate() const { return targetSampleRate; } + ResamplingQuality getQuality() const { return quality; } + void setQuality(const ResamplingQuality value) { + quality = value; + reset(); + }; T &getNestedPlugin() { return plugin; } @@ -310,7 +399,8 @@ class Resample : public Plugin { private: T plugin; - float targetSampleRate = 44100.0f; + float targetSampleRate = 8000.0f; + ResamplingQuality quality = ResamplingQuality::WindowedSinc; double resamplerRatio = 1.0; double inverseResamplerRatio = 1.0; @@ -318,11 +408,11 @@ class Resample : public Plugin { juce::AudioBuffer inputReservoir; int samplesInInputReservoir = 0; - std::vector nativeToTargetResamplers; + std::vector nativeToTargetResamplers; juce::AudioBuffer resampledBuffer; int cleanSamplesInResampledBuffer = 0; int processedSamplesInResampledBuffer = 0; - std::vector targetToNativeResamplers; + std::vector targetToNativeResamplers; juce::AudioBuffer outputBuffer; int samplesInOutputBuffer = 0; @@ -342,45 +432,124 @@ class Resample : public Plugin { } }; - inline void init_resample(py::module &m) { - py::class_, float>, Plugin>(m, "Resample") - .def(py::init([](float targetSampleRate) { - auto plugin = std::make_unique, float>>(); - plugin->setTargetSampleRate(targetSampleRate); - return plugin; + py::class_, float>, Plugin> resample( + m, "Resample", + "A plugin that downsamples the input audio to the given sample rate, " + "then upsamples it again. Various quality settings will produce audible " + "distortion effects."); + + py::enum_(resample, "Quality") + .value("ZeroOrderHold", ResamplingQuality::ZeroOrderHold, + "The lowest quality and fastest resampling method, with lots of " + "audible artifacts.") + .value("Linear", ResamplingQuality::Linear, + "A resampling method slightly less noisy than the simplest " + "method, but not by much.") + .value("CatmullRom", ResamplingQuality::CatmullRom, + "A moderately good-sounding resampling method which is fast to " + "run.") + .value("Lagrange", ResamplingQuality::Lagrange, + "A moderately good-sounding resampling method which is slow to " + "run.") + .value("WindowedSinc", ResamplingQuality::WindowedSinc, + "The highest quality and slowest resampling method, with no " + "audible artifacts.") + .export_values(); + + resample + .def(py::init([](float targetSampleRate, ResamplingQuality quality) { + auto resampler = + std::make_unique, float>>(); + resampler->setTargetSampleRate(targetSampleRate); + resampler->setQuality(quality); + return resampler; }), - py::arg("target_sample_rate") = 8000.0) - .def("__repr__", [](const Resample, float> &plugin) { - std::ostringstream ss; - ss << ""; - return ss.str(); - }); + py::arg("target_sample_rate") = 8000.0, + py::arg("quality") = ResamplingQuality::WindowedSinc) + .def("__repr__", + [](const Resample, float> &plugin) { + std::ostringstream ss; + ss << ""; + return ss.str(); + }) + .def_property("target_sample_rate", + &Resample, float>::getTargetSampleRate, + &Resample, float>::setTargetSampleRate) + .def_property("quality", &Resample, float>::getQuality, + &Resample, float>::setQuality); } - - /** - * An internal test plugin that does nothing but add latency to the resampled signal. + * An internal test plugin that does nothing but add latency to the resampled + * signal. */ inline void init_resample_with_latency(py::module &m) { py::class_, Plugin>(m, "ResampleWithLatency") - .def(py::init([](float targetSampleRate, int internalLatency) { + .def(py::init([](float targetSampleRate, int internalLatency, + ResamplingQuality quality) { auto plugin = std::make_unique>(); plugin->setTargetSampleRate(targetSampleRate); - plugin->getNestedPlugin().getDSP().setMaximumDelayInSamples(internalLatency); + plugin->getNestedPlugin().getDSP().setMaximumDelayInSamples( + internalLatency); plugin->getNestedPlugin().getDSP().setDelay(internalLatency); + plugin->setQuality(quality); return plugin; }), - py::arg("target_sample_rate") = 8000.0, py::arg("internal_latency") = 1024) + py::arg("target_sample_rate") = 8000.0, + py::arg("internal_latency") = 1024, + py::arg("quality") = ResamplingQuality::WindowedSinc) .def("__repr__", [](Resample &plugin) { std::ostringstream ss; ss << ""; return ss.str(); diff --git a/tests/test_resample.py b/tests/test_resample.py index 7980aafa..4de8024d 100644 --- a/tests/test_resample.py +++ b/tests/test_resample.py @@ -21,6 +21,14 @@ from pedalboard_native._internal import ResampleWithLatency +TOLERANCE_PER_QUALITY = { + Resample.Quality.ZeroOrderHold: 0.65, + Resample.Quality.Linear: 0.35, + Resample.Quality.CatmullRom: 0.15, + Resample.Quality.Lagrange: 0.14, + Resample.Quality.WindowedSinc: 0.12, +} + @pytest.mark.parametrize("fundamental_hz", [440]) @pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) @@ -47,8 +55,113 @@ def test_resample( if num_channels == 2: sine_wave = np.stack([sine_wave, sine_wave]) + plugin = plugin_class(target_sample_rate) + output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + + np.testing.assert_allclose(sine_wave, output, atol=0.12) + + +@pytest.mark.parametrize("fundamental_hz", [440]) +@pytest.mark.parametrize("sample_rate_multiple", [1, 2, 3, 4, 5]) +@pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) +@pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) +@pytest.mark.parametrize("duration", [0.5]) +@pytest.mark.parametrize("num_channels", [1, 2]) +@pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) +def test_identical_with_zero_order_hold( + fundamental_hz: float, + sample_rate_multiple: float, + sample_rate: float, + buffer_size: int, + duration: float, + num_channels: int, + plugin_class, +): + noise = np.random.rand(int(duration * sample_rate)) if num_channels == 2: - np.testing.assert_allclose(sine_wave[0], sine_wave[1]) + noise = np.stack([noise, noise]) - plugin = plugin_class(target_sample_rate) + plugin = plugin_class( + sample_rate * sample_rate_multiple, quality=Resample.Quality.ZeroOrderHold + ) + output = plugin.process(noise, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(noise, output, atol=1e-9) + + +@pytest.mark.parametrize("fundamental_hz", [440]) +@pytest.mark.parametrize("sample_rate_multiple", [1 / 2, 1, 2]) +@pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) +@pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) +@pytest.mark.parametrize("duration", [0.5]) +@pytest.mark.parametrize("num_channels", [1, 2]) +@pytest.mark.parametrize( + "quality", + [ + Resample.Quality.ZeroOrderHold, + Resample.Quality.Linear, + Resample.Quality.Lagrange, + Resample.Quality.CatmullRom, + Resample.Quality.WindowedSinc, + ], +) +@pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) +def test_all_quality_levels( + fundamental_hz: float, + sample_rate_multiple: float, + sample_rate: float, + buffer_size: int, + duration: float, + num_channels: int, + quality: Resample.Quality, + plugin_class, +): + samples = np.arange(duration * sample_rate) + sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + if num_channels == 2: + sine_wave = np.stack([sine_wave, sine_wave]) + + plugin = plugin_class(sample_rate * sample_rate_multiple, quality=quality) + output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(sine_wave, output, atol=0.35) + + +@pytest.mark.parametrize("fundamental_hz", [10]) +@pytest.mark.parametrize("sample_rate", [100, 384_123.45]) +@pytest.mark.parametrize("target_sample_rate", [100, 384_123.45]) +@pytest.mark.parametrize("buffer_size", [1, 1_000_000]) +@pytest.mark.parametrize("duration", [1.0]) +@pytest.mark.parametrize("num_channels", [1, 2]) +@pytest.mark.parametrize( + "quality", + [ + Resample.Quality.ZeroOrderHold, + Resample.Quality.Linear, + Resample.Quality.Lagrange, + Resample.Quality.CatmullRom, + Resample.Quality.WindowedSinc, + ], +) +@pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) +def test_extreme_resampling( + fundamental_hz: float, + sample_rate: float, + target_sample_rate: float, + buffer_size: int, + duration: float, + num_channels: int, + quality: Resample.Quality, + plugin_class, +): + samples = np.arange(duration * sample_rate) + sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + # Fade the sine wave in at the start and out at the end to remove any transients: + fade_duration = int(sample_rate * 0.1) + sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) + sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) + if num_channels == 2: + sine_wave = np.stack([sine_wave, sine_wave]) + + plugin = plugin_class(target_sample_rate, quality=quality) output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + + np.testing.assert_allclose(sine_wave, output, atol=TOLERANCE_PER_QUALITY[quality]) From 3def00e76a77b4ec27e79efea1f5828216eedde1 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 6 Feb 2022 02:08:46 -0500 Subject: [PATCH 13/22] Move most GSMCompressor buffering logic to utility classes. --- .../plugin_templates/FixedBlockSizePlugin.h | 350 ++++++++++++++++++ pedalboard/plugin_templates/ForceMono.h | 84 +++++ .../plugin_templates/PrimeWithSilence.h | 7 +- pedalboard/plugin_templates/Resample.h | 89 +++-- pedalboard/plugins/GSMCompressor.h | 327 ++++------------ pedalboard/python_bindings.cpp | 9 +- tests/test_fixed_size_blocks.py | 55 +++ tests/test_gsm_compressor.py | 49 ++- tests/test_resample.py | 62 +++- 9 files changed, 713 insertions(+), 319 deletions(-) create mode 100644 pedalboard/plugin_templates/FixedBlockSizePlugin.h create mode 100644 pedalboard/plugin_templates/ForceMono.h create mode 100644 tests/test_fixed_size_blocks.py diff --git a/pedalboard/plugin_templates/FixedBlockSizePlugin.h b/pedalboard/plugin_templates/FixedBlockSizePlugin.h new file mode 100644 index 00000000..a9a6e888 --- /dev/null +++ b/pedalboard/plugin_templates/FixedBlockSizePlugin.h @@ -0,0 +1,350 @@ +/* + * pedalboard + * Copyright 2022 Spotify AB + * + * Licensed under the GNU Public License, Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.gnu.org/licenses/gpl-3.0.html + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../JuceHeader.h" +#include "../Plugin.h" +#include "../plugins/AddLatency.h" +#include + +namespace Pedalboard { + +/** + * A template class that wraps a Pedalboard plugin, + * but ensures that its process() function is only ever passed a fixed + * block size. This block size can be set in the prepare() method, or as a + * template argument. + */ +template +class FixedBlockSize : public Plugin { +public: + virtual ~FixedBlockSize(){}; + + virtual void prepare(const juce::dsp::ProcessSpec &spec) { + if (lastSpec.sampleRate != spec.sampleRate || + lastSpec.maximumBlockSize != spec.maximumBlockSize || + lastSpec.numChannels != spec.numChannels) { + if (spec.maximumBlockSize % blockSize == 0) { + // We need much less intermediate memory in this case: + inputBuffer.setSize(spec.numChannels, blockSize); + outputBuffer.clear(); + inStreamLatency = 0; + } else { + inputBuffer.setSize(spec.numChannels, + blockSize * 2 + spec.maximumBlockSize * 2); + outputBuffer.setSize(spec.numChannels, + blockSize * 2 + spec.maximumBlockSize * 2); + // Add enough latency to the stream to allow us to process an entire + // block: + inStreamLatency = blockSize; + } + lastSpec = spec; + } + + // Tell the delegate plugin that its maximum block + // size is the fixed size we'll be sending in: + juce::dsp::ProcessSpec newSpec = spec; + newSpec.maximumBlockSize = blockSize; + plugin.prepare(newSpec); + } + + virtual int + process(const juce::dsp::ProcessContextReplacing &context) { + auto ioBlock = context.getOutputBlock(); + + if (lastSpec.maximumBlockSize % blockSize == 0) { + // The best case scenario: the input is evenly divisible + // by the fixed block size, so we need no buffers! + int samplesOutput = 0; + for (int i = 0; i < ioBlock.getNumSamples(); i += blockSize) { + unsigned long samplesAvailable = + std::min((unsigned long)blockSize, ioBlock.getNumSamples() - i); + if (samplesAvailable < blockSize) + break; + + juce::dsp::AudioBlock subBlock = + ioBlock.getSubBlock(i, blockSize); + juce::dsp::ProcessContextReplacing subContext(subBlock); + int samplesOutputThisBlock = plugin.process(subContext); + + if (samplesOutputThisBlock != blockSize) { + throw std::runtime_error( + "FixedBlockSize currently requires wrapped plugins to impart no " + "delay. This is an internal Pedalboard error and should be " + "reported."); + } + + if (samplesOutput > 0 && samplesOutputThisBlock < blockSize) { + throw std::runtime_error("Plugin that returns fixed-size blocks " + "returned too few samples!"); + } + samplesOutput += samplesOutputThisBlock; + } + + int remainderInSamples = ioBlock.getNumSamples() % blockSize; + if (remainderInSamples > 0) { + if (samplesProcessed > 0) { + // We're at the end of our buffer, so pad with zeros. + + int offset = ioBlock.getNumSamples() - remainderInSamples; + + // Copy the remainder into inputBuffer: + juce::dsp::AudioBlock inputBlock(inputBuffer); + juce::dsp::AudioBlock subBlock = + inputBlock.getSubBlock(0, blockSize); + + subBlock.clear(); + subBlock.copyFrom(ioBlock.getSubBlock(offset, remainderInSamples)); + + juce::dsp::ProcessContextReplacing subContext(subBlock); + int samplesOutputThisBlock = plugin.process(subContext); + if (samplesOutputThisBlock != blockSize) { + throw std::runtime_error( + "FixedBlockSize currently requires wrapped plugins to impart " + "no delay. This is an internal Pedalboard error and should be " + "reported."); + } + + if (samplesOutput > 0 && samplesOutputThisBlock < blockSize) { + throw std::runtime_error( + "Plugin that returns fixed-size blocks returned too few " + "samples! This is an internal Pedalboard error and should be " + "reported."); + } + + // Copy the output back into ioBlock, right-aligned: + ioBlock + .getSubBlock(ioBlock.getNumSamples() - remainderInSamples, + remainderInSamples) + .copyFrom(subBlock); + + samplesOutput += remainderInSamples; + } else { + jassertfalse; + } + } + + samplesProcessed += samplesOutput; + return samplesOutput; + } else { + // We have to render three parts: + // 1) Push as many samples as possible into inputBuffer + int samplesToAddToInputBuffer = + std::min((int)inputBuffer.getNumSamples() - (int)inputBufferSamples, + (int)ioBlock.getNumSamples()); + + ioBlock.copyTo(inputBuffer, 0, inputBufferSamples, + samplesToAddToInputBuffer); + inputBufferSamples += samplesToAddToInputBuffer; + + // 2) Copy the output from the previous render call into the ioBlock + int samplesOutput = 0; + + int minimumSamplesToOutput = ioBlock.getNumSamples(); + if (ioBlock.getNumSamples() == lastSpec.maximumBlockSize) { + minimumSamplesToOutput += inStreamLatency; + } + if (outputBufferSamples >= minimumSamplesToOutput) { + ioBlock.copyFrom(outputBuffer, 0, 0, ioBlock.getNumSamples()); + outputBufferSamples -= ioBlock.getNumSamples(); + + // Move the remainder of the output buffer to the left: + if (outputBufferSamples > 0) { + for (int i = 0; i < outputBuffer.getNumChannels(); i++) { + std::memmove(outputBuffer.getWritePointer(i), + outputBuffer.getWritePointer(i) + + ioBlock.getNumSamples(), + sizeof(SampleType) * outputBufferSamples); + } + } + + samplesOutput = ioBlock.getNumSamples(); + } + + // 3) If the input buffer is large enough, process! + int samplesProcessed = 0; + juce::dsp::AudioBlock inputBlock(inputBuffer); + for (int i = 0; i < inputBufferSamples; i += blockSize) { + int samplesAvailable = std::min(blockSize, inputBufferSamples - i); + if (samplesAvailable < blockSize) + break; + + juce::dsp::AudioBlock subBlock = + inputBlock.getSubBlock(i, blockSize); + juce::dsp::ProcessContextReplacing subContext(subBlock); + int samplesProcessedThisBlock = plugin.process(subContext); + if (samplesProcessedThisBlock != blockSize) { + throw std::runtime_error( + "FixedBlockSize currently requires wrapped plugins to impart no " + "delay. This is an internal Pedalboard error and should be " + "reported."); + } + + if (samplesProcessed > 0 && samplesProcessedThisBlock < blockSize) { + throw std::runtime_error( + "Plugin that returns fixed-size blocks returned too few samples! " + "This is an internal Pedalboard error and should be reported."); + } + samplesProcessed += samplesProcessedThisBlock; + } + + // Copy the newly-processed data into the output buffer: + inputBlock.copyTo(outputBuffer, 0, outputBufferSamples, samplesProcessed); + outputBufferSamples += samplesProcessed; + + if (!(outputBufferSamples <= outputBuffer.getNumSamples())) { + throw std::runtime_error("Output buffer overrun! This is an internal " + "Pedalboard error and should be reported."); + } + + // ... and move the remaining input data to the left of the input buffer: + inputBlock.move(samplesProcessed, 0, + inputBufferSamples - samplesProcessed); + inputBufferSamples -= samplesProcessed; + + // ... and copy the remaining output buffer if we can: + if (samplesOutput == 0 && outputBufferSamples >= minimumSamplesToOutput) { + ioBlock.copyFrom(outputBuffer, 0, 0, ioBlock.getNumSamples()); + outputBufferSamples -= ioBlock.getNumSamples(); + + // Move the remainder of the output buffer to the left: + if (outputBufferSamples > 0) { + for (int i = 0; i < outputBuffer.getNumChannels(); i++) { + std::memmove(outputBuffer.getWritePointer(i), + outputBuffer.getWritePointer(i) + + ioBlock.getNumSamples(), + sizeof(SampleType) * outputBufferSamples); + } + } + + samplesOutput = ioBlock.getNumSamples(); + } + + samplesProcessed += samplesOutput; + return samplesOutput; + } + } + + virtual void reset() { + inputBufferSamples = 0; + outputBufferSamples = 0; + + inStreamLatency = 0; + samplesProcessed = 0; + lastSpec = {0}; + plugin.reset(); + + inputBuffer.clear(); + outputBuffer.clear(); + } + + T &getNestedPlugin() { return plugin; } + + void setFixedBlockSize(int newBlockSize) { + blockSize = newBlockSize; + reset(); + } + + int getFixedBlockSize() const { return blockSize; } + +private: + T plugin; + unsigned int blockSize = DefaultBlockSize; + int inStreamLatency = 0; + + juce::AudioBuffer inputBuffer; + unsigned int inputBufferSamples = 0; + + juce::AudioBuffer outputBuffer; + unsigned int outputBufferSamples = 0; + + unsigned int samplesProcessed = 0; +}; + +// TODO: Add plugin wrappers to make mono plugins stereo (and/or multichannel), +// or to mixdown to mono. + +/** + * A test plugin used to verify the behaviour of the FixedBlockSize wrapper. + */ +class ExpectsFixedBlockSize : public AddLatency { +public: + virtual ~ExpectsFixedBlockSize(){}; + + virtual void prepare(const juce::dsp::ProcessSpec &spec) { + if (spec.maximumBlockSize != expectedBlockSize) { + throw std::runtime_error("Expected maximum block size of exactly " + + std::to_string(expectedBlockSize) + "!"); + } + AddLatency::prepare(spec); + this->getDSP().setMaximumDelayInSamples(1024); + this->getDSP().setDelay(1024); + } + + virtual int + process(const juce::dsp::ProcessContextReplacing &context) { + if (context.getInputBlock().getNumSamples() != expectedBlockSize) { + throw std::runtime_error("Expected maximum block size of exactly " + + std::to_string(expectedBlockSize) + "!"); + } + return AddLatency::process(context); + } + + virtual void reset() { AddLatency::reset(); } + + void setExpectedBlockSize(int newExpectedBlockSize) { + expectedBlockSize = newExpectedBlockSize; + } + +private: + int expectedBlockSize = 0; +}; + +class FixedSizeBlockTestPlugin : public FixedBlockSize { +public: + void setExpectedBlockSize(int newExpectedBlockSize) { + setFixedBlockSize(newExpectedBlockSize); + getNestedPlugin().setExpectedBlockSize(newExpectedBlockSize); + } + + int getExpectedBlockSize() const { return getFixedBlockSize(); } + +private: + int expectedBlockSize = 0; +}; + +inline void init_fixed_size_block_test_plugin(py::module &m) { + py::class_(m, "FixedSizeBlockTestPlugin") + .def(py::init([](int expectedBlockSize) { + auto plugin = new FixedSizeBlockTestPlugin(); + plugin->setExpectedBlockSize(expectedBlockSize); + return plugin; + }), + py::arg("expected_block_size") = 160) + .def("__repr__", [](const FixedSizeBlockTestPlugin &plugin) { + std::ostringstream ss; + ss << ""; + return ss.str(); + }); +} + +} // namespace Pedalboard \ No newline at end of file diff --git a/pedalboard/plugin_templates/ForceMono.h b/pedalboard/plugin_templates/ForceMono.h new file mode 100644 index 00000000..7af20aab --- /dev/null +++ b/pedalboard/plugin_templates/ForceMono.h @@ -0,0 +1,84 @@ +/* + * pedalboard + * Copyright 2022 Spotify AB + * + * Licensed under the GNU Public License, Version 3.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.gnu.org/licenses/gpl-3.0.html + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../JuceHeader.h" +#include "../Plugin.h" +#include + +namespace Pedalboard { + +/** + * A template class that wraps a Pedalboard plugin, + * but ensures that its process() function is only ever passed a mono signal. + */ +template +class ForceMono : public Plugin { +public: + virtual ~ForceMono(){}; + + virtual void prepare(const juce::dsp::ProcessSpec &spec) { + juce::dsp::ProcessSpec newSpec = spec; + newSpec.numChannels = 1; + plugin.prepare(newSpec); + } + + virtual int + process(const juce::dsp::ProcessContextReplacing &context) { + auto ioBlock = context.getOutputBlock(); + + // Mix all channels to mono first, if necessary. + if (ioBlock.getNumChannels() > 1) { + float channelVolume = 1.0f / ioBlock.getNumChannels(); + for (int i = 0; i < ioBlock.getNumChannels(); i++) { + ioBlock.getSingleChannelBlock(i) *= channelVolume; + } + + // Copy all of the latter channels into the first channel, + // which will be used for processing: + auto firstChannel = ioBlock.getSingleChannelBlock(0); + for (int i = 1; i < ioBlock.getNumChannels(); i++) { + firstChannel += ioBlock.getSingleChannelBlock(i); + } + } + + juce::dsp::AudioBlock monoBlock = + ioBlock.getSingleChannelBlock(0); + juce::dsp::ProcessContextReplacing subContext(monoBlock); + int samplesProcessed = plugin.process(monoBlock); + + // Copy the mono signal back out to all other channels: + if (ioBlock.getNumChannels() > 1) { + auto firstChannel = ioBlock.getSingleChannelBlock(0); + for (int i = 1; i < ioBlock.getNumChannels(); i++) { + ioBlock.getSingleChannelBlock(i).copyFrom(firstChannel); + } + } + + return samplesProcessed; + } + + virtual void reset() { plugin.reset(); } + + T &getNestedPlugin() { return plugin; } + +private: + T plugin; +}; + +} // namespace Pedalboard \ No newline at end of file diff --git a/pedalboard/plugin_templates/PrimeWithSilence.h b/pedalboard/plugin_templates/PrimeWithSilence.h index 043ba6aa..ac02b319 100644 --- a/pedalboard/plugin_templates/PrimeWithSilence.h +++ b/pedalboard/plugin_templates/PrimeWithSilence.h @@ -28,7 +28,8 @@ namespace Pedalboard { * A dummy plugin that buffers audio data internally, used to test Pedalboard's * automatic latency compensation. */ -template +template class PrimeWithSilence : public JucePlugin> { @@ -39,6 +40,8 @@ class PrimeWithSilence JucePlugin>::prepare(spec); + this->getDSP().setMaximumDelayInSamples(silenceLengthSamples); + this->getDSP().setDelay(silenceLengthSamples); plugin.prepare(spec); } @@ -82,7 +85,7 @@ class PrimeWithSilence private: T plugin; int samplesOutput = 0; - int silenceLengthSamples = 0; + int silenceLengthSamples = DefaultSilenceLengthSamples; }; /** diff --git a/pedalboard/plugin_templates/Resample.h b/pedalboard/plugin_templates/Resample.h index a95019bb..db3bf918 100644 --- a/pedalboard/plugin_templates/Resample.h +++ b/pedalboard/plugin_templates/Resample.h @@ -120,7 +120,8 @@ template class Passthrough : public Plugin { * resampled audio and its sampleRate and maximumBlockSize parameters * are adjusted accordingly. */ -template , typename SampleType = float> +template , typename SampleType = float, + int DefaultSampleRate = 8000> class Resample : public Plugin { public: virtual ~Resample(){}; @@ -129,7 +130,7 @@ class Resample : public Plugin { bool specChanged = lastSpec.sampleRate != spec.sampleRate || lastSpec.maximumBlockSize < spec.maximumBlockSize || lastSpec.numChannels != spec.numChannels; - if (specChanged) { + if (specChanged || nativeToTargetResamplers.empty()) { reset(); nativeToTargetResamplers.resize(spec.numChannels); @@ -146,12 +147,6 @@ class Resample : public Plugin { maximumBlockSizeInTargetSampleRate = std::ceil(spec.maximumBlockSize / resamplerRatio); - const juce::dsp::ProcessSpec subSpec = { - .numChannels = spec.numChannels, - .sampleRate = targetSampleRate, - .maximumBlockSize = maximumBlockSizeInTargetSampleRate}; - plugin.prepare(subSpec); - // Store the remainder of the input: any samples that weren't consumed in // one pushSamples() call but would be consumable in the next one. inputReservoir.setSize(spec.numChannels, @@ -176,6 +171,12 @@ class Resample : public Plugin { lastSpec = spec; } + + const juce::dsp::ProcessSpec subSpec = { + .numChannels = spec.numChannels, + .sampleRate = targetSampleRate, + .maximumBlockSize = maximumBlockSizeInTargetSampleRate}; + plugin.prepare(subSpec); } int process(const juce::dsp::ProcessContextReplacing &context) @@ -376,6 +377,8 @@ class Resample : public Plugin { T &getNestedPlugin() { return plugin; } virtual void reset() override final { + plugin.reset(); + nativeToTargetResamplers.clear(); targetToNativeResamplers.clear(); @@ -399,7 +402,7 @@ class Resample : public Plugin { private: T plugin; - float targetSampleRate = 8000.0f; + float targetSampleRate = (float)DefaultSampleRate; ResamplingQuality quality = ResamplingQuality::WindowedSinc; double resamplerRatio = 1.0; @@ -523,37 +526,43 @@ inline void init_resample_with_latency(py::module &m) { py::arg("target_sample_rate") = 8000.0, py::arg("internal_latency") = 1024, py::arg("quality") = ResamplingQuality::WindowedSinc) - .def("__repr__", [](Resample &plugin) { - std::ostringstream ss; - ss << ""; - return ss.str(); - }); + .def("__repr__", + [](Resample &plugin) { + std::ostringstream ss; + ss << ""; + return ss.str(); + }) + .def_property("target_sample_rate", + &Resample::getTargetSampleRate, + &Resample::setTargetSampleRate) + .def_property("quality", &Resample::getQuality, + &Resample::setQuality); } } // namespace Pedalboard \ No newline at end of file diff --git a/pedalboard/plugins/GSMCompressor.h b/pedalboard/plugins/GSMCompressor.h index 0ad25069..2f403c93 100644 --- a/pedalboard/plugins/GSMCompressor.h +++ b/pedalboard/plugins/GSMCompressor.h @@ -16,6 +16,11 @@ */ #include "../Plugin.h" +#include "../plugin_templates/FixedBlockSizePlugin.h" +#include "../plugin_templates/ForceMono.h" +#include "../plugin_templates/PrimeWithSilence.h" +#include "../plugin_templates/Resample.h" + extern "C" { #include } @@ -48,9 +53,9 @@ class GSMWrapper { gsm _gsm = nullptr; }; -class GSMCompressor : public Plugin { +class GSMCompressorInternal : public Plugin { public: - virtual ~GSMCompressor(){}; + virtual ~GSMCompressorInternal(){}; virtual void prepare(const juce::dsp::ProcessSpec &spec) override { bool specChanged = lastSpec.sampleRate != spec.sampleRate || @@ -59,17 +64,10 @@ class GSMCompressor : public Plugin { if (!encoder || specChanged) { reset(); - resamplerRatio = spec.sampleRate / GSM_SAMPLE_RATE; - inverseResamplerRatio = GSM_SAMPLE_RATE / spec.sampleRate; - - gsmFrameSizeInNativeSampleRate = GSM_FRAME_SIZE_SAMPLES * resamplerRatio; - int maximumBlockSizeInGSMSampleRate = - spec.maximumBlockSize / resamplerRatio; - - // Store the remainder of the input: any samples that weren't consumed in - // one pushSamples() call but would be consumable in the next one. - inputReservoir.setSize(1, (int)std::ceil(resamplerRatio) + - spec.maximumBlockSize); + if (spec.sampleRate != GSM_SAMPLE_RATE) { + throw std::runtime_error("GSMCompressor plugin must be run at " + + std::to_string(GSM_SAMPLE_RATE) + "Hz!"); + } if (!encoder.getContext()) { throw std::runtime_error("Failed to initialize GSM encoder."); @@ -78,25 +76,6 @@ class GSMCompressor : public Plugin { throw std::runtime_error("Failed to initialize GSM decoder."); } - inStreamLatency = 0; - - // Add the resamplers' latencies so the output is properly aligned: - inStreamLatency += nativeToGSMResampler.getBaseLatency() * resamplerRatio; - inStreamLatency += gsmToNativeResampler.getBaseLatency() * resamplerRatio; - - resampledBuffer.setSize(1, maximumBlockSizeInGSMSampleRate + - GSM_FRAME_SIZE_SAMPLES + - (inStreamLatency / resamplerRatio)); - outputBuffer.setSize(1, spec.maximumBlockSize + - gsmFrameSizeInNativeSampleRate + - inStreamLatency); - - // Feed one GSM frame's worth of silence at the start so that we - // can tolerate different buffer sizes without underrunning any buffers. - std::vector silence(gsmFrameSizeInNativeSampleRate); - inStreamLatency += silence.size(); - pushSamples(silence.data(), silence.size()); - lastSpec = spec; } } @@ -105,269 +84,87 @@ class GSMCompressor : public Plugin { const juce::dsp::ProcessContextReplacing &context) override final { auto ioBlock = context.getOutputBlock(); - // Mix all channels to mono first, if necessary; GSM (in reality) is - // mono-only. - if (ioBlock.getNumChannels() > 1) { - float channelVolume = 1.0f / ioBlock.getNumChannels(); - for (int i = 0; i < ioBlock.getNumChannels(); i++) { - ioBlock.getSingleChannelBlock(i) *= channelVolume; - } - - // Copy all of the latter channels into the first channel, - // which will be used for processing: - auto firstChannel = ioBlock.getSingleChannelBlock(0); - for (int i = 1; i < ioBlock.getNumChannels(); i++) { - firstChannel += ioBlock.getSingleChannelBlock(i); - } - } - - // Actually do the GSM processing! - pushSamples(ioBlock.getChannelPointer(0), ioBlock.getNumSamples()); - int samplesOutput = - pullSamples(ioBlock.getChannelPointer(0), ioBlock.getNumSamples()); - - // Copy the mono signal back out to all other channels: - if (ioBlock.getNumChannels() > 1) { - auto firstChannel = ioBlock.getSingleChannelBlock(0); - for (int i = 1; i < ioBlock.getNumChannels(); i++) { - ioBlock.getSingleChannelBlock(i).copyFrom(firstChannel); - } + if (ioBlock.getNumSamples() != GSM_FRAME_SIZE_SAMPLES) { + throw std::runtime_error("GSMCompressor plugin must be passed exactly " + + std::to_string(GSM_FRAME_SIZE_SAMPLES) + + " at a time."); } - samplesProduced += samplesOutput; - int samplesToReturn = std::min((long)(samplesProduced - inStreamLatency), - (long)ioBlock.getNumSamples()); - if (samplesToReturn < 0) - samplesToReturn = 0; - - return samplesToReturn; - } - - /* - * Return the number of samples needed for this - * plugin to return a single GSM frame's worth of audio. - */ - int spaceAvailableInResampledBuffer() const { - return resampledBuffer.getNumSamples() - samplesInResampledBuffer; - } - - int spaceAvailableInOutputBuffer() const { - return outputBuffer.getNumSamples() - samplesInOutputBuffer; - } - - /* - * Push a certain number of input samples into the internal buffer(s) - * of this plugin, as GSM coding processes audio 160 samples at a time. - */ - void pushSamples(float *inputSamples, int numInputSamples) { - float expectedOutputSamples = numInputSamples / resamplerRatio; - - if (spaceAvailableInResampledBuffer() < expectedOutputSamples) { + if (ioBlock.getNumChannels() != 1) { throw std::runtime_error( - "More samples were provided than can be buffered! This is an " - "internal Pedalboard error and should be reported. Buffer had " + - std::to_string(samplesInResampledBuffer) + "/" + - std::to_string(resampledBuffer.getNumSamples()) + - " samples at 8kHz, but was provided " + - std::to_string(expectedOutputSamples) + "."); + "GSMCompressor plugin must be passed mono input!"); } - float *resampledBufferPointer = - resampledBuffer.getWritePointer(0) + samplesInResampledBuffer; + // Convert samples to signed 16-bit integer first, + // then pass to the GSM Encoder, then immediately back + // around to the GSM decoder. + short frame[GSM_FRAME_SIZE_SAMPLES]; - int samplesUsed = 0; - if (samplesInInputReservoir) { - // Copy the input samples into the input reservoir and use that as the - // resampler's input: - expectedOutputSamples += (float)samplesInInputReservoir / resamplerRatio; - inputReservoir.copyFrom(0, samplesInInputReservoir, inputSamples, - numInputSamples); - samplesUsed = nativeToGSMResampler.process( - resamplerRatio, inputReservoir.getReadPointer(0), - resampledBufferPointer, expectedOutputSamples); + juce::AudioDataConverters::convertFloatToInt16LE( + ioBlock.getChannelPointer(0), frame, GSM_FRAME_SIZE_SAMPLES); - if (samplesUsed < numInputSamples + samplesInInputReservoir) { - // Take the missing samples and put them at the start of the input - // reservoir for next time: - int unusedInputSampleCount = - (numInputSamples + samplesInInputReservoir) - samplesUsed; - inputReservoir.copyFrom(0, 0, - inputReservoir.getReadPointer(0) + samplesUsed, - unusedInputSampleCount); - samplesInInputReservoir = unusedInputSampleCount; - } else { - samplesInInputReservoir = 0; - } - } else { - samplesUsed = nativeToGSMResampler.process(resamplerRatio, inputSamples, - resampledBufferPointer, - expectedOutputSamples); + // Actually do the GSM processing! + gsm_frame encodedFrame; - if (samplesUsed < numInputSamples) { - // Take the missing samples and put them at the start of the input - // reservoir for next time: - int unusedInputSampleCount = numInputSamples - samplesUsed; - inputReservoir.copyFrom(0, 0, inputSamples + samplesUsed, - unusedInputSampleCount); - samplesInInputReservoir = unusedInputSampleCount; - } + gsm_encode(encoder.getContext(), frame, encodedFrame); + if (gsm_decode(decoder.getContext(), encodedFrame, frame) < 0) { + throw std::runtime_error("GSM decoder could not decode frame!"); } - samplesInResampledBuffer += expectedOutputSamples; - - performEncodeAndDecode(); - } - - int pullSamples(float *outputSamples, int maxOutputSamples) { - performEncodeAndDecode(); + juce::AudioDataConverters::convertInt16LEToFloat( + frame, ioBlock.getChannelPointer(0), GSM_FRAME_SIZE_SAMPLES); - // Copy the data out of outputBuffer and into the provided pointer, at - // the right side of the buffer: - int samplesToCopy = std::min(samplesInOutputBuffer, maxOutputSamples); - int offsetInOutput = maxOutputSamples - samplesToCopy; - juce::FloatVectorOperations::copy(outputSamples + offsetInOutput, - outputBuffer.getWritePointer(0), - samplesToCopy); - samplesInOutputBuffer -= samplesToCopy; - - // Move remaining samples to the left side of the output buffer: - std::memmove((char *)outputBuffer.getWritePointer(0), - (char *)(outputBuffer.getWritePointer(0) + samplesToCopy), - samplesInOutputBuffer * sizeof(float)); - - performEncodeAndDecode(); - - return samplesToCopy; - } - - void performEncodeAndDecode() { - while (samplesInResampledBuffer >= GSM_FRAME_SIZE_SAMPLES) { - float *encodeBuffer = resampledBuffer.getWritePointer(0); - - // Convert samples to signed 16-bit integer first, - // then pass to the GSM Encoder, then immediately back - // around to the GSM decoder. - short frame[GSM_FRAME_SIZE_SAMPLES]; - - juce::AudioDataConverters::convertFloatToInt16LE(encodeBuffer, frame, - GSM_FRAME_SIZE_SAMPLES); - - // Actually do the GSM encoding/decoding: - gsm_frame encodedFrame; - - gsm_encode(encoder.getContext(), frame, encodedFrame); - if (gsm_decode(decoder.getContext(), encodedFrame, frame) < 0) { - throw std::runtime_error("GSM decoder could not decode frame!"); - } - - if (spaceAvailableInOutputBuffer() < gsmFrameSizeInNativeSampleRate) { - throw std::runtime_error( - "Not enough space in output buffer to store a GSM frame! Needed " + - std::to_string(gsmFrameSizeInNativeSampleRate) + - " samples but only had " + - std::to_string(spaceAvailableInOutputBuffer()) + - " samples available. This is " - "an internal Pedalboard error and should be reported."); - } - float *outputBufferPointer = - outputBuffer.getWritePointer(0) + samplesInOutputBuffer; - - juce::AudioDataConverters::convertInt16LEToFloat( - frame, gsmOutputFrame + samplesInGsmOutputFrame, - GSM_FRAME_SIZE_SAMPLES); - samplesInGsmOutputFrame += GSM_FRAME_SIZE_SAMPLES; - - // Resample back up to the native sample rate and store in outputBuffer, - // using gsmOutputFrame as a temporary buffer to store up to 1 extra - // sample to compensate for rounding errors: - int expectedOutputSamples = samplesInGsmOutputFrame * resamplerRatio; - int samplesConsumed = gsmToNativeResampler.process( - inverseResamplerRatio, gsmOutputFrame, outputBufferPointer, - expectedOutputSamples); - std::memmove((char *)gsmOutputFrame, - (char *)(gsmOutputFrame + samplesConsumed), - (samplesInGsmOutputFrame - samplesConsumed) * sizeof(float)); - - samplesInGsmOutputFrame -= samplesConsumed; - samplesInOutputBuffer += expectedOutputSamples; - - // Now that we're done with this chunk of resampledBuffer, move its - // contents to the left: - int samplesRemainingInResampledBuffer = - samplesInResampledBuffer - GSM_FRAME_SIZE_SAMPLES; - std::memmove( - (char *)resampledBuffer.getWritePointer(0), - (char *)(resampledBuffer.getWritePointer(0) + GSM_FRAME_SIZE_SAMPLES), - samplesRemainingInResampledBuffer * sizeof(float)); - samplesInResampledBuffer -= GSM_FRAME_SIZE_SAMPLES; - } + return GSM_FRAME_SIZE_SAMPLES; } void reset() override final { encoder.reset(); decoder.reset(); - nativeToGSMResampler.reset(); - gsmToNativeResampler.reset(); - - resampledBuffer.clear(); - outputBuffer.clear(); - inputReservoir.clear(); - - samplesInResampledBuffer = 0; - samplesInOutputBuffer = 0; - samplesInInputReservoir = 0; - samplesInGsmOutputFrame = 0; - - samplesProduced = 0; - inStreamLatency = 0; } -protected: - virtual int getLatencyHint() override { return inStreamLatency; } - -private: static constexpr size_t GSM_FRAME_SIZE_SAMPLES = 160; - static constexpr float GSM_SAMPLE_RATE = 8000; - - double resamplerRatio = 1.0; - double inverseResamplerRatio = 1.0; - float gsmFrameSizeInNativeSampleRate; - - juce::AudioBuffer inputReservoir; - int samplesInInputReservoir = 0; - - juce::Interpolators::Lagrange nativeToGSMResampler; - juce::AudioBuffer resampledBuffer; - int samplesInResampledBuffer = 0; + static constexpr int GSM_SAMPLE_RATE = 8000; +private: GSMWrapper encoder; GSMWrapper decoder; - - juce::Interpolators::Lagrange gsmToNativeResampler; - float gsmOutputFrame[GSM_FRAME_SIZE_SAMPLES + 1]; - int samplesInGsmOutputFrame = 0; - - juce::AudioBuffer outputBuffer; - int samplesInOutputBuffer = 0; - - int samplesProduced = 0; - int inStreamLatency = 0; }; +using GSMCompressor = ForceMono< + Resample, + float, 1600>, + float, GSMCompressorInternal::GSM_SAMPLE_RATE>>; + inline void init_gsm_compressor(py::module &m) { py::class_( m, "GSMCompressor", "Apply an GSM compressor to emulate the sound of a GSM (\"2G\") cellular " "phone connection. This plugin internally resamples the input audio to " "8kHz.") - .def(py::init([]() { return new GSMCompressor(); })) - .def("__repr__", [](const GSMCompressor &plugin) { - std::ostringstream ss; - ss << ""; - return ss.str(); - }); + .def(py::init([](ResamplingQuality quality) { + auto plugin = std::make_unique(); + plugin->getNestedPlugin().setQuality(quality); + return plugin; + }), + py::arg("quality") = ResamplingQuality::WindowedSinc) + .def("__repr__", + [](const GSMCompressor &plugin) { + std::ostringstream ss; + ss << ""; + return ss.str(); + }) + .def_property( + "quality", + [](GSMCompressor &plugin) { + return plugin.getNestedPlugin().getQuality(); + }, + [](GSMCompressor &plugin, ResamplingQuality quality) { + return plugin.getNestedPlugin().setQuality(quality); + }); } }; // namespace Pedalboard \ No newline at end of file diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index eff491f8..b69ebe9f 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -33,6 +33,7 @@ namespace py = pybind11; #include "process.h" #include "plugin_templates/Resample.h" +#include "plugin_templates/PrimeWithSilence.h" #include "plugins/AddLatency.h" #include "plugins/Chorus.h" @@ -141,13 +142,18 @@ PYBIND11_MODULE(pedalboard_native, m) { py::arg("reset") = true); plugin.attr("__call__") = plugin.attr("process"); + // Publicly accessible plugins: init_chorus(m); init_compressor(m); init_convolution(m); init_delay(m); init_distortion(m); init_gain(m); + + // Init Resample before GSMCompressor, which uses Resample::Quality: + init_resample(m); init_gsm_compressor(m); + init_highpass(m); init_invert(m); init_ladderfilter(m); @@ -157,7 +163,6 @@ PYBIND11_MODULE(pedalboard_native, m) { init_noisegate(m); init_phaser(m); init_pitch_shift(m); - init_resample(m); init_reverb(m); init_external_plugins(m); @@ -167,4 +172,6 @@ PYBIND11_MODULE(pedalboard_native, m) { init_add_latency(internal); init_prime_with_silence_test_plugin(internal); init_resample_with_latency(internal); + init_prime_with_silence_test_plugin(internal); + init_fixed_size_block_test_plugin(internal); }; diff --git a/tests/test_fixed_size_blocks.py b/tests/test_fixed_size_blocks.py new file mode 100644 index 00000000..a7c7eca6 --- /dev/null +++ b/tests/test_fixed_size_blocks.py @@ -0,0 +1,55 @@ +#! /usr/bin/env python +# +# Copyright 2022 Spotify AB +# +# Licensed under the GNU Public License, Version 3.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.gnu.org/licenses/gpl-3.0.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +import numpy as np +from pedalboard_native._internal import FixedSizeBlockTestPlugin + + +@pytest.mark.parametrize("sample_rate", [22050, 44100, 48000]) +@pytest.mark.parametrize("buffer_size", [1, 16, 40, 128, 160, 8192, 8193]) +@pytest.mark.parametrize("fixed_buffer_size", [1, 16, 40, 128, 160, 8192, 8193]) +@pytest.mark.parametrize("num_channels", [1, 2]) +def test_fixed_size_blocks_plugin(sample_rate, buffer_size, fixed_buffer_size, num_channels): + num_seconds = 5.0 + noise = np.random.rand(int(num_seconds * sample_rate)) + if num_channels == 2: + noise = np.stack([noise, noise]) + plugin = FixedSizeBlockTestPlugin(fixed_buffer_size) + output = plugin.process(noise, sample_rate, buffer_size=buffer_size) + try: + np.testing.assert_allclose(noise, output) + except AssertionError: + import matplotlib.pyplot as plt + + if num_channels == 2: + noise = noise[0] + output = output[0] + + for cut in (buffer_size * 2, len(noise) // 200, len(noise)): + fig, ax = plt.subplots(3) + ax[0].plot(noise[:cut]) + ax[0].set_title("Input") + ax[1].plot(output[:cut]) + ax[1].set_title("Output") + ax[2].plot(np.abs(noise - output)[:cut]) + ax[2].set_title("Diff") + ax[2].set_ylim(0, 1) + plt.savefig(f"{sample_rate}-{buffer_size}-{cut}.png", dpi=300) + plt.clf() + + raise diff --git a/tests/test_gsm_compressor.py b/tests/test_gsm_compressor.py index 9408cc2f..c03cfe31 100644 --- a/tests/test_gsm_compressor.py +++ b/tests/test_gsm_compressor.py @@ -17,7 +17,7 @@ import pytest import numpy as np -from pedalboard import GSMCompressor +from pedalboard import GSMCompressor, Resample # GSM is a _very_ lossy codec: GSM_ABSOLUTE_TOLERANCE = 0.75 @@ -34,6 +34,10 @@ def generate_sine_at( ) -> np.ndarray: samples = np.arange(num_seconds * sample_rate) sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + # Fade the sine wave in at the start and out at the end to remove any transients: + fade_duration = int(sample_rate * 0.1) + sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) + sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) if num_channels == 2: return np.stack([sine_wave, sine_wave]) return sine_wave @@ -42,27 +46,56 @@ def generate_sine_at( @pytest.mark.parametrize("fundamental_hz", [440.0]) @pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 32000, 32001, 44100, 48000]) @pytest.mark.parametrize("buffer_size", [1, 32, 160, 8192]) -@pytest.mark.parametrize("duration", [1.0, 3.0, 30.0]) +@pytest.mark.parametrize("duration", [1.0]) +@pytest.mark.parametrize( + "quality", + [ + Resample.Quality.ZeroOrderHold, + Resample.Quality.Linear, + Resample.Quality.Lagrange, + Resample.Quality.CatmullRom, + Resample.Quality.WindowedSinc, + ], +) @pytest.mark.parametrize("num_channels", [1, 2]) def test_gsm_compressor( - fundamental_hz: float, sample_rate: float, buffer_size: int, duration: float, num_channels: int + fundamental_hz: float, + sample_rate: float, + buffer_size: int, + duration: float, + quality: Resample.Quality, + num_channels: int, ): signal = ( generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) * SINE_WAVE_VOLUME ) - compressed = GSMCompressor()(signal, sample_rate, buffer_size=buffer_size) - np.testing.assert_allclose(signal, compressed, atol=GSM_ABSOLUTE_TOLERANCE) + output = GSMCompressor(quality=quality)(signal, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(signal, output, atol=GSM_ABSOLUTE_TOLERANCE) @pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 32000, 32001, 44100, 48000]) -@pytest.mark.parametrize("num_channels", [1]) # , 2]) -def test_gsm_compressor_invariant_to_buffer_size(sample_rate: float, num_channels: int): +@pytest.mark.parametrize( + "quality", + [ + Resample.Quality.ZeroOrderHold, + Resample.Quality.Linear, + Resample.Quality.Lagrange, + Resample.Quality.CatmullRom, + Resample.Quality.WindowedSinc, + ], +) +@pytest.mark.parametrize("num_channels", [1, 2]) +def test_gsm_compressor_invariant_to_buffer_size( + sample_rate: float, + quality: Resample.Quality, + num_channels: int, +): fundamental_hz = 400.0 duration = 3.0 signal = generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) compressed = [ - GSMCompressor()(signal, sample_rate, buffer_size=buffer_size) + GSMCompressor(quality=quality)(signal, sample_rate, buffer_size=buffer_size) for buffer_size in (1, 32, 8192) ] for a, b in zip(compressed, compressed[1:]): diff --git a/tests/test_resample.py b/tests/test_resample.py index 4de8024d..c7293961 100644 --- a/tests/test_resample.py +++ b/tests/test_resample.py @@ -31,8 +31,8 @@ @pytest.mark.parametrize("fundamental_hz", [440]) -@pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) -@pytest.mark.parametrize("target_sample_rate", [8000, 22050, 44100, 48000, 1234.56]) +@pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 44100, 48000]) +@pytest.mark.parametrize("target_sample_rate", [8000, 11025, 22050, 44100, 48000, 1234.56]) @pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) @pytest.mark.parametrize("duration", [0.5, 1.23456]) @pytest.mark.parametrize("num_channels", [1, 2]) @@ -58,7 +58,41 @@ def test_resample( plugin = plugin_class(target_sample_rate) output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) - np.testing.assert_allclose(sine_wave, output, atol=0.12) + np.testing.assert_allclose(sine_wave, output, atol=0.16) + + +@pytest.mark.parametrize("fundamental_hz", [440]) +@pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 44100, 48000]) +@pytest.mark.parametrize("target_sample_rate", [8000, 11025, 22050, 44100, 48000, 1234.56]) +@pytest.mark.parametrize("duration", [0.5, 1.23456]) +@pytest.mark.parametrize("num_channels", [1, 2]) +@pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) +def test_resample_invariant_to_buffer_size( + fundamental_hz: float, + sample_rate: float, + target_sample_rate: float, + duration: float, + num_channels: int, + plugin_class, +): + samples = np.arange(duration * sample_rate) + sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + # Fade the sine wave in at the start and out at the end to remove any transients: + fade_duration = int(sample_rate * 0.1) + sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) + sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) + if num_channels == 2: + sine_wave = np.stack([sine_wave, sine_wave]) + + plugin = plugin_class(target_sample_rate) + + buffer_sizes = [1, 32, 128, 8192, 96000] + outputs = [ + plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + for buffer_size in buffer_sizes + ] + + np.testing.assert_allclose(a, b, atol=1e-3) @pytest.mark.parametrize("fundamental_hz", [440]) @@ -165,3 +199,25 @@ def test_extreme_resampling( output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) np.testing.assert_allclose(sine_wave, output, atol=TOLERANCE_PER_QUALITY[quality]) + + +@pytest.mark.parametrize("num_channels", [1, 2]) +def test_quality_can_change( + num_channels: int, + fundamental_hz: float = 440, + sample_rate: float = 8000, + buffer_size: int = 96000, + duration: float = 0.5, +): + samples = np.arange(duration * sample_rate) + sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + if num_channels == 2: + sine_wave = np.stack([sine_wave, sine_wave]) + + plugin = Resample(sample_rate) + output1 = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(sine_wave, output1, atol=0.35) + + plugin.quality = Resample.Quality.ZeroOrderHold + output2 = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(sine_wave, output2, atol=0.75) From c1b3d18c715df463c1c0a7dc7713321b16880735 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 6 Feb 2022 14:59:10 -0500 Subject: [PATCH 14/22] Fix bugs in FixedBlockSize. --- .../plugin_templates/FixedBlockSizePlugin.h | 111 +++++-------- pedalboard/plugin_templates/Resample.h | 11 +- tests/test_fixed_size_blocks.py | 38 +---- tests/test_gsm_compressor.py | 25 +-- ..._prime_with_silence.py => test_priming.py} | 0 tests/test_resample.py | 150 +++++------------- tests/utils.py | 25 +++ 7 files changed, 125 insertions(+), 235 deletions(-) rename tests/{test_prime_with_silence.py => test_priming.py} (100%) create mode 100644 tests/utils.py diff --git a/pedalboard/plugin_templates/FixedBlockSizePlugin.h b/pedalboard/plugin_templates/FixedBlockSizePlugin.h index a9a6e888..14b4d5cd 100644 --- a/pedalboard/plugin_templates/FixedBlockSizePlugin.h +++ b/pedalboard/plugin_templates/FixedBlockSizePlugin.h @@ -83,61 +83,36 @@ class FixedBlockSize : public Plugin { juce::dsp::ProcessContextReplacing subContext(subBlock); int samplesOutputThisBlock = plugin.process(subContext); - if (samplesOutputThisBlock != blockSize) { - throw std::runtime_error( - "FixedBlockSize currently requires wrapped plugins to impart no " - "delay. This is an internal Pedalboard error and should be " - "reported."); - } - if (samplesOutput > 0 && samplesOutputThisBlock < blockSize) { - throw std::runtime_error("Plugin that returns fixed-size blocks " - "returned too few samples!"); + throw std::runtime_error("Plugin that using FixedBlockSize " + "returned too few samples! This is an internal Pedalboard error and should be reported."); } samplesOutput += samplesOutputThisBlock; } int remainderInSamples = ioBlock.getNumSamples() % blockSize; if (remainderInSamples > 0) { - if (samplesProcessed > 0) { - // We're at the end of our buffer, so pad with zeros. - - int offset = ioBlock.getNumSamples() - remainderInSamples; - - // Copy the remainder into inputBuffer: - juce::dsp::AudioBlock inputBlock(inputBuffer); - juce::dsp::AudioBlock subBlock = - inputBlock.getSubBlock(0, blockSize); - - subBlock.clear(); - subBlock.copyFrom(ioBlock.getSubBlock(offset, remainderInSamples)); - - juce::dsp::ProcessContextReplacing subContext(subBlock); - int samplesOutputThisBlock = plugin.process(subContext); - if (samplesOutputThisBlock != blockSize) { - throw std::runtime_error( - "FixedBlockSize currently requires wrapped plugins to impart " - "no delay. This is an internal Pedalboard error and should be " - "reported."); - } + // We're at the end of our buffer, so pad with zeros. + int offset = ioBlock.getNumSamples() - remainderInSamples; - if (samplesOutput > 0 && samplesOutputThisBlock < blockSize) { - throw std::runtime_error( - "Plugin that returns fixed-size blocks returned too few " - "samples! This is an internal Pedalboard error and should be " - "reported."); - } + // Copy the remainder into inputBuffer: + juce::dsp::AudioBlock inputBlock(inputBuffer); + juce::dsp::AudioBlock subBlock = + inputBlock.getSubBlock(0, blockSize); - // Copy the output back into ioBlock, right-aligned: - ioBlock - .getSubBlock(ioBlock.getNumSamples() - remainderInSamples, - remainderInSamples) - .copyFrom(subBlock); + subBlock.clear(); + subBlock.copyFrom(ioBlock.getSubBlock(offset, remainderInSamples)); - samplesOutput += remainderInSamples; - } else { - jassertfalse; - } + juce::dsp::ProcessContextReplacing subContext(subBlock); + int samplesOutputThisBlock = plugin.process(subContext); + + // Copy the output back into ioBlock, right-aligned: + ioBlock + .getSubBlock(ioBlock.getNumSamples() - remainderInSamples, + remainderInSamples) + .copyFrom(subBlock); + + samplesOutput += remainderInSamples; } samplesProcessed += samplesOutput; @@ -145,13 +120,12 @@ class FixedBlockSize : public Plugin { } else { // We have to render three parts: // 1) Push as many samples as possible into inputBuffer - int samplesToAddToInputBuffer = - std::min((int)inputBuffer.getNumSamples() - (int)inputBufferSamples, - (int)ioBlock.getNumSamples()); + if (inputBuffer.getNumSamples() - inputBufferSamples < ioBlock.getNumSamples()) { + throw std::runtime_error("Input buffer overflow! This is an internal Pedalboard error and should be reported."); + } - ioBlock.copyTo(inputBuffer, 0, inputBufferSamples, - samplesToAddToInputBuffer); - inputBufferSamples += samplesToAddToInputBuffer; + ioBlock.copyTo(inputBuffer, 0, inputBufferSamples, ioBlock.getNumSamples()); + inputBufferSamples += ioBlock.getNumSamples(); // 2) Copy the output from the previous render call into the ioBlock int samplesOutput = 0; @@ -179,6 +153,7 @@ class FixedBlockSize : public Plugin { // 3) If the input buffer is large enough, process! int samplesProcessed = 0; + int inputSamplesConsumed = 0; juce::dsp::AudioBlock inputBlock(inputBuffer); for (int i = 0; i < inputBufferSamples; i += blockSize) { int samplesAvailable = std::min(blockSize, inputBufferSamples - i); @@ -189,36 +164,28 @@ class FixedBlockSize : public Plugin { inputBlock.getSubBlock(i, blockSize); juce::dsp::ProcessContextReplacing subContext(subBlock); int samplesProcessedThisBlock = plugin.process(subContext); - if (samplesProcessedThisBlock != blockSize) { - throw std::runtime_error( - "FixedBlockSize currently requires wrapped plugins to impart no " - "delay. This is an internal Pedalboard error and should be " - "reported."); - } + inputSamplesConsumed += blockSize; - if (samplesProcessed > 0 && samplesProcessedThisBlock < blockSize) { - throw std::runtime_error( - "Plugin that returns fixed-size blocks returned too few samples! " - "This is an internal Pedalboard error and should be reported."); + if (samplesProcessedThisBlock > 0) { + // Move the output to the left side of the buffer: + inputBlock.move(i + blockSize - samplesProcessedThisBlock, samplesProcessed, samplesProcessedThisBlock); } + samplesProcessed += samplesProcessedThisBlock; } // Copy the newly-processed data into the output buffer: + if (outputBuffer.getNumSamples() < outputBufferSamples + samplesProcessed) { + throw std::runtime_error("Output buffer overflow! This is an internal Pedalboard error and should be reported."); + } inputBlock.copyTo(outputBuffer, 0, outputBufferSamples, samplesProcessed); outputBufferSamples += samplesProcessed; - if (!(outputBufferSamples <= outputBuffer.getNumSamples())) { - throw std::runtime_error("Output buffer overrun! This is an internal " - "Pedalboard error and should be reported."); - } - // ... and move the remaining input data to the left of the input buffer: - inputBlock.move(samplesProcessed, 0, - inputBufferSamples - samplesProcessed); - inputBufferSamples -= samplesProcessed; + inputBlock.move(inputSamplesConsumed, 0, inputBufferSamples - inputSamplesConsumed); + inputBufferSamples -= inputSamplesConsumed; - // ... and copy the remaining output buffer if we can: + // ... and try to output the remaining output buffer contents if we now have enough: if (samplesOutput == 0 && outputBufferSamples >= minimumSamplesToOutput) { ioBlock.copyFrom(outputBuffer, 0, 0, ioBlock.getNumSamples()); outputBufferSamples -= ioBlock.getNumSamples(); @@ -293,8 +260,8 @@ class ExpectsFixedBlockSize : public AddLatency { std::to_string(expectedBlockSize) + "!"); } AddLatency::prepare(spec); - this->getDSP().setMaximumDelayInSamples(1024); - this->getDSP().setDelay(1024); + this->getDSP().setMaximumDelayInSamples(10); + this->getDSP().setDelay(10); } virtual int diff --git a/pedalboard/plugin_templates/Resample.h b/pedalboard/plugin_templates/Resample.h index db3bf918..3de2ed13 100644 --- a/pedalboard/plugin_templates/Resample.h +++ b/pedalboard/plugin_templates/Resample.h @@ -136,10 +136,11 @@ class Resample : public Plugin { nativeToTargetResamplers.resize(spec.numChannels); targetToNativeResamplers.resize(spec.numChannels); - // Set the quality on each resampler: for (int i = 0; i < spec.numChannels; i++) { nativeToTargetResamplers[i].setQuality(quality); + nativeToTargetResamplers[i].reset(); targetToNativeResamplers[i].setQuality(quality); + targetToNativeResamplers[i].reset(); } resamplerRatio = spec.sampleRate / targetSampleRate; @@ -221,12 +222,8 @@ class Resample : public Plugin { int unusedInputSampleCount = (ioBlock.getNumSamples() + samplesInInputReservoir) - samplesUsed; - for (size_t c = 0; c < ioBlock.getNumChannels(); c++) { - inputReservoir.copyFrom( - c, 0, inputReservoir.getReadPointer(c) + samplesUsed, - unusedInputSampleCount); - } - + juce::dsp::AudioBlock inputReservoirBlock(inputReservoir); + inputReservoirBlock.move(samplesUsed, 0, unusedInputSampleCount); samplesInInputReservoir = unusedInputSampleCount; } else { samplesInInputReservoir = 0; diff --git a/tests/test_fixed_size_blocks.py b/tests/test_fixed_size_blocks.py index a7c7eca6..60474117 100644 --- a/tests/test_fixed_size_blocks.py +++ b/tests/test_fixed_size_blocks.py @@ -18,38 +18,16 @@ import pytest import numpy as np from pedalboard_native._internal import FixedSizeBlockTestPlugin +from .utils import generate_sine_at -@pytest.mark.parametrize("sample_rate", [22050, 44100, 48000]) -@pytest.mark.parametrize("buffer_size", [1, 16, 40, 128, 160, 8192, 8193]) -@pytest.mark.parametrize("fixed_buffer_size", [1, 16, 40, 128, 160, 8192, 8193]) +@pytest.mark.parametrize("sample_rate", [22050, 44100]) +@pytest.mark.parametrize("buffer_size", [1, 64, 65, 128, 8192, 8193]) +@pytest.mark.parametrize("fixed_buffer_size", [1, 64, 65, 128, 8192, 8193]) @pytest.mark.parametrize("num_channels", [1, 2]) def test_fixed_size_blocks_plugin(sample_rate, buffer_size, fixed_buffer_size, num_channels): - num_seconds = 5.0 - noise = np.random.rand(int(num_seconds * sample_rate)) - if num_channels == 2: - noise = np.stack([noise, noise]) - plugin = FixedSizeBlockTestPlugin(fixed_buffer_size) - output = plugin.process(noise, sample_rate, buffer_size=buffer_size) - try: - np.testing.assert_allclose(noise, output) - except AssertionError: - import matplotlib.pyplot as plt - - if num_channels == 2: - noise = noise[0] - output = output[0] + signal = generate_sine_at(sample_rate, num_seconds=1.0, num_channels=num_channels) - for cut in (buffer_size * 2, len(noise) // 200, len(noise)): - fig, ax = plt.subplots(3) - ax[0].plot(noise[:cut]) - ax[0].set_title("Input") - ax[1].plot(output[:cut]) - ax[1].set_title("Output") - ax[2].plot(np.abs(noise - output)[:cut]) - ax[2].set_title("Diff") - ax[2].set_ylim(0, 1) - plt.savefig(f"{sample_rate}-{buffer_size}-{cut}.png", dpi=300) - plt.clf() - - raise + plugin = FixedSizeBlockTestPlugin(fixed_buffer_size) + output = plugin.process(signal, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(signal, output) \ No newline at end of file diff --git a/tests/test_gsm_compressor.py b/tests/test_gsm_compressor.py index c03cfe31..05596d08 100644 --- a/tests/test_gsm_compressor.py +++ b/tests/test_gsm_compressor.py @@ -18,6 +18,7 @@ import pytest import numpy as np from pedalboard import GSMCompressor, Resample +from .utils import generate_sine_at # GSM is a _very_ lossy codec: GSM_ABSOLUTE_TOLERANCE = 0.75 @@ -26,26 +27,10 @@ SINE_WAVE_VOLUME = 0.9 -def generate_sine_at( - sample_rate: float, - fundamental_hz: float = 440.0, - num_seconds: float = 3.0, - num_channels: int = 1, -) -> np.ndarray: - samples = np.arange(num_seconds * sample_rate) - sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) - # Fade the sine wave in at the start and out at the end to remove any transients: - fade_duration = int(sample_rate * 0.1) - sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) - sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) - if num_channels == 2: - return np.stack([sine_wave, sine_wave]) - return sine_wave - @pytest.mark.parametrize("fundamental_hz", [440.0]) -@pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 32000, 32001, 44100, 48000]) -@pytest.mark.parametrize("buffer_size", [1, 32, 160, 8192]) +@pytest.mark.parametrize("sample_rate", [8000, 11025, 32001.2345, 44100, 48000]) +@pytest.mark.parametrize("buffer_size", [1, 32, 160, 1_000_000]) @pytest.mark.parametrize("duration", [1.0]) @pytest.mark.parametrize( "quality", @@ -73,7 +58,7 @@ def test_gsm_compressor( np.testing.assert_allclose(signal, output, atol=GSM_ABSOLUTE_TOLERANCE) -@pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 32000, 32001, 44100, 48000]) +@pytest.mark.parametrize("sample_rate", [8000, 44100]) @pytest.mark.parametrize( "quality", [ @@ -96,7 +81,7 @@ def test_gsm_compressor_invariant_to_buffer_size( compressed = [ GSMCompressor(quality=quality)(signal, sample_rate, buffer_size=buffer_size) - for buffer_size in (1, 32, 8192) + for buffer_size in (1, 32, 7000, 8192) ] for a, b in zip(compressed, compressed[1:]): np.testing.assert_allclose(a, b) diff --git a/tests/test_prime_with_silence.py b/tests/test_priming.py similarity index 100% rename from tests/test_prime_with_silence.py rename to tests/test_priming.py diff --git a/tests/test_resample.py b/tests/test_resample.py index c7293961..5273b118 100644 --- a/tests/test_resample.py +++ b/tests/test_resample.py @@ -1,6 +1,6 @@ #! /usr/bin/env python # -# Copyright 2021 Spotify AB +# Copyright 2022 Spotify AB # # Licensed under the GNU Public License, Version 3.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,23 +19,26 @@ import numpy as np from pedalboard import Pedalboard, Resample from pedalboard_native._internal import ResampleWithLatency - +from .utils import generate_sine_at TOLERANCE_PER_QUALITY = { Resample.Quality.ZeroOrderHold: 0.65, Resample.Quality.Linear: 0.35, - Resample.Quality.CatmullRom: 0.15, - Resample.Quality.Lagrange: 0.14, - Resample.Quality.WindowedSinc: 0.12, + Resample.Quality.CatmullRom: 0.16, + Resample.Quality.Lagrange: 0.16, + Resample.Quality.WindowedSinc: 0.151, } +DURATIONS = [0.345678] + @pytest.mark.parametrize("fundamental_hz", [440]) @pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 44100, 48000]) -@pytest.mark.parametrize("target_sample_rate", [8000, 11025, 22050, 44100, 48000, 1234.56]) -@pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) -@pytest.mark.parametrize("duration", [0.5, 1.23456]) +@pytest.mark.parametrize("target_sample_rate", [8000, 11025, 12345.67, 22050, 44100, 48000]) +@pytest.mark.parametrize("buffer_size", [1, 32, 8192, 1_000_000]) +@pytest.mark.parametrize("duration", DURATIONS) @pytest.mark.parametrize("num_channels", [1, 2]) +@pytest.mark.parametrize("quality", TOLERANCE_PER_QUALITY.keys()) @pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) def test_resample( fundamental_hz: float, @@ -44,28 +47,22 @@ def test_resample( buffer_size: int, duration: float, num_channels: int, + quality: Resample.Quality, plugin_class, ): - samples = np.arange(duration * sample_rate) - sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) - # Fade the sine wave in at the start and out at the end to remove any transients: - fade_duration = int(sample_rate * 0.1) - sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) - sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) - if num_channels == 2: - sine_wave = np.stack([sine_wave, sine_wave]) - - plugin = plugin_class(target_sample_rate) + sine_wave = generate_sine_at(sample_rate, fundamental_hz, num_channels=num_channels) + plugin = plugin_class(target_sample_rate, quality=quality) output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) - - np.testing.assert_allclose(sine_wave, output, atol=0.16) + np.testing.assert_allclose(sine_wave, output, atol=TOLERANCE_PER_QUALITY[quality]) + @pytest.mark.parametrize("fundamental_hz", [440]) -@pytest.mark.parametrize("sample_rate", [8000, 11025, 22050, 44100, 48000]) -@pytest.mark.parametrize("target_sample_rate", [8000, 11025, 22050, 44100, 48000, 1234.56]) -@pytest.mark.parametrize("duration", [0.5, 1.23456]) +@pytest.mark.parametrize("sample_rate", [1234.56, 8000, 11025, 48000]) +@pytest.mark.parametrize("target_sample_rate", [1234.56, 8000, 11025, 48000]) +@pytest.mark.parametrize("duration", DURATIONS) @pytest.mark.parametrize("num_channels", [1, 2]) +@pytest.mark.parametrize("quality", TOLERANCE_PER_QUALITY.keys()) @pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) def test_resample_invariant_to_buffer_size( fundamental_hz: float, @@ -73,36 +70,30 @@ def test_resample_invariant_to_buffer_size( target_sample_rate: float, duration: float, num_channels: int, + quality: Resample.Quality, plugin_class, ): - samples = np.arange(duration * sample_rate) - sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) - # Fade the sine wave in at the start and out at the end to remove any transients: - fade_duration = int(sample_rate * 0.1) - sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) - sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) - if num_channels == 2: - sine_wave = np.stack([sine_wave, sine_wave]) - - plugin = plugin_class(target_sample_rate) + sine_wave = generate_sine_at(sample_rate, fundamental_hz, num_channels=num_channels) + plugin = plugin_class(target_sample_rate, quality=quality) - buffer_sizes = [1, 32, 128, 8192, 96000] + buffer_sizes = [1, 7000, 8192, 1_000_000] outputs = [ plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) for buffer_size in buffer_sizes ] - np.testing.assert_allclose(a, b, atol=1e-3) + for a, b in zip(outputs, outputs[1:]): + np.testing.assert_allclose(a, b) @pytest.mark.parametrize("fundamental_hz", [440]) -@pytest.mark.parametrize("sample_rate_multiple", [1, 2, 3, 4, 5]) -@pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) -@pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) -@pytest.mark.parametrize("duration", [0.5]) +@pytest.mark.parametrize("sample_rate_multiple", [1, 2, 3, 4, 20]) +@pytest.mark.parametrize("sample_rate", [8000, 44100, 48000]) +@pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 1_000_000]) +@pytest.mark.parametrize("duration", DURATIONS) @pytest.mark.parametrize("num_channels", [1, 2]) @pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) -def test_identical_with_zero_order_hold( +def test_identical_noise_with_zero_order_hold( fundamental_hz: float, sample_rate_multiple: float, sample_rate: float, @@ -119,62 +110,16 @@ def test_identical_with_zero_order_hold( sample_rate * sample_rate_multiple, quality=Resample.Quality.ZeroOrderHold ) output = plugin.process(noise, sample_rate, buffer_size=buffer_size) - np.testing.assert_allclose(noise, output, atol=1e-9) - - -@pytest.mark.parametrize("fundamental_hz", [440]) -@pytest.mark.parametrize("sample_rate_multiple", [1 / 2, 1, 2]) -@pytest.mark.parametrize("sample_rate", [8000, 22050, 44100, 48000]) -@pytest.mark.parametrize("buffer_size", [1, 32, 128, 8192, 96000]) -@pytest.mark.parametrize("duration", [0.5]) -@pytest.mark.parametrize("num_channels", [1, 2]) -@pytest.mark.parametrize( - "quality", - [ - Resample.Quality.ZeroOrderHold, - Resample.Quality.Linear, - Resample.Quality.Lagrange, - Resample.Quality.CatmullRom, - Resample.Quality.WindowedSinc, - ], -) -@pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) -def test_all_quality_levels( - fundamental_hz: float, - sample_rate_multiple: float, - sample_rate: float, - buffer_size: int, - duration: float, - num_channels: int, - quality: Resample.Quality, - plugin_class, -): - samples = np.arange(duration * sample_rate) - sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) - if num_channels == 2: - sine_wave = np.stack([sine_wave, sine_wave]) - - plugin = plugin_class(sample_rate * sample_rate_multiple, quality=quality) - output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) - np.testing.assert_allclose(sine_wave, output, atol=0.35) + np.testing.assert_allclose(noise, output) @pytest.mark.parametrize("fundamental_hz", [10]) @pytest.mark.parametrize("sample_rate", [100, 384_123.45]) @pytest.mark.parametrize("target_sample_rate", [100, 384_123.45]) @pytest.mark.parametrize("buffer_size", [1, 1_000_000]) -@pytest.mark.parametrize("duration", [1.0]) +@pytest.mark.parametrize("duration", DURATIONS) @pytest.mark.parametrize("num_channels", [1, 2]) -@pytest.mark.parametrize( - "quality", - [ - Resample.Quality.ZeroOrderHold, - Resample.Quality.Linear, - Resample.Quality.Lagrange, - Resample.Quality.CatmullRom, - Resample.Quality.WindowedSinc, - ], -) +@pytest.mark.parametrize("quality", TOLERANCE_PER_QUALITY.keys()) @pytest.mark.parametrize("plugin_class", [Resample, ResampleWithLatency]) def test_extreme_resampling( fundamental_hz: float, @@ -186,18 +131,9 @@ def test_extreme_resampling( quality: Resample.Quality, plugin_class, ): - samples = np.arange(duration * sample_rate) - sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) - # Fade the sine wave in at the start and out at the end to remove any transients: - fade_duration = int(sample_rate * 0.1) - sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) - sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) - if num_channels == 2: - sine_wave = np.stack([sine_wave, sine_wave]) - + sine_wave = generate_sine_at(sample_rate, fundamental_hz, num_channels=num_channels) plugin = plugin_class(target_sample_rate, quality=quality) output = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) - np.testing.assert_allclose(sine_wave, output, atol=TOLERANCE_PER_QUALITY[quality]) @@ -207,17 +143,19 @@ def test_quality_can_change( fundamental_hz: float = 440, sample_rate: float = 8000, buffer_size: int = 96000, - duration: float = 0.5, + duration: float = DURATIONS[0], ): - samples = np.arange(duration * sample_rate) - sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) - if num_channels == 2: - sine_wave = np.stack([sine_wave, sine_wave]) + sine_wave = generate_sine_at(sample_rate, fundamental_hz, num_channels=num_channels) plugin = Resample(sample_rate) output1 = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) - np.testing.assert_allclose(sine_wave, output1, atol=0.35) + np.testing.assert_allclose(sine_wave, output1, atol=TOLERANCE_PER_QUALITY[plugin.quality]) + original_quality = plugin.quality plugin.quality = Resample.Quality.ZeroOrderHold output2 = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) - np.testing.assert_allclose(sine_wave, output2, atol=0.75) + np.testing.assert_allclose(sine_wave, output2, atol=TOLERANCE_PER_QUALITY[plugin.quality]) + + plugin.quality = original_quality + output3 = plugin.process(sine_wave, sample_rate, buffer_size=buffer_size) + np.testing.assert_allclose(output1, output3) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..55ba88b3 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,25 @@ +import numpy as np + + +TEST_SINE_WAVE_CACHE = {} + + +def generate_sine_at( + sample_rate: float, + fundamental_hz: float = 440.0, + num_seconds: float = 3.0, + num_channels: int = 1, +) -> np.ndarray: + cache_key = "-".join([str(x) for x in [sample_rate, fundamental_hz, num_seconds, num_channels]]) + if cache_key not in TEST_SINE_WAVE_CACHE: + samples = np.arange(num_seconds * sample_rate) + sine_wave = np.sin(2 * np.pi * fundamental_hz * samples / sample_rate) + # Fade the sine wave in at the start and out at the end to remove any transients: + fade_duration = int(sample_rate * 0.1) + sine_wave[:fade_duration] *= np.linspace(0, 1, fade_duration) + sine_wave[-fade_duration:] *= np.linspace(1, 0, fade_duration) + if num_channels == 2: + TEST_SINE_WAVE_CACHE[cache_key] = np.stack([sine_wave, sine_wave]) + else: + TEST_SINE_WAVE_CACHE[cache_key] = sine_wave + return TEST_SINE_WAVE_CACHE[cache_key] From 57cf05a39fbcb73abe28f21cf054fd88c5c818d2 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 6 Feb 2022 15:58:30 -0500 Subject: [PATCH 15/22] Fix remaining GSMFullRateCompressor issues. --- .../plugin_templates/FixedBlockSizePlugin.h | 43 ++++++++++-------- ...SMCompressor.h => GSMFullRateCompressor.h} | 45 +++++++++++-------- pedalboard/python_bindings.cpp | 7 ++- tests/test_gsm_compressor.py | 7 ++- 4 files changed, 58 insertions(+), 44 deletions(-) rename pedalboard/plugins/{GSMCompressor.h => GSMFullRateCompressor.h} (76%) diff --git a/pedalboard/plugin_templates/FixedBlockSizePlugin.h b/pedalboard/plugin_templates/FixedBlockSizePlugin.h index 14b4d5cd..b6967193 100644 --- a/pedalboard/plugin_templates/FixedBlockSizePlugin.h +++ b/pedalboard/plugin_templates/FixedBlockSizePlugin.h @@ -84,8 +84,10 @@ class FixedBlockSize : public Plugin { int samplesOutputThisBlock = plugin.process(subContext); if (samplesOutput > 0 && samplesOutputThisBlock < blockSize) { - throw std::runtime_error("Plugin that using FixedBlockSize " - "returned too few samples! This is an internal Pedalboard error and should be reported."); + throw std::runtime_error( + "Plugin that using FixedBlockSize " + "returned too few samples! This is an internal Pedalboard error " + "and should be reported."); } samplesOutput += samplesOutputThisBlock; } @@ -109,7 +111,7 @@ class FixedBlockSize : public Plugin { // Copy the output back into ioBlock, right-aligned: ioBlock .getSubBlock(ioBlock.getNumSamples() - remainderInSamples, - remainderInSamples) + remainderInSamples) .copyFrom(subBlock); samplesOutput += remainderInSamples; @@ -120,21 +122,20 @@ class FixedBlockSize : public Plugin { } else { // We have to render three parts: // 1) Push as many samples as possible into inputBuffer - if (inputBuffer.getNumSamples() - inputBufferSamples < ioBlock.getNumSamples()) { - throw std::runtime_error("Input buffer overflow! This is an internal Pedalboard error and should be reported."); + if (inputBuffer.getNumSamples() - inputBufferSamples < + ioBlock.getNumSamples()) { + throw std::runtime_error("Input buffer overflow! This is an internal " + "Pedalboard error and should be reported."); } - ioBlock.copyTo(inputBuffer, 0, inputBufferSamples, ioBlock.getNumSamples()); + ioBlock.copyTo(inputBuffer, 0, inputBufferSamples, + ioBlock.getNumSamples()); inputBufferSamples += ioBlock.getNumSamples(); // 2) Copy the output from the previous render call into the ioBlock int samplesOutput = 0; - int minimumSamplesToOutput = ioBlock.getNumSamples(); - if (ioBlock.getNumSamples() == lastSpec.maximumBlockSize) { - minimumSamplesToOutput += inStreamLatency; - } - if (outputBufferSamples >= minimumSamplesToOutput) { + if (outputBufferSamples >= ioBlock.getNumSamples()) { ioBlock.copyFrom(outputBuffer, 0, 0, ioBlock.getNumSamples()); outputBufferSamples -= ioBlock.getNumSamples(); @@ -168,25 +169,31 @@ class FixedBlockSize : public Plugin { if (samplesProcessedThisBlock > 0) { // Move the output to the left side of the buffer: - inputBlock.move(i + blockSize - samplesProcessedThisBlock, samplesProcessed, samplesProcessedThisBlock); + inputBlock.move(i + blockSize - samplesProcessedThisBlock, + samplesProcessed, samplesProcessedThisBlock); } - + samplesProcessed += samplesProcessedThisBlock; } // Copy the newly-processed data into the output buffer: - if (outputBuffer.getNumSamples() < outputBufferSamples + samplesProcessed) { - throw std::runtime_error("Output buffer overflow! This is an internal Pedalboard error and should be reported."); + if (outputBuffer.getNumSamples() < + outputBufferSamples + samplesProcessed) { + throw std::runtime_error("Output buffer overflow! This is an internal " + "Pedalboard error and should be reported."); } inputBlock.copyTo(outputBuffer, 0, outputBufferSamples, samplesProcessed); outputBufferSamples += samplesProcessed; // ... and move the remaining input data to the left of the input buffer: - inputBlock.move(inputSamplesConsumed, 0, inputBufferSamples - inputSamplesConsumed); + inputBlock.move(inputSamplesConsumed, 0, + inputBufferSamples - inputSamplesConsumed); inputBufferSamples -= inputSamplesConsumed; - // ... and try to output the remaining output buffer contents if we now have enough: - if (samplesOutput == 0 && outputBufferSamples >= minimumSamplesToOutput) { + // ... and try to output the remaining output buffer contents if we now + // have enough: + if (samplesOutput == 0 && + outputBufferSamples >= ioBlock.getNumSamples()) { ioBlock.copyFrom(outputBuffer, 0, 0, ioBlock.getNumSamples()); outputBufferSamples -= ioBlock.getNumSamples(); diff --git a/pedalboard/plugins/GSMCompressor.h b/pedalboard/plugins/GSMFullRateCompressor.h similarity index 76% rename from pedalboard/plugins/GSMCompressor.h rename to pedalboard/plugins/GSMFullRateCompressor.h index 2f403c93..9773ed4f 100644 --- a/pedalboard/plugins/GSMCompressor.h +++ b/pedalboard/plugins/GSMFullRateCompressor.h @@ -53,9 +53,9 @@ class GSMWrapper { gsm _gsm = nullptr; }; -class GSMCompressorInternal : public Plugin { +class GSMFullRateCompressorInternal : public Plugin { public: - virtual ~GSMCompressorInternal(){}; + virtual ~GSMFullRateCompressorInternal(){}; virtual void prepare(const juce::dsp::ProcessSpec &spec) override { bool specChanged = lastSpec.sampleRate != spec.sampleRate || @@ -130,39 +130,48 @@ class GSMCompressorInternal : public Plugin { GSMWrapper decoder; }; -using GSMCompressor = ForceMono< - Resample, - float, 1600>, - float, GSMCompressorInternal::GSM_SAMPLE_RATE>>; - -inline void init_gsm_compressor(py::module &m) { - py::class_( - m, "GSMCompressor", - "Apply an GSM compressor to emulate the sound of a GSM (\"2G\") cellular " +/** + * Use the GSMFullRateCompressorInternal plugin, but: + * - ensure that it only ever sees fixed-size blocks of 160 samples + * - prime the input with a single block of silence + * - resample whatever input sample rate is provided down to 8kHz + * - only provide mono input to the plugin, and copy the mono signal + * back to stereo if necessary + */ +using GSMFullRateCompressor = ForceMono, + float, GSMFullRateCompressorInternal::GSM_FRAME_SIZE_SAMPLES>, + float, GSMFullRateCompressorInternal::GSM_SAMPLE_RATE>>; + +inline void init_gsm_full_rate_compressor(py::module &m) { + py::class_( + m, "GSMFullRateCompressor", + "Apply an GSM Full Rate compressor to emulate the sound of a GSM Full " + "Rate (\"2G\") cellular " "phone connection. This plugin internally resamples the input audio to " "8kHz.") .def(py::init([](ResamplingQuality quality) { - auto plugin = std::make_unique(); + auto plugin = std::make_unique(); plugin->getNestedPlugin().setQuality(quality); return plugin; }), py::arg("quality") = ResamplingQuality::WindowedSinc) .def("__repr__", - [](const GSMCompressor &plugin) { + [](const GSMFullRateCompressor &plugin) { std::ostringstream ss; - ss << ""; return ss.str(); }) .def_property( "quality", - [](GSMCompressor &plugin) { + [](GSMFullRateCompressor &plugin) { return plugin.getNestedPlugin().getQuality(); }, - [](GSMCompressor &plugin, ResamplingQuality quality) { + [](GSMFullRateCompressor &plugin, ResamplingQuality quality) { return plugin.getNestedPlugin().setQuality(quality); }); } diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index b69ebe9f..f3d6b251 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -41,7 +41,7 @@ namespace py = pybind11; #include "plugins/Convolution.h" #include "plugins/Delay.h" #include "plugins/Distortion.h" -#include "plugins/GSMCompressor.h" +#include "plugins/GSMFullRateCompressor.h" #include "plugins/Gain.h" #include "plugins/HighpassFilter.h" #include "plugins/Invert.h" @@ -150,9 +150,9 @@ PYBIND11_MODULE(pedalboard_native, m) { init_distortion(m); init_gain(m); - // Init Resample before GSMCompressor, which uses Resample::Quality: + // Init Resample before GSMFullRateCompressor, which uses Resample::Quality: init_resample(m); - init_gsm_compressor(m); + init_gsm_full_rate_compressor(m); init_highpass(m); init_invert(m); @@ -172,6 +172,5 @@ PYBIND11_MODULE(pedalboard_native, m) { init_add_latency(internal); init_prime_with_silence_test_plugin(internal); init_resample_with_latency(internal); - init_prime_with_silence_test_plugin(internal); init_fixed_size_block_test_plugin(internal); }; diff --git a/tests/test_gsm_compressor.py b/tests/test_gsm_compressor.py index 05596d08..b8cd4cc0 100644 --- a/tests/test_gsm_compressor.py +++ b/tests/test_gsm_compressor.py @@ -17,7 +17,7 @@ import pytest import numpy as np -from pedalboard import GSMCompressor, Resample +from pedalboard import GSMFullRateCompressor, Resample from .utils import generate_sine_at # GSM is a _very_ lossy codec: @@ -27,7 +27,6 @@ SINE_WAVE_VOLUME = 0.9 - @pytest.mark.parametrize("fundamental_hz", [440.0]) @pytest.mark.parametrize("sample_rate", [8000, 11025, 32001.2345, 44100, 48000]) @pytest.mark.parametrize("buffer_size", [1, 32, 160, 1_000_000]) @@ -54,7 +53,7 @@ def test_gsm_compressor( signal = ( generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) * SINE_WAVE_VOLUME ) - output = GSMCompressor(quality=quality)(signal, sample_rate, buffer_size=buffer_size) + output = GSMFullRateCompressor(quality=quality)(signal, sample_rate, buffer_size=buffer_size) np.testing.assert_allclose(signal, output, atol=GSM_ABSOLUTE_TOLERANCE) @@ -80,7 +79,7 @@ def test_gsm_compressor_invariant_to_buffer_size( signal = generate_sine_at(sample_rate, fundamental_hz, duration, num_channels) compressed = [ - GSMCompressor(quality=quality)(signal, sample_rate, buffer_size=buffer_size) + GSMFullRateCompressor(quality=quality)(signal, sample_rate, buffer_size=buffer_size) for buffer_size in (1, 32, 7000, 8192) ] for a, b in zip(compressed, compressed[1:]): From 55a1e1b9416316fc8345f7b6032deb9b7f796104 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 6 Feb 2022 15:58:47 -0500 Subject: [PATCH 16/22] Fix MSVC incompatibility by removing toast command-line. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4360afae..ed685343 100644 --- a/setup.py +++ b/setup.py @@ -106,9 +106,8 @@ 'vendors/lame/', ] - # libgsm -ALL_SOURCE_PATHS += list(Path("vendors/libgsm/src").glob("*.c")) +ALL_SOURCE_PATHS += [p for p in Path("vendors/libgsm/src").glob("*.c") if 'toast' not in p.name] ALL_INCLUDES += ['vendors/libgsm/inc'] @@ -232,6 +231,7 @@ def patch_compile(original_compile): """ On GCC/Clang, we want to pass different arguments when compiling C files vs C++ files. """ + def new_compile(obj, src, ext, cc_args, extra_postargs, *args, **kwargs): _cc_args = cc_args From 5c5e41b8f224abcc650981d05ba458b6a9b95759 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 6 Feb 2022 17:47:38 -0500 Subject: [PATCH 17/22] Rename to FixedBlockSize.h. --- .../{FixedBlockSizePlugin.h => FixedBlockSize.h} | 0 pedalboard/python_bindings.cpp | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) rename pedalboard/plugin_templates/{FixedBlockSizePlugin.h => FixedBlockSize.h} (100%) diff --git a/pedalboard/plugin_templates/FixedBlockSizePlugin.h b/pedalboard/plugin_templates/FixedBlockSize.h similarity index 100% rename from pedalboard/plugin_templates/FixedBlockSizePlugin.h rename to pedalboard/plugin_templates/FixedBlockSize.h diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index f3d6b251..ebc24fd0 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -34,6 +34,7 @@ namespace py = pybind11; #include "plugin_templates/Resample.h" #include "plugin_templates/PrimeWithSilence.h" +#include "plugin_templates/FixedBlockSize.h" #include "plugins/AddLatency.h" #include "plugins/Chorus.h" @@ -54,8 +55,6 @@ namespace py = pybind11; #include "plugins/PitchShift.h" #include "plugins/Reverb.h" -#include "plugin_templates/PrimeWithSilence.h" - using namespace Pedalboard; static constexpr int DEFAULT_BUFFER_SIZE = 8192; From 858131871e156fe20512ff57e2763114e6012c33 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Sun, 6 Feb 2022 17:48:30 -0500 Subject: [PATCH 18/22] Rename fixed block size test. --- tests/{test_fixed_size_blocks.py => test_fixed_block_size.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_fixed_size_blocks.py => test_fixed_block_size.py} (100%) diff --git a/tests/test_fixed_size_blocks.py b/tests/test_fixed_block_size.py similarity index 100% rename from tests/test_fixed_size_blocks.py rename to tests/test_fixed_block_size.py From 35b963a2b2d95441988d92ea283504d0cd90084c Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Mon, 7 Feb 2022 23:50:18 -0500 Subject: [PATCH 19/22] Rename to GSMFullRateCompressor in README. --- README.md | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 81160251..50ac474f 100644 --- a/README.md +++ b/README.md @@ -17,21 +17,13 @@ Internally at Spotify, `pedalboard` is used for [data augmentation](https://en.w ## Features - - Built-in support for a number of basic audio transformations: - - `Convolution` - - `Compressor` - - `Chorus` - - `Distortion` - - `Gain` - - `HighpassFilter` - - `LadderFilter` - - `Limiter` - - `LowpassFilter` - - `Phaser` - - `PitchShift` (provided by Chris Cannam's [Rubber Band Library](https://github.com/breakfastquay/rubberband)) - - `Reverb` - - Built-in lossy compression algorithms: - - `GSMCompressor`, an implementation of the GSM ("2G") lossy voice compression codec + - Built-in support for a number of basic audio transformations, including: + - Guitar-style effects: `Chorus`, `Distortion`, `Phaser` + - Loudness and dynamic range effects: `Compressor`, `Gain`, `Limiter` + - Equalizers and filters: `HighpassFilter`, `LadderFilter`, `LowpassFilter` + - Spatial effects: `Convolution`, `Delay`, `Reverb` + - Pitch effects: `PitchShift` + - Lossy compression: `GSMFullRateCompressor`, `MP3Compressor` - Supports VST3® plugins on macOS, Windows, and Linux (`pedalboard.load_plugin`) - Supports Audio Units on macOS - Strong thread-safety, memory usage, and speed guarantees @@ -244,6 +236,6 @@ Not yet, either - although the underlying framework (JUCE) supports passing MIDI - The [VST3 SDK](https://github.com/steinbergmedia/vst3sdk), bundled with JUCE, is owned by [Steinberg® Media Technologies GmbH](https://www.steinberg.net/en/home.html) and licensed under the GPLv3. - The `PitchShift` plugin uses [the Rubber Band Library](https://github.com/breakfastquay/rubberband), which is [dual-licensed under a commercial license](https://breakfastquay.com/technology/license.html) and the GPLv2 (or newer). - The `MP3Compressor` plugin uses [`libmp3lame` from the LAME project](https://lame.sourceforge.io/), which is [licensed under the LGPLv2](https://github.com/lameproject/lame/blob/master/README) and [upgraded to the GPLv3 for inclusion in this project (as permitted by the LGPLv2)](https://www.gnu.org/licenses/gpl-faq.html#AllCompatibility). - - The `GSMCompressor` plugin uses [`libgsm`](http://quut.com/gsm/), which is [licensed under the ISC license](https://github.com/timothytylee/libgsm/blob/master/COPYRIGHT) and [compatible with the GPLv3](https://www.gnu.org/licenses/license-list.en.html#ISC). + - The `GSMFullRateCompressor` plugin uses [`libgsm`](http://quut.com/gsm/), which is [licensed under the ISC license](https://github.com/timothytylee/libgsm/blob/master/COPYRIGHT) and [compatible with the GPLv3](https://www.gnu.org/licenses/license-list.en.html#ISC). _VST is a registered trademark of Steinberg Media Technologies GmbH._ From 176c3d16f96dfdd5855f5d18e79f8b0d2379b037 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Tue, 8 Feb 2022 00:48:56 -0500 Subject: [PATCH 20/22] Fixed renamed import. --- pedalboard/plugins/GSMFullRateCompressor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pedalboard/plugins/GSMFullRateCompressor.h b/pedalboard/plugins/GSMFullRateCompressor.h index 9773ed4f..2c43c0f8 100644 --- a/pedalboard/plugins/GSMFullRateCompressor.h +++ b/pedalboard/plugins/GSMFullRateCompressor.h @@ -16,7 +16,7 @@ */ #include "../Plugin.h" -#include "../plugin_templates/FixedBlockSizePlugin.h" +#include "../plugin_templates/FixedBlockSize.h" #include "../plugin_templates/ForceMono.h" #include "../plugin_templates/PrimeWithSilence.h" #include "../plugin_templates/Resample.h" From 9ba99b9abec5d66d52b090d5a0a50f9c5f572d38 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Tue, 8 Feb 2022 00:49:38 -0500 Subject: [PATCH 21/22] Whoops. --- pedalboard/plugin_templates/ForceMono.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/pedalboard/plugin_templates/ForceMono.h b/pedalboard/plugin_templates/ForceMono.h index 0dc3230a..4fa185e3 100644 --- a/pedalboard/plugin_templates/ForceMono.h +++ b/pedalboard/plugin_templates/ForceMono.h @@ -21,11 +21,8 @@ #include "../Plugin.h" #include -<<<<<<< HEAD -======= #include "../plugins/AddLatency.h" ->>>>>>> origin/master namespace Pedalboard { /** From f000972b76f4e24aa909b1e6cc6c3657b4ea5ba0 Mon Sep 17 00:00:00 2001 From: Peter Sobot Date: Tue, 8 Feb 2022 01:02:51 -0500 Subject: [PATCH 22/22] Remove redundant resample initialization. --- pedalboard/python_bindings.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/pedalboard/python_bindings.cpp b/pedalboard/python_bindings.cpp index 66b4d604..8a3402c1 100644 --- a/pedalboard/python_bindings.cpp +++ b/pedalboard/python_bindings.cpp @@ -163,7 +163,6 @@ PYBIND11_MODULE(pedalboard_native, m) { init_noisegate(m); init_phaser(m); init_pitch_shift(m); - init_resample(m); init_reverb(m); init_external_plugins(m);