Merge pull request #8 from jorshi/feat/synth-update

Feat/synth update

.github/workflows/cmake.yml

@@ -12,6 +12,7 @@ jobs:
     build:
         name: Test plugin on ${{ matrix.os }}
         runs-on: ${{ matrix.os }}
+        timeout-minutes: 15
         strategy:
           fail-fast: false # show all errors for each platform (vs. cancel jobs on error)
           matrix:

.gitignore

@@ -56,4 +56,5 @@ modules/libtorch/
 # Files related to release packaging
 Presets/
 prepare_plugin_release.sh
+remove_library_presets.sh
 SignedPlugins/

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9" CACHE STRING "Minimum OS X deployment target")
 set(CMAKE_CXX_STANDARD 17)
-project(TorchDrum VERSION 0.0.1)
+project(TorchDrum VERSION 0.0.2)
 
 # Adding submodules
 add_subdirectory(modules)

source/FeatureExtraction/FeatureExtraction.cpp

@@ -13,6 +13,7 @@ void FeatureExtraction::prepare(double sr, int fs, int hs)
     // Prepare spectral extractor
     spectralExtractor.prepare(sr, fs);
     spectralResults.resize(1);
+    spectralResults.assign(1, 0.0);
 }
 
 void FeatureExtraction::process(const juce::AudioBuffer<float>& buffer, FeatureExtractionResults& results)

source/FeatureExtraction/FeatureExtraction.h

@@ -32,6 +32,12 @@ struct FeatureExtractionResults
 {
     FeatureValue<float> rmsMean;
     FeatureValue<float> spectralCentroidMean;
+
+    void reset()
+    {
+        rmsMean.reset();
+        spectralCentroidMean.reset();
+    }
 };
 
 class FeatureExtraction

source/FeatureExtraction/SpectralExtractor.h

@@ -16,13 +16,17 @@ class SpectralExtractor
 
     void prepare(double sr, int size)
     {
+        // Obtain the write lock - this will block until the lock is acquired
+        const juce::ScopedWriteLock writeLock(fftLock);
+
+        isPrepared = false;
         sampleRate = sr;
         fftSize = size;
 
         // Initialize FFT
         int fftOrder = std::log2(fftSize);
         fft = std::make_unique<juce::dsp::FFT>(fftOrder);
-        fftBuffer.resize(fftSize);
+        fftBuffer.resize(fftSize * 2);
         fftWindow.resize(fftSize);
 
         // Initialize window function
@@ -31,18 +35,25 @@ class SpectralExtractor
             fftSize,
             juce::dsp::WindowingFunction<float>::hann,
             false);
+
+        isPrepared = true;
     }
 
     void process(const juce::AudioBuffer<float>& buffer, std::vector<float>& results)
     {
         jassert(buffer.getNumChannels() == 1 && buffer.getNumSamples() == fftSize);
 
+        // Don't do anything if the FFT is not initialized or updating
+        const juce::ScopedTryReadLock readLock(fftLock);
+        if (! isPrepared || ! readLock.isLocked())
+            return;
+
         // Apply window function and copy to FFT buffer
         for (int i = 0; i < fftSize; ++i)
             fftBuffer[i] = buffer.getSample(0, i) * fftWindow[i];
 
         // Perform FFT
-        fft->performFrequencyOnlyForwardTransform(fftBuffer.data());
+        fft->performFrequencyOnlyForwardTransform(fftBuffer.data(), true);
 
         // Calculate spectral centroid
         jassert(results.size() >= 1);
@@ -90,4 +101,7 @@ class SpectralExtractor
     std::unique_ptr<juce::dsp::FFT> fft;
     std::vector<float> fftBuffer;
     std::vector<float> fftWindow;
+
+    juce::ReadWriteLock fftLock;
+    std::atomic<bool> isPrepared { false };
 };

source/Parameters.h

@@ -2,18 +2,31 @@
 
 #include <juce_audio_processors/juce_audio_processors.h>
 
-struct Parameters
+class Parameters
 {
+public:
+    Parameters()
+    {
+        parameters.push_back(sensitivity);
+    }
+
     void add(juce::AudioProcessor& processor) const
     {
-        processor.addParameter(gain);
-        processor.addParameter(enable);
+        processor.addParameter(sensitivity);
+    }
+
+    // Free parameters -- this is here to support unit testing.
+    // Parameters will be owned by the AudioProcessor in the application.
+    void freeParameters()
+    {
+        for (auto* param : parameters)
+            delete param;
     }
 
     //Raw pointers. They will be owned by either the processor or the APVTS (if you use it)
-    juce::AudioParameterFloat* gain =
-        new juce::AudioParameterFloat({ "Gain", 1 }, "Gain", 0.f, 1.f, 0.5f);
+    juce::AudioParameterFloat* sensitivity =
+        new juce::AudioParameterFloat({ "sensitivity", 1 }, "Sensitivity", 0.f, 4.f, 1.0f);
 
-    juce::AudioParameterBool* enable =
-        new juce::AudioParameterBool({ "Enable", 1 }, "Enable", true);
+private:
+    std::vector<juce::RangedAudioParameter*> parameters;
 };

source/PluginEditor.cpp

@@ -11,6 +11,7 @@ TorchDrumEditor::TorchDrumEditor(TorchDrumProcessor& p)
 
     addAndMakeVisible(editor);
     addAndMakeVisible(loadModelButton);
+    addAndMakeVisible(resetNormButton);
 
     // Setup the load model button
     int flags = juce::FileBrowserComponent::openMode;
@@ -24,6 +25,12 @@ TorchDrumEditor::TorchDrumEditor(TorchDrumProcessor& p)
                                  });
     };
 
+    // Setup the reset normalizer button
+    resetNormButton.onClick = [this]
+    {
+        processor.getSynthController().resetFeatureNormalizers();
+    };
+
     setSize(400, 500);
 }
 
@@ -47,6 +54,7 @@ void TorchDrumEditor::resized()
     editor.setBounds(area);
 
     loadModelButton.setBounds(25, 20, 100, 50);
+    resetNormButton.setBounds(150, 20, 150, 50);
 }
 
 juce::File TorchDrumEditor::getPresetFolder()

source/PluginEditor.h

@@ -27,5 +27,6 @@ class TorchDrumEditor : public juce::AudioProcessorEditor
     TorchDrumProcessor& processor;
     juce::GenericAudioProcessorEditor editor { processor };
     juce::TextButton loadModelButton { "Load Model" };
+    juce::TextButton resetNormButton { "Reset Normalizer" };
     std::unique_ptr<juce::FileChooser> fileChooser;
 };

source/PluginProcessor.cpp

@@ -1,7 +1,7 @@
 #include "PluginProcessor.h"
 #include "PluginEditor.h"
 
-TorchDrumProcessor::TorchDrumProcessor() : synthController(drumSynth)
+TorchDrumProcessor::TorchDrumProcessor() : synthController(drumSynth, parameters)
 {
     // Add synthesizer parameters
     drumSynth.getParameters().add(*this);

source/Synth/DrumSynth.cpp

@@ -28,6 +28,8 @@ DrumSynth::DrumSynth()
     parameters.addCallback(6, [this](float value)
                            { noiseEnv.setDecayMs(value); });
     parameters.addCallback(7, [this](float value)
+                           { tonalGain.setGainDecibels(value); });
+    parameters.addCallback(8, [this](float value)
                            { noiseGain.setGainDecibels(value); });
 }
 
@@ -47,6 +49,7 @@ float DrumSynth::process()
     // Sinusoidal signal
     float y = osc.process(freqEnv.process());
     y = y * ampEnv.process();
+    y = tonalGain.process(y);
 
     // Noise signal
     float n = noise.process() * noiseEnv.process();

source/Synth/DrumSynth.h

@@ -41,6 +41,7 @@ class DrumSynth : public SynthWithParameterBase<DrumSynthParameters>
     WhiteNoise noise;
 
     Tanh waveshaper;
+    Gain tonalGain;
     Gain noiseGain;
     Gain gain;
 };

source/Synth/DrumSynthParameters.h

@@ -23,6 +23,7 @@ struct DrumSynthParameters : public SynthParameterBase
         addParameter(tanhGain);
         addParameter(outGain);
         addParameter(noiseEnvDecay);
+        addParameter(tonalGain);
         addParameter(noiseGain);
     }
 
@@ -47,6 +48,9 @@ struct DrumSynthParameters : public SynthParameterBase
     juce::AudioParameterFloat* noiseEnvDecay =
         new juce::AudioParameterFloat({ "noise_env_decay", 1 }, "Noise Env Decay", 10.f, 2000.f, 500.f);
 
+    juce::AudioParameterFloat* tonalGain =
+        new juce::AudioParameterFloat({ "tonal_gain", 1 }, "Tonal Gain", -60.0f, 6.0f, 0.f);
+
     juce::AudioParameterFloat* noiseGain =
         new juce::AudioParameterFloat({ "noise_gain", 1 }, "Noise Gain", -60.0f, 6.0f, 0.f);
 };

source/Synth/SynthParameterBase.h

@@ -41,14 +41,15 @@ struct SynthParameterBase
         }
     }
 
-    void updateAllParametersWithModulation(const std::vector<double>& modulation)
+    void updateAllParametersWithModulation(const std::vector<double>& modulation, float sensitivity = 1.0f)
     {
         jassert(modulation.size() == parameters.size());
         for (int i = 0; i < parameters.size(); ++i)
         {
             auto* param = parameters[i];
             auto& callback = callbacks[i];
-            float value = juce::jlimit(0.0f, 1.0f, param->getValue() + (float) modulation[i]);
+            float modAmount = (float) modulation[i] * sensitivity;
+            float value = juce::jlimit(0.0f, 1.0f, param->getValue() + modAmount);
             callback(param->convertFrom0to1(value));
         }
     }

source/SynthController.cpp

@@ -1,6 +1,7 @@
 #include "SynthController.h"
 
-SynthController::SynthController(SynthBase& synth) : synth(synth)
+SynthController::SynthController(SynthBase& synth, Parameters& params)
+    : synth(synth), parameters(params)
 {
 }
 
@@ -25,11 +26,11 @@ void SynthController::prepare(double sr, int samplesPerBlock)
 
     // Prepare input and output features for NN
     size_t numSynthParams = synth.getParameters().parameters.size();
-    neuralInput.resize(12);
+    neuralInput.resize(2);
     neuralOutput.resize(numSynthParams);
 
     // Load the neural network model
-    neuralMapper.setInOutFeatures(12, numSynthParams);
+    neuralMapper.setInOutFeatures(2, numSynthParams);
 
     // Update synth parameters with the current patch
     neuralMapper.getCurrentPatch(synth.getParameters().parameters);
@@ -60,14 +61,15 @@ void SynthController::process(float x)
         copySamplesToFeatureBuffer();
         featureExtraction.process(featureBuffer, features);
 
-        // TODO: map features to neural network input -- for now, just use random values
-        for (int i = 0; i < neuralInput.size(); ++i)
-            neuralInput[i] = features.rmsMean.getNormalized();
+        // Input features to the neural network
+        neuralInput[0] = features.rmsMean.getNormalized();
+        neuralInput[1] = features.spectralCentroidMean.getNormalized();
 
+        // Process the neural network
         neuralMapper.process(neuralInput, neuralOutput);
 
-        // TODO: calculate synth parameters, and trigger synth
-        synth.getParameters().updateAllParametersWithModulation(neuralOutput);
+        // Calculate synth parameters, and trigger synth
+        synth.getParameters().updateAllParametersWithModulation(neuralOutput, parameters.sensitivity->get());
         synth.trigger();
     }
 }

source/SynthController.h

@@ -11,6 +11,7 @@
 #include "NeuralNetwork.h"
 #endif
 #include "OnsetDetection.h"
+#include "Parameters.h"
 #include "Synth/SynthBase.h"
 #include <juce_audio_utils/juce_audio_utils.h>
 
@@ -22,7 +23,7 @@ class SynthController
 {
 public:
     SynthController() = delete;
-    SynthController(SynthBase& synth);
+    SynthController(SynthBase& synth, Parameters& parameters);
     ~SynthController() {}
 
     // Prepare the contoller with sample rate and block size
@@ -42,13 +43,17 @@ class SynthController
     // before triggering the synthesizer
     bool getIsOnset() const { return isOnset; }
 
+    // Reset the feature normalizers to their initial state
+    void resetFeatureNormalizers() { features.reset(); }
+
 private:
     // Add a sample to the circular audio buffer
     void addSampleToBuffer(float x);
     void copySamplesToFeatureBuffer();
 
     double sampleRate;
     SynthBase& synth;
+    Parameters& parameters;
 
     OnsetDetection onsetDetection;
     bool isOnset = false;

test/test_feature_extraction.py

@@ -5,9 +5,11 @@
 @pytest.fixture
 def controller(torchdrum):
     synth = torchdrum.DrumSynth()
-    sc = torchdrum.SynthController(synth)
+    parameters = torchdrum.Parameters()
+    sc = torchdrum.SynthController(synth, parameters)
     yield sc
     synth.getParameters().freeParameters()
+    parameters.freeParameters()
 
 
 SR = 48000

test/test_spectral_extractor.py

@@ -12,9 +12,11 @@
 @pytest.fixture
 def controller(torchdrum):
     synth = torchdrum.DrumSynth()
-    sc = torchdrum.SynthController(synth)
+    parameters = torchdrum.Parameters()
+    sc = torchdrum.SynthController(synth, parameters)
     yield sc
     synth.getParameters().freeParameters()
+    parameters.freeParameters()
 
 
 def test_spectral_extractor_init(torchdrum):
@@ -32,7 +34,7 @@ def test_spectral_extractor_prepare(torchdrum):
 
     fft = extractor.getFFT()
     assert fft.getSize() == fft_size
-    assert extractor.getFFTBuffer().size() == fft_size
+    assert extractor.getFFTBuffer().size() == fft_size * 2
 
     # Confirm the window function is set
     window = extractor.getFFTWindow()
@@ -66,7 +68,7 @@ def test_spectral_extractor_process_fft(torchdrum, controller):
     extractor.process(buffer, results)
 
     fftBuffer = extractor.getFFTBuffer()
-    assert fftBuffer.size() == fft_size
+    assert fftBuffer.size() == fft_size * 2
     results = torch.tensor(fftBuffer)
 
     # Compare to the expected FFT from torch

test/test_synth_controller.py

@@ -5,9 +5,11 @@
 @pytest.fixture
 def controller(torchdrum):
     synth = torchdrum.DrumSynth()
-    sc = torchdrum.SynthController(synth)
+    parameters = torchdrum.Parameters()
+    sc = torchdrum.SynthController(synth, parameters)
     yield sc
     synth.getParameters().freeParameters()
+    parameters.freeParameters()
 
 
 def test_synth_controller_init(controller, torchdrum):