Upload Artefacts [Don't Merge]

.github/workflows/cmake.yml

@@ -60,3 +60,9 @@ jobs:
         - name: Test with pytest
           run: |
             CONFIG=Release pytest
+
+        - name: Upload build artefacts
+          uses: actions/upload-artifact@v3
+          with:
+            name: VST3
+            path: build/TorchDrum_artefacts/Release/VST3/TorchDrum.vst3

.gitignore

@@ -49,3 +49,6 @@ notebooks/
 
 # libtorch module
 modules/libtorch/
+
+# Mac OSX
+.DS_Store

CMakeLists.txt

@@ -25,6 +25,8 @@ endif()
 
 set(BaseTargetName TorchDrum)
 
+option(COPY_PLUGIN_AFTER_BUILD "Copy plugin to build directory after building" ON)
+
 juce_add_plugin("${BaseTargetName}"
         # VERSION ...                               # Set this if the plugin version is different to the project version
         # ICON_BIG ...                              # ICON_* arguments specify a path to an image file to use as an icon for the Standalone
@@ -35,7 +37,7 @@ juce_add_plugin("${BaseTargetName}"
         NEEDS_MIDI_OUTPUT FALSE
         IS_MIDI_EFFECT FALSE
         EDITOR_WANTS_KEYBOARD_FOCUS TRUE
-        COPY_PLUGIN_AFTER_BUILD TRUE
+        COPY_PLUGIN_AFTER_BUILD ${COPY_PLUGIN_AFTER_BUILD}
         PLUGIN_MANUFACTURER_CODE C4dM
         PLUGIN_CODE c4td
         FORMATS ${JUCE_FORMATS}

source/CMakeLists.txt

@@ -1,10 +1,10 @@
+add_subdirectory(FeatureExtraction)
 add_subdirectory(Synth)
 
 target_sources(${BaseTargetName} PRIVATE
         PluginProcessor.cpp
         PluginEditor.cpp
         Biquad.cpp
-        FeatureExtraction.cpp
         NeuralNetwork.cpp
         OnsetDetection.cpp
         SynthController.cpp)

source/FeatureExtraction/CMakeLists.txt

@@ -0,0 +1,2 @@
+target_sources(${BaseTargetName} PRIVATE
+        FeatureExtraction.cpp)

source/FeatureExtraction/FeatureExtraction.cpp

@@ -0,0 +1,36 @@
+#include "FeatureExtraction.h"
+
+FeatureExtraction::FeatureExtraction()
+{
+}
+
+void FeatureExtraction::prepare(double sr, int fs, int hs)
+{
+    sampleRate = sr;
+    frameSize = fs;
+    hopSize = hs;
+}
+
+void FeatureExtraction::process(const juce::AudioBuffer<float>& buffer, FeatureExtractionResults& results)
+{
+    jassert(buffer.getNumChannels() == 1 && buffer.getNumSamples() == frameSize);
+
+    // Calculate RMS
+    double rms = 0.0;
+    auto* audio = buffer.getReadPointer(0);
+    for (int i = 0; i < buffer.getNumSamples(); ++i)
+    {
+        rms += audio[i] * audio[i];
+    }
+
+    // Normalize RMS
+    rms /= buffer.getNumSamples();
+    rms = std::sqrt(rms);
+
+    // Convert to dB with epsilon to avoid log(0) and floor at -80 dB
+    rms = 20.0f * std::log10(rms + 1e-8f);
+    rms = std::max(rms, -80.0);
+
+    // Update the results
+    results.rmsMean.set(rms, true);
+}

source/FeatureExtraction/FeatureExtraction.h

@@ -0,0 +1,56 @@
+/**
+ * FeatureExtraction.h
+ * Performs audio feature extraction on an input buffer
+ *
+ * TODO: Implement feature extraction based on Rodrigo Constanzo's SP-Tools
+ * - RMS Loudness (should this include the perceptual filtering?)
+ * - Spectral Centroid
+ * - Spectral Flatness
+ * - Pitch?
+ *
+ * These should be calculated on a per-frame basis and then take the mean and first
+ * order difference (mean of?).
+ * Spectal features can all be calculated within a single class that uses the built-in
+ * JUCE FFT class.
+ * Pre-calculate a window function for the FFT. So we should know the window size and
+ * hop size at time of initialization.
+ *
+ * Implement the feature normalizers -- maybe this is a separate class that holds the
+ * min and max values for each feature and then normalizes the input feature to a
+ * range of 0 to 1. This could have a method to perform a rolling normalization, where
+ * the min and max values are updated over time. (This should be able to be frozen and
+ * also reset).
+**/
+
+#pragma once
+
+#include "FeatureValue.h"
+#include <juce_audio_utils/juce_audio_utils.h>
+
+struct FeatureExtractionResults
+{
+    FeatureValue<float> rmsMean;
+};
+
+class FeatureExtraction
+{
+public:
+    FeatureExtraction();
+    ~FeatureExtraction() {}
+
+    // Prepare the feature extraction with sample rate
+    void prepare(double sr, int frameSize, int hopSize);
+
+    // Process a buffer of audio samples and store the results
+    void process(const juce::AudioBuffer<float>& buffer, FeatureExtractionResults& results);
+
+    // Getters
+    double getSampleRate() const { return sampleRate; }
+    int getFrameSize() const { return frameSize; }
+    int getHopSize() const { return hopSize; }
+
+private:
+    double sampleRate;
+    int frameSize;
+    int hopSize;
+};

source/FeatureExtraction/FeatureValue.h

@@ -0,0 +1,67 @@
+/**
+ * FeatureValue.h
+ *
+ * A class to hold a single feature value and support normalization.
+*/
+
+#pragma once
+
+#include <juce_core/juce_core.h>
+#include <limits>
+
+template <typename T>
+class FeatureValue
+{
+public:
+    FeatureValue() = default;
+    ~FeatureValue() = default;
+
+    // Set the value
+    void set(T newValue, bool updateMinMax = false)
+    {
+        value = newValue;
+        if (updateMinMax)
+        {
+            if (value < minValue)
+                minValue = value;
+
+            if (value > maxValue)
+                maxValue = value;
+        }
+    }
+
+    // Set min and max values
+    void setMinMax(T newMinValue, T newMaxValue)
+    {
+        minValue = newMinValue;
+        maxValue = newMaxValue;
+    }
+
+    // Get the raw value
+    T getRawValue() const { return value; }
+
+    // Get the normalized value
+    T getNormalized() const
+    {
+        if (minValue == maxValue || minValue == std::numeric_limits<T>::max())
+            return 0.5;
+
+        T normalized = (value - minValue) / (maxValue - minValue);
+        return juce::jlimit(static_cast<T>(0.0), static_cast<T>(1.0), normalized);
+    }
+
+    // Get the minmax values
+    std::pair<T, T> getMinMax() const { return std::make_pair(minValue, maxValue); }
+
+    // Reset the min and max values
+    void reset()
+    {
+        minValue = std::numeric_limits<T>::max();
+        maxValue = std::numeric_limits<T>::lowest();
+    }
+
+private:
+    T value;
+    T minValue = std::numeric_limits<T>::max();
+    T maxValue = std::numeric_limits<T>::lowest();
+};

source/SynthController.cpp

@@ -18,6 +18,11 @@ void SynthController::prepare(double sr, int samplesPerBlock)
     isOnset = false;
     elapsedSamples = 0;
 
+    // Prepare feature extraction
+    featureExtraction.prepare(sampleRate, ONSET_WINDOW_SIZE, ONSET_WINDOW_SIZE / 4);
+    featureBuffer.clear();
+    featureBuffer.setSize(1, ONSET_WINDOW_SIZE);
+
     // Prepare input and output features for NN
     size_t numSynthParams = synth.getParameters().parameters.size();
     neuralInput.resize(3);
@@ -52,11 +57,12 @@ void SynthController::process(float x)
 
         // TODO: Create a featureBuffer to transfer the create number of samples to
         // from the cicular buffer and pass that to featureExtraction.process()
-        featureExtraction.process(buffer, featureExtractionResults);
+        copySamplesToFeatureBuffer();
+        featureExtraction.process(featureBuffer, features);
 
         // TODO: map features to neural network input -- for now, just use random values
         for (int i = 0; i < neuralInput.size(); ++i)
-            neuralInput[i] = random.nextFloat();
+            neuralInput[i] = features.rmsMean.getNormalized();
 
         neuralMapper.process(neuralInput, neuralOutput);
 
@@ -79,3 +85,17 @@ void SynthController::addSampleToBuffer(float x)
     currentSample = (currentSample + 1) % buffer.getNumSamples();
     jassert(currentSample < buffer.getNumSamples());
 }
+
+void SynthController::copySamplesToFeatureBuffer()
+{
+    int samplePtr = currentSample - ONSET_WINDOW_SIZE;
+    if (samplePtr < 0)
+        samplePtr += buffer.getNumSamples();
+
+    for (int i = 0; i < ONSET_WINDOW_SIZE; ++i)
+    {
+        featureBuffer.setSample(0, i, buffer.getSample(0, samplePtr++));
+        if (samplePtr >= buffer.getNumSamples())
+            samplePtr = 0;
+    }
+}

source/SynthController.h

@@ -4,7 +4,7 @@
 */
 #pragma once
 
-#include "FeatureExtraction.h"
+#include "FeatureExtraction/FeatureExtraction.h"
 #if TORCHDRUMLIB_BUILD
 #include "Utils/NeuralNetworkMock.h"
 #else
@@ -34,8 +34,9 @@ class SynthController
     // Update the neural network model
     void updateModel(const std::string& path);
 
-    // Get the audio buffer
+    // Getters for audio buffers
     const juce::AudioBuffer<float>& getBuffer() const { return buffer; }
+    const juce::AudioBuffer<float>& getFeatureBuffer() const { return featureBuffer; }
 
     // Indicate whether we're in the period after an detected onset but
     // before triggering the synthesizer
@@ -44,6 +45,7 @@ class SynthController
 private:
     // Add a sample to the circular audio buffer
     void addSampleToBuffer(float x);
+    void copySamplesToFeatureBuffer();
 
     double sampleRate;
     SynthBase& synth;
@@ -57,7 +59,8 @@ class SynthController
     int currentSample = 0;
 
     FeatureExtraction featureExtraction;
-    FeatureExtractionResults featureExtractionResults;
+    FeatureExtractionResults features;
+    juce::AudioBuffer<float> featureBuffer;
 
     // Neural network for mapping features to synthesizer parameters
     NeuralNetwork neuralMapper;

source/TorchDrumLib.cpp

@@ -1,7 +1,7 @@
 #include "TorchDrumLib.h"
 
 #include "Biquad.cpp"
-#include "FeatureExtraction.cpp"
+#include "FeatureExtraction/FeatureExtraction.cpp"
 #include "OnsetDetection.cpp"
 #include "Synth/DrumSynth.cpp"
 #include "Synth/Modules/Envelope.cpp"

source/TorchDrumLib.h

@@ -6,7 +6,8 @@ Files to include in dynamic library
 
 #include "Biquad.h"
 #include "EnvelopeFollower.h"
-#include "FeatureExtraction.h"
+#include "FeatureExtraction/FeatureExtraction.h"
+#include "FeatureExtraction/FeatureValue.h"
 #include "OnsetDetection.h"
 #include "Synth/DrumSynth.h"
 #include "Synth/DrumSynthParameters.h"

test/test_feature_extraction.py

@@ -1,3 +1,50 @@
-def test_faeture_extraction_init(torchdrum):
+import numpy as np
+import pytest
+
+
+@pytest.fixture
+def controller(torchdrum):
+    synth = torchdrum.DrumSynth()
+    sc = torchdrum.SynthController(synth)
+    yield sc
+    synth.getParameters().freeParameters()
+
+
+SR = 48000
+
+
+def test_feature_extraction_init(torchdrum):
     fe = torchdrum.FeatureExtraction()
     assert isinstance(fe, torchdrum.FeatureExtraction)
+
+
+def test_feature_extraction_prepare(torchdrum):
+    fe = torchdrum.FeatureExtraction()
+    fe.prepare(SR, 512, 256)
+    assert fe.getSampleRate() == SR
+    assert fe.getFrameSize() == 512
+    assert fe.getHopSize() == 256
+
+
+def test_feature_extraction_process(torchdrum, controller):
+    buffer = controller.getFeatureBuffer()
+    buffer.setSize(1, 512)
+
+    results = torchdrum.FeatureExtractionResults()
+
+    extractor = torchdrum.FeatureExtraction()
+    extractor.prepare(SR, 512, 256)
+
+    # Set the buffer to all zeros
+    for i in range(buffer.getNumSamples()):
+        buffer.setSample(0, i, 0.0)
+
+    extractor.process(buffer, results)
+    assert results.rmsMean.getRawValue() == -80.0
+
+    # Set the buffer to all ones
+    for i in range(buffer.getNumSamples()):
+        buffer.setSample(0, i, 1.0)
+
+    extractor.process(buffer, results)
+    assert np.isclose(results.rmsMean.getRawValue(), -0.0, atol=1e-6)

test/test_feature_value.py

@@ -0,0 +1,117 @@
+import numpy as np
+
+
+def test_feature_value_init_double(torchdrum):
+    fv = torchdrum.FeatureValue["double"]()
+    assert isinstance(fv, torchdrum.FeatureValue["double"])
+
+
+def test_feature_value_init_float(torchdrum):
+    fv = torchdrum.FeatureValue["float"]()
+    assert isinstance(fv, torchdrum.FeatureValue["float"])
+
+
+def test_feature_value_set_once_get_normalized_no_update(torchdrum):
+    fv = torchdrum.FeatureValue["float"]()
+    fv.set(10.0)
+
+    # Getting the normalized value without updating the minmax range
+    # will return a value of 0.5
+    assert fv.getNormalized() == 0.5
+
+
+def test_feature_value_set_get_normalized(torchdrum):
+    fv = torchdrum.FeatureValue["float"]()
+    fv.set(10.0, True)
+
+    # Getting the normalized value after an initial set will return
+    # a value of 0.5 since there is only one value in the minmax range
+    assert fv.getNormalized() == 0.5
+
+    # A second value means that the minmax range will now have two values for norm
+    fv.set(5.0, True)
+    assert fv.getNormalized() == 0.0
+
+    fv.set(20.0, True)
+    assert fv.getNormalized() == 1.0
+
+    fv.set(10.0, True)
+    assert np.isclose(fv.getNormalized(), (10.0 - 5.0) / (20.0 - 5.0))
+
+    fv.set(-10.0, True)
+    assert fv.getNormalized() == 0.0
+
+    fv.set(5.0, True)
+    assert fv.getNormalized() == (5.0 - -10.0) / (20.0 - -10.0)
+
+
+def test_feature_value_set_get_normalized_no_update(torchdrum):
+    fv = torchdrum.FeatureValue["float"]()
+
+    # The first value will set minmax and will return 0.5
+    fv.set(25.0, True)
+    assert fv.getNormalized() == 0.5
+
+    fv.set(10.0, True)
+    assert fv.getNormalized() == 0.0
+
+    # Now that the minmax range is set, calling getNormalized without updating
+    # will use the stored minmax range and a value outside of the range will
+    # be clamped to 0.0 or 1.0
+    fv.set(30.0)
+    assert fv.getNormalized() == 1.0
+
+    fv.set(5.0)
+    assert fv.getNormalized() == 0.0
+
+    fv.set(15.0)
+    assert np.isclose(fv.getNormalized(), (15.0 - 10.0) / (25.0 - 10.0))
+
+
+def test_feature_value_reset(torchdrum):
+    fv = torchdrum.FeatureValue["float"]()
+
+    # The first value will set minmax and will return 0.5
+    fv.set(20.0, True)
+    assert fv.getNormalized() == 0.5
+
+    fv.set(10.0, True)
+    assert fv.getNormalized() == 0.0
+
+    # Reset
+    fv.reset()
+    fv.set(10.0)
+    assert fv.getNormalized() == 0.5
+
+    fv.set(20.0)
+    assert fv.getNormalized() == 0.5
+
+
+def test_feature_value_set_min_max(torchdrum):
+    fv = torchdrum.FeatureValue["float"]()
+
+    fv.setMinMax(100.0, 200.0)
+
+    fv.set(110.0)
+    assert np.isclose(fv.getNormalized(), 0.1)
+
+    fv.set(120.0)
+    assert np.isclose(fv.getNormalized(), 0.2)
+
+
+def test_feature_get_min_max_double(torchdrum):
+    fv = torchdrum.FeatureValue["float"]()
+
+    x = fv.getMinMax()
+    assert x[0] == np.finfo(np.float32).max
+    assert x[1] == np.finfo(np.float32).min
+
+    fv.set(-10.0, True)
+    x = fv.getMinMax()
+    assert x[0] == -10.0
+    assert x[1] == -10.0
+
+    fv.set(20.0, True)
+    x = fv.getMinMax()
+    assert x[0] == -10.0
+    assert x[1] == 20.0