Merge pull request #9 from jorshi/feat/synth-update

Synth Update

.github/workflows/cmake.yml

@@ -1,62 +1,63 @@
-name: CI
-
-on:
-    push:
-      branches: ["main"]
-    pull_request:
-
-permissions:
-    contents: read
-
-jobs:
-    build:
-        name: Test plugin on ${{ matrix.os }}
-        runs-on: ${{ matrix.os }}
-        strategy:
-          fail-fast: false # show all errors for each platform (vs. cancel jobs on error)
-          matrix:
-            os: [ubuntu-latest, macOS-latest]
-
-        steps:
-        - name: Install Linux Deps
-          if: runner.os == 'Linux'
-          run: |
-            sudo apt-get update
-            sudo apt install libasound2-dev libcurl4-openssl-dev libx11-dev libxinerama-dev libxext-dev libfreetype6-dev libwebkit2gtk-4.0-dev libglu1-mesa-dev libjack-jackd2-dev lv2-dev
-            sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 9
-
-        - name: Get latest CMake
-          uses: lukka/get-cmake@latest
-
-        - name: Checkout code
-          uses: actions/checkout@v2
-          with:
-            submodules: recursive
-
-        - name: Set up Python 3.10 # Required for torch and unit tests
-          uses: actions/setup-python@v4
-          with:
-            python-version: "3.10"
-            cache: "pip" # cache pip dependencies
-
-        - name: Install Python Dependencies
-          run: |
-            python -m pip install --upgrade pip
-            pip install torch==2.1.1 --index-url https://download.pytorch.org/whl/cpu
-            pip install -r requirements.txt
-
-        - name: Configure
-          shell: bash
-          working-directory: ${{github.workspace}}
-          run: |
-            export TORCHLIB_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'`
-            cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$TORCHLIB_PATH -DBUILD_TORCHDRUM_LIB=ON
-
-        - name: Build
-          shell: bash
-          working-directory: ${{github.workspace}}
-          run: cmake --build build --config Release --parallel 4
-
-        - name: Test with pytest
-          run: |
-            CONFIG=Release pytest
+# name: CI
+
+# on:
+#     push:
+#       branches: ["main"]
+#     pull_request:
+
+# permissions:
+#     contents: read
+
+# jobs:
+#     build:
+#         name: Test plugin on ${{ matrix.os }}
+#         runs-on: ${{ matrix.os }}
+#         timeout-minutes: 15
+#         strategy:
+#           fail-fast: false # show all errors for each platform (vs. cancel jobs on error)
+#           matrix:
+#             os: [ubuntu-latest, macOS-latest]
+
+#         steps:
+#         - name: Install Linux Deps
+#           if: runner.os == 'Linux'
+#           run: |
+#             sudo apt-get update
+#             sudo apt install libasound2-dev libcurl4-openssl-dev libx11-dev libxinerama-dev libxext-dev libfreetype6-dev libwebkit2gtk-4.0-dev libglu1-mesa-dev libjack-jackd2-dev lv2-dev
+#             sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 9
+
+#         - name: Get latest CMake
+#           uses: lukka/get-cmake@latest
+
+#         - name: Checkout code
+#           uses: actions/checkout@v2
+#           with:
+#             submodules: recursive
+
+#         - name: Set up Python 3.10 # Required for torch and unit tests
+#           uses: actions/setup-python@v4
+#           with:
+#             python-version: "3.10"
+#             cache: "pip" # cache pip dependencies
+
+#         - name: Install Python Dependencies
+#           run: |
+#             python -m pip install --upgrade pip
+#             pip install torch==2.1.1 --index-url https://download.pytorch.org/whl/cpu
+#             pip install -r requirements.txt
+
+#         - name: Configure
+#           shell: bash
+#           working-directory: ${{github.workspace}}
+#           run: |
+#             export TORCHLIB_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'`
+#             cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$TORCHLIB_PATH -DBUILD_TORCHDRUM_LIB=ON
+
+#         - name: Build
+#           shell: bash
+#           working-directory: ${{github.workspace}}
+#           run: cmake --build build --config Release --parallel 4
+
+#         - name: Test with pytest
+#           run: |
+#             CONFIG=Release pytest

.gitignore

@@ -49,3 +49,12 @@ notebooks/
 
 # libtorch module
 modules/libtorch/
+
+# OS X
+.DS_Store
+
+# Files related to release packaging
+Presets/
+prepare_plugin_release.sh
+remove_library_presets.sh
+SignedPlugins/

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9" CACHE STRING "Minimum OS X deployment target")
 set(CMAKE_CXX_STANDARD 17)
-project(TorchDrum VERSION 0.0.1)
+project(TorchDrum VERSION 0.0.2)
 
 # Adding submodules
 add_subdirectory(modules)

source/CMakeLists.txt

@@ -1,10 +1,10 @@
+add_subdirectory(FeatureExtraction)
 add_subdirectory(Synth)
 
 target_sources(${BaseTargetName} PRIVATE
         PluginProcessor.cpp
         PluginEditor.cpp
         Biquad.cpp
-        FeatureExtraction.cpp
         NeuralNetwork.cpp
         OnsetDetection.cpp
         SynthController.cpp)

source/FeatureExtraction/CMakeLists.txt

@@ -0,0 +1,2 @@
+target_sources(${BaseTargetName} PRIVATE
+        FeatureExtraction.cpp)

source/FeatureExtraction/FeatureExtraction.cpp

@@ -0,0 +1,45 @@
+#include "FeatureExtraction.h"
+
+FeatureExtraction::FeatureExtraction()
+{
+}
+
+void FeatureExtraction::prepare(double sr, int fs, int hs)
+{
+    sampleRate = sr;
+    frameSize = fs;
+    hopSize = hs;
+
+    // Prepare spectral extractor
+    spectralExtractor.prepare(sr, fs);
+    spectralResults.resize(1);
+    spectralResults.assign(1, 0.0);
+}
+
+void FeatureExtraction::process(const juce::AudioBuffer<float>& buffer, FeatureExtractionResults& results)
+{
+    jassert(buffer.getNumChannels() == 1 && buffer.getNumSamples() == frameSize);
+
+    // Calculate RMS
+    double rms = 0.0;
+    auto* audio = buffer.getReadPointer(0);
+    for (int i = 0; i < buffer.getNumSamples(); ++i)
+    {
+        rms += audio[i] * audio[i];
+    }
+
+    // Normalize RMS
+    rms /= buffer.getNumSamples();
+    rms = std::sqrt(rms);
+
+    // Convert to dB with epsilon to avoid log(0) and floor at -80 dB
+    rms = 20.0f * std::log10(rms + 1e-8f);
+    rms = std::max(rms, -80.0);
+
+    // Update the results
+    results.rmsMean.set(rms, true);
+
+    // Calculate spectral centroid
+    spectralExtractor.process(buffer, spectralResults);
+    results.spectralCentroidMean.set(spectralResults[0], true);
+}

source/FeatureExtraction/FeatureExtraction.h

@@ -0,0 +1,68 @@
+/**
+ * FeatureExtraction.h
+ * Performs audio feature extraction on an input buffer
+ *
+ * TODO: Implement feature extraction based on Rodrigo Constanzo's SP-Tools
+ * - RMS Loudness (should this include the perceptual filtering?)
+ * - Spectral Centroid
+ * - Spectral Flatness
+ * - Pitch?
+ *
+ * These should be calculated on a per-frame basis and then take the mean and first
+ * order difference (mean of?).
+ * Spectal features can all be calculated within a single class that uses the built-in
+ * JUCE FFT class.
+ * Pre-calculate a window function for the FFT. So we should know the window size and
+ * hop size at time of initialization.
+ *
+ * Implement the feature normalizers -- maybe this is a separate class that holds the
+ * min and max values for each feature and then normalizes the input feature to a
+ * range of 0 to 1. This could have a method to perform a rolling normalization, where
+ * the min and max values are updated over time. (This should be able to be frozen and
+ * also reset).
+**/
+
+#pragma once
+
+#include "FeatureValue.h"
+#include "SpectralExtractor.h"
+#include <juce_audio_utils/juce_audio_utils.h>
+
+struct FeatureExtractionResults
+{
+    FeatureValue<float> rmsMean;
+    FeatureValue<float> spectralCentroidMean;
+
+    void reset()
+    {
+        rmsMean.reset();
+        spectralCentroidMean.reset();
+    }
+};
+
+class FeatureExtraction
+{
+public:
+    FeatureExtraction();
+    ~FeatureExtraction() {}
+
+    // Prepare the feature extraction with sample rate
+    void prepare(double sr, int frameSize, int hopSize);
+
+    // Process a buffer of audio samples and store the results
+    void process(const juce::AudioBuffer<float>& buffer, FeatureExtractionResults& results);
+
+    // Getters
+    double getSampleRate() const { return sampleRate; }
+    int getFrameSize() const { return frameSize; }
+    int getHopSize() const { return hopSize; }
+
+private:
+    double sampleRate;
+    int frameSize;
+    int hopSize;
+
+    // Spectral features
+    SpectralExtractor spectralExtractor;
+    std::vector<float> spectralResults;
+};

source/FeatureExtraction/FeatureValue.h

@@ -0,0 +1,67 @@
+/**
+ * FeatureValue.h
+ *
+ * A class to hold a single feature value and support normalization.
+*/
+
+#pragma once
+
+#include <juce_core/juce_core.h>
+#include <limits>
+
+template <typename T>
+class FeatureValue
+{
+public:
+    FeatureValue() = default;
+    ~FeatureValue() = default;
+
+    // Set the value
+    void set(T newValue, bool updateMinMax = false)
+    {
+        value = newValue;
+        if (updateMinMax)
+        {
+            if (value < minValue)
+                minValue = value;
+
+            if (value > maxValue)
+                maxValue = value;
+        }
+    }
+
+    // Set min and max values
+    void setMinMax(T newMinValue, T newMaxValue)
+    {
+        minValue = newMinValue;
+        maxValue = newMaxValue;
+    }
+
+    // Get the raw value
+    T getRawValue() const { return value; }
+
+    // Get the normalized value
+    T getNormalized() const
+    {
+        if (minValue == maxValue || minValue == std::numeric_limits<T>::max())
+            return 0.5;
+
+        T normalized = (value - minValue) / (maxValue - minValue);
+        return juce::jlimit(static_cast<T>(0.0), static_cast<T>(1.0), normalized);
+    }
+
+    // Get the minmax values
+    std::pair<T, T> getMinMax() const { return std::make_pair(minValue, maxValue); }
+
+    // Reset the min and max values
+    void reset()
+    {
+        minValue = std::numeric_limits<T>::max();
+        maxValue = std::numeric_limits<T>::lowest();
+    }
+
+private:
+    T value;
+    T minValue = std::numeric_limits<T>::max();
+    T maxValue = std::numeric_limits<T>::lowest();
+};