Merge pull request #64 from spotify/psobot/handle-plugin-latency

Compensate for plugin latency.

pedalboard/ExternalPlugin.h

@@ -514,6 +514,7 @@ template <typename ExternalPluginType> class ExternalPlugin : public Plugin {
 
       // Force prepare() to be called again later by invalidating lastSpec:
       lastSpec.maximumBlockSize = 0;
+      samplesProvided = 0;
     }
   }
 
@@ -543,8 +544,8 @@ template <typename ExternalPluginType> class ExternalPlugin : public Plugin {
     }
   }
 
-  void
-  process(const juce::dsp::ProcessContextReplacing<float> &context) override {
+  int process(
+      const juce::dsp::ProcessContextReplacing<float> &context) override {
 
     if (pluginInstance) {
       juce::MidiBuffer emptyMidiBuffer;
@@ -614,7 +615,17 @@ template <typename ExternalPluginType> class ExternalPlugin : public Plugin {
                                            pluginBufferChannelCount,
                                            outputBlock.getNumSamples());
       pluginInstance->processBlock(audioBuffer, emptyMidiBuffer);
+      samplesProvided += outputBlock.getNumSamples();
+
+      // To compensate for any latency added by the plugin,
+      // only tell Pedalboard to use the last _n_ samples.
+      long usableSamplesProduced =
+          samplesProvided - pluginInstance->getLatencySamples();
+      return static_cast<int>(
+          std::min(usableSamplesProduced, (long)outputBlock.getNumSamples()));
     }
+
+    return 0;
   }
 
   std::vector<juce::AudioProcessorParameter *> getParameters() const {
@@ -634,6 +645,12 @@ template <typename ExternalPluginType> class ExternalPlugin : public Plugin {
     return nullptr;
   }
 
+  virtual int getLatencyHint() override {
+    if (!pluginInstance)
+      return 0;
+    return pluginInstance->getLatencySamples();
+  }
+
 private:
   constexpr static int ExternalLoadSampleRate = 44100,
                        ExternalLoadMaximumBlockSize = 8192;
@@ -642,6 +659,8 @@ template <typename ExternalPluginType> class ExternalPlugin : public Plugin {
   juce::AudioPluginFormatManager pluginFormatManager;
   std::unique_ptr<juce::AudioPluginInstance> pluginInstance;
 
+  long samplesProvided = 0;
+
   ExternalPluginReloadType reloadType = ExternalPluginReloadType::Unknown;
 };
 

pedalboard/JucePlugin.h

@@ -58,12 +58,13 @@ template <typename DSPType> class JucePlugin : public Plugin {
     }
   }
 
-  void process(
-      const juce::dsp::ProcessContextReplacing<float> &context) override final {
+  int process(
+      const juce::dsp::ProcessContextReplacing<float> &context) override {
     dspBlock.process(context);
+    return context.getOutputBlock().getNumSamples();
   }
 
-  void reset() override final { dspBlock.reset(); }
+  void reset() override { dspBlock.reset(); }
 
   DSPType &getDSP() { return dspBlock; };
 

pedalboard/Plugin.h

@@ -28,13 +28,48 @@ class Plugin {
 public:
   virtual ~Plugin(){};
 
+  /**
+   * Prepare the data structures that will be necessary for this plugin to
+   * process audio at the provided sample rate, maximum block size, and number
+   * of channels.
+   */
   virtual void prepare(const juce::dsp::ProcessSpec &spec) = 0;
 
-  virtual void
+  /**
+   * Process a single buffer of audio through this plugin.
+   * Returns the number of samples that were output.
+   *
+   * If less than a whole buffer of audio was output, the samples that
+   * were produced should be right-aligned in the buffer
+   * (i.e.: they should come last).
+   */
+  virtual int
   process(const juce::dsp::ProcessContextReplacing<float> &context) = 0;
 
+  /**
+   * Reset this plugin's state, clearing any internal buffers or delay lines.
+   */
   virtual void reset() = 0;
 
+  /**
+   * Get the number of samples of latency introduced by this plugin.
+   * This is the number of samples that must be provided to the plugin
+   * before meaningful output will be returned.
+   * Pedalboard will automatically compensate for this latency when processing
+   * by using the return value from the process() call, but this hint can
+   * make processing more efficient.
+   *
+   * This function will only be called after prepare(), so it can take into
+   * account variables like the current sample rate, maximum block size, and
+   * other plugin parameters.
+   *
+   * Returning a value from getLatencyHint() that is larger than necessary will
+   * allocate that many extra samples during processing, increasing memory
+   * usage. Returning a value that is too small will cause memory to be
+   * reallocated during rendering, impacting rendering speed.
+   */
+  virtual int getLatencyHint() { return 0; }
+
   // A mutex to gate access to this plugin, as its internals may not be
   // thread-safe. Note: use std::lock or std::scoped_lock when locking multiple
   // plugins to avoid deadlocking.

pedalboard/RubberbandPlugin.h

@@ -1,18 +1,54 @@
+/*
+ * pedalboard
+ * Copyright 2022 Spotify AB
+ *
+ * Licensed under the GNU Public License, Version 3.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 #include "../vendors/rubberband/rubberband/RubberBandStretcher.h"
 #include "Plugin.h"
 
 using namespace RubberBand;
 
 namespace Pedalboard {
-class RubberbandPlugin : public Plugin
+
 /*
-Base class for rubberband plugins.
-*/
-{
+ * Base class for all Rubber Band-derived plugins.
+ */
+class RubberbandPlugin : public Plugin {
 public:
   virtual ~RubberbandPlugin(){};
 
-  void process(
+  virtual void prepare(const juce::dsp::ProcessSpec &spec) override {
+    bool specChanged = lastSpec.sampleRate != spec.sampleRate ||
+                       lastSpec.maximumBlockSize < spec.maximumBlockSize ||
+                       spec.numChannels != lastSpec.numChannels;
+
+    if (!rbPtr || specChanged) {
+      auto stretcherOptions = RubberBandStretcher::OptionProcessRealTime |
+                              RubberBandStretcher::OptionThreadingNever |
+                              RubberBandStretcher::OptionChannelsTogether |
+                              RubberBandStretcher::OptionPitchHighQuality;
+      rbPtr = std::make_unique<RubberBandStretcher>(
+          spec.sampleRate, spec.numChannels, stretcherOptions);
+      rbPtr->setMaxProcessSize(spec.maximumBlockSize);
+
+      lastSpec = spec;
+      reset();
+    }
+  }
+
+  int process(
       const juce::dsp::ProcessContextReplacing<float> &context) override final {
     if (rbPtr) {
       auto inBlock = context.getInputBlock();
@@ -34,8 +70,9 @@ Base class for rubberband plugins.
       }
 
       // Rubberband expects all channel data with one float array per channel
-      processSamples(inChannels, outChannels, len, numChannels);
+      return processSamples(inChannels, outChannels, len, numChannels);
     }
+    return 0;
   }
 
   void reset() override final {
@@ -45,8 +82,8 @@ Base class for rubberband plugins.
   }
 
 private:
-  void processSamples(const float *const *inBlock, float **outBlock,
-                      size_t samples, size_t numChannels) {
+  int processSamples(const float *const *inBlock, float **outBlock,
+                     size_t samples, size_t numChannels) {
     // Push all of the input samples into RubberBand:
     rbPtr->process(inBlock, samples, false);
 
@@ -76,10 +113,23 @@ Base class for rubberband plugins.
     }
 
     // Pull the next audio data out of Rubberband:
-    rbPtr->retrieve(outBlock, samplesToPull);
+    return rbPtr->retrieve(outBlock, samplesToPull);
   }
 
 protected:
+  virtual int getLatencyHint() override {
+    if (!rbPtr)
+      return 0;
+
+    initialSamplesRequired =
+        std::max(initialSamplesRequired,
+                 (int)(rbPtr->getSamplesRequired() + rbPtr->getLatency() +
+                       lastSpec.maximumBlockSize));
+
+    return initialSamplesRequired;
+  }
+
   std::unique_ptr<RubberBandStretcher> rbPtr;
+  int initialSamplesRequired = 0;
 };
 }; // namespace Pedalboard

pedalboard/plugins/DelayLine.h

@@ -0,0 +1,67 @@
+/*
+ * pedalboard
+ * Copyright 2022 Spotify AB
+ *
+ * Licensed under the GNU Public License, Version 3.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "../JucePlugin.h"
+
+namespace Pedalboard {
+
+/**
+ * A dummy plugin that buffers audio data internally, used to test Pedalboard's
+ * automatic delay compensation.
+ */
+class DelayLine : public JucePlugin<juce::dsp::DelayLine<
+                      float, juce::dsp::DelayLineInterpolationTypes::None>> {
+public:
+  virtual ~DelayLine(){};
+
+  virtual void reset() override {
+    getDSP().reset();
+    samplesProvided = 0;
+  }
+
+  virtual int
+  process(const juce::dsp::ProcessContextReplacing<float> &context) override {
+    getDSP().process(context);
+    int blockSize = context.getInputBlock().getNumSamples();
+    samplesProvided += blockSize;
+
+    return std::min((int)blockSize,
+                    std::max(0, (int)(samplesProvided - getDSP().getDelay())));
+  }
+
+  virtual int getLatencyHint() override { return getDSP().getDelay(); }
+
+private:
+  int samplesProvided = 0;
+};
+
+inline void init_delay_line(py::module &m) {
+  py::class_<DelayLine, Plugin>(
+      m, "DelayLine",
+      "A dummy plugin that delays input audio for the given number of samples "
+      "before passing it back to the output. Used internally to test "
+      "Pedalboard's automatic latency compensation. Probably not useful as a "
+      "real effect.")
+      .def(py::init([](int samples) {
+             auto dl = new DelayLine();
+             dl->getDSP().setMaximumDelayInSamples(samples);
+             dl->getDSP().setDelay(samples);
+             return dl;
+           }),
+           py::arg("samples") = 44100);
+}
+}; // namespace Pedalboard

pedalboard/plugins/PitchShift.h

@@ -23,41 +23,38 @@ namespace py = pybind11;
 #include "../RubberbandPlugin.h"
 
 namespace Pedalboard {
-class PitchShift : public RubberbandPlugin
 /*
 Modifies pitch of an audio without affecting duration
 */
+class PitchShift : public RubberbandPlugin
+
 {
 private:
   double _scaleFactor = 1.0;
 
+  // Allow pitch shifting by up to 6 octaves up or down:
+  static constexpr float MIN_SCALE_FACTOR = 0.015625;
+  static constexpr float MAX_SCALE_FACTOR = 64.0;
+
 public:
   void setScaleFactor(double scale) {
-    if (scale <= 0) {
-      throw std::range_error("Pitch scale must be a value greater than 0.0.");
+    if (scale < MIN_SCALE_FACTOR || scale > MAX_SCALE_FACTOR) {
+      throw std::range_error("Scale factor must be a value between " +
+                             std::to_string(MIN_SCALE_FACTOR) + " and " +
+                             std::to_string(MAX_SCALE_FACTOR) + ".");
     }
+
     _scaleFactor = scale;
+
     if (rbPtr)
       rbPtr->setPitchScale(_scaleFactor);
   }
 
   double getScaleFactor() { return _scaleFactor; }
 
   void prepare(const juce::dsp::ProcessSpec &spec) override final {
-    bool specChanged = lastSpec.sampleRate != spec.sampleRate ||
-                       lastSpec.maximumBlockSize < spec.maximumBlockSize ||
-                       spec.numChannels != lastSpec.numChannels;
-
-    if (!rbPtr || specChanged) {
-      auto stretcherOptions = RubberBandStretcher::OptionProcessRealTime |
-                              RubberBandStretcher::OptionThreadingNever;
-      rbPtr = std::make_unique<RubberBandStretcher>(
-          spec.sampleRate, spec.numChannels, stretcherOptions);
-      rbPtr->setMaxProcessSize(spec.maximumBlockSize);
-      rbPtr->setPitchScale(_scaleFactor);
-      rbPtr->reset();
-      lastSpec = spec;
-    }
+    RubberbandPlugin::prepare(spec);
+    rbPtr->setPitchScale(_scaleFactor);
   }
 };