Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/peaks #202

Merged
merged 21 commits into from
Dec 8, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions FlucomaClients.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ add_client(BufSTFT clients/nrt/BufSTFTClient.hpp CLASS NRTThreadedBufferSTFTClie
add_client(BufScale clients/nrt/BufScaleClient.hpp CLASS NRTThreadedBufferScaleClient )
add_client(BufSelect clients/nrt/BufSelectClient.hpp CLASS NRTThreadingSelectClient )
add_client(BufSelectEvery clients/nrt/BufSelectEveryClient.hpp CLASS NRTThreadingSelectEveryClient )
add_client(BufSineFeature clients/rt/SineFeatureClient.hpp CLASS NRTThreadedSineFeatureClient )
add_client(BufSines clients/rt/SinesClient.hpp CLASS NRTThreadedSinesClient )
add_client(BufSpectralShape clients/rt/SpectralShapeClient.hpp CLASS NRTThreadedSpectralShapeClient )
add_client(BufStats clients/nrt/BufStatsClient.hpp CLASS NRTThreadedBufferStatsClient )
Expand All @@ -132,6 +133,7 @@ add_client(OnsetFeature clients/rt/OnsetFeatureClient.hpp CLASS RTOnsetFeatureCl
add_client(OnsetSlice clients/rt/OnsetSliceClient.hpp CLASS RTOnsetSliceClient )
add_client(Pitch clients/rt/PitchClient.hpp CLASS RTPitchClient )
add_client(STFTPass clients/rt/BaseSTFTClient.hpp CLASS RTSTFTPassClient NOINSTALL)
add_client(SineFeature clients/rt/SineFeatureClient.hpp CLASS RTSineFeatureClient )
add_client(Sines clients/rt/SinesClient.hpp CLASS RTSinesClient )
add_client(SpectralShape clients/rt/SpectralShapeClient.hpp CLASS RTSpectralShapeClient )
add_kr_in_client(Stats clients/rt/RunningStatsClient.hpp CLASS RunningStatsClient )
Expand Down
3 changes: 2 additions & 1 deletion include/algorithms/public/SineExtraction.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ class SineExtraction
mWindowTransform(maxFFT, alloc)
{}

void init(index windowSize, index fftSize, index transformSize, Allocator& alloc)
void init(index windowSize, index fftSize, index transformSize,
Allocator& alloc)
{
mBins = fftSize / 2 + 1;
mCurrentFrame = 0;
Expand Down
89 changes: 89 additions & 0 deletions include/algorithms/public/SineFeature.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/)
Copyright 2017-2019 University of Huddersfield.
Licensed under the BSD-3 License.
See license.md file in the project root for full license information.
This project has received funding from the European Research Council (ERC)
under the European Union’s Horizon 2020 research and innovation programme
(grant agreement No 725899).
*/

#pragma once

#include "WindowFuncs.hpp"
#include "../util/AlgorithmUtils.hpp"
#include "../util/FFT.hpp"
#include "../util/FluidEigenMappings.hpp"
#include "../util/PartialTracking.hpp"
#include "../util/PeakDetection.hpp"
#include "../../data/FluidIndex.hpp"
#include "../../data/FluidMemory.hpp"
#include "../../data/TensorTypes.hpp"
#include <Eigen/Core>
#include <cmath>
#include <queue>

namespace fluid {
namespace algorithm {


class SineFeature
{
using ArrayXd = Eigen::ArrayXd;
using VectorXd = Eigen::VectorXd;
using ArrayXcd = Eigen::ArrayXcd;
template <typename T>
using vector = rt::vector<T>;

public:
SineFeature(Allocator&) {}

void init(index windowSize, index fftSize)
{
mBins = fftSize / 2 + 1;
mScale = 1.0 / (windowSize / 4.0); // scale to original amplitude
mInitialized = true;
}

index processFrame(const ComplexVectorView in, RealVectorView freqOut,
RealVectorView magOut, double sampleRate,
double detectionThreshold, index sortBy, Allocator& alloc)
{
assert(mInitialized);
using namespace Eigen;
index fftSize = 2 * (mBins - 1);
ScopedEigenMap<ArrayXcd> frame(in.size(), alloc);
frame = _impl::asEigen<Array>(in);

ScopedEigenMap<ArrayXd> mag(in.size(), alloc);
mag = frame.abs().real();
mag = mag * mScale;
ScopedEigenMap<ArrayXd> logMagIn(in.size(), alloc);
logMagIn = 20 * mag.max(epsilon).log10();

auto tmpPeaks =
mPeakDetection.process(logMagIn, 0, detectionThreshold, true, sortBy);

index maxNumOut = std::min<index>(freqOut.size(), asSigned(tmpPeaks.size()));

double ratio = sampleRate / fftSize;
std::transform(tmpPeaks.begin(), tmpPeaks.begin() + maxNumOut,
freqOut.begin(),
[ratio](auto peak) { return peak.first * ratio; });

std::transform(tmpPeaks.begin(), tmpPeaks.begin() + maxNumOut,
magOut.begin(), [](auto peak) { return peak.second; });

return maxNumOut;
}

bool initialized() { return mInitialized; }

private:
PeakDetection mPeakDetection;
index mBins{513};
double mScale{1.0};
bool mInitialized{false};
};
} // namespace algorithm
} // namespace fluid
187 changes: 187 additions & 0 deletions include/clients/rt/SineFeatureClient.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
/*
Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/)
Copyright 2017-2019 University of Huddersfield.
Licensed under the BSD-3 License.
See license.md file in the project root for full license information.
This project has received funding from the European Research Council (ERC)
under the European Union’s Horizon 2020 research and innovation programme
(grant agreement No 725899).
*/
#pragma once

#include "../common/AudioClient.hpp"
#include "../common/BufferedProcess.hpp"
#include "../common/FluidBaseClient.hpp"
#include "../common/FluidNRTClientWrapper.hpp"
#include "../common/ParameterConstraints.hpp"
#include "../common/ParameterSet.hpp"
#include "../common/ParameterTrackChanges.hpp"
#include "../common/ParameterTypes.hpp"
#include "../../algorithms/public/SineFeature.hpp"
#include <tuple>

namespace fluid {
namespace client {
namespace sinefeature {

enum SineFeatureParamIndex {
kNPeaks,
kDetectionThreshold,
kSortBy,
kLogFreq,
kLogMag,
kFFT
};

constexpr auto SineFeatureParams = defineParameters(
LongParamRuntimeMax<Primary>("numPeaks", "Number of Sinusoidal Peaks", 10,
Min(1), FrameSizeUpperLimit<kFFT>()),
FloatParam("detectionThreshold", "Peak Detection Threshold", -96, Min(-144),
tremblap marked this conversation as resolved.
Show resolved Hide resolved
Max(0)),
EnumParam("order", "Sort Peaks Output", 0, "Frequencies", "Amplitudes"),
EnumParam("freqUnit", "Units for Frequencies", 0, "Hz", "MIDI"),
EnumParam("magUnit", "Units for Magnitudes", 0, "Amp", "dB"),
FFTParam("fftSettings", "FFT Settings", 1024, -1, -1));

class SineFeatureClient : public FluidBaseClient,
public AudioIn,
public ControlOut
{

public:
using ParamDescType = decltype(SineFeatureParams);

using ParamSetViewType = ParameterSetView<ParamDescType>;
std::reference_wrapper<ParamSetViewType> mParams;

template <size_t N>
auto& get() const
{
return mParams.get().template get<N>();
}

void setParams(ParamSetViewType& p)
{
mParams = p;
controlChannelsOut({2, get<kNPeaks>(), get<kNPeaks>().max()});
}

static constexpr auto& getParameterDescriptors() { return SineFeatureParams; }

SineFeatureClient(ParamSetViewType& p, FluidContext& c)
: mParams(p), mSTFTBufferedProcess{get<kFFT>(), 1, 2, c.hostVectorSize(),
c.allocator()},
mSineFeature{c.allocator()},
mPeaks(get<kNPeaks>().max(), c.allocator()),
mMags(get<kNPeaks>().max(), c.allocator())
{
audioChannelsIn(1);
controlChannelsOut({2, get<kNPeaks>(), get<kNPeaks>().max()});
setInputLabels({"Audio Input"});
setOutputLabels({"Peak Frequencies", "Peak Magnitudes"});
}

template <typename T>
void process(std::vector<HostVector<T>>& input,
std::vector<HostVector<T>>& output, FluidContext& c)
{
if (!input[0].data() || !output[0].data()) return;

index nPeaks = get<kNPeaks>();

if (!mSineFeature.initialized() ||
mTrackValues.changed(get<kFFT>().winSize(), get<kFFT>().fftSize(),
nPeaks, sampleRate()))
{
mSineFeature.init(get<kFFT>().winSize(), get<kFFT>().fftSize());
controlChannelsOut({2, nPeaks});
}

if (mHostSizeTracker.changed(c.hostVectorSize()))
{
mSTFTBufferedProcess = STFTBufferedProcess<false>(
get<kFFT>(), 1, 0, c.hostVectorSize(), c.allocator());
}

auto peaks = mPeaks(Slice(0, nPeaks));
auto mags = mMags(Slice(0, nPeaks));

mSTFTBufferedProcess.processInput(
get<kFFT>(), input, c, [&](ComplexMatrixView in) {
mNumPeaks = mSineFeature.processFrame(
in.row(0), peaks, mags, sampleRate(), get<kDetectionThreshold>(),
get<kSortBy>(), c.allocator());
});

auto validPeaks = mPeaks(Slice(0, mNumPeaks));
auto validMags = mMags(Slice(0, mNumPeaks));

if (get<kLogFreq>())
{
std::transform(validPeaks.begin(), validPeaks.end(), output[0].begin(),
[](auto peak) {
constexpr auto ratio = 1 / 440.0;
return peak == 0 ? -999
: 69 + (12 * std::log2(peak * ratio));
});
output[0](Slice(mNumPeaks, get<kNPeaks>().max() - mNumPeaks)).fill(-999);
}
else
{
output[0](Slice(0, mNumPeaks)) <<= validPeaks;
output[0](Slice(mNumPeaks, get<kNPeaks>().max() - mNumPeaks)).fill(0);
}

if (get<kLogMag>())
{
output[1](Slice(0, mNumPeaks)) <<= validMags;
output[1](Slice(mNumPeaks, get<kNPeaks>().max() - mNumPeaks)).fill(-144);
}
else
{
std::transform(validMags.begin(), validMags.end(), output[1].begin(),
[](auto peak) { return std::pow(10, (peak / 20)); });
output[1](Slice(mNumPeaks, get<kNPeaks>().max() - mNumPeaks)).fill(0);
}
}

index latency() { return get<kFFT>().winSize(); }

void reset(FluidContext&)
{
mSTFTBufferedProcess.reset();
mSineFeature.init(get<kFFT>().winSize(), get<kFFT>().fftSize());
}

AnalysisSize analysisSettings()
{
return {get<kFFT>().winSize(), get<kFFT>().hopSize()};
}

private:
STFTBufferedProcess<false> mSTFTBufferedProcess;
algorithm::SineFeature mSineFeature;
ParameterTrackChanges<index, index, index, double> mTrackValues;
ParameterTrackChanges<index> mHostSizeTracker;
FluidTensor<double, 1> mPeaks;
FluidTensor<double, 1> mMags;
index mNumPeaks;
};

} // namespace sinefeature
using RTSineFeatureClient = ClientWrapper<sinefeature::SineFeatureClient>;

auto constexpr NRTSineFeatureParams =
makeNRTParams<sinefeature::SineFeatureClient>(
InputBufferParam("source", "Source Buffer"),
BufferParam("frequency", "Peak Frequencies Buffer"),
BufferParam("magnitude", "Peak Magnitudes Buffer"));

using NRTSineFeatureClient = NRTControlAdaptor<sinefeature::SineFeatureClient,
decltype(NRTSineFeatureParams),
NRTSineFeatureParams, 1, 2>;

using NRTThreadedSineFeatureClient = NRTThreadingAdaptor<NRTSineFeatureClient>;

} // namespace client
} // namespace fluid