Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache frq for MOD+ #1090

Merged
merged 1 commit into from
Apr 20, 2024
Merged

Cache frq for MOD+ #1090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions OpenUtau.Core/Classic/ClassicSinger.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public class ClassicSinger : USinger {
OtoWatcher otoWatcher;

public bool? UseFilenameAsAlias { get => voicebank.UseFilenameAsAlias; set => voicebank.UseFilenameAsAlias = value; }
public Dictionary<string, Frq> Frqs { get; set; } = new Dictionary<string, Frq>();

public ClassicSinger(Voicebank voicebank) {
this.voicebank = voicebank;
Expand Down
90 changes: 90 additions & 0 deletions OpenUtau.Core/Classic/Frq.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,107 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using NAudio.Wave;
using OpenUtau.Core;
using OpenUtau.Core.Ustx;

namespace OpenUtau.Classic {
public class OtoFrq {
public double[] toneDiffFix = new double[0];
public double[] toneDiffStretch = new double[0];
public int hopSize;
public bool loaded = false;

public OtoFrq(UOto oto, Dictionary<string, Frq> dict) {
if (!dict.TryGetValue(oto.File, out var frq)) {
frq = new Frq();
if (frq.Load(oto.File)){
dict.Add(oto.File, frq);
} else {
frq = null;
}
}
if(frq != null && frq.wavSampleLength != - 1) {
this.hopSize = frq.hopSize;

if (frq.wavSampleLength == 0) {
try {
using (var waveStream = Core.Format.Wave.OpenFile(oto.File)) {
var sampleProvider = waveStream.ToSampleProvider();
if (sampleProvider.WaveFormat.SampleRate == 44100) {
frq.wavSampleLength = Core.Format.Wave.GetSamples(sampleProvider).Length;
} else {
frq.wavSampleLength = -1;
}
}
} catch {
frq.wavSampleLength = - 1;
}
}

if (frq.wavSampleLength > 0) {
int offset = (int)Math.Floor(oto.Offset * 44100 / 1000 / frq.hopSize); // frq samples
int consonant = (int)Math.Floor((oto.Offset + oto.Consonant) * 44100 / 1000 / frq.hopSize);
int cutoff = oto.Cutoff < 0 ?
(int)Math.Floor((oto.Offset - oto.Cutoff) * 44100 / 1000 / frq.hopSize)
: frq.wavSampleLength - (int)Math.Floor(oto.Cutoff * 44100 / 1000 / frq.hopSize);
var completionF0 = Completion(frq.f0);
var averageTone = MusicMath.FreqToTone(frq.averageF0);
toneDiffFix = completionF0.Skip(offset).Take(consonant - offset).Select(f => MusicMath.FreqToTone(f) - averageTone).ToArray();
toneDiffStretch = completionF0.Skip(consonant).Take(cutoff - consonant).Select(f => MusicMath.FreqToTone(f) - averageTone).ToArray();

loaded = true;
}
}
}

private double[] Completion(double[] frqs) {
var list = new List<double>();
for (int i = 0; i < frqs.Length; i++) {
if (frqs[i] <= 60) {
int min = i - 1;
double minFrq = 0;
while (min >= 0) {
if (frqs[min] > 60) {
minFrq = frqs[min];
break;
}
min--;
}
int max = i + 1;
double maxFrq = 0;
while (max < frqs.Length) {
if (frqs[max] > 60) {
maxFrq = frqs[max];
break;
}
max++;
}
if (minFrq <= 60) {
list.Add(maxFrq);
} else if (maxFrq <= 60) {
list.Add(minFrq);
} else {
list.Add(MusicMath.Linear(min, max, minFrq, maxFrq, i));
}
} else {
list.Add(frqs[i]);
}
}
return list.ToArray();
}
}

public class Frq {
public const int kHopSize = 256;

public int hopSize;
public double averageF0;
public double[] f0 = new double[0];
public double[] amp = new double[0];
public int wavSampleLength = 0;

/// <summary>
/// If the wav path is null (machine learning voicebank), return false.
Expand Down
116 changes: 60 additions & 56 deletions OpenUtau.Core/Render/RenderPhrase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Numerics;
using K4os.Hash.xxHash;
using OpenUtau.Classic;
using OpenUtau.Core.Ustx;
using Serilog;

Expand Down Expand Up @@ -299,69 +300,72 @@ internal RenderPhrase(UProject project, UTrack track, UVoicePart part, IEnumerab
}
}
// Mod plus
if (track.TryGetExpDescriptor(project, Format.Ustx.MODP, out var modp) && renderer.SupportsExpression(modp)) {
if (track.TryGetExpDescriptor(project, Format.Ustx.MODP, out var modp) && renderer.SupportsExpression(modp) && singer is ClassicSinger cSinger) {
foreach (var phoneme in phonemes) {
var mod = phoneme.GetExpression(project, track, Format.Ustx.MODP).Item1;
if (mod == 0) {
var phonemeModp = phoneme.GetExpression(project, track, Format.Ustx.MODP).Item1;
if (phonemeModp == 0) {
continue;
}

try {
if (phoneme.TryGetFrq(out var frqFix, out var frqStretch, out double average, out int hopSize)) {
UTempo[] noteTempos = project.timeAxis.TemposBetweenTicks(part.position + phoneme.position, part.position + phoneme.End);
var tempo = noteTempos[0].bpm; // compromise 妥協！
var frqIntervalTick = MusicMath.TempoMsToTick(tempo, (double)1 * 1000 / 44100 * hopSize);
double consonantStretch = Math.Pow(2f, 1.0f - phoneme.GetExpression(project, track, Format.Ustx.VEL).Item1 / 100f);

var preutter = MusicMath.TempoMsToTick(tempo, Math.Min(phoneme.preutter, phoneme.oto.Preutter * consonantStretch));
int startIndex = Math.Max(0, (int)Math.Floor((phoneme.position - pitchStart - preutter) / pitchInterval));
int position = (int)Math.Round((double)((phoneme.position - pitchStart) / pitchInterval));
int startStretch = position + (int)Math.Round(MusicMath.TempoMsToTick(tempo, (phoneme.oto.Consonant - phoneme.oto.Preutter) * consonantStretch) / pitchInterval);
int endIndex = Math.Min(pitches.Length, (int)Math.Ceiling(phoneme.End - pitchStart - MusicMath.TempoMsToTick(tempo, phoneme.tailIntrude - phoneme.tailOverlap)) / pitchInterval);

frqFix = frqFix.Select(f => f - average).ToArray();
frqStretch = frqStretch.Select(f => f - average).ToArray();
double stretch = 1;
if (frqStretch.Length * frqIntervalTick < ((double)endIndex - startStretch) * pitchInterval) {
stretch = ((double)endIndex - startStretch) * pitchInterval / (frqStretch.Length * frqIntervalTick);
}
var env0 = new Vector2(0, 0);
var env1 = new Vector2((phoneme.envelope.data[1].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env3 = new Vector2((phoneme.envelope.data[3].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env4 = new Vector2(1, 0);

for (int i = 0; startStretch + i <= endIndex; i++) {
var pit = startStretch + i;
if (pit >= pitches.Length) break;
var frq = i * (pitchInterval / frqIntervalTick) / stretch;
var frqMin = Math.Clamp((int)Math.Floor(frq), 0, frqStretch.Length - 1);
var frqMax = Math.Clamp((int)Math.Ceiling(frq), 0, frqStretch.Length - 1);
var diff = MusicMath.Linear(frqMin, frqMax, frqStretch[frqMin], frqStretch[frqMax], frq);
diff = diff * mod / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
}
for (int i = 0; startStretch + i - 1 >= startIndex; i--) {
var pit = startStretch + i - 1;
if (pit > endIndex || pit >= pitches.Length) continue;
var frq = frqFix.Length + (i * (pitchInterval / frqIntervalTick) / consonantStretch);
var frqMin = Math.Clamp((int)Math.Floor(frq), 0, frqFix.Length - 1);
var frqMax = Math.Clamp((int)Math.Ceiling(frq), 0, frqFix.Length - 1);
var diff = MusicMath.Linear(frqMin, frqMax, frqFix[frqMin], frqFix[frqMax], frq);
diff = diff * mod / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
if (phoneme.oto.Frq == null) {
phoneme.oto.Frq = new OtoFrq(phoneme.oto, cSinger.Frqs);
}
if (phoneme.oto.Frq.loaded == false) {
continue;
}
var frq = phoneme.oto.Frq;
UTempo[] noteTempos = project.timeAxis.TemposBetweenTicks(part.position + phoneme.position, part.position + phoneme.End);
var tempo = noteTempos[0].bpm; // compromise 妥協！
var frqIntervalTick = MusicMath.TempoMsToTick(tempo, (double)1 * 1000 / 44100 * frq.hopSize);
double consonantStretch = Math.Pow(2f, 1.0f - phoneme.GetExpression(project, track, Format.Ustx.VEL).Item1 / 100f);

var preutter = MusicMath.TempoMsToTick(tempo, Math.Min(phoneme.preutter, phoneme.oto.Preutter * consonantStretch));
int startIndex = Math.Max(0, (int)Math.Floor((phoneme.position - pitchStart - preutter) / pitchInterval));
int position = (int)Math.Round((double)((phoneme.position - pitchStart) / pitchInterval));
int startStretch = position + (int)Math.Round(MusicMath.TempoMsToTick(tempo, (phoneme.oto.Consonant - phoneme.oto.Preutter) * consonantStretch) / pitchInterval);
int endIndex = Math.Min(pitches.Length, (int)Math.Ceiling(phoneme.End - pitchStart - MusicMath.TempoMsToTick(tempo, phoneme.tailIntrude - phoneme.tailOverlap)) / pitchInterval);

double stretch = 1;
if (frq.toneDiffStretch.Length * frqIntervalTick < ((double)endIndex - startStretch) * pitchInterval) {
stretch = ((double)endIndex - startStretch) * pitchInterval / (frq.toneDiffStretch.Length * frqIntervalTick);
}
var env0 = new Vector2(0, 0);
var env1 = new Vector2((phoneme.envelope.data[1].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env3 = new Vector2((phoneme.envelope.data[3].X - phoneme.envelope.data[0].X) / (phoneme.envelope.data[4].X - phoneme.envelope.data[0].X), 100);
var env4 = new Vector2(1, 0);

for (int i = 0; startStretch + i <= endIndex; i++) {
var pit = startStretch + i;
if (pit >= pitches.Length) break;
var frqPoint = i * (pitchInterval / frqIntervalTick) / stretch;
var frqPointMin = Math.Clamp((int)Math.Floor(frqPoint), 0, frq.toneDiffStretch.Length - 1);
var frqPointMax = Math.Clamp((int)Math.Ceiling(frqPoint), 0, frq.toneDiffStretch.Length - 1);
var diff = MusicMath.Linear(frqPointMin, frqPointMax, frq.toneDiffStretch[frqPointMin], frq.toneDiffStretch[frqPointMax], frqPoint);
diff = diff * phonemeModp / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
}
for (int i = 0; startStretch + i - 1 >= startIndex; i--) {
var pit = startStretch + i - 1;
if (pit > endIndex || pit >= pitches.Length) continue;
var frqPoint = frq.toneDiffFix.Length + (i * (pitchInterval / frqIntervalTick) / consonantStretch);
var frqPointMin = Math.Clamp((int)Math.Floor(frqPoint), 0, frq.toneDiffFix.Length - 1);
var frqPointMax = Math.Clamp((int)Math.Ceiling(frqPoint), 0, frq.toneDiffFix.Length - 1);
var diff = MusicMath.Linear(frqPointMin, frqPointMax, frq.toneDiffFix[frqPointMin], frq.toneDiffFix[frqPointMax], frqPoint);
diff = diff * phonemeModp / 100;
diff = Fade(diff, pit);
pitches[pit] = pitches[pit] + (float)(diff * 100);
}
double Fade(double diff, int pit) {
var percentage = (double)(pit - startIndex) / (endIndex - startIndex);
if (phoneme.Next != null && phoneme.End == phoneme.Next.position && percentage > env3.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env3.X, env4.X, env3.Y, env4.Y, percentage), 0, 100) / 100;
}
double Fade(double diff, int pit) {
var percentage = (double)(pit - startIndex) / (endIndex - startIndex);
if (phoneme.Next != null && phoneme.End == phoneme.Next.position && percentage > env3.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env3.X, env4.X, env3.Y, env4.Y, percentage), 0, 100) / 100;
}
if (phoneme.Prev != null && phoneme.Prev.End == phoneme.position && percentage < env1.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env0.X, env1.X, env0.Y, env1.Y, percentage), 0, 100) / 100;
}
return diff;
if (phoneme.Prev != null && phoneme.Prev.End == phoneme.position && percentage < env1.X) {
diff = diff * Math.Clamp(MusicMath.Linear(env0.X, env1.X, env0.Y, env1.Y, percentage), 0, 100) / 100;
}
return diff;
}
} catch(Exception e) {
Log.Error(e, "Failed to compute mod plus.");
Expand Down
74 changes: 0 additions & 74 deletions OpenUtau.Core/Ustx/UPhoneme.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using NAudio.Wave;
using OpenUtau.Classic;
using SharpCompress;
using YamlDotNet.Serialization;

namespace OpenUtau.Core.Ustx {
Expand Down Expand Up @@ -237,77 +234,6 @@ public string GetVoiceColor(UProject project, UTrack track) {
}
return track.VoiceColorExp.options[index];
}

public bool TryGetFrq(out double[] frqFix, out double[] frqStretch, out double average, out int hopSize) {
frqFix = new double[0];
frqStretch = new double[0];
average = 0;
hopSize = 0;

var frq = new Frq();
if (frq.Load(oto.File)) {
average = MusicMath.FreqToTone(frq.averageF0); // 1 = 1tone
hopSize = frq.hopSize;

int wavLength;
using (var waveStream = Format.Wave.OpenFile(oto.File)) {
var sampleProvider = waveStream.ToSampleProvider();
if (sampleProvider.WaveFormat.SampleRate != 44100) {
return false;
}
wavLength = Format.Wave.GetSamples(sampleProvider).Length;
}

int offset = (int)Math.Floor(oto.Offset * 44100 / 1000 / frq.hopSize); // frq samples
int consonant = (int)Math.Floor((oto.Offset + oto.Consonant) * 44100 / 1000 / frq.hopSize);
int cutoff = oto.Cutoff < 0 ?
(int)Math.Floor((oto.Offset - oto.Cutoff) * 44100 / 1000 / frq.hopSize)
: wavLength - (int)Math.Floor(oto.Cutoff * 44100 / 1000 / frq.hopSize);
var avr = average;
var f0 = Completion(frq.f0);
frqFix = f0.Skip(offset).Take(consonant - offset).Select(f => MusicMath.FreqToTone(f)).ToArray();
frqStretch = f0.Skip(consonant).Take(cutoff - consonant).Select(f => MusicMath.FreqToTone(f)).ToArray();

double[] Completion(double[] frqs) {
var list = new List<double>();
for (int i = 0; i < frqs.Length; i++) {
if (frqs[i] <= 0) {
int min = i - 1;
double minFrq = 0;
while (min >= 0) {
if (frqs[min] > 0) {
minFrq = frqs[min];
break;
}
min--;
}
int max = i + 1;
double maxFrq = 0;
while (max < frqs.Length) {
if (frqs[max] > 0) {
maxFrq = frqs[max];
break;
}
max++;
}
if(minFrq <= 0) {
list.Add(maxFrq);
} else if (maxFrq <= 0) {
list.Add(minFrq);
} else {
list.Add(MusicMath.Linear(min, max, minFrq, maxFrq, i));
}
} else {
list.Add(frqs[i]);
}
}
return list.ToArray();
}
return true;
} else {
return false;
}
}
}

public class UEnvelope {
Expand Down
1 change: 1 addition & 0 deletions OpenUtau.Core/Ustx/USinger.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ public double Overlap {
NotifyPropertyChanged(nameof(Overlap));
}
}
public OtoFrq Frq { get;set; }
public List<string> SearchTerms { get; private set; }

public event PropertyChangedEventHandler PropertyChanged;
Expand Down