From 6c74724acbb4e79c26fa39dc24b2ed4eacf11c6e Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sat, 24 Aug 2024 08:25:26 +0900 Subject: [PATCH 01/15] =?UTF-8?q?=E3=83=95=E3=83=AC=E3=83=BC=E3=82=BA?= =?UTF-8?q?=E3=83=AC=E3=83=B3=E3=83=80=E3=83=A9=E3=83=BC=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E3=80=81=E3=83=AA=E3=83=95=E3=82=A1=E3=82=AF=E3=82=BF?= =?UTF-8?q?=E3=83=AA=E3=83=B3=E3=82=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sing/SequencerPhraseIndicator.vue | 9 +- src/components/Sing/SequencerPitch.vue | 25 +- src/sing/domain.ts | 33 +- src/sing/phraseRendering.ts | 733 ++++++++++++++++ src/store/singing.ts | 814 ++++++------------ src/store/type.ts | 126 ++- tests/unit/lib/selectPriorPhrase.spec.ts | 44 +- 7 files changed, 1129 insertions(+), 655 deletions(-) create mode 100644 src/sing/phraseRendering.ts diff --git a/src/components/Sing/SequencerPhraseIndicator.vue b/src/components/Sing/SequencerPhraseIndicator.vue index 85049ed190..4994188884 100644 --- a/src/components/Sing/SequencerPhraseIndicator.vue +++ b/src/components/Sing/SequencerPhraseIndicator.vue @@ -6,15 +6,16 @@ import { computed } from "vue"; import { useStore } from "@/store"; import { getOrThrow } from "@/helpers/mapHelper"; -import { PhraseSourceHash, PhraseState } from "@/store/type"; +import { PhraseKey, PhraseState } from "@/store/type"; const props = defineProps<{ - phraseKey: PhraseSourceHash; + phraseKey: PhraseKey; isInSelectedTrack: boolean; }>(); const store = useStore(); const classNames: Record = { + SINGER_IS_NOT_SET: "singer-is-not-set", WAITING_TO_BE_RENDERED: "waiting-to-be-rendered", NOW_RENDERING: "now-rendering", COULD_NOT_RENDER: "could-not-render", @@ -43,6 +44,10 @@ const className = computed(() => { } } +.singer-is-not-set { + visibility: hidden; +} + .waiting-to-be-rendered { @include tint-if-in-other-track( "background-color", diff --git a/src/components/Sing/SequencerPitch.vue b/src/components/Sing/SequencerPitch.vue index e03c86fd0e..65ce6330c4 100644 --- a/src/components/Sing/SequencerPitch.vue +++ b/src/components/Sing/SequencerPitch.vue @@ -30,6 +30,7 @@ import { ExhaustiveError } from "@/type/utility"; import { createLogger } from "@/domain/frontend/log"; import { getLast } from "@/sing/utility"; import { getOrThrow } from "@/helpers/mapHelper"; +import { FrameAudioQuery } from "@/openapi"; type PitchLine = { readonly color: Color; @@ -57,19 +58,29 @@ const previewPitchEdit = computed(() => props.previewPitchEdit); const selectedTrackId = computed(() => store.getters.SELECTED_TRACK_ID); const editFrameRate = computed(() => store.state.editFrameRate); const singingGuidesInSelectedTrack = computed(() => { - const singingGuides = []; + const singingGuides: { + query: FrameAudioQuery; + frameRate: number; + startTime: number; + }[] = []; for (const phrase of store.state.phrases.values()) { if (phrase.trackId !== selectedTrackId.value) { continue; } - if (phrase.singingGuideKey == undefined) { + if (phrase.queryKey == undefined) { continue; } - const singingGuide = getOrThrow( - store.state.singingGuides, - phrase.singingGuideKey, - ); - singingGuides.push(singingGuide); + const track = store.state.tracks.get(phrase.trackId); + if (track == undefined || track.singer == undefined) { + continue; + } + const phraseQuery = getOrThrow(store.state.phraseQueries, phrase.queryKey); + const engineManifest = store.state.engineManifests[track.singer.engineId]; + singingGuides.push({ + startTime: phrase.startTime, + query: phraseQuery, + frameRate: engineManifest.frameRate, + }); } return singingGuides; }); diff --git a/src/sing/domain.ts b/src/sing/domain.ts index 6d84530b73..27e851d820 100644 --- a/src/sing/domain.ts +++ b/src/sing/domain.ts @@ -4,17 +4,12 @@ import { Note, Phrase, PhraseSource, - SingingGuide, - SingingGuideSource, - SingingVoiceSource, Tempo, TimeSignature, - phraseSourceHashSchema, + PhraseKey, Track, - singingGuideSourceHashSchema, - singingVoiceSourceHashSchema, } from "@/store/type"; -import { FramePhoneme } from "@/openapi"; +import { FrameAudioQuery, FramePhoneme } from "@/openapi"; import { TrackId } from "@/type/preload"; const BEAT_TYPES = [2, 4, 8, 16]; @@ -379,23 +374,9 @@ export function isValidPitchEditData(pitchEditData: number[]) { ); } -export const calculatePhraseSourceHash = async (phraseSource: PhraseSource) => { +export const calculatePhraseKey = async (phraseSource: PhraseSource) => { const hash = await calculateHash(phraseSource); - return phraseSourceHashSchema.parse(hash); -}; - -export const calculateSingingGuideSourceHash = async ( - singingGuideSource: SingingGuideSource, -) => { - const hash = await calculateHash(singingGuideSource); - return singingGuideSourceHashSchema.parse(hash); -}; - -export const calculateSingingVoiceSourceHash = async ( - singingVoiceSource: SingingVoiceSource, -) => { - const hash = await calculateHash(singingVoiceSource); - return singingVoiceSourceHashSchema.parse(hash); + return PhraseKey(hash); }; export function getStartTicksOfPhrase(phrase: Phrase) { @@ -469,7 +450,11 @@ export function convertToFramePhonemes(phonemes: FramePhoneme[]) { } export function applyPitchEdit( - singingGuide: SingingGuide, + singingGuide: { + query: FrameAudioQuery; + frameRate: number; + startTime: number; + }, pitchEditData: number[], editFrameRate: number, ) { diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts new file mode 100644 index 0000000000..44e904252f --- /dev/null +++ b/src/sing/phraseRendering.ts @@ -0,0 +1,733 @@ +import { + FrameAudioQueryKey, + Note, + PhraseKey, + Singer, + SingingVoice, + SingingVoiceKey, + Tempo, + Track, + SingingVolumeKey, + SingingVolume, +} from "@/store/type"; +import { + FrameAudioQuery, + FramePhoneme, + Note as NoteForRequestToEngine, +} from "@/openapi"; +import { applyPitchEdit, decibelToLinear, tickToSecond } from "@/sing/domain"; +import { calculateHash, linearInterpolation } from "@/sing/utility"; +import { EngineId, StyleId, TrackId } from "@/type/preload"; +import { createLogger } from "@/domain/frontend/log"; +import { cloneWithUnwrapProxy } from "@/helpers/cloneWithUnwrapProxy"; +import { getOrThrow } from "@/helpers/mapHelper"; + +const logger = createLogger("store/singing"); + +// リクエスト用のノーツ（と休符）を作成する +const createNotesForRequestToEngine = ( + firstRestDuration: number, + lastRestDurationSeconds: number, + notes: Note[], + tempos: Tempo[], + tpqn: number, + frameRate: number, +) => { + const notesForRequestToEngine: NoteForRequestToEngine[] = []; + + // 先頭の休符を変換 + const firstRestStartSeconds = tickToSecond( + notes[0].position - firstRestDuration, + tempos, + tpqn, + ); + const firstRestStartFrame = Math.round(firstRestStartSeconds * frameRate); + const firstRestEndSeconds = tickToSecond(notes[0].position, tempos, tpqn); + const firstRestEndFrame = Math.round(firstRestEndSeconds * frameRate); + notesForRequestToEngine.push({ + key: undefined, + frameLength: firstRestEndFrame - firstRestStartFrame, + lyric: "", + }); + + // ノートを変換 + for (const note of notes) { + const noteOnSeconds = tickToSecond(note.position, tempos, tpqn); + const noteOnFrame = Math.round(noteOnSeconds * frameRate); + const noteOffSeconds = tickToSecond( + note.position + note.duration, + tempos, + tpqn, + ); + const noteOffFrame = Math.round(noteOffSeconds * frameRate); + notesForRequestToEngine.push({ + key: note.noteNumber, + frameLength: noteOffFrame - noteOnFrame, + lyric: note.lyric, + }); + } + + // 末尾に休符を追加 + const lastRestFrameLength = Math.round(lastRestDurationSeconds * frameRate); + notesForRequestToEngine.push({ + key: undefined, + frameLength: lastRestFrameLength, + lyric: "", + }); + + // frameLengthが1以上になるようにする + for (let i = 0; i < notesForRequestToEngine.length; i++) { + const frameLength = notesForRequestToEngine[i].frameLength; + const frameToShift = Math.max(0, 1 - frameLength); + notesForRequestToEngine[i].frameLength += frameToShift; + if (i < notesForRequestToEngine.length - 1) { + notesForRequestToEngine[i + 1].frameLength -= frameToShift; + } + } + + return notesForRequestToEngine; +}; + +const shiftKeyOfNotes = (notes: NoteForRequestToEngine[], keyShift: number) => { + for (const note of notes) { + if (note.key != undefined) { + note.key += keyShift; + } + } +}; + +const getPhonemes = (query: FrameAudioQuery) => { + return query.phonemes.map((value) => value.phoneme).join(" "); +}; + +const shiftPitch = (f0: number[], pitchShift: number) => { + for (let i = 0; i < f0.length; i++) { + f0[i] *= Math.pow(2, pitchShift / 12); + } +}; + +const shiftVolume = (volume: number[], volumeShift: number) => { + for (let i = 0; i < volume.length; i++) { + volume[i] *= decibelToLinear(volumeShift); + } +}; + +// 歌とpauの呼吸音が重ならないようにvolumeを制御する +// fadeOutDurationSecondsが0の場合は即座にvolumeを0にする +const muteLastPauSection = ( + volume: number[], + phonemes: FramePhoneme[], + frameRate: number, + fadeOutDurationSeconds: number, +) => { + const lastPhoneme = phonemes.at(-1); + if (lastPhoneme == undefined || lastPhoneme.phoneme !== "pau") { + throw new Error("No pau exists at the end."); + } + + let lastPauStartFrame = 0; + for (let i = 0; i < phonemes.length - 1; i++) { + lastPauStartFrame += phonemes[i].frameLength; + } + + const lastPauFrameLength = lastPhoneme.frameLength; + let fadeOutFrameLength = Math.round(fadeOutDurationSeconds * frameRate); + fadeOutFrameLength = Math.max(0, fadeOutFrameLength); + fadeOutFrameLength = Math.min(lastPauFrameLength, fadeOutFrameLength); + + // フェードアウト処理を行う + if (fadeOutFrameLength === 1) { + volume[lastPauStartFrame] *= 0.5; + } else { + for (let i = 0; i < fadeOutFrameLength; i++) { + volume[lastPauStartFrame + i] *= linearInterpolation( + 0, + 1, + fadeOutFrameLength - 1, + 0, + i, + ); + } + } + // 音量を0にする + for (let i = fadeOutFrameLength; i < lastPauFrameLength; i++) { + volume[lastPauStartFrame + i] = 0; + } +}; + +const singingTeacherStyleId = StyleId(6000); // TODO: 設定できるようにする +const lastRestDurationSeconds = 0.5; // TODO: 設定できるようにする +const fadeOutDurationSeconds = 0.15; // TODO: 設定できるようにする + +type Snapshot = Readonly<{ + tpqn: number; + tempos: Tempo[]; + tracks: Map; + engineFrameRates: Map; + editFrameRate: number; +}>; + +type Phrase = Readonly<{ + firstRestDuration: number; + notes: Note[]; + startTime: number; + queryKey: { + get: () => FrameAudioQueryKey | undefined; + set: (value: FrameAudioQueryKey | undefined) => void; + }; + singingVolumeKey: { + get: () => SingingVolumeKey | undefined; + set: (value: SingingVolumeKey | undefined) => void; + }; + singingVoiceKey: { + get: () => SingingVoiceKey | undefined; + set: (value: SingingVoiceKey | undefined) => void; + }; +}>; + +type ExternalDependencies = Readonly<{ + queryCache: Map; + singingVolumeCache: Map; + singingVoiceCache: Map; + + phrases: { + get: (phraseKey: PhraseKey) => Phrase; + }; + phraseQueries: { + get: (queryKey: FrameAudioQueryKey) => FrameAudioQuery; + set: (queryKey: FrameAudioQueryKey, query: FrameAudioQuery) => void; + delete: (queryKey: FrameAudioQueryKey) => void; + }; + phraseSingingVolumes: { + get: (singingVolumeKey: SingingVolumeKey) => SingingVolume; + set: ( + singingVolumeKey: SingingVolumeKey, + singingVolume: SingingVolume, + ) => void; + delete: (singingVolumeKey: SingingVolumeKey) => void; + }; + phraseSingingVoices: { + set: (singingVoiceKey: SingingVoiceKey, singingVoice: SingingVoice) => void; + delete: (singingVoiceKey: SingingVoiceKey) => void; + }; + + fetchQuery: ( + engineId: EngineId, + notes: NoteForRequestToEngine[], + ) => Promise; + fetchSingFrameVolume: ( + notes: NoteForRequestToEngine[], + query: FrameAudioQuery, + engineId: EngineId, + styleId: StyleId, + ) => Promise; + synthesizeSingingVoice: ( + singer: Singer, + query: FrameAudioQuery, + ) => Promise; +}>; + +type Context = Readonly<{ + snapshot: Snapshot; + trackId: TrackId; + phraseKey: PhraseKey; + externalDependencies: ExternalDependencies; +}>; + +// クエリ生成ステージ + +type QuerySource = Readonly<{ + engineId: EngineId; + engineFrameRate: number; + tpqn: number; + tempos: Tempo[]; + firstRestDuration: number; + notes: Note[]; + keyRangeAdjustment: number; +}>; + +type QueryGenerationStage = Readonly<{ + id: "queryGeneration"; + shouldBeExecuted: (context: Context) => Promise; + deleteExecutionResult: (context: Context) => void; + execute: (context: Context) => Promise; +}>; + +const generateQuerySource = (context: Context): QuerySource => { + const phrases = context.externalDependencies.phrases; + + const track = getOrThrow(context.snapshot.tracks, context.trackId); + if (track.singer == undefined) { + throw new Error("track.singer is undefined."); + } + const engineFrameRate = getOrThrow( + context.snapshot.engineFrameRates, + track.singer.engineId, + ); + const phrase = phrases.get(context.phraseKey); + return { + engineId: track.singer.engineId, + engineFrameRate, + tpqn: context.snapshot.tpqn, + tempos: context.snapshot.tempos, + firstRestDuration: phrase.firstRestDuration, + notes: phrase.notes, + keyRangeAdjustment: track.keyRangeAdjustment, + }; +}; + +const calculateQueryKey = async (querySource: QuerySource) => { + const hash = await calculateHash(querySource); + return FrameAudioQueryKey(hash); +}; + +const generateQuery = async ( + querySource: QuerySource, + externalDependencies: ExternalDependencies, +) => { + const notesForRequestToEngine = createNotesForRequestToEngine( + querySource.firstRestDuration, + lastRestDurationSeconds, + querySource.notes, + querySource.tempos, + querySource.tpqn, + querySource.engineFrameRate, + ); + + shiftKeyOfNotes(notesForRequestToEngine, -querySource.keyRangeAdjustment); + + const query = await externalDependencies.fetchQuery( + querySource.engineId, + notesForRequestToEngine, + ); + + shiftPitch(query.f0, querySource.keyRangeAdjustment); + return query; +}; + +const queryGenerationStage: QueryGenerationStage = { + id: "queryGeneration", + shouldBeExecuted: async (context: Context) => { + const phrases = context.externalDependencies.phrases; + + const track = getOrThrow(context.snapshot.tracks, context.trackId); + if (track.singer == undefined) { + return false; + } + const phrase = phrases.get(context.phraseKey); + const phraseQueryKey = phrase.queryKey.get(); + const querySource = generateQuerySource(context); + const queryKey = await calculateQueryKey(querySource); + return phraseQueryKey == undefined || phraseQueryKey !== queryKey; + }, + deleteExecutionResult: (context: Context) => { + const phrases = context.externalDependencies.phrases; + const phraseQueries = context.externalDependencies.phraseQueries; + + const phrase = phrases.get(context.phraseKey); + const phraseQueryKey = phrase.queryKey.get(); + if (phraseQueryKey != undefined) { + phraseQueries.delete(phraseQueryKey); + phrase.queryKey.set(undefined); + } + }, + execute: async (context: Context) => { + const phrases = context.externalDependencies.phrases; + const phraseQueries = context.externalDependencies.phraseQueries; + const queryCache = context.externalDependencies.queryCache; + + const querySource = generateQuerySource(context); + const queryKey = await calculateQueryKey(querySource); + + let query = queryCache.get(queryKey); + if (query != undefined) { + logger.info(`Loaded query from cache.`); + } else { + query = await generateQuery(querySource, context.externalDependencies); + const phonemes = getPhonemes(query); + logger.info(`Generated query. phonemes: ${phonemes}`); + queryCache.set(queryKey, query); + } + + const phrase = phrases.get(context.phraseKey); + const phraseQueryKey = phrase.queryKey.get(); + if (phraseQueryKey != undefined) { + phraseQueries.delete(phraseQueryKey); + } + phraseQueries.set(queryKey, query); + phrase.queryKey.set(queryKey); + }, +}; + +// 歌唱ボリューム生成ステージ + +type SingingVolumeSource = Readonly<{ + engineId: EngineId; + engineFrameRate: number; + tpqn: number; + tempos: Tempo[]; + firstRestDuration: number; + notes: Note[]; + keyRangeAdjustment: number; + volumeRangeAdjustment: number; + queryForVolumeGeneration: FrameAudioQuery; +}>; + +type SingingVolumeGenerationStage = Readonly<{ + id: "singingVolumeGeneration"; + shouldBeExecuted: (context: Context) => Promise; + deleteExecutionResult: (context: Context) => void; + execute: (context: Context) => Promise; +}>; + +const generateSingingVolumeSource = (context: Context): SingingVolumeSource => { + const phrases = context.externalDependencies.phrases; + const phraseQueries = context.externalDependencies.phraseQueries; + + const track = getOrThrow(context.snapshot.tracks, context.trackId); + if (track.singer == undefined) { + throw new Error("track.singer is undefined."); + } + const engineFrameRate = getOrThrow( + context.snapshot.engineFrameRates, + track.singer.engineId, + ); + const phrase = phrases.get(context.phraseKey); + const phraseQueryKey = phrase.queryKey.get(); + if (phraseQueryKey == undefined) { + throw new Error("phraseQueryKey is undefined."); + } + const query = phraseQueries.get(phraseQueryKey); + const clonedQuery = cloneWithUnwrapProxy(query); + applyPitchEdit( + { + query: clonedQuery, + startTime: phrase.startTime, + frameRate: engineFrameRate, + }, + track.pitchEditData, + context.snapshot.editFrameRate, + ); + return { + engineId: track.singer.engineId, + engineFrameRate, + tpqn: context.snapshot.tpqn, + tempos: context.snapshot.tempos, + firstRestDuration: phrase.firstRestDuration, + notes: phrase.notes, + keyRangeAdjustment: track.keyRangeAdjustment, + volumeRangeAdjustment: track.volumeRangeAdjustment, + queryForVolumeGeneration: clonedQuery, + }; +}; + +const calculateSingingVolumeKey = async ( + singingVolumeSource: SingingVolumeSource, +) => { + const hash = await calculateHash(singingVolumeSource); + return SingingVolumeKey(hash); +}; + +const generateSingingVolume = async ( + singingVolumeSource: SingingVolumeSource, + externalDependencies: ExternalDependencies, +) => { + const notesForRequestToEngine = createNotesForRequestToEngine( + singingVolumeSource.firstRestDuration, + lastRestDurationSeconds, + singingVolumeSource.notes, + singingVolumeSource.tempos, + singingVolumeSource.tpqn, + singingVolumeSource.engineFrameRate, + ); + const queryForVolumeGeneration = singingVolumeSource.queryForVolumeGeneration; + + shiftKeyOfNotes( + notesForRequestToEngine, + -singingVolumeSource.keyRangeAdjustment, + ); + shiftPitch( + queryForVolumeGeneration.f0, + -singingVolumeSource.keyRangeAdjustment, + ); + + const singingVolume = await externalDependencies.fetchSingFrameVolume( + notesForRequestToEngine, + queryForVolumeGeneration, + singingVolumeSource.engineId, + singingTeacherStyleId, + ); + + shiftVolume(singingVolume, singingVolumeSource.volumeRangeAdjustment); + + // 末尾のpauの区間の音量を0にする + muteLastPauSection( + singingVolume, + queryForVolumeGeneration.phonemes, + singingVolumeSource.engineFrameRate, + fadeOutDurationSeconds, + ); + return singingVolume; +}; + +const singingVolumeGenerationStage: SingingVolumeGenerationStage = { + id: "singingVolumeGeneration", + shouldBeExecuted: async (context: Context) => { + const phrases = context.externalDependencies.phrases; + + const track = getOrThrow(context.snapshot.tracks, context.trackId); + if (track.singer == undefined) { + return false; + } + const singingVolumeSource = generateSingingVolumeSource(context); + const singingVolumeKey = + await calculateSingingVolumeKey(singingVolumeSource); + const phrase = phrases.get(context.phraseKey); + const phraseSingingVolumeKey = phrase.singingVolumeKey.get(); + return ( + phraseSingingVolumeKey == undefined || + phraseSingingVolumeKey !== singingVolumeKey + ); + }, + deleteExecutionResult: (context: Context) => { + const phrases = context.externalDependencies.phrases; + const phraseSingingVolumes = + context.externalDependencies.phraseSingingVolumes; + + const phrase = phrases.get(context.phraseKey); + const phraseSingingVolumeKey = phrase.singingVolumeKey.get(); + if (phraseSingingVolumeKey != undefined) { + phraseSingingVolumes.delete(phraseSingingVolumeKey); + phrase.singingVolumeKey.set(undefined); + } + }, + execute: async (context: Context) => { + const phrases = context.externalDependencies.phrases; + const phraseSingingVolumes = + context.externalDependencies.phraseSingingVolumes; + const singingVolumeCache = context.externalDependencies.singingVolumeCache; + + const singingVolumeSource = generateSingingVolumeSource(context); + const singingVolumeKey = + await calculateSingingVolumeKey(singingVolumeSource); + + let singingVolume = singingVolumeCache.get(singingVolumeKey); + if (singingVolume != undefined) { + logger.info(`Loaded singing volume from cache.`); + } else { + singingVolume = await generateSingingVolume( + singingVolumeSource, + context.externalDependencies, + ); + logger.info(`Generated singing volume.`); + singingVolumeCache.set(singingVolumeKey, singingVolume); + } + + const phrase = phrases.get(context.phraseKey); + const phraseSingingVolumeKey = phrase.singingVolumeKey.get(); + if (phraseSingingVolumeKey != undefined) { + phraseSingingVolumes.delete(phraseSingingVolumeKey); + } + phraseSingingVolumes.set(singingVolumeKey, singingVolume); + phrase.singingVolumeKey.set(singingVolumeKey); + }, +}; + +// 音声合成ステージ + +type SingingVoiceSource = Readonly<{ + singer: Singer; + queryForSingingVoiceSynthesis: FrameAudioQuery; +}>; + +type SingingVoiceSynthesisStage = Readonly<{ + id: "singingVoiceSynthesis"; + shouldBeExecuted: (context: Context) => Promise; + deleteExecutionResult: (context: Context) => void; + execute: (context: Context) => Promise; +}>; + +const generateSingingVoiceSource = (context: Context): SingingVoiceSource => { + const phrases = context.externalDependencies.phrases; + const phraseQueries = context.externalDependencies.phraseQueries; + const phraseSingingVolumes = + context.externalDependencies.phraseSingingVolumes; + + const track = getOrThrow(context.snapshot.tracks, context.trackId); + if (track.singer == undefined) { + throw new Error("track.singer is undefined."); + } + const engineFrameRate = getOrThrow( + context.snapshot.engineFrameRates, + track.singer.engineId, + ); + const phrase = phrases.get(context.phraseKey); + const phraseQueryKey = phrase.queryKey.get(); + const phraseSingingVolumeKey = phrase.singingVolumeKey.get(); + if (phraseQueryKey == undefined) { + throw new Error("phraseQueryKey is undefined."); + } + if (phraseSingingVolumeKey == undefined) { + throw new Error("phraseSingingVolumeKey is undefined."); + } + const query = phraseQueries.get(phraseQueryKey); + const singingVolume = phraseSingingVolumes.get(phraseSingingVolumeKey); + const clonedQuery = cloneWithUnwrapProxy(query); + const clonedSingingVolume = cloneWithUnwrapProxy(singingVolume); + applyPitchEdit( + { + query: clonedQuery, + startTime: phrase.startTime, + frameRate: engineFrameRate, + }, + track.pitchEditData, + context.snapshot.editFrameRate, + ); + clonedQuery.volume = clonedSingingVolume; + return { + singer: track.singer, + queryForSingingVoiceSynthesis: clonedQuery, + }; +}; + +const calculateSingingVoiceKey = async ( + singingVoiceSource: SingingVoiceSource, +) => { + const hash = await calculateHash(singingVoiceSource); + return SingingVoiceKey(hash); +}; + +const synthesizeSingingVoice = async ( + singingVoiceSource: SingingVoiceSource, + externalDependencies: ExternalDependencies, +) => { + const singingVoice = await externalDependencies.synthesizeSingingVoice( + singingVoiceSource.singer, + singingVoiceSource.queryForSingingVoiceSynthesis, + ); + return singingVoice; +}; + +const singingVoiceSynthesisStage: SingingVoiceSynthesisStage = { + id: "singingVoiceSynthesis", + shouldBeExecuted: async (context: Context) => { + const phrases = context.externalDependencies.phrases; + + const track = getOrThrow(context.snapshot.tracks, context.trackId); + if (track.singer == undefined) { + return false; + } + const singingVoiceSource = generateSingingVoiceSource(context); + const singingVoiceKey = await calculateSingingVoiceKey(singingVoiceSource); + const phrase = phrases.get(context.phraseKey); + const phraseSingingVoiceKey = phrase.singingVoiceKey.get(); + return ( + phraseSingingVoiceKey == undefined || + phraseSingingVoiceKey !== singingVoiceKey + ); + }, + deleteExecutionResult: (context: Context) => { + const phrases = context.externalDependencies.phrases; + const phraseSingingVoices = + context.externalDependencies.phraseSingingVoices; + + const phrase = phrases.get(context.phraseKey); + const phraseSingingVoiceKey = phrase.singingVoiceKey.get(); + if (phraseSingingVoiceKey != undefined) { + phraseSingingVoices.delete(phraseSingingVoiceKey); + phrase.singingVoiceKey.set(undefined); + } + }, + execute: async (context: Context) => { + const phrases = context.externalDependencies.phrases; + const phraseSingingVoices = + context.externalDependencies.phraseSingingVoices; + const singingVoiceCache = context.externalDependencies.singingVoiceCache; + + const singingVoiceSource = generateSingingVoiceSource(context); + const singingVoiceKey = await calculateSingingVoiceKey(singingVoiceSource); + + let singingVoice = singingVoiceCache.get(singingVoiceKey); + if (singingVoice != undefined) { + logger.info(`Loaded singing voice from cache.`); + } else { + singingVoice = await synthesizeSingingVoice( + singingVoiceSource, + context.externalDependencies, + ); + logger.info(`Generated singing voice.`); + singingVoiceCache.set(singingVoiceKey, singingVoice); + } + + const phrase = phrases.get(context.phraseKey); + const phraseSingingVoiceKey = phrase.singingVoiceKey.get(); + if (phraseSingingVoiceKey != undefined) { + phraseSingingVoices.delete(phraseSingingVoiceKey); + } + phraseSingingVoices.set(singingVoiceKey, singingVoice); + phrase.singingVoiceKey.set(singingVoiceKey); + }, +}; + +// フレーズレンダラー + +const stages = [ + queryGenerationStage, + singingVolumeGenerationStage, + singingVoiceSynthesisStage, +] as const; + +export type PhraseRenderStageId = (typeof stages)[number]["id"]; + +export const createPhraseRenderer = ( + externalDependencies: ExternalDependencies, +) => { + return { + getFirstRenderStageId: () => { + return stages[0].id; + }, + determineStartStage: async ( + snapshot: Snapshot, + trackId: TrackId, + phraseKey: PhraseKey, + ) => { + const context: Context = { + snapshot, + trackId, + phraseKey, + externalDependencies, + }; + for (const stage of stages) { + if (await stage.shouldBeExecuted(context)) { + return stage.id; + } + } + return undefined; + }, + render: async ( + snapshot: Snapshot, + trackId: TrackId, + phraseKey: PhraseKey, + startStageId: PhraseRenderStageId, + ) => { + const context: Context = { + snapshot, + trackId, + phraseKey, + externalDependencies, + }; + const startStageIndex = stages.findIndex((value) => { + return value.id === startStageId; + }); + if (startStageIndex === -1) { + throw new Error("Stage not found."); + } + for (let i = stages.length - 1; i >= startStageIndex; i--) { + stages[i].deleteExecutionResult(context); + } + for (let i = startStageIndex; i < stages.length; i++) { + await stages[i].execute(context); + } + }, + } as const; +}; diff --git a/src/store/singing.ts b/src/store/singing.ts index 7369744dc6..37a052ef51 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -14,14 +14,15 @@ import { Singer, Phrase, transformCommandStore, - SingingGuide, SingingVoice, - SingingGuideSourceHash, - SingingVoiceSourceHash, SequencerEditTarget, - PhraseSourceHash, + PhraseKey, Track, SequenceId, + FrameAudioQueryKey, + SingingVolumeKey, + SingingVolume, + SingingVoiceKey, } from "./type"; import { DEFAULT_PROJECT_NAME, sanitizeFileName } from "./utility"; import { @@ -57,13 +58,9 @@ import { isValidVolumeRangeAdjustment, secondToTick, tickToSecond, - calculateSingingGuideSourceHash, - calculateSingingVoiceSourceHash, - decibelToLinear, - applyPitchEdit, VALUE_INDICATING_NO_DATA, isValidPitchEditData, - calculatePhraseSourceHash, + calculatePhraseKey, isValidTempos, isValidTimeSignatures, isValidTpqn, @@ -86,7 +83,6 @@ import { import { AnimationTimer, createPromiseThatResolvesWhen, - linearInterpolation, round, } from "@/sing/utility"; import { getWorkaroundKeyRangeAdjustment } from "@/sing/workaroundKeyRangeAdjustment"; @@ -98,6 +94,10 @@ import { ufProjectToVoicevox } from "@/sing/utaformatixProject/toVoicevox"; import { uuid4 } from "@/helpers/random"; import { convertToWavFileData } from "@/sing/convertToWavFileData"; import { generateWriteErrorMessage } from "@/helpers/fileHelper"; +import { + PhraseRenderStageId, + createPhraseRenderer, +} from "@/sing/phraseRendering"; const logger = createLogger("store/singing"); @@ -156,9 +156,8 @@ const offlineRenderTracks = async ( withLimiter: boolean, multiTrackEnabled: boolean, tracks: Map, - phrases: Map, - singingGuides: Map, - singingVoices: Map, + phrases: Map, + singingVoices: Map, ) => { const offlineAudioContext = new OfflineAudioContext( numberOfChannels, @@ -182,21 +181,16 @@ const offlineRenderTracks = async ( } for (const phrase of phrases.values()) { - if ( - phrase.singingGuideKey == undefined || - phrase.singingVoiceKey == undefined || - phrase.state !== "PLAYABLE" - ) { + if (phrase.singingVoiceKey == undefined || phrase.state !== "PLAYABLE") { continue; } - const singingGuide = getOrThrow(singingGuides, phrase.singingGuideKey); const singingVoice = getOrThrow(singingVoices, phrase.singingVoiceKey); // TODO: この辺りの処理を共通化する const audioEvents = await generateAudioEvents( offlineAudioContext, - singingGuide.startTime, - singingVoice.blob, + phrase.startTime, + singingVoice, ); const audioPlayer = new AudioPlayer(offlineAudioContext); const audioSequence: AudioSequence = { @@ -249,12 +243,13 @@ if (window.AudioContext) { } const playheadPosition = new FrequentlyUpdatedState(0); -const singingVoices = new Map(); +const phraseSingingVoices = new Map(); const sequences = new Map(); const animationTimer = new AnimationTimer(); -const singingGuideCache = new Map(); -const singingVoiceCache = new Map(); +const queryCache = new Map(); +const singingVolumeCache = new Map(); +const singingVoiceCache = new Map(); const initialTrackId = TrackId(crypto.randomUUID()); @@ -419,7 +414,8 @@ export const singingStoreState: SingingStoreState = { editFrameRate: DEPRECATED_DEFAULT_EDIT_FRAME_RATE, phrases: new Map(), - singingGuides: new Map(), + phraseQueries: new Map(), + phraseSingingVolumes: new Map(), sequencerZoomX: 0.5, sequencerZoomY: 0.75, sequencerSnapType: 16, @@ -876,20 +872,37 @@ export const singingStore = createPartialStore({ }, }, - SET_SINGING_GUIDE_KEY_TO_PHRASE: { + SET_QUERY_KEY_TO_PHRASE: { + mutation( + state, + { + phraseKey, + queryKey, + }: { + phraseKey: PhraseKey; + queryKey: FrameAudioQueryKey | undefined; + }, + ) { + const phrase = getOrThrow(state.phrases, phraseKey); + + phrase.queryKey = queryKey; + }, + }, + + SET_SINGING_VOLUME_KEY_TO_PHRASE: { mutation( state, { phraseKey, - singingGuideKey, + singingVolumeKey, }: { - phraseKey: PhraseSourceHash; - singingGuideKey: SingingGuideSourceHash | undefined; + phraseKey: PhraseKey; + singingVolumeKey: SingingVolumeKey | undefined; }, ) { const phrase = getOrThrow(state.phrases, phraseKey); - phrase.singingGuideKey = singingGuideKey; + phrase.singingVolumeKey = singingVolumeKey; }, }, @@ -900,8 +913,8 @@ export const singingStore = createPartialStore({ phraseKey, singingVoiceKey, }: { - phraseKey: PhraseSourceHash; - singingVoiceKey: SingingVoiceSourceHash | undefined; + phraseKey: PhraseKey; + singingVoiceKey: SingingVoiceKey | undefined; }, ) { const phrase = getOrThrow(state.phrases, phraseKey); @@ -917,7 +930,7 @@ export const singingStore = createPartialStore({ phraseKey, sequenceId, }: { - phraseKey: PhraseSourceHash; + phraseKey: PhraseKey; sequenceId: SequenceId | undefined; }, ) { @@ -927,27 +940,45 @@ export const singingStore = createPartialStore({ }, }, - SET_SINGING_GUIDE: { + SET_PHRASE_QUERY: { mutation( state, { - singingGuideKey, - singingGuide, + queryKey, + query, }: { - singingGuideKey: SingingGuideSourceHash; - singingGuide: SingingGuide; + queryKey: FrameAudioQueryKey; + query: FrameAudioQuery; }, ) { - state.singingGuides.set(singingGuideKey, singingGuide); + state.phraseQueries.set(queryKey, query); }, }, - DELETE_SINGING_GUIDE: { + DELETE_PHRASE_QUERY: { + mutation(state, { queryKey }: { queryKey: FrameAudioQueryKey }) { + state.phraseQueries.delete(queryKey); + }, + }, + + SET_PHRASE_SINGING_VOLUME: { + mutation( + state, + { + singingVolumeKey, + singingVolume, + }: { singingVolumeKey: SingingVolumeKey; singingVolume: SingingVolume }, + ) { + state.phraseSingingVolumes.set(singingVolumeKey, singingVolume); + }, + }, + + DELETE_PHRASE_SINGING_VOLUME: { mutation( state, - { singingGuideKey }: { singingGuideKey: SingingGuideSourceHash }, + { singingVolumeKey }: { singingVolumeKey: SingingVolumeKey }, ) { - state.singingGuides.delete(singingGuideKey); + state.phraseSingingVolumes.delete(singingVolumeKey); }, }, @@ -1333,6 +1364,19 @@ export const singingStore = createPartialStore({ return phraseFirstRestDuration; }; + const calculatePhraseStartTime = ( + phraseFirstRestDuration: number, + phraseNotes: Note[], + tempos: Tempo[], + tpqn: number, + ) => { + return tickToSecond( + phraseNotes[0].position - phraseFirstRestDuration, + tempos, + tpqn, + ); + }; + const searchPhrases = async ( notes: Note[], tempos: Tempo[], @@ -1340,7 +1384,7 @@ export const singingStore = createPartialStore({ phraseFirstRestMinDurationSeconds: number, trackId: TrackId, ) => { - const foundPhrases = new Map(); + const foundPhrases = new Map(); let phraseNotes: Note[] = []; let prevPhraseLastNote: Note | undefined = undefined; @@ -1365,14 +1409,22 @@ export const singingStore = createPartialStore({ tempos, tpqn, ); - const notesHash = await calculatePhraseSourceHash({ + const phraseStartTime = calculatePhraseStartTime( + phraseFirstRestDuration, + phraseNotes, + tempos, + tpqn, + ); + const phraseKey = await calculatePhraseKey({ firstRestDuration: phraseFirstRestDuration, notes: phraseNotes, + startTime: phraseStartTime, trackId, }); - foundPhrases.set(notesHash, { + foundPhrases.set(phraseKey, { firstRestDuration: phraseFirstRestDuration, notes: phraseNotes, + startTime: phraseStartTime, state: "WAITING_TO_BE_RENDERED", trackId, }); @@ -1386,89 +1438,6 @@ export const singingStore = createPartialStore({ return foundPhrases; }; - // リクエスト用のノーツ（と休符）を作成する - const createNotesForRequestToEngine = ( - firstRestDuration: number, - lastRestDurationSeconds: number, - notes: Note[], - tempos: Tempo[], - tpqn: number, - frameRate: number, - ) => { - const notesForRequestToEngine: NoteForRequestToEngine[] = []; - - // 先頭の休符を変換 - const firstRestStartSeconds = tickToSecond( - notes[0].position - firstRestDuration, - tempos, - tpqn, - ); - const firstRestStartFrame = Math.round( - firstRestStartSeconds * frameRate, - ); - const firstRestEndSeconds = tickToSecond( - notes[0].position, - tempos, - tpqn, - ); - const firstRestEndFrame = Math.round(firstRestEndSeconds * frameRate); - notesForRequestToEngine.push({ - key: undefined, - frameLength: firstRestEndFrame - firstRestStartFrame, - lyric: "", - }); - - // ノートを変換 - for (const note of notes) { - const noteOnSeconds = tickToSecond(note.position, tempos, tpqn); - const noteOnFrame = Math.round(noteOnSeconds * frameRate); - const noteOffSeconds = tickToSecond( - note.position + note.duration, - tempos, - tpqn, - ); - const noteOffFrame = Math.round(noteOffSeconds * frameRate); - notesForRequestToEngine.push({ - key: note.noteNumber, - frameLength: noteOffFrame - noteOnFrame, - lyric: note.lyric, - }); - } - - // 末尾に休符を追加 - const lastRestFrameLength = Math.round( - lastRestDurationSeconds * frameRate, - ); - notesForRequestToEngine.push({ - key: undefined, - frameLength: lastRestFrameLength, - lyric: "", - }); - - // frameLengthが1以上になるようにする - for (let i = 0; i < notesForRequestToEngine.length; i++) { - const frameLength = notesForRequestToEngine[i].frameLength; - const frameToShift = Math.max(0, 1 - frameLength); - notesForRequestToEngine[i].frameLength += frameToShift; - if (i < notesForRequestToEngine.length - 1) { - notesForRequestToEngine[i + 1].frameLength -= frameToShift; - } - } - - return notesForRequestToEngine; - }; - - const shiftKeyOfNotes = ( - notes: NoteForRequestToEngine[], - keyShift: number, - ) => { - for (const note of notes) { - if (note.key != undefined) { - note.key += keyShift; - } - } - }; - const singingTeacherStyleId = StyleId(6000); // TODO: 設定できるようにする const fetchQuery = async ( @@ -1500,78 +1469,10 @@ export const singingStore = createPartialStore({ } }; - const getPhonemes = (frameAudioQuery: FrameAudioQuery) => { - return frameAudioQuery.phonemes.map((value) => value.phoneme).join(" "); - }; - - const shiftGuidePitch = ( - frameAudioQuery: FrameAudioQuery, - pitchShift: number, - ) => { - frameAudioQuery.f0 = frameAudioQuery.f0.map((value) => { - return value * Math.pow(2, pitchShift / 12); - }); - }; - - const shiftGuideVolume = ( - frameAudioQuery: FrameAudioQuery, - volumeShift: number, + const synthesizeSingingVoice = async ( + singer: Singer, + query: FrameAudioQuery, ) => { - frameAudioQuery.volume = frameAudioQuery.volume.map((value) => { - return value * decibelToLinear(volumeShift); - }); - }; - - // 歌とpauの呼吸音が重ならないようにvolumeを制御する - // fadeOutDurationSecondsが0の場合は即座にvolumeを0にする - const muteLastPauSection = ( - frameAudioQuery: FrameAudioQuery, - frameRate: number, - fadeOutDurationSeconds: number, - ) => { - const lastPhoneme = frameAudioQuery.phonemes.at(-1); - if (lastPhoneme == undefined || lastPhoneme.phoneme !== "pau") { - throw new Error("No pau exists at the end."); - } - - let lastPauStartFrame = 0; - for (let i = 0; i < frameAudioQuery.phonemes.length - 1; i++) { - lastPauStartFrame += frameAudioQuery.phonemes[i].frameLength; - } - - const lastPauFrameLength = lastPhoneme.frameLength; - let fadeOutFrameLength = Math.round(fadeOutDurationSeconds * frameRate); - fadeOutFrameLength = Math.max(0, fadeOutFrameLength); - fadeOutFrameLength = Math.min(lastPauFrameLength, fadeOutFrameLength); - - // フェードアウト処理を行う - if (fadeOutFrameLength === 1) { - frameAudioQuery.volume[lastPauStartFrame] *= 0.5; - } else { - for (let i = 0; i < fadeOutFrameLength; i++) { - frameAudioQuery.volume[lastPauStartFrame + i] *= - linearInterpolation(0, 1, fadeOutFrameLength - 1, 0, i); - } - } - // 音量を0にする - for (let i = fadeOutFrameLength; i < lastPauFrameLength; i++) { - frameAudioQuery.volume[lastPauStartFrame + i] = 0; - } - }; - - const calculateStartTime = ( - phrase: Phrase, - tempos: Tempo[], - tpqn: number, - ) => { - return tickToSecond( - phrase.notes[0].position - phrase.firstRestDuration, - tempos, - tpqn, - ); - }; - - const synthesize = async (singer: Singer, query: FrameAudioQuery) => { if (!getters.IS_ENGINE_READY(singer.engineId)) { throw new Error("Engine not ready."); } @@ -1605,63 +1506,138 @@ export const singingStore = createPartialStore({ * @param phraseKey フレーズのキー * @returns シーケンスID */ - const getPhraseSequenceId = (phraseKey: PhraseSourceHash) => { + const getPhraseSequenceId = (phraseKey: PhraseKey) => { return getOrThrow(state.phrases, phraseKey).sequenceId; }; + /** + * フレーズが持つ歌声のキーを取得する。 + * @param phraseKey フレーズのキー + * @returns 歌声のキー + */ + const getPhraseSingingVoiceKey = (phraseKey: PhraseKey) => { + return getOrThrow(state.phrases, phraseKey).singingVoiceKey; + }; + const render = async () => { if (!audioContext) { throw new Error("audioContext is undefined."); } const audioContextRef = audioContext; - // レンダリング中に変更される可能性のあるデータをコピーする - const tracks = cloneWithUnwrapProxy(state.tracks); + const firstRestMinDurationSeconds = 0.12; - const overlappingNoteIdsMap = new Map( - [...tracks.keys()].map((trackId) => [ - trackId, - getters.OVERLAPPING_NOTE_IDS(trackId), - ]), - ); + // レンダリング中に変更される可能性のあるデータをコピーする + const snapshot = { + tpqn: state.tpqn, + tempos: cloneWithUnwrapProxy(state.tempos), + tracks: cloneWithUnwrapProxy(state.tracks), + trackOverlappingNoteIds: new Map( + [...state.tracks.keys()].map((trackId) => [ + trackId, + getters.OVERLAPPING_NOTE_IDS(trackId), + ]), + ), + engineFrameRates: new Map( + Object.entries(state.engineManifests).map( + ([engineId, engineManifest]) => [ + engineId as EngineId, + engineManifest.frameRate, + ], + ), + ), + editFrameRate: state.editFrameRate, + }; - const singerAndFrameRates = new Map( - [...tracks].map(([trackId, track]) => [ - trackId, - track.singer - ? { - singer: track.singer, - frameRate: - state.engineManifests[track.singer.engineId].frameRate, - } - : undefined, - ]), - ); - const tpqn = state.tpqn; - const tempos = state.tempos.map((value) => ({ ...value })); - const editFrameRate = state.editFrameRate; + const phraseRenderer = createPhraseRenderer({ + queryCache, + singingVolumeCache, + singingVoiceCache, + phrases: { + get: (phraseKey: PhraseKey) => ({ + firstRestDuration: getOrThrow(state.phrases, phraseKey) + .firstRestDuration, + notes: getOrThrow(state.phrases, phraseKey).notes, + startTime: getOrThrow(state.phrases, phraseKey).startTime, + queryKey: { + get: () => getOrThrow(state.phrases, phraseKey).queryKey, + set: (value) => + mutations.SET_QUERY_KEY_TO_PHRASE({ + phraseKey, + queryKey: value, + }), + }, + singingVolumeKey: { + get: () => + getOrThrow(state.phrases, phraseKey).singingVolumeKey, + set: (value) => + mutations.SET_SINGING_VOLUME_KEY_TO_PHRASE({ + phraseKey, + singingVolumeKey: value, + }), + }, + singingVoiceKey: { + get: () => getOrThrow(state.phrases, phraseKey).singingVoiceKey, + set: (value) => + mutations.SET_SINGING_VOICE_KEY_TO_PHRASE({ + phraseKey, + singingVoiceKey: value, + }), + }, + }), + }, + phraseQueries: { + get: (queryKey) => getOrThrow(state.phraseQueries, queryKey), + set: (queryKey, query) => + mutations.SET_PHRASE_QUERY({ queryKey, query }), + delete: (queryKey) => mutations.DELETE_PHRASE_QUERY({ queryKey }), + }, + phraseSingingVolumes: { + get: (singingVolumeKey) => + getOrThrow(state.phraseSingingVolumes, singingVolumeKey), + set: (singingVolumeKey, singingVolume) => + mutations.SET_PHRASE_SINGING_VOLUME({ + singingVolumeKey, + singingVolume, + }), + delete: (singingVolumeKey) => + mutations.DELETE_PHRASE_SINGING_VOLUME({ singingVolumeKey }), + }, + phraseSingingVoices: { + set: (singingVoiceKey, singingVoice) => + phraseSingingVoices.set(singingVoiceKey, singingVoice), + delete: (singingVoiceKey) => + phraseSingingVoices.delete(singingVoiceKey), + }, + fetchQuery, + fetchSingFrameVolume: (notes, query, engineId, styleId) => + actions.FETCH_SING_FRAME_VOLUME({ + notes, + frameAudioQuery: query, + engineId, + styleId, + }), + synthesizeSingingVoice, + }); - const firstRestMinDurationSeconds = 0.12; - const lastRestDurationSeconds = 0.5; - const fadeOutDurationSeconds = 0.15; + const renderStartStageIds = new Map(); // フレーズを更新する - const foundPhrases = new Map(); - for (const [trackId, track] of tracks) { - if (!track.singer) { - continue; - } - + const foundPhrases = new Map(); + for (const [trackId, track] of snapshot.tracks) { // 重なっているノートを削除する - const overlappingNoteIds = getOrThrow(overlappingNoteIdsMap, trackId); + const overlappingNoteIds = getOrThrow( + snapshot.trackOverlappingNoteIds, + trackId, + ); const notes = track.notes.filter( (value) => !overlappingNoteIds.has(value.id), ); const phrases = await searchPhrases( notes, - tempos, - tpqn, + snapshot.tempos, + snapshot.tpqn, firstRestMinDurationSeconds, trackId, ); @@ -1670,8 +1646,8 @@ export const singingStore = createPartialStore({ } } - const phrases = new Map(); - const disappearedPhraseKeys = new Set(); + const phrases = new Map(); + const disappearedPhraseKeys = new Set(); for (const phraseKey of state.phrases.keys()) { if (!foundPhrases.has(phraseKey)) { @@ -1681,82 +1657,32 @@ export const singingStore = createPartialStore({ } for (const [phraseKey, foundPhrase] of foundPhrases) { const existingPhrase = state.phrases.get(phraseKey); - if (!existingPhrase) { + if (existingPhrase == undefined) { // 新しいフレーズの場合 + const renderStartStageId = phraseRenderer.getFirstRenderStageId(); + renderStartStageIds.set(phraseKey, renderStartStageId); phrases.set(phraseKey, foundPhrase); - continue; - } - - const track = getOrThrow(tracks, existingPhrase.trackId); - - const singerAndFrameRate = getOrThrow( - singerAndFrameRates, - existingPhrase.trackId, - ); - - // すでに存在するフレーズの場合 - // 再レンダリングする必要があるかどうかをチェックする - // シンガーが未設定の場合、とりあえず常に再レンダリングする - // 音声合成を行う必要がある場合、singingVoiceKeyをundefinedにする - // 歌い方の推論も行う必要がある場合、singingGuideKeyとsingingVoiceKeyをundefinedにする - // TODO: リファクタリングする - const phrase = { ...existingPhrase }; - if (!singerAndFrameRate || phrase.state === "COULD_NOT_RENDER") { - if (phrase.singingGuideKey != undefined) { - phrase.singingGuideKey = undefined; - } - if (phrase.singingVoiceKey != undefined) { - phrase.singingVoiceKey = undefined; - } - } else if (phrase.singingGuideKey != undefined) { - const calculatedHash = await calculateSingingGuideSourceHash({ - engineId: singerAndFrameRate.singer.engineId, - tpqn, - tempos, - firstRestDuration: phrase.firstRestDuration, - lastRestDurationSeconds, - notes: phrase.notes, - keyRangeAdjustment: track.keyRangeAdjustment, - volumeRangeAdjustment: track.volumeRangeAdjustment, - frameRate: singerAndFrameRate.frameRate, - }); - const hash = phrase.singingGuideKey; - if (hash !== calculatedHash) { - phrase.singingGuideKey = undefined; - if (phrase.singingVoiceKey != undefined) { - phrase.singingVoiceKey = undefined; - } - } else if (phrase.singingVoiceKey != undefined) { - let singingGuide = getOrThrow( - state.singingGuides, - phrase.singingGuideKey, - ); - - // 歌い方をコピーして、ピッチ編集を適用する - singingGuide = structuredClone(toRaw(singingGuide)); - applyPitchEdit(singingGuide, track.pitchEditData, editFrameRate); - - const calculatedHash = await calculateSingingVoiceSourceHash({ - singer: singerAndFrameRate.singer, - frameAudioQuery: singingGuide.query, - }); - const hash = phrase.singingVoiceKey; - if (hash !== calculatedHash) { - phrase.singingVoiceKey = undefined; + } else { + // すでに存在するフレーズの場合 + const phrase = { ...existingPhrase }; + const track = getOrThrow(snapshot.tracks, phrase.trackId); + if (track.singer == undefined) { + phrase.state = "SINGER_IS_NOT_SET"; + } else { + const renderStartStageId = + phrase.state === "COULD_NOT_RENDER" + ? phraseRenderer.getFirstRenderStageId() + : await phraseRenderer.determineStartStage( + snapshot, + foundPhrase.trackId, + phraseKey, + ); + if (renderStartStageId != undefined) { + renderStartStageIds.set(phraseKey, renderStartStageId); + phrase.state = "WAITING_TO_BE_RENDERED"; } } - } - - phrases.set(phraseKey, phrase); - } - - // フレーズのstateを更新する - for (const phrase of phrases.values()) { - if ( - phrase.singingGuideKey == undefined || - phrase.singingVoiceKey == undefined - ) { - phrase.state = "WAITING_TO_BE_RENDERED"; + phrases.set(phraseKey, phrase); } } @@ -1765,76 +1691,53 @@ export const singingStore = createPartialStore({ const phraseSequenceId = getPhraseSequenceId(phraseKey); if (phraseSequenceId != undefined) { deleteSequence(phraseSequenceId); - mutations.SET_SEQUENCE_ID_TO_PHRASE({ - phraseKey, - sequenceId: undefined, - }); } } - // 使われていない歌い方と歌声を削除する - const singingGuideKeysInUse = new Set( - [...phrases.values()] - .map((value) => value.singingGuideKey) - .filter((value) => value != undefined), - ); - const singingVoiceKeysInUse = new Set( - [...phrases.values()] - .map((value) => value.singingVoiceKey) - .filter((value) => value != undefined), - ); - const existingSingingGuideKeys = new Set(state.singingGuides.keys()); - const existingSingingVoiceKeys = new Set(singingVoices.keys()); - const singingGuideKeysToDelete = existingSingingGuideKeys.difference( - singingGuideKeysInUse, - ); - const singingVoiceKeysToDelete = existingSingingVoiceKeys.difference( - singingVoiceKeysInUse, - ); - for (const singingGuideKey of singingGuideKeysToDelete) { - mutations.DELETE_SINGING_GUIDE({ singingGuideKey }); - } - for (const singingVoiceKey of singingVoiceKeysToDelete) { - singingVoices.delete(singingVoiceKey); - } - mutations.SET_PHRASES({ phrases }); logger.info("Phrases updated."); // 各フレーズのレンダリングを行う + for (const [phraseKey, phrase] of state.phrases.entries()) { + if ( + phrase.state === "SINGER_IS_NOT_SET" || + phrase.state === "WAITING_TO_BE_RENDERED" + ) { + // シーケンスが存在する場合は、シーケンスを削除する + // TODO: ピッチを編集したときは行わないようにする + const phraseSequenceId = getPhraseSequenceId(phraseKey); + if (phraseSequenceId != undefined) { + deleteSequence(phraseSequenceId); + mutations.SET_SEQUENCE_ID_TO_PHRASE({ + phraseKey, + sequenceId: undefined, + }); + } + + // ノートシーケンスを作成して登録し、プレビュー音が鳴るようにする + const noteEvents = generateNoteEvents( + phrase.notes, + snapshot.tempos, + snapshot.tpqn, + ); + const polySynth = new PolySynth(audioContextRef); + const sequenceId = SequenceId(uuid4()); + registerSequence(sequenceId, { + type: "note", + instrument: polySynth, + noteEvents, + trackId: phrase.trackId, + }); + mutations.SET_SEQUENCE_ID_TO_PHRASE({ phraseKey, sequenceId }); + } + } const phrasesToBeRendered = new Map( [...state.phrases.entries()].filter(([, phrase]) => { return phrase.state === "WAITING_TO_BE_RENDERED"; }), ); - for (const [phraseKey, phrase] of phrasesToBeRendered) { - // シーケンスが存在する場合は、シーケンスを削除する - // TODO: ピッチを編集したときは行わないようにする - - const phraseSequenceId = getPhraseSequenceId(phraseKey); - if (phraseSequenceId != undefined) { - deleteSequence(phraseSequenceId); - mutations.SET_SEQUENCE_ID_TO_PHRASE({ - phraseKey, - sequenceId: undefined, - }); - } - - // ノートシーケンスを作成して登録し、プレビュー音が鳴るようにする - - const noteEvents = generateNoteEvents(phrase.notes, tempos, tpqn); - const polySynth = new PolySynth(audioContextRef); - const sequenceId = SequenceId(uuid4()); - registerSequence(sequenceId, { - type: "note", - instrument: polySynth, - noteEvents, - trackId: phrase.trackId, - }); - mutations.SET_SEQUENCE_ID_TO_PHRASE({ phraseKey, sequenceId }); - } while (phrasesToBeRendered.size > 0) { if (startRenderingRequested() || stopRenderingRequested()) { return; @@ -1845,175 +1748,20 @@ export const singingStore = createPartialStore({ ); phrasesToBeRendered.delete(phraseKey); - const track = getOrThrow(tracks, phrase.trackId); - - const singerAndFrameRate = getOrThrow( - singerAndFrameRates, - phrase.trackId, - ); - - // シンガーが未設定の場合は、歌い方の生成や音声合成は行わない - - if (!singerAndFrameRate) { - mutations.SET_STATE_TO_PHRASE({ - phraseKey, - phraseState: "PLAYABLE", - }); - continue; - } - mutations.SET_STATE_TO_PHRASE({ phraseKey, phraseState: "NOW_RENDERING", }); try { - // リクエスト（クエリ生成と音量生成）用のノーツを作る - const notesForRequestToEngine = createNotesForRequestToEngine( - phrase.firstRestDuration, - lastRestDurationSeconds, - phrase.notes, - tempos, - tpqn, - singerAndFrameRate.frameRate, - ); - - // リクエスト用のノーツのキーのシフトを行う - shiftKeyOfNotes(notesForRequestToEngine, -track.keyRangeAdjustment); - - // 歌い方が存在する場合、歌い方を取得する - // 歌い方が存在しない場合、キャッシュがあれば取得し、なければ歌い方を生成する - - let singingGuide: SingingGuide | undefined; - if (phrase.singingGuideKey != undefined) { - singingGuide = getOrThrow( - state.singingGuides, - phrase.singingGuideKey, - ); - } else { - const singingGuideSourceHash = - await calculateSingingGuideSourceHash({ - engineId: singerAndFrameRate.singer.engineId, - tpqn, - tempos, - firstRestDuration: phrase.firstRestDuration, - lastRestDurationSeconds, - notes: phrase.notes, - keyRangeAdjustment: track.keyRangeAdjustment, - volumeRangeAdjustment: track.volumeRangeAdjustment, - frameRate: singerAndFrameRate.frameRate, - }); - - const singingGuideKey = singingGuideSourceHash; - const cachedSingingGuide = singingGuideCache.get(singingGuideKey); - if (cachedSingingGuide) { - singingGuide = cachedSingingGuide; - - logger.info(`Loaded singing guide from cache.`); - } else { - // クエリを生成する - const query = await fetchQuery( - singerAndFrameRate.singer.engineId, - notesForRequestToEngine, - ); - - const phonemes = getPhonemes(query); - logger.info(`Fetched frame audio query. phonemes: ${phonemes}`); - - // ピッチのシフトを行う - shiftGuidePitch(query, track.keyRangeAdjustment); - - // フレーズの開始時刻を計算する - const startTime = calculateStartTime(phrase, tempos, tpqn); - - singingGuide = { - query, - frameRate: singerAndFrameRate.frameRate, - startTime, - }; - - singingGuideCache.set(singingGuideKey, singingGuide); - } - mutations.SET_SINGING_GUIDE({ singingGuideKey, singingGuide }); - mutations.SET_SINGING_GUIDE_KEY_TO_PHRASE({ - phraseKey, - singingGuideKey, - }); - } - - // ピッチ編集を適用する前に、歌い方をコピーする - singingGuide = structuredClone(toRaw(singingGuide)); - - // ピッチ編集を適用する - applyPitchEdit(singingGuide, track.pitchEditData, editFrameRate); - - // 歌声のキャッシュがあれば取得し、なければ音声合成を行う - - let singingVoice: SingingVoice | undefined; - - const singingVoiceSourceHash = - await calculateSingingVoiceSourceHash({ - singer: singerAndFrameRate.singer, - frameAudioQuery: singingGuide.query, - }); - - const singingVoiceKey = singingVoiceSourceHash; - const cachedSingingVoice = singingVoiceCache.get(singingVoiceKey); - if (cachedSingingVoice) { - singingVoice = cachedSingingVoice; - - logger.info(`Loaded singing voice from cache.`); - } else { - // 音量生成用のクエリを作る - // ピッチ編集を適用したクエリをコピーし、 - // f0をもう一度シフトして、元の（クエリ生成時の）高さに戻す - const queryForVolumeGeneration = structuredClone( - singingGuide.query, - ); - shiftGuidePitch( - queryForVolumeGeneration, - -track.keyRangeAdjustment, - ); - - // 音量を生成して、生成した音量を歌い方のクエリにセットする - // 音量値はAPIを叩く毎に変わるので、calc hashしたあとに音量を取得している - const volumes = await actions.FETCH_SING_FRAME_VOLUME({ - notes: notesForRequestToEngine, - frameAudioQuery: queryForVolumeGeneration, - styleId: singingTeacherStyleId, - engineId: singerAndFrameRate.singer.engineId, - }); - singingGuide.query.volume = volumes; - - // 音量のシフトを行う - shiftGuideVolume(singingGuide.query, track.volumeRangeAdjustment); - - // 末尾のpauの区間の音量を0にする - muteLastPauSection( - singingGuide.query, - singerAndFrameRate.frameRate, - fadeOutDurationSeconds, - ); - - // 音声合成を行う - const blob = await synthesize( - singerAndFrameRate.singer, - singingGuide.query, - ); - - logger.info(`Synthesized.`); - - singingVoice = { blob }; - singingVoiceCache.set(singingVoiceKey, singingVoice); - } - singingVoices.set(singingVoiceKey, singingVoice); - mutations.SET_SINGING_VOICE_KEY_TO_PHRASE({ + await phraseRenderer.render( + snapshot, + phrase.trackId, phraseKey, - singingVoiceKey, - }); + getOrThrow(renderStartStageIds, phraseKey), + ); // シーケンスが存在する場合、シーケンスを削除する - const phraseSequenceId = getPhraseSequenceId(phraseKey); if (phraseSequenceId != undefined) { deleteSequence(phraseSequenceId); @@ -2024,11 +1772,18 @@ export const singingStore = createPartialStore({ } // オーディオシーケンスを作成して登録する - + const singingVoiceKey = getPhraseSingingVoiceKey(phraseKey); + if (singingVoiceKey == undefined) { + throw new Error("singingVoiceKey is undefined."); + } + const singingVoice = getOrThrow( + phraseSingingVoices, + singingVoiceKey, + ); const audioEvents = await generateAudioEvents( audioContextRef, - singingGuide.startTime, - singingVoice.blob, + phrase.startTime, + singingVoice, ); const audioPlayer = new AudioPlayer(audioContext); const sequenceId = SequenceId(uuid4()); @@ -2205,8 +1960,7 @@ export const singingStore = createPartialStore({ state.experimentalSetting.enableMultiTrack, state.tracks, state.phrases, - state.singingGuides, - singingVoiceCache, + phraseSingingVoices, ); const waveFileData = convertToWavFileData(audioBuffer); diff --git a/src/store/type.ts b/src/store/type.ts index f3b54213f3..cf64e0cbfd 100644 --- a/src/store/type.ts +++ b/src/store/type.ts @@ -738,65 +738,38 @@ export type Singer = z.infer; export type Track = z.infer; export type PhraseState = + | "SINGER_IS_NOT_SET" | "WAITING_TO_BE_RENDERED" | "NOW_RENDERING" | "COULD_NOT_RENDER" | "PLAYABLE"; /** - * 歌い方 + * 歌唱ボリューム */ -export type SingingGuide = { - query: FrameAudioQuery; - frameRate: number; - startTime: number; -}; - -/** - * 歌い方のソース（歌い方を生成するために必要なデータ） - */ -export type SingingGuideSource = { - engineId: EngineId; - tpqn: number; - tempos: Tempo[]; - firstRestDuration: number; - lastRestDurationSeconds: number; - notes: Note[]; - keyRangeAdjustment: number; - volumeRangeAdjustment: number; - frameRate: number; -}; +export type SingingVolume = number[]; /** * 歌声 */ -export type SingingVoice = { - blob: Blob; -}; +export type SingingVoice = Blob; -/** - * 歌声のソース（歌声を合成するために必要なデータ） - */ -export type SingingVoiceSource = { - singer: Singer; - frameAudioQuery: FrameAudioQuery; -}; +const frameAudioQueryKeySchema = z.string().brand<"FrameAudioQueryKey">(); +export type FrameAudioQueryKey = z.infer; +export const FrameAudioQueryKey = (id: string): FrameAudioQueryKey => + frameAudioQueryKeySchema.parse(id); -export const singingGuideSourceHashSchema = z - .string() - .brand<"SingingGuideSourceHash">(); -export type SingingGuideSourceHash = z.infer< - typeof singingGuideSourceHashSchema ->; +const singingVolumeKeySchema = z.string().brand<"SingingVolumeKey">(); +export type SingingVolumeKey = z.infer; +export const SingingVolumeKey = (id: string): SingingVolumeKey => + singingVolumeKeySchema.parse(id); -export const singingVoiceSourceHashSchema = z - .string() - .brand<"SingingVoiceSourceHash">(); -export type SingingVoiceSourceHash = z.infer< - typeof singingVoiceSourceHashSchema ->; +const singingVoiceKeySchema = z.string().brand<"SingingVoiceKey">(); +export type SingingVoiceKey = z.infer; +export const SingingVoiceKey = (id: string): SingingVoiceKey => + singingVoiceKeySchema.parse(id); -export const sequenceIdSchema = z.string().brand<"SequenceId">(); +const sequenceIdSchema = z.string().brand<"SequenceId">(); export type SequenceId = z.infer; export const SequenceId = (id: string): SequenceId => sequenceIdSchema.parse(id); @@ -806,12 +779,14 @@ export const SequenceId = (id: string): SequenceId => */ export type Phrase = { firstRestDuration: number; - trackId: TrackId; notes: Note[]; + startTime: number; state: PhraseState; - singingGuideKey?: SingingGuideSourceHash; - singingVoiceKey?: SingingVoiceSourceHash; + queryKey?: FrameAudioQueryKey; + singingVolumeKey?: SingingVolumeKey; + singingVoiceKey?: SingingVoiceKey; sequenceId?: SequenceId; + trackId: TrackId; // NOTE: state.tracksと同期していないので使用する際は注意 }; /** @@ -819,12 +794,14 @@ export type Phrase = { */ export type PhraseSource = { firstRestDuration: number; - trackId: TrackId; notes: Note[]; + startTime: number; + trackId: TrackId; }; -export const phraseSourceHashSchema = z.string().brand<"PhraseSourceHash">(); -export type PhraseSourceHash = z.infer; +const phraseKeySchema = z.string().brand<"PhraseKey">(); +export type PhraseKey = z.infer; +export const PhraseKey = (id: string): PhraseKey => phraseKeySchema.parse(id); export type SequencerEditTarget = "NOTE" | "PITCH"; @@ -836,8 +813,9 @@ export type SingingStoreState = { trackOrder: TrackId[]; _selectedTrackId: TrackId; editFrameRate: number; - phrases: Map; - singingGuides: Map; + phrases: Map; + phraseQueries: Map; + phraseSingingVolumes: Map; sequencerZoomX: number; sequencerZoomY: number; sequencerSnapType: number; @@ -980,46 +958,64 @@ export type SingingStoreTypes = { }; SET_PHRASES: { - mutation: { phrases: Map }; + mutation: { phrases: Map }; }; SET_STATE_TO_PHRASE: { mutation: { - phraseKey: PhraseSourceHash; + phraseKey: PhraseKey; phraseState: PhraseState; }; }; - SET_SINGING_GUIDE_KEY_TO_PHRASE: { + SET_QUERY_KEY_TO_PHRASE: { mutation: { - phraseKey: PhraseSourceHash; - singingGuideKey: SingingGuideSourceHash | undefined; + phraseKey: PhraseKey; + queryKey: FrameAudioQueryKey | undefined; + }; + }; + + SET_SINGING_VOLUME_KEY_TO_PHRASE: { + mutation: { + phraseKey: PhraseKey; + singingVolumeKey: SingingVolumeKey | undefined; }; }; SET_SINGING_VOICE_KEY_TO_PHRASE: { mutation: { - phraseKey: PhraseSourceHash; - singingVoiceKey: SingingVoiceSourceHash | undefined; + phraseKey: PhraseKey; + singingVoiceKey: SingingVoiceKey | undefined; }; }; SET_SEQUENCE_ID_TO_PHRASE: { mutation: { - phraseKey: PhraseSourceHash; + phraseKey: PhraseKey; sequenceId: SequenceId | undefined; }; }; - SET_SINGING_GUIDE: { + SET_PHRASE_QUERY: { + mutation: { + queryKey: FrameAudioQueryKey; + query: FrameAudioQuery; + }; + }; + + DELETE_PHRASE_QUERY: { + mutation: { queryKey: FrameAudioQueryKey }; + }; + + SET_PHRASE_SINGING_VOLUME: { mutation: { - singingGuideKey: SingingGuideSourceHash; - singingGuide: SingingGuide; + singingVolumeKey: SingingVolumeKey; + singingVolume: SingingVolume; }; }; - DELETE_SINGING_GUIDE: { - mutation: { singingGuideKey: SingingGuideSourceHash }; + DELETE_PHRASE_SINGING_VOLUME: { + mutation: { singingVolumeKey: SingingVolumeKey }; }; SELECTED_TRACK: { diff --git a/tests/unit/lib/selectPriorPhrase.spec.ts b/tests/unit/lib/selectPriorPhrase.spec.ts index 8dc7d0f4f2..7509155583 100644 --- a/tests/unit/lib/selectPriorPhrase.spec.ts +++ b/tests/unit/lib/selectPriorPhrase.spec.ts @@ -1,11 +1,11 @@ import { it, expect } from "vitest"; +import { Phrase, PhraseKey, PhraseState } from "@/store/type"; import { - Phrase, - PhraseSourceHash, - PhraseState, - phraseSourceHashSchema, -} from "@/store/type"; -import { DEFAULT_TPQN, selectPriorPhrase } from "@/sing/domain"; + DEFAULT_BPM, + DEFAULT_TPQN, + selectPriorPhrase, + tickToSecond, +} from "@/sing/domain"; import { NoteId, TrackId } from "@/type/preload"; import { uuid4 } from "@/helpers/random"; @@ -29,30 +29,20 @@ const createPhrase = ( lyric: "ド", }, ], + startTime: tickToSecond( + start * DEFAULT_TPQN - firstRestDuration * DEFAULT_TPQN, + [{ position: 0, bpm: DEFAULT_BPM }], + DEFAULT_TPQN, + ), state, }; }; -const basePhrases = new Map([ - [ - phraseSourceHashSchema.parse("1"), - createPhrase(0, 0, 1, "WAITING_TO_BE_RENDERED"), - ], - [ - phraseSourceHashSchema.parse("2"), - createPhrase(0, 1, 2, "WAITING_TO_BE_RENDERED"), - ], - [ - phraseSourceHashSchema.parse("3"), - createPhrase(0, 2, 3, "WAITING_TO_BE_RENDERED"), - ], - [ - phraseSourceHashSchema.parse("4"), - createPhrase(0, 3, 4, "WAITING_TO_BE_RENDERED"), - ], - [ - phraseSourceHashSchema.parse("5"), - createPhrase(0, 4, 5, "WAITING_TO_BE_RENDERED"), - ], +const basePhrases = new Map([ + [PhraseKey("1"), createPhrase(0, 0, 1, "WAITING_TO_BE_RENDERED")], + [PhraseKey("2"), createPhrase(0, 1, 2, "WAITING_TO_BE_RENDERED")], + [PhraseKey("3"), createPhrase(0, 2, 3, "WAITING_TO_BE_RENDERED")], + [PhraseKey("4"), createPhrase(0, 3, 4, "WAITING_TO_BE_RENDERED")], + [PhraseKey("5"), createPhrase(0, 4, 5, "WAITING_TO_BE_RENDERED")], ]); it("しっかり優先順位に従って探している", () => { From b9f7c7f6f5cc35eab67a78190f83ade676c25feb Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 25 Aug 2024 11:40:01 +0900 Subject: [PATCH 02/15] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=81=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sing/phraseRendering.ts | 144 +++++++++++++++++++++++++++--------- src/store/singing.ts | 7 +- 2 files changed, 112 insertions(+), 39 deletions(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index 44e904252f..9cde9fb9a2 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -22,9 +22,11 @@ import { createLogger } from "@/domain/frontend/log"; import { cloneWithUnwrapProxy } from "@/helpers/cloneWithUnwrapProxy"; import { getOrThrow } from "@/helpers/mapHelper"; -const logger = createLogger("store/singing"); +const logger = createLogger("sing/phraseRendering"); -// リクエスト用のノーツ（と休符）を作成する +/** + * リクエスト用のノーツ（と休符）を作成する。 + */ const createNotesForRequestToEngine = ( firstRestDuration: number, lastRestDurationSeconds: number, @@ -112,8 +114,10 @@ const shiftVolume = (volume: number[], volumeShift: number) => { } }; -// 歌とpauの呼吸音が重ならないようにvolumeを制御する -// fadeOutDurationSecondsが0の場合は即座にvolumeを0にする +/** + * 末尾のpauの区間のvolumeを0にする。（歌とpauの呼吸音が重ならないようにする） + * fadeOutDurationSecondsが0の場合は即座にvolumeを0にする。 + */ const muteLastPauSection = ( volume: number[], phonemes: FramePhoneme[], @@ -159,6 +163,9 @@ const singingTeacherStyleId = StyleId(6000); // TODO: 設定できるように const lastRestDurationSeconds = 0.5; // TODO: 設定できるようにする const fadeOutDurationSeconds = 0.15; // TODO: 設定できるようにする +/** + * フレーズレンダリングに必要なデータのスナップショット + */ type Snapshot = Readonly<{ tpqn: number; tempos: Tempo[]; @@ -167,6 +174,9 @@ type Snapshot = Readonly<{ editFrameRate: number; }>; +/** + * フレーズ + */ type Phrase = Readonly<{ firstRestDuration: number; notes: Note[]; @@ -185,6 +195,9 @@ type Phrase = Readonly<{ }; }>; +/** + * フレーズレンダリングで必要となるキャッシュや関数 + */ type ExternalDependencies = Readonly<{ queryCache: Map; singingVolumeCache: Map; @@ -227,6 +240,9 @@ type ExternalDependencies = Readonly<{ ) => Promise; }>; +/** + * フレーズレンダリングのコンテキスト + */ type Context = Readonly<{ snapshot: Snapshot; trackId: TrackId; @@ -234,8 +250,37 @@ type Context = Readonly<{ externalDependencies: ExternalDependencies; }>; +/** + * フレーズレンダリングのステージ + */ +type Stage = Readonly<{ + id: "queryGeneration" | "singingVolumeGeneration" | "singingVoiceSynthesis"; + + /** + * このステージが実行されるべきかを判定する。 + * @param context コンテキスト + * @returns 実行が必要かどうかのブール値 + */ + shouldBeExecuted: (context: Context) => Promise; + + /** + * 前回の処理結果を削除する。 + * @param context コンテキスト + */ + deleteExecutionResult: (context: Context) => void; + + /** + * ステージの処理を実行する。 + * @param context コンテキスト + */ + execute: (context: Context) => Promise; +}>; + // クエリ生成ステージ +/** + * クエリの生成に必要なデータ + */ type QuerySource = Readonly<{ engineId: EngineId; engineFrameRate: number; @@ -246,13 +291,6 @@ type QuerySource = Readonly<{ keyRangeAdjustment: number; }>; -type QueryGenerationStage = Readonly<{ - id: "queryGeneration"; - shouldBeExecuted: (context: Context) => Promise; - deleteExecutionResult: (context: Context) => void; - execute: (context: Context) => Promise; -}>; - const generateQuerySource = (context: Context): QuerySource => { const phrases = context.externalDependencies.phrases; @@ -305,7 +343,7 @@ const generateQuery = async ( return query; }; -const queryGenerationStage: QueryGenerationStage = { +const queryGenerationStage: Stage = { id: "queryGeneration", shouldBeExecuted: async (context: Context) => { const phrases = context.externalDependencies.phrases; @@ -361,6 +399,9 @@ const queryGenerationStage: QueryGenerationStage = { // 歌唱ボリューム生成ステージ +/** + * 歌唱ボリュームの生成に必要なデータ + */ type SingingVolumeSource = Readonly<{ engineId: EngineId; engineFrameRate: number; @@ -373,13 +414,6 @@ type SingingVolumeSource = Readonly<{ queryForVolumeGeneration: FrameAudioQuery; }>; -type SingingVolumeGenerationStage = Readonly<{ - id: "singingVolumeGeneration"; - shouldBeExecuted: (context: Context) => Promise; - deleteExecutionResult: (context: Context) => void; - execute: (context: Context) => Promise; -}>; - const generateSingingVolumeSource = (context: Context): SingingVolumeSource => { const phrases = context.externalDependencies.phrases; const phraseQueries = context.externalDependencies.phraseQueries; @@ -459,8 +493,6 @@ const generateSingingVolume = async ( ); shiftVolume(singingVolume, singingVolumeSource.volumeRangeAdjustment); - - // 末尾のpauの区間の音量を0にする muteLastPauSection( singingVolume, queryForVolumeGeneration.phonemes, @@ -470,7 +502,7 @@ const generateSingingVolume = async ( return singingVolume; }; -const singingVolumeGenerationStage: SingingVolumeGenerationStage = { +const singingVolumeGenerationStage: Stage = { id: "singingVolumeGeneration", shouldBeExecuted: async (context: Context) => { const phrases = context.externalDependencies.phrases; @@ -533,20 +565,16 @@ const singingVolumeGenerationStage: SingingVolumeGenerationStage = { }, }; -// 音声合成ステージ +// 歌唱音声合成ステージ +/** + * 歌唱音声の合成に必要なデータ + */ type SingingVoiceSource = Readonly<{ singer: Singer; queryForSingingVoiceSynthesis: FrameAudioQuery; }>; -type SingingVoiceSynthesisStage = Readonly<{ - id: "singingVoiceSynthesis"; - shouldBeExecuted: (context: Context) => Promise; - deleteExecutionResult: (context: Context) => void; - execute: (context: Context) => Promise; -}>; - const generateSingingVoiceSource = (context: Context): SingingVoiceSource => { const phrases = context.externalDependencies.phrases; const phraseQueries = context.externalDependencies.phraseQueries; @@ -608,7 +636,7 @@ const synthesizeSingingVoice = async ( return singingVoice; }; -const singingVoiceSynthesisStage: SingingVoiceSynthesisStage = { +const singingVoiceSynthesisStage: Stage = { id: "singingVoiceSynthesis", shouldBeExecuted: async (context: Context) => { const phrases = context.externalDependencies.phrases; @@ -671,17 +699,59 @@ const singingVoiceSynthesisStage: SingingVoiceSynthesisStage = { // フレーズレンダラー -const stages = [ +export type PhraseRenderStageId = Stage["id"]; + +export type PhraseRenderer = Readonly<{ + /** + * レンダリング処理の開始点となる最初のステージのIDを返す。 + * @returns ステージID + */ + getFirstRenderStageId: () => PhraseRenderStageId; + + /** + * レンダリングがどのステージから開始されるべきかを判断する。 + * すべてのステージがスキップ可能な場合、undefinedが返される。 + * @param snapshot スナップショット + * @param trackId トラックID + * @param phraseKey フレーズキー + * @returns ステージID または undefined + */ + determineStartStage: ( + snapshot: Snapshot, + trackId: TrackId, + phraseKey: PhraseKey, + ) => Promise; + + /** + * 指定されたステージからレンダリング処理を開始する。 + * レンダリング処理を開始する前に、前回のレンダリング処理結果の削除が行われる。 + * @param snapshot スナップショット + * @param trackId トラックID + * @param phraseKey フレーズキー + * @param startStageId 開始ステージID + */ + render: ( + snapshot: Snapshot, + trackId: TrackId, + phraseKey: PhraseKey, + startStageId: PhraseRenderStageId, + ) => Promise; +}>; + +const stages: readonly Stage[] = [ queryGenerationStage, singingVolumeGenerationStage, singingVoiceSynthesisStage, -] as const; - -export type PhraseRenderStageId = (typeof stages)[number]["id"]; +]; +/** + * フレーズレンダラーを作成する。 + * @param externalDependencies レンダリング処理で必要となるキャッシュや関数 + * @returns フレーズレンダラー + */ export const createPhraseRenderer = ( externalDependencies: ExternalDependencies, -) => { +): PhraseRenderer => { return { getFirstRenderStageId: () => { return stages[0].id; @@ -729,5 +799,5 @@ export const createPhraseRenderer = ( await stages[i].execute(context); } }, - } as const; + }; }; diff --git a/src/store/singing.ts b/src/store/singing.ts index 37a052ef51..0579e0ec1f 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -1527,7 +1527,7 @@ export const singingStore = createPartialStore({ const firstRestMinDurationSeconds = 0.12; - // レンダリング中に変更される可能性のあるデータをコピーする + // レンダリング中に変更される可能性のあるデータのコピー const snapshot = { tpqn: state.tpqn, tempos: cloneWithUnwrapProxy(state.tempos), @@ -1547,7 +1547,7 @@ export const singingStore = createPartialStore({ ), ), editFrameRate: state.editFrameRate, - }; + } as const; const phraseRenderer = createPhraseRenderer({ queryCache, @@ -1669,6 +1669,8 @@ export const singingStore = createPartialStore({ if (track.singer == undefined) { phrase.state = "SINGER_IS_NOT_SET"; } else { + // どのステージから開始するかを決める + // phrase.stateがCOULD_NOT_RENDERだった場合は最初からレンダリングし直す const renderStartStageId = phrase.state === "COULD_NOT_RENDER" ? phraseRenderer.getFirstRenderStageId() @@ -1754,6 +1756,7 @@ export const singingStore = createPartialStore({ }); try { + // フレーズのレンダリングを行う await phraseRenderer.render( snapshot, phrase.trackId, From 246aea29d1f593bb52aa619a5c7969d100fc3052 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:00:20 +0900 Subject: [PATCH 03/15] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sing/phraseRendering.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index 9cde9fb9a2..c5a709934a 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -196,7 +196,7 @@ type Phrase = Readonly<{ }>; /** - * フレーズレンダリングで必要となるキャッシュや関数 + * フレーズレンダリングで必要となる外部のキャッシュや関数 */ type ExternalDependencies = Readonly<{ queryCache: Map; @@ -746,7 +746,7 @@ const stages: readonly Stage[] = [ /** * フレーズレンダラーを作成する。 - * @param externalDependencies レンダリング処理で必要となるキャッシュや関数 + * @param externalDependencies レンダリング処理で必要となる外部のキャッシュや関数 * @returns フレーズレンダラー */ export const createPhraseRenderer = ( From 1c74f03687315a12edc516599b95c26fbc966725 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 25 Aug 2024 14:16:42 +0900 Subject: [PATCH 04/15] =?UTF-8?q?=E3=83=90=E3=82=B0=E3=81=8C=E3=81=82?= =?UTF-8?q?=E3=81=A3=E3=81=9F=E3=81=AE=E3=81=A7=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/singing.ts | 46 +++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/src/store/singing.ts b/src/store/singing.ts index 0579e0ec1f..dd247fdc5b 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -1656,36 +1656,30 @@ export const singingStore = createPartialStore({ } } for (const [phraseKey, foundPhrase] of foundPhrases) { + // 新しいフレーズまたは既存のフレーズの場合 const existingPhrase = state.phrases.get(phraseKey); - if (existingPhrase == undefined) { - // 新しいフレーズの場合 - const renderStartStageId = phraseRenderer.getFirstRenderStageId(); - renderStartStageIds.set(phraseKey, renderStartStageId); - phrases.set(phraseKey, foundPhrase); + const phrase = existingPhrase ?? foundPhrase; + const track = getOrThrow(snapshot.tracks, phrase.trackId); + if (track.singer == undefined) { + phrase.state = "SINGER_IS_NOT_SET"; } else { - // すでに存在するフレーズの場合 - const phrase = { ...existingPhrase }; - const track = getOrThrow(snapshot.tracks, phrase.trackId); - if (track.singer == undefined) { - phrase.state = "SINGER_IS_NOT_SET"; - } else { - // どのステージから開始するかを決める - // phrase.stateがCOULD_NOT_RENDERだった場合は最初からレンダリングし直す - const renderStartStageId = - phrase.state === "COULD_NOT_RENDER" - ? phraseRenderer.getFirstRenderStageId() - : await phraseRenderer.determineStartStage( - snapshot, - foundPhrase.trackId, - phraseKey, - ); - if (renderStartStageId != undefined) { - renderStartStageIds.set(phraseKey, renderStartStageId); - phrase.state = "WAITING_TO_BE_RENDERED"; - } + // 新しいフレーズの場合は最初からレンダリングする + // phrase.stateがCOULD_NOT_RENDERだった場合は最初からレンダリングし直す + // 既存のフレーズの場合は適切なレンダリング開始ステージを決定する + const renderStartStageId = + existingPhrase == undefined || phrase.state === "COULD_NOT_RENDER" + ? phraseRenderer.getFirstRenderStageId() + : await phraseRenderer.determineStartStage( + snapshot, + foundPhrase.trackId, + phraseKey, + ); + if (renderStartStageId != undefined) { + renderStartStageIds.set(phraseKey, renderStartStageId); + phrase.state = "WAITING_TO_BE_RENDERED"; } - phrases.set(phraseKey, phrase); } + phrases.set(phraseKey, phrase); } // 無くなったフレーズのシーケンスを削除する From 837026d30cc7bc4710e185a2ce875bf713340a99 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 25 Aug 2024 17:56:38 +0900 Subject: [PATCH 05/15] =?UTF-8?q?clone=E3=82=92=E3=81=97=E3=81=A6=E3=81=84?= =?UTF-8?q?=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=AE=E3=81=A7=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/singing.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/store/singing.ts b/src/store/singing.ts index dd247fdc5b..a9027b401a 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -1658,7 +1658,10 @@ export const singingStore = createPartialStore({ for (const [phraseKey, foundPhrase] of foundPhrases) { // 新しいフレーズまたは既存のフレーズの場合 const existingPhrase = state.phrases.get(phraseKey); - const phrase = existingPhrase ?? foundPhrase; + const phrase = + existingPhrase == undefined + ? foundPhrase + : cloneWithUnwrapProxy(existingPhrase); const track = getOrThrow(snapshot.tracks, phrase.trackId); if (track.singer == undefined) { phrase.state = "SINGER_IS_NOT_SET"; From c6407f01e5431862cb224088acf5d42c9cc7139f Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sat, 31 Aug 2024 09:50:19 +0900 Subject: [PATCH 06/15] =?UTF-8?q?EditorFrameAudioQuery=E3=82=92=E4=BD=BF?= =?UTF-8?q?=E3=81=86=E3=82=88=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/Sing/SequencerPitch.vue | 13 ++----- src/sing/domain.ts | 38 +++++++++--------- src/sing/phraseRendering.ts | 54 +++++++++++--------------- src/store/singing.ts | 13 ++++--- src/store/type.ts | 32 +++++++++------ 5 files changed, 71 insertions(+), 79 deletions(-) diff --git a/src/components/Sing/SequencerPitch.vue b/src/components/Sing/SequencerPitch.vue index 65ce6330c4..5f39338458 100644 --- a/src/components/Sing/SequencerPitch.vue +++ b/src/components/Sing/SequencerPitch.vue @@ -30,7 +30,7 @@ import { ExhaustiveError } from "@/type/utility"; import { createLogger } from "@/domain/frontend/log"; import { getLast } from "@/sing/utility"; import { getOrThrow } from "@/helpers/mapHelper"; -import { FrameAudioQuery } from "@/openapi"; +import { EditorFrameAudioQuery } from "@/store/type"; type PitchLine = { readonly color: Color; @@ -59,8 +59,7 @@ const selectedTrackId = computed(() => store.getters.SELECTED_TRACK_ID); const editFrameRate = computed(() => store.state.editFrameRate); const singingGuidesInSelectedTrack = computed(() => { const singingGuides: { - query: FrameAudioQuery; - frameRate: number; + query: EditorFrameAudioQuery; startTime: number; }[] = []; for (const phrase of store.state.phrases.values()) { @@ -70,16 +69,10 @@ const singingGuidesInSelectedTrack = computed(() => { if (phrase.queryKey == undefined) { continue; } - const track = store.state.tracks.get(phrase.trackId); - if (track == undefined || track.singer == undefined) { - continue; - } const phraseQuery = getOrThrow(store.state.phraseQueries, phrase.queryKey); - const engineManifest = store.state.engineManifests[track.singer.engineId]; singingGuides.push({ startTime: phrase.startTime, query: phraseQuery, - frameRate: engineManifest.frameRate, }); } return singingGuides; @@ -276,7 +269,7 @@ const generateOriginalPitchData = () => { const tempData = []; for (const singingGuide of singingGuidesInSelectedTrack.value) { // TODO: 補間を行うようにする - if (singingGuide.frameRate !== frameRate) { + if (singingGuide.query.frameRate !== frameRate) { throw new Error( "The frame rate between the singing guide and the edit does not match.", ); diff --git a/src/sing/domain.ts b/src/sing/domain.ts index 27e851d820..55efbcc89a 100644 --- a/src/sing/domain.ts +++ b/src/sing/domain.ts @@ -8,8 +8,9 @@ import { TimeSignature, PhraseKey, Track, + EditorFrameAudioQuery, } from "@/store/type"; -import { FrameAudioQuery, FramePhoneme } from "@/openapi"; +import { FramePhoneme } from "@/openapi"; import { TrackId } from "@/type/preload"; const BEAT_TYPES = [2, 4, 8, 16]; @@ -450,24 +451,21 @@ export function convertToFramePhonemes(phonemes: FramePhoneme[]) { } export function applyPitchEdit( - singingGuide: { - query: FrameAudioQuery; - frameRate: number; - startTime: number; - }, + phraseQuery: EditorFrameAudioQuery, + phraseStartTime: number, pitchEditData: number[], editFrameRate: number, ) { - // 歌い方のフレームレートと編集フレームレートが一致しない場合はエラー + // フレーズのクエリのフレームレートと編集フレームレートが一致しない場合はエラー // TODO: 補間するようにする - if (singingGuide.frameRate !== editFrameRate) { + if (phraseQuery.frameRate !== editFrameRate) { throw new Error( - "The frame rate between the singing guide and the edit data does not match.", + "The frame rate between the phrase query and the edit data does not match.", ); } const unvoicedPhonemes = UNVOICED_PHONEMES; - const f0 = singingGuide.query.f0; - const phonemes = singingGuide.query.phonemes; + const f0 = phraseQuery.f0; + const phonemes = phraseQuery.phonemes; // 各フレームの音素の配列を生成する const framePhonemes = convertToFramePhonemes(phonemes); @@ -475,21 +473,21 @@ export function applyPitchEdit( throw new Error("f0.length and framePhonemes.length do not match."); } - // 歌い方の開始フレームと終了フレームを計算する - const singingGuideFrameLength = f0.length; - const singingGuideStartFrame = Math.round( - singingGuide.startTime * singingGuide.frameRate, + // フレーズのクエリの開始フレームと終了フレームを計算する + const phraseQueryFrameLength = f0.length; + const phraseQueryStartFrame = Math.round( + phraseStartTime * phraseQuery.frameRate, ); - const singingGuideEndFrame = singingGuideStartFrame + singingGuideFrameLength; + const phraseQueryEndFrame = phraseQueryStartFrame + phraseQueryFrameLength; // ピッチ編集をf0に適用する - const startFrame = Math.max(0, singingGuideStartFrame); - const endFrame = Math.min(pitchEditData.length, singingGuideEndFrame); + const startFrame = Math.max(0, phraseQueryStartFrame); + const endFrame = Math.min(pitchEditData.length, phraseQueryEndFrame); for (let i = startFrame; i < endFrame; i++) { - const phoneme = framePhonemes[i - singingGuideStartFrame]; + const phoneme = framePhonemes[i - phraseQueryStartFrame]; const voiced = !unvoicedPhonemes.includes(phoneme); if (voiced && pitchEditData[i] !== VALUE_INDICATING_NO_DATA) { - f0[i - singingGuideStartFrame] = pitchEditData[i]; + f0[i - phraseQueryStartFrame] = pitchEditData[i]; } } } diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index c5a709934a..4fa9bf4510 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -1,5 +1,4 @@ import { - FrameAudioQueryKey, Note, PhraseKey, Singer, @@ -9,6 +8,8 @@ import { Track, SingingVolumeKey, SingingVolume, + EditorFrameAudioQueryKey, + EditorFrameAudioQuery, } from "@/store/type"; import { FrameAudioQuery, @@ -98,7 +99,7 @@ const shiftKeyOfNotes = (notes: NoteForRequestToEngine[], keyShift: number) => { } }; -const getPhonemes = (query: FrameAudioQuery) => { +const getPhonemes = (query: EditorFrameAudioQuery) => { return query.phonemes.map((value) => value.phoneme).join(" "); }; @@ -182,8 +183,8 @@ type Phrase = Readonly<{ notes: Note[]; startTime: number; queryKey: { - get: () => FrameAudioQueryKey | undefined; - set: (value: FrameAudioQueryKey | undefined) => void; + get: () => EditorFrameAudioQueryKey | undefined; + set: (value: EditorFrameAudioQueryKey | undefined) => void; }; singingVolumeKey: { get: () => SingingVolumeKey | undefined; @@ -199,7 +200,7 @@ type Phrase = Readonly<{ * フレーズレンダリングで必要となる外部のキャッシュや関数 */ type ExternalDependencies = Readonly<{ - queryCache: Map; + queryCache: Map; singingVolumeCache: Map; singingVoiceCache: Map; @@ -207,9 +208,12 @@ type ExternalDependencies = Readonly<{ get: (phraseKey: PhraseKey) => Phrase; }; phraseQueries: { - get: (queryKey: FrameAudioQueryKey) => FrameAudioQuery; - set: (queryKey: FrameAudioQueryKey, query: FrameAudioQuery) => void; - delete: (queryKey: FrameAudioQueryKey) => void; + get: (queryKey: EditorFrameAudioQueryKey) => EditorFrameAudioQuery; + set: ( + queryKey: EditorFrameAudioQueryKey, + query: EditorFrameAudioQuery, + ) => void; + delete: (queryKey: EditorFrameAudioQueryKey) => void; }; phraseSingingVolumes: { get: (singingVolumeKey: SingingVolumeKey) => SingingVolume; @@ -316,13 +320,13 @@ const generateQuerySource = (context: Context): QuerySource => { const calculateQueryKey = async (querySource: QuerySource) => { const hash = await calculateHash(querySource); - return FrameAudioQueryKey(hash); + return EditorFrameAudioQueryKey(hash); }; const generateQuery = async ( querySource: QuerySource, externalDependencies: ExternalDependencies, -) => { +): Promise => { const notesForRequestToEngine = createNotesForRequestToEngine( querySource.firstRestDuration, lastRestDurationSeconds, @@ -340,7 +344,7 @@ const generateQuery = async ( ); shiftPitch(query.f0, querySource.keyRangeAdjustment); - return query; + return { ...query, frameRate: querySource.engineFrameRate }; }; const queryGenerationStage: Stage = { @@ -411,7 +415,7 @@ type SingingVolumeSource = Readonly<{ notes: Note[]; keyRangeAdjustment: number; volumeRangeAdjustment: number; - queryForVolumeGeneration: FrameAudioQuery; + queryForVolumeGeneration: EditorFrameAudioQuery; }>; const generateSingingVolumeSource = (context: Context): SingingVolumeSource => { @@ -422,10 +426,6 @@ const generateSingingVolumeSource = (context: Context): SingingVolumeSource => { if (track.singer == undefined) { throw new Error("track.singer is undefined."); } - const engineFrameRate = getOrThrow( - context.snapshot.engineFrameRates, - track.singer.engineId, - ); const phrase = phrases.get(context.phraseKey); const phraseQueryKey = phrase.queryKey.get(); if (phraseQueryKey == undefined) { @@ -434,17 +434,14 @@ const generateSingingVolumeSource = (context: Context): SingingVolumeSource => { const query = phraseQueries.get(phraseQueryKey); const clonedQuery = cloneWithUnwrapProxy(query); applyPitchEdit( - { - query: clonedQuery, - startTime: phrase.startTime, - frameRate: engineFrameRate, - }, + clonedQuery, + phrase.startTime, track.pitchEditData, context.snapshot.editFrameRate, ); return { engineId: track.singer.engineId, - engineFrameRate, + engineFrameRate: query.frameRate, tpqn: context.snapshot.tpqn, tempos: context.snapshot.tempos, firstRestDuration: phrase.firstRestDuration, @@ -572,7 +569,7 @@ const singingVolumeGenerationStage: Stage = { */ type SingingVoiceSource = Readonly<{ singer: Singer; - queryForSingingVoiceSynthesis: FrameAudioQuery; + queryForSingingVoiceSynthesis: EditorFrameAudioQuery; }>; const generateSingingVoiceSource = (context: Context): SingingVoiceSource => { @@ -585,10 +582,6 @@ const generateSingingVoiceSource = (context: Context): SingingVoiceSource => { if (track.singer == undefined) { throw new Error("track.singer is undefined."); } - const engineFrameRate = getOrThrow( - context.snapshot.engineFrameRates, - track.singer.engineId, - ); const phrase = phrases.get(context.phraseKey); const phraseQueryKey = phrase.queryKey.get(); const phraseSingingVolumeKey = phrase.singingVolumeKey.get(); @@ -603,11 +596,8 @@ const generateSingingVoiceSource = (context: Context): SingingVoiceSource => { const clonedQuery = cloneWithUnwrapProxy(query); const clonedSingingVolume = cloneWithUnwrapProxy(singingVolume); applyPitchEdit( - { - query: clonedQuery, - startTime: phrase.startTime, - frameRate: engineFrameRate, - }, + clonedQuery, + phrase.startTime, track.pitchEditData, context.snapshot.editFrameRate, ); diff --git a/src/store/singing.ts b/src/store/singing.ts index a9027b401a..b140d3af7d 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -19,10 +19,11 @@ import { PhraseKey, Track, SequenceId, - FrameAudioQueryKey, SingingVolumeKey, SingingVolume, SingingVoiceKey, + EditorFrameAudioQueryKey, + EditorFrameAudioQuery, } from "./type"; import { DEFAULT_PROJECT_NAME, sanitizeFileName } from "./utility"; import { @@ -247,7 +248,7 @@ const phraseSingingVoices = new Map(); const sequences = new Map(); const animationTimer = new AnimationTimer(); -const queryCache = new Map(); +const queryCache = new Map(); const singingVolumeCache = new Map(); const singingVoiceCache = new Map(); @@ -880,7 +881,7 @@ export const singingStore = createPartialStore({ queryKey, }: { phraseKey: PhraseKey; - queryKey: FrameAudioQueryKey | undefined; + queryKey: EditorFrameAudioQueryKey | undefined; }, ) { const phrase = getOrThrow(state.phrases, phraseKey); @@ -947,8 +948,8 @@ export const singingStore = createPartialStore({ queryKey, query, }: { - queryKey: FrameAudioQueryKey; - query: FrameAudioQuery; + queryKey: EditorFrameAudioQueryKey; + query: EditorFrameAudioQuery; }, ) { state.phraseQueries.set(queryKey, query); @@ -956,7 +957,7 @@ export const singingStore = createPartialStore({ }, DELETE_PHRASE_QUERY: { - mutation(state, { queryKey }: { queryKey: FrameAudioQueryKey }) { + mutation(state, { queryKey }: { queryKey: EditorFrameAudioQueryKey }) { state.phraseQueries.delete(queryKey); }, }, diff --git a/src/store/type.ts b/src/store/type.ts index cf64e0cbfd..e63bba6a64 100644 --- a/src/store/type.ts +++ b/src/store/type.ts @@ -744,6 +744,11 @@ export type PhraseState = | "COULD_NOT_RENDER" | "PLAYABLE"; +/** + * エディタ用のFrameAudioQuery + */ +export type EditorFrameAudioQuery = FrameAudioQuery & { frameRate: number }; + /** * 歌唱ボリューム */ @@ -754,10 +759,15 @@ export type SingingVolume = number[]; */ export type SingingVoice = Blob; -const frameAudioQueryKeySchema = z.string().brand<"FrameAudioQueryKey">(); -export type FrameAudioQueryKey = z.infer; -export const FrameAudioQueryKey = (id: string): FrameAudioQueryKey => - frameAudioQueryKeySchema.parse(id); +const editorFrameAudioQueryKeySchema = z + .string() + .brand<"EditorFrameAudioQueryKey">(); +export type EditorFrameAudioQueryKey = z.infer< + typeof editorFrameAudioQueryKeySchema +>; +export const EditorFrameAudioQueryKey = ( + id: string, +): EditorFrameAudioQueryKey => editorFrameAudioQueryKeySchema.parse(id); const singingVolumeKeySchema = z.string().brand<"SingingVolumeKey">(); export type SingingVolumeKey = z.infer; @@ -782,7 +792,7 @@ export type Phrase = { notes: Note[]; startTime: number; state: PhraseState; - queryKey?: FrameAudioQueryKey; + queryKey?: EditorFrameAudioQueryKey; singingVolumeKey?: SingingVolumeKey; singingVoiceKey?: SingingVoiceKey; sequenceId?: SequenceId; @@ -790,7 +800,7 @@ export type Phrase = { }; /** - * フレーズのソース + * フレーズの生成に必要なデータ */ export type PhraseSource = { firstRestDuration: number; @@ -814,7 +824,7 @@ export type SingingStoreState = { _selectedTrackId: TrackId; editFrameRate: number; phrases: Map; - phraseQueries: Map; + phraseQueries: Map; phraseSingingVolumes: Map; sequencerZoomX: number; sequencerZoomY: number; @@ -971,7 +981,7 @@ export type SingingStoreTypes = { SET_QUERY_KEY_TO_PHRASE: { mutation: { phraseKey: PhraseKey; - queryKey: FrameAudioQueryKey | undefined; + queryKey: EditorFrameAudioQueryKey | undefined; }; }; @@ -998,13 +1008,13 @@ export type SingingStoreTypes = { SET_PHRASE_QUERY: { mutation: { - queryKey: FrameAudioQueryKey; - query: FrameAudioQuery; + queryKey: EditorFrameAudioQueryKey; + query: EditorFrameAudioQuery; }; }; DELETE_PHRASE_QUERY: { - mutation: { queryKey: FrameAudioQueryKey }; + mutation: { queryKey: EditorFrameAudioQueryKey }; }; SET_PHRASE_SINGING_VOLUME: { From 2e579c8ab929957516a03ca3af5a3b8381175046 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sat, 31 Aug 2024 09:59:01 +0900 Subject: [PATCH 07/15] =?UTF-8?q?editFrameRate=E3=82=92editorFrameRate?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/Sing/ScoreSequencer.vue | 8 ++++---- src/components/Sing/SequencerPitch.vue | 6 +++--- src/sing/domain.ts | 10 +++++----- src/sing/phraseRendering.ts | 6 +++--- src/store/singing.ts | 6 +++--- src/store/type.ts | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/components/Sing/ScoreSequencer.vue b/src/components/Sing/ScoreSequencer.vue index f8efa45c89..ec9bb4c12a 100644 --- a/src/components/Sing/ScoreSequencer.vue +++ b/src/components/Sing/ScoreSequencer.vue @@ -360,7 +360,7 @@ const phraseInfosInOtherTracks = computed(() => { const ctrlKey = useCommandOrControlKey(); const editTarget = computed(() => state.sequencerEditTarget); -const editFrameRate = computed(() => state.editFrameRate); +const editorFrameRate = computed(() => state.editorFrameRate); const scrollBarWidth = ref(12); const sequencerBody = ref(null); @@ -601,7 +601,7 @@ const previewDrawPitch = () => { if (previewPitchEdit.value.type !== "draw") { throw new Error("previewPitchEdit.value.type is not draw."); } - const frameRate = editFrameRate.value; + const frameRate = editorFrameRate.value; const cursorBaseX = (scrollX.value + cursorX.value) / zoomX.value; const cursorBaseY = (scrollY.value + cursorY.value) / zoomY.value; const cursorTicks = baseXToTick(cursorBaseX, tpqn.value); @@ -675,7 +675,7 @@ const previewErasePitch = () => { if (previewPitchEdit.value.type !== "erase") { throw new Error("previewPitchEdit.value.type is not erase."); } - const frameRate = editFrameRate.value; + const frameRate = editorFrameRate.value; const cursorBaseX = (scrollX.value + cursorX.value) / zoomX.value; const cursorTicks = baseXToTick(cursorBaseX, tpqn.value); const cursorSeconds = tickToSecond(cursorTicks, tempos.value, tpqn.value); @@ -827,7 +827,7 @@ const startPreview = (event: MouseEvent, mode: PreviewMode, note?: Note) => { } else if (editTarget.value === "PITCH") { // 編集ターゲットがピッチのときの処理 - const frameRate = editFrameRate.value; + const frameRate = editorFrameRate.value; const cursorTicks = baseXToTick(cursorBaseX, tpqn.value); const cursorSeconds = tickToSecond(cursorTicks, tempos.value, tpqn.value); const cursorFrame = Math.round(cursorSeconds * frameRate); diff --git a/src/components/Sing/SequencerPitch.vue b/src/components/Sing/SequencerPitch.vue index 5f39338458..a5d911cc1e 100644 --- a/src/components/Sing/SequencerPitch.vue +++ b/src/components/Sing/SequencerPitch.vue @@ -56,7 +56,7 @@ const pitchEditData = computed(() => { }); const previewPitchEdit = computed(() => props.previewPitchEdit); const selectedTrackId = computed(() => store.getters.SELECTED_TRACK_ID); -const editFrameRate = computed(() => store.state.editFrameRate); +const editorFrameRate = computed(() => store.state.editorFrameRate); const singingGuidesInSelectedTrack = computed(() => { const singingGuides: { query: EditorFrameAudioQuery; @@ -263,7 +263,7 @@ const setPitchDataToPitchLine = async ( const generateOriginalPitchData = () => { const unvoicedPhonemes = UNVOICED_PHONEMES; - const frameRate = editFrameRate.value; // f0（元のピッチ）は編集フレームレートで表示する + const frameRate = editorFrameRate.value; // f0（元のピッチ）はエディターのフレームレートで表示する // 選択中のトラックで使われている歌い方のf0を結合してピッチデータを生成する const tempData = []; @@ -316,7 +316,7 @@ const generateOriginalPitchData = () => { }; const generatePitchEditData = () => { - const frameRate = editFrameRate.value; + const frameRate = editorFrameRate.value; const tempData = [...pitchEditData.value]; // プレビュー中のピッチ編集があれば、適用する diff --git a/src/sing/domain.ts b/src/sing/domain.ts index 55efbcc89a..9acfef90c6 100644 --- a/src/sing/domain.ts +++ b/src/sing/domain.ts @@ -293,7 +293,7 @@ export const DEFAULT_BEAT_TYPE = 4; export const SEQUENCER_MIN_NUM_MEASURES = 32; // マルチエンジン対応のために将来的に廃止予定で、利用は非推奨 -export const DEPRECATED_DEFAULT_EDIT_FRAME_RATE = 93.75; +export const DEPRECATED_DEFAULT_EDITOR_FRAME_RATE = 93.75; export const VALUE_INDICATING_NO_DATA = -1; @@ -454,13 +454,13 @@ export function applyPitchEdit( phraseQuery: EditorFrameAudioQuery, phraseStartTime: number, pitchEditData: number[], - editFrameRate: number, + editorFrameRate: number, ) { - // フレーズのクエリのフレームレートと編集フレームレートが一致しない場合はエラー + // フレーズのクエリのフレームレートとエディターのフレームレートが一致しない場合はエラー // TODO: 補間するようにする - if (phraseQuery.frameRate !== editFrameRate) { + if (phraseQuery.frameRate !== editorFrameRate) { throw new Error( - "The frame rate between the phrase query and the edit data does not match.", + "The frame rate between the phrase query and the editor does not match.", ); } const unvoicedPhonemes = UNVOICED_PHONEMES; diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index 4fa9bf4510..b1e27e357e 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -172,7 +172,7 @@ type Snapshot = Readonly<{ tempos: Tempo[]; tracks: Map; engineFrameRates: Map; - editFrameRate: number; + editorFrameRate: number; }>; /** @@ -437,7 +437,7 @@ const generateSingingVolumeSource = (context: Context): SingingVolumeSource => { clonedQuery, phrase.startTime, track.pitchEditData, - context.snapshot.editFrameRate, + context.snapshot.editorFrameRate, ); return { engineId: track.singer.engineId, @@ -599,7 +599,7 @@ const generateSingingVoiceSource = (context: Context): SingingVoiceSource => { clonedQuery, phrase.startTime, track.pitchEditData, - context.snapshot.editFrameRate, + context.snapshot.editorFrameRate, ); clonedQuery.volume = clonedSingingVolume; return { diff --git a/src/store/singing.ts b/src/store/singing.ts index b140d3af7d..b3efc0adec 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -66,7 +66,7 @@ import { isValidTimeSignatures, isValidTpqn, DEFAULT_TPQN, - DEPRECATED_DEFAULT_EDIT_FRAME_RATE, + DEPRECATED_DEFAULT_EDITOR_FRAME_RATE, createDefaultTrack, createDefaultTempo, createDefaultTimeSignature, @@ -413,7 +413,7 @@ export const singingStoreState: SingingStoreState = { */ _selectedTrackId: initialTrackId, - editFrameRate: DEPRECATED_DEFAULT_EDIT_FRAME_RATE, + editorFrameRate: DEPRECATED_DEFAULT_EDITOR_FRAME_RATE, phrases: new Map(), phraseQueries: new Map(), phraseSingingVolumes: new Map(), @@ -1547,7 +1547,7 @@ export const singingStore = createPartialStore({ ], ), ), - editFrameRate: state.editFrameRate, + editorFrameRate: state.editorFrameRate, } as const; const phraseRenderer = createPhraseRenderer({ diff --git a/src/store/type.ts b/src/store/type.ts index e63bba6a64..0c5ff101b0 100644 --- a/src/store/type.ts +++ b/src/store/type.ts @@ -822,7 +822,7 @@ export type SingingStoreState = { tracks: Map; trackOrder: TrackId[]; _selectedTrackId: TrackId; - editFrameRate: number; + editorFrameRate: number; phrases: Map; phraseQueries: Map; phraseSingingVolumes: Map; From 54129d3f3484e9975340485b1d330c5c16ca8f29 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sat, 31 Aug 2024 20:51:36 +0900 Subject: [PATCH 08/15] =?UTF-8?q?PhraseRenderStageId=E3=82=92=E4=B8=8A?= =?UTF-8?q?=E3=81=A7=E5=AE=9A=E7=BE=A9=E3=81=97=E3=81=A6=E3=80=81Stage?= =?UTF-8?q?=E3=81=A7=E3=81=9D=E3=82=8C=E3=82=92=E4=BD=BF=E3=81=86=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sing/phraseRendering.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index b1e27e357e..28fa9492a4 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -254,11 +254,16 @@ type Context = Readonly<{ externalDependencies: ExternalDependencies; }>; +export type PhraseRenderStageId = + | "queryGeneration" + | "singingVolumeGeneration" + | "singingVoiceSynthesis"; + /** * フレーズレンダリングのステージ */ type Stage = Readonly<{ - id: "queryGeneration" | "singingVolumeGeneration" | "singingVoiceSynthesis"; + id: PhraseRenderStageId; /** * このステージが実行されるべきかを判定する。 @@ -689,8 +694,6 @@ const singingVoiceSynthesisStage: Stage = { // フレーズレンダラー -export type PhraseRenderStageId = Stage["id"]; - export type PhraseRenderer = Readonly<{ /** * レンダリング処理の開始点となる最初のステージのIDを返す。 From f79db809cc3aebc95eebfe01f2844dc18f867079 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 1 Sep 2024 13:48:08 +0900 Subject: [PATCH 09/15] =?UTF-8?q?Stage=E3=82=92BaseStage=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sing/phraseRendering.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index 28fa9492a4..bfa3b9dbe8 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -262,7 +262,7 @@ export type PhraseRenderStageId = /** * フレーズレンダリングのステージ */ -type Stage = Readonly<{ +type BaseStage = Readonly<{ id: PhraseRenderStageId; /** @@ -352,7 +352,7 @@ const generateQuery = async ( return { ...query, frameRate: querySource.engineFrameRate }; }; -const queryGenerationStage: Stage = { +const queryGenerationStage: BaseStage = { id: "queryGeneration", shouldBeExecuted: async (context: Context) => { const phrases = context.externalDependencies.phrases; @@ -504,7 +504,7 @@ const generateSingingVolume = async ( return singingVolume; }; -const singingVolumeGenerationStage: Stage = { +const singingVolumeGenerationStage: BaseStage = { id: "singingVolumeGeneration", shouldBeExecuted: async (context: Context) => { const phrases = context.externalDependencies.phrases; @@ -631,7 +631,7 @@ const synthesizeSingingVoice = async ( return singingVoice; }; -const singingVoiceSynthesisStage: Stage = { +const singingVoiceSynthesisStage: BaseStage = { id: "singingVoiceSynthesis", shouldBeExecuted: async (context: Context) => { const phrases = context.externalDependencies.phrases; @@ -731,7 +731,7 @@ export type PhraseRenderer = Readonly<{ ) => Promise; }>; -const stages: readonly Stage[] = [ +const stages: readonly BaseStage[] = [ queryGenerationStage, singingVolumeGenerationStage, singingVoiceSynthesisStage, From 159babd62cd7f42e97346384f4196cd818436c7a Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 1 Sep 2024 13:58:27 +0900 Subject: [PATCH 10/15] =?UTF-8?q?externalDependencies=E3=81=8B=E3=82=89?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=81=99=E3=82=8B=E3=81=A8=E3=81=93=E3=82=8D?= =?UTF-8?q?=E3=82=92=E5=88=86=E5=89=B2=E4=BB=A3=E5=85=A5=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sing/phraseRendering.ts | 44 +++++++++++++------------------------ 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index bfa3b9dbe8..ba063cb98e 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -301,7 +301,7 @@ type QuerySource = Readonly<{ }>; const generateQuerySource = (context: Context): QuerySource => { - const phrases = context.externalDependencies.phrases; + const { phrases } = context.externalDependencies; const track = getOrThrow(context.snapshot.tracks, context.trackId); if (track.singer == undefined) { @@ -355,7 +355,7 @@ const generateQuery = async ( const queryGenerationStage: BaseStage = { id: "queryGeneration", shouldBeExecuted: async (context: Context) => { - const phrases = context.externalDependencies.phrases; + const { phrases } = context.externalDependencies; const track = getOrThrow(context.snapshot.tracks, context.trackId); if (track.singer == undefined) { @@ -368,8 +368,7 @@ const queryGenerationStage: BaseStage = { return phraseQueryKey == undefined || phraseQueryKey !== queryKey; }, deleteExecutionResult: (context: Context) => { - const phrases = context.externalDependencies.phrases; - const phraseQueries = context.externalDependencies.phraseQueries; + const { phrases, phraseQueries } = context.externalDependencies; const phrase = phrases.get(context.phraseKey); const phraseQueryKey = phrase.queryKey.get(); @@ -379,9 +378,7 @@ const queryGenerationStage: BaseStage = { } }, execute: async (context: Context) => { - const phrases = context.externalDependencies.phrases; - const phraseQueries = context.externalDependencies.phraseQueries; - const queryCache = context.externalDependencies.queryCache; + const { phrases, phraseQueries, queryCache } = context.externalDependencies; const querySource = generateQuerySource(context); const queryKey = await calculateQueryKey(querySource); @@ -424,8 +421,7 @@ type SingingVolumeSource = Readonly<{ }>; const generateSingingVolumeSource = (context: Context): SingingVolumeSource => { - const phrases = context.externalDependencies.phrases; - const phraseQueries = context.externalDependencies.phraseQueries; + const { phrases, phraseQueries } = context.externalDependencies; const track = getOrThrow(context.snapshot.tracks, context.trackId); if (track.singer == undefined) { @@ -507,7 +503,7 @@ const generateSingingVolume = async ( const singingVolumeGenerationStage: BaseStage = { id: "singingVolumeGeneration", shouldBeExecuted: async (context: Context) => { - const phrases = context.externalDependencies.phrases; + const { phrases } = context.externalDependencies; const track = getOrThrow(context.snapshot.tracks, context.trackId); if (track.singer == undefined) { @@ -524,9 +520,7 @@ const singingVolumeGenerationStage: BaseStage = { ); }, deleteExecutionResult: (context: Context) => { - const phrases = context.externalDependencies.phrases; - const phraseSingingVolumes = - context.externalDependencies.phraseSingingVolumes; + const { phrases, phraseSingingVolumes } = context.externalDependencies; const phrase = phrases.get(context.phraseKey); const phraseSingingVolumeKey = phrase.singingVolumeKey.get(); @@ -536,10 +530,8 @@ const singingVolumeGenerationStage: BaseStage = { } }, execute: async (context: Context) => { - const phrases = context.externalDependencies.phrases; - const phraseSingingVolumes = - context.externalDependencies.phraseSingingVolumes; - const singingVolumeCache = context.externalDependencies.singingVolumeCache; + const { phrases, phraseSingingVolumes, singingVolumeCache } = + context.externalDependencies; const singingVolumeSource = generateSingingVolumeSource(context); const singingVolumeKey = @@ -578,10 +570,8 @@ type SingingVoiceSource = Readonly<{ }>; const generateSingingVoiceSource = (context: Context): SingingVoiceSource => { - const phrases = context.externalDependencies.phrases; - const phraseQueries = context.externalDependencies.phraseQueries; - const phraseSingingVolumes = - context.externalDependencies.phraseSingingVolumes; + const { phrases, phraseQueries, phraseSingingVolumes } = + context.externalDependencies; const track = getOrThrow(context.snapshot.tracks, context.trackId); if (track.singer == undefined) { @@ -634,7 +624,7 @@ const synthesizeSingingVoice = async ( const singingVoiceSynthesisStage: BaseStage = { id: "singingVoiceSynthesis", shouldBeExecuted: async (context: Context) => { - const phrases = context.externalDependencies.phrases; + const { phrases } = context.externalDependencies; const track = getOrThrow(context.snapshot.tracks, context.trackId); if (track.singer == undefined) { @@ -650,9 +640,7 @@ const singingVoiceSynthesisStage: BaseStage = { ); }, deleteExecutionResult: (context: Context) => { - const phrases = context.externalDependencies.phrases; - const phraseSingingVoices = - context.externalDependencies.phraseSingingVoices; + const { phrases, phraseSingingVoices } = context.externalDependencies; const phrase = phrases.get(context.phraseKey); const phraseSingingVoiceKey = phrase.singingVoiceKey.get(); @@ -662,10 +650,8 @@ const singingVoiceSynthesisStage: BaseStage = { } }, execute: async (context: Context) => { - const phrases = context.externalDependencies.phrases; - const phraseSingingVoices = - context.externalDependencies.phraseSingingVoices; - const singingVoiceCache = context.externalDependencies.singingVoiceCache; + const { phrases, phraseSingingVoices, singingVoiceCache } = + context.externalDependencies; const singingVoiceSource = generateSingingVoiceSource(context); const singingVoiceKey = await calculateSingingVoiceKey(singingVoiceSource); From bbd7df5af05c404af2eaceaec4fca4c6b11ad3a3 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 1 Sep 2024 14:04:06 +0900 Subject: [PATCH 11/15] =?UTF-8?q?phrase=E3=81=AE=E4=B8=80=E6=99=82?= =?UTF-8?q?=E5=A4=89=E6=95=B0=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/singing.ts | 65 +++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/src/store/singing.ts b/src/store/singing.ts index b3efc0adec..971f88ecdf 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -1555,37 +1555,40 @@ export const singingStore = createPartialStore({ singingVolumeCache, singingVoiceCache, phrases: { - get: (phraseKey: PhraseKey) => ({ - firstRestDuration: getOrThrow(state.phrases, phraseKey) - .firstRestDuration, - notes: getOrThrow(state.phrases, phraseKey).notes, - startTime: getOrThrow(state.phrases, phraseKey).startTime, - queryKey: { - get: () => getOrThrow(state.phrases, phraseKey).queryKey, - set: (value) => - mutations.SET_QUERY_KEY_TO_PHRASE({ - phraseKey, - queryKey: value, - }), - }, - singingVolumeKey: { - get: () => - getOrThrow(state.phrases, phraseKey).singingVolumeKey, - set: (value) => - mutations.SET_SINGING_VOLUME_KEY_TO_PHRASE({ - phraseKey, - singingVolumeKey: value, - }), - }, - singingVoiceKey: { - get: () => getOrThrow(state.phrases, phraseKey).singingVoiceKey, - set: (value) => - mutations.SET_SINGING_VOICE_KEY_TO_PHRASE({ - phraseKey, - singingVoiceKey: value, - }), - }, - }), + get: (phraseKey: PhraseKey) => { + const phrase = getOrThrow(state.phrases, phraseKey); + return { + firstRestDuration: phrase.firstRestDuration, + notes: phrase.notes, + startTime: phrase.startTime, + queryKey: { + get: () => getOrThrow(state.phrases, phraseKey).queryKey, + set: (value) => + mutations.SET_QUERY_KEY_TO_PHRASE({ + phraseKey, + queryKey: value, + }), + }, + singingVolumeKey: { + get: () => + getOrThrow(state.phrases, phraseKey).singingVolumeKey, + set: (value) => + mutations.SET_SINGING_VOLUME_KEY_TO_PHRASE({ + phraseKey, + singingVolumeKey: value, + }), + }, + singingVoiceKey: { + get: () => + getOrThrow(state.phrases, phraseKey).singingVoiceKey, + set: (value) => + mutations.SET_SINGING_VOICE_KEY_TO_PHRASE({ + phraseKey, + singingVoiceKey: value, + }), + }, + }; + }, }, phraseQueries: { get: (queryKey) => getOrThrow(state.phraseQueries, queryKey), From a4284f7354cd6e78b72adc8bcf0edbe8f297c515 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 1 Sep 2024 14:29:00 +0900 Subject: [PATCH 12/15] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0=E3=83=BB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sing/phraseRendering.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index ba063cb98e..679f989762 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -1,3 +1,8 @@ +/** + * フレーズごとに音声合成するフレーズレンダラーと、それに必要な処理。 + * レンダリングが必要かどうかの判定やキャッシュの作成も行う。 + */ + import { Note, PhraseKey, @@ -682,14 +687,17 @@ const singingVoiceSynthesisStage: BaseStage = { export type PhraseRenderer = Readonly<{ /** - * レンダリング処理の開始点となる最初のステージのIDを返す。 + * 一番最初のステージのIDを返す。 + * 一度もレンダリングを行っていないフレーズは、 + * この（一番最初の）ステージからレンダリング処理を開始する必要がある。 * @returns ステージID */ getFirstRenderStageId: () => PhraseRenderStageId; /** - * レンダリングがどのステージから開始されるべきかを判断する。 - * すべてのステージがスキップ可能な場合、undefinedが返される。 + * レンダリングが必要なフレーズかどうかを判断し、 + * レンダリングが必要であればどのステージから開始されるべきかを判断して、そのステージのIDを返す。 + * レンダリングが必要ない場合、undefinedが返される。 * @param snapshot スナップショット * @param trackId トラックID * @param phraseKey フレーズキー @@ -702,7 +710,7 @@ export type PhraseRenderer = Readonly<{ ) => Promise; /** - * 指定されたステージからレンダリング処理を開始する。 + * 指定されたフレーズのレンダリング処理を、指定されたステージから開始する。 * レンダリング処理を開始する前に、前回のレンダリング処理結果の削除が行われる。 * @param snapshot スナップショット * @param trackId トラックID From 5c1f14388acd9edbbc9b2120d4b7eff53aa288f9 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 1 Sep 2024 14:31:45 +0900 Subject: [PATCH 13/15] Update src/sing/phraseRendering.ts Co-authored-by: Hiroshiba --- src/sing/phraseRendering.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index 679f989762..b551cce375 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -265,7 +265,8 @@ export type PhraseRenderStageId = | "singingVoiceSynthesis"; /** - * フレーズレンダリングのステージ + * フレーズレンダリングのステージのインターフェイス。 + * フレーズレンダラー内で順に実行される。 */ type BaseStage = Readonly<{ id: PhraseRenderStageId; From c09eb5f6c3aceb35383feda0085aab3ee7b3b925 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 1 Sep 2024 14:31:56 +0900 Subject: [PATCH 14/15] Update src/sing/phraseRendering.ts Co-authored-by: Hiroshiba --- src/sing/phraseRendering.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index b551cce375..9e53a7a62d 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -686,6 +686,10 @@ const singingVoiceSynthesisStage: BaseStage = { // フレーズレンダラー +/** + * フレーズレンダラー。 + * 各フレーズごとに、ステージを進めながらレンダリング処理を行う。 + */ export type PhraseRenderer = Readonly<{ /** * 一番最初のステージのIDを返す。 From e035919e2e3a90994309adc9ba236a20c4700191 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Tue, 3 Sep 2024 23:59:47 +0900 Subject: [PATCH 15/15] =?UTF-8?q?EditorFrameAudioQuery=E3=82=92=E4=BD=BF?= =?UTF-8?q?=E3=81=86=E3=82=88=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sing/phraseRendering.ts | 16 +++++++--------- src/store/singing.ts | 20 +++++++++++++------- src/store/type.ts | 2 +- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/sing/phraseRendering.ts b/src/sing/phraseRendering.ts index 9e53a7a62d..dd4aba5545 100644 --- a/src/sing/phraseRendering.ts +++ b/src/sing/phraseRendering.ts @@ -16,11 +16,7 @@ import { EditorFrameAudioQueryKey, EditorFrameAudioQuery, } from "@/store/type"; -import { - FrameAudioQuery, - FramePhoneme, - Note as NoteForRequestToEngine, -} from "@/openapi"; +import { FramePhoneme, Note as NoteForRequestToEngine } from "@/openapi"; import { applyPitchEdit, decibelToLinear, tickToSecond } from "@/sing/domain"; import { calculateHash, linearInterpolation } from "@/sing/utility"; import { EngineId, StyleId, TrackId } from "@/type/preload"; @@ -235,17 +231,18 @@ type ExternalDependencies = Readonly<{ fetchQuery: ( engineId: EngineId, + engineFrameRate: number, notes: NoteForRequestToEngine[], - ) => Promise; + ) => Promise; fetchSingFrameVolume: ( notes: NoteForRequestToEngine[], - query: FrameAudioQuery, + query: EditorFrameAudioQuery, engineId: EngineId, styleId: StyleId, ) => Promise; synthesizeSingingVoice: ( singer: Singer, - query: FrameAudioQuery, + query: EditorFrameAudioQuery, ) => Promise; }>; @@ -351,11 +348,12 @@ const generateQuery = async ( const query = await externalDependencies.fetchQuery( querySource.engineId, + querySource.engineFrameRate, notesForRequestToEngine, ); shiftPitch(query.f0, querySource.keyRangeAdjustment); - return { ...query, frameRate: querySource.engineFrameRate }; + return query; }; const queryGenerationStage: BaseStage = { diff --git a/src/store/singing.ts b/src/store/singing.ts index 971f88ecdf..9dc9aa71ea 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -33,7 +33,7 @@ import { StyleId, TrackId, } from "@/type/preload"; -import { FrameAudioQuery, Note as NoteForRequestToEngine } from "@/openapi"; +import { Note as NoteForRequestToEngine } from "@/openapi"; import { ResultError, getValueOrThrow } from "@/type/result"; import { AudioEvent, @@ -1443,6 +1443,7 @@ export const singingStore = createPartialStore({ const fetchQuery = async ( engineId: EngineId, + engineFrameRate: number, notesForRequestToEngine: NoteForRequestToEngine[], ) => { try { @@ -1452,12 +1453,17 @@ export const singingStore = createPartialStore({ const instance = await actions.INSTANTIATE_ENGINE_CONNECTOR({ engineId, }); - return await instance.invoke( + const query = await instance.invoke( "singFrameAudioQuerySingFrameAudioQueryPost", )({ score: { notes: notesForRequestToEngine }, speaker: singingTeacherStyleId, }); + const editorQuery: EditorFrameAudioQuery = { + ...query, + frameRate: engineFrameRate, + }; + return editorQuery; } catch (error) { const lyrics = notesForRequestToEngine .map((value) => value.lyric) @@ -1472,7 +1478,7 @@ export const singingStore = createPartialStore({ const synthesizeSingingVoice = async ( singer: Singer, - query: FrameAudioQuery, + query: EditorFrameAudioQuery, ) => { if (!getters.IS_ENGINE_READY(singer.engineId)) { throw new Error("Engine not ready."); @@ -1617,7 +1623,7 @@ export const singingStore = createPartialStore({ fetchSingFrameVolume: (notes, query, engineId, styleId) => actions.FETCH_SING_FRAME_VOLUME({ notes, - frameAudioQuery: query, + query, engineId, styleId, }), @@ -1868,12 +1874,12 @@ export const singingStore = createPartialStore({ { actions }, { notes, - frameAudioQuery, + query, engineId, styleId, }: { notes: NoteForRequestToEngine[]; - frameAudioQuery: FrameAudioQuery; + query: EditorFrameAudioQuery; engineId: EngineId; styleId: StyleId; }, @@ -1886,7 +1892,7 @@ export const singingStore = createPartialStore({ score: { notes, }, - frameAudioQuery, + frameAudioQuery: query, }, speaker: styleId, }); diff --git a/src/store/type.ts b/src/store/type.ts index 0c5ff101b0..70d080a5e5 100644 --- a/src/store/type.ts +++ b/src/store/type.ts @@ -1072,7 +1072,7 @@ export type SingingStoreTypes = { FETCH_SING_FRAME_VOLUME: { action(palyoad: { notes: NoteForRequestToEngine[]; - frameAudioQuery: FrameAudioQuery; + query: EditorFrameAudioQuery; engineId: EngineId; styleId: StyleId; }): Promise;