ソング：フレーズのレンダリング処理をリファクタリング (#2248)

* フレーズレンダラーを追加、リファクタリング

* 修正、コメントを追加

* コメントを修正

* バグがあったので修正

* cloneをしていなかったので修正

* EditorFrameAudioQueryを使うようにした

* editFrameRateをeditorFrameRateに変更

* PhraseRenderStageIdを上で定義して、Stageでそれを使うようにした

* StageをBaseStageに変更

* externalDependenciesから取得するところを分割代入に変更

* phraseの一時変数を追加

* コメントを追加・修正

* Update src/sing/phraseRendering.ts

Co-authored-by: Hiroshiba <hihokaruta@gmail.com>

* Update src/sing/phraseRendering.ts

Co-authored-by: Hiroshiba <hihokaruta@gmail.com>

* EditorFrameAudioQueryを使うようにした

---------

Co-authored-by: Hiroshiba <hihokaruta@gmail.com>

src/components/Sing/ScoreSequencer.vue

@@ -360,7 +360,7 @@ const phraseInfosInOtherTracks = computed(() => {
 
 const ctrlKey = useCommandOrControlKey();
 const editTarget = computed(() => state.sequencerEditTarget);
-const editFrameRate = computed(() => state.editFrameRate);
+const editorFrameRate = computed(() => state.editorFrameRate);
 const scrollBarWidth = ref(12);
 const sequencerBody = ref<HTMLElement | null>(null);
 
@@ -601,7 +601,7 @@ const previewDrawPitch = () => {
  if (previewPitchEdit.value.type !== "draw") {
  throw new Error("previewPitchEdit.value.type is not draw.");
  }
- const frameRate = editFrameRate.value;
+ const frameRate = editorFrameRate.value;
  const cursorBaseX = (scrollX.value + cursorX.value) / zoomX.value;
  const cursorBaseY = (scrollY.value + cursorY.value) / zoomY.value;
  const cursorTicks = baseXToTick(cursorBaseX, tpqn.value);
@@ -675,7 +675,7 @@ const previewErasePitch = () => {
  if (previewPitchEdit.value.type !== "erase") {
  throw new Error("previewPitchEdit.value.type is not erase.");
  }
- const frameRate = editFrameRate.value;
+ const frameRate = editorFrameRate.value;
  const cursorBaseX = (scrollX.value + cursorX.value) / zoomX.value;
  const cursorTicks = baseXToTick(cursorBaseX, tpqn.value);
  const cursorSeconds = tickToSecond(cursorTicks, tempos.value, tpqn.value);
@@ -827,7 +827,7 @@ const startPreview = (event: MouseEvent, mode: PreviewMode, note?: Note) => {
  } else if (editTarget.value === "PITCH") {
  // 編集ターゲットがピッチのときの処理
 
- const frameRate = editFrameRate.value;
+ const frameRate = editorFrameRate.value;
  const cursorTicks = baseXToTick(cursorBaseX, tpqn.value);
  const cursorSeconds = tickToSecond(cursorTicks, tempos.value, tpqn.value);
  const cursorFrame = Math.round(cursorSeconds * frameRate);

src/components/Sing/SequencerPhraseIndicator.vue

@@ -6,15 +6,16 @@
 import { computed } from "vue";
 import { useStore } from "@/store";
 import { getOrThrow } from "@/helpers/mapHelper";
-import { PhraseSourceHash, PhraseState } from "@/store/type";
+import { PhraseKey, PhraseState } from "@/store/type";
 
 const props = defineProps<{
- phraseKey: PhraseSourceHash;
+ phraseKey: PhraseKey;
  isInSelectedTrack: boolean;
 }>();
 
 const store = useStore();
 const classNames: Record<PhraseState, string> = {
+ SINGER_IS_NOT_SET: "singer-is-not-set",
  WAITING_TO_BE_RENDERED: "waiting-to-be-rendered",
  NOW_RENDERING: "now-rendering",
  COULD_NOT_RENDER: "could-not-render",
@@ -43,6 +44,10 @@ const className = computed(() => {
  }
 }
 
+.singer-is-not-set {
+ visibility: hidden;
+}
+
 .waiting-to-be-rendered {
  @include tint-if-in-other-track(
  "background-color",

src/components/Sing/SequencerPitch.vue

@@ -30,6 +30,7 @@ import { ExhaustiveError } from "@/type/utility";
 import { createLogger } from "@/domain/frontend/log";
 import { getLast } from "@/sing/utility";
 import { getOrThrow } from "@/helpers/mapHelper";
+import { EditorFrameAudioQuery } from "@/store/type";
 
 type PitchLine = {
  readonly color: Color;
@@ -55,21 +56,24 @@ const pitchEditData = computed(() => {
 });
 const previewPitchEdit = computed(() => props.previewPitchEdit);
 const selectedTrackId = computed(() => store.getters.SELECTED_TRACK_ID);
-const editFrameRate = computed(() => store.state.editFrameRate);
+const editorFrameRate = computed(() => store.state.editorFrameRate);
 const singingGuidesInSelectedTrack = computed(() => {
- const singingGuides = [];
+ const singingGuides: {
+ query: EditorFrameAudioQuery;
+ startTime: number;
+ }[] = [];
  for (const phrase of store.state.phrases.values()) {
  if (phrase.trackId !== selectedTrackId.value) {
  continue;
  }
- if (phrase.singingGuideKey == undefined) {
+ if (phrase.queryKey == undefined) {
  continue;
  }
- const singingGuide = getOrThrow(
-  store.state.singingGuides,
- phrase.singingGuideKey,
- );
- singingGuides.push(singingGuide);
+ const phraseQuery = getOrThrow(store.state.phraseQueries, phrase.queryKey);
+ singingGuides.push({
+ startTime: phrase.startTime,
+  query: phraseQuery,
+ });
  }
  return singingGuides;
 });
@@ -259,13 +263,13 @@ const setPitchDataToPitchLine = async (
 
 const generateOriginalPitchData = () => {
  const unvoicedPhonemes = UNVOICED_PHONEMES;
- const frameRate = editFrameRate.value; // f0（元のピッチ）は編集フレームレートで表示する
+ const frameRate = editorFrameRate.value; // f0（元のピッチ）はエディターのフレームレートで表示する
 
  // 選択中のトラックで使われている歌い方のf0を結合してピッチデータを生成する
  const tempData = [];
  for (const singingGuide of singingGuidesInSelectedTrack.value) {
  // TODO: 補間を行うようにする
- if (singingGuide.frameRate !== frameRate) {
+ if (singingGuide.query.frameRate !== frameRate) {
  throw new Error(
  "The frame rate between the singing guide and the edit does not match.",
  );
@@ -312,7 +316,7 @@ const generateOriginalPitchData = () => {
 };
 
 const generatePitchEditData = () => {
- const frameRate = editFrameRate.value;
+ const frameRate = editorFrameRate.value;
 
  const tempData = [...pitchEditData.value];
  // プレビュー中のピッチ編集があれば、適用する

src/sing/domain.ts

@@ -4,15 +4,11 @@ import {
  Note,
  Phrase,
  PhraseSource,
- SingingGuide,
- SingingGuideSource,
- SingingVoiceSource,
  Tempo,
  TimeSignature,
- phraseSourceHashSchema,
+ PhraseKey,
  Track,
- singingGuideSourceHashSchema,
- singingVoiceSourceHashSchema,
+ EditorFrameAudioQuery,
 } from "@/store/type";
 import { FramePhoneme } from "@/openapi";
 import { TrackId } from "@/type/preload";
@@ -297,7 +293,7 @@ export const DEFAULT_BEAT_TYPE = 4;
 export const SEQUENCER_MIN_NUM_MEASURES = 32;
 
 // マルチエンジン対応のために将来的に廃止予定で、利用は非推奨
-export const DEPRECATED_DEFAULT_EDIT_FRAME_RATE = 93.75;
+export const DEPRECATED_DEFAULT_EDITOR_FRAME_RATE = 93.75;
 
 export const VALUE_INDICATING_NO_DATA = -1;
 
@@ -379,23 +375,9 @@ export function isValidPitchEditData(pitchEditData: number[]) {
  );
 }
 
-export const calculatePhraseSourceHash = async (phraseSource: PhraseSource) => {
+export const calculatePhraseKey = async (phraseSource: PhraseSource) => {
  const hash = await calculateHash(phraseSource);
- return phraseSourceHashSchema.parse(hash);
-};
-
-export const calculateSingingGuideSourceHash = async (
- singingGuideSource: SingingGuideSource,
-) => {
- const hash = await calculateHash(singingGuideSource);
- return singingGuideSourceHashSchema.parse(hash);
-};
-
-export const calculateSingingVoiceSourceHash = async (
- singingVoiceSource: SingingVoiceSource,
-) => {
- const hash = await calculateHash(singingVoiceSource);
- return singingVoiceSourceHashSchema.parse(hash);
+ return PhraseKey(hash);
 };
 
 export function getStartTicksOfPhrase(phrase: Phrase) {
@@ -469,42 +451,43 @@ export function convertToFramePhonemes(phonemes: FramePhoneme[]) {
 }
 
 export function applyPitchEdit(
- singingGuide: SingingGuide,
+ phraseQuery: EditorFrameAudioQuery,
+ phraseStartTime: number,
  pitchEditData: number[],
- editFrameRate: number,
+ editorFrameRate: number,
 ) {
- // 歌い方のフレームレートと編集フレームレートが一致しない場合はエラー
+ // フレーズのクエリのフレームレートとエディターのフレームレートが一致しない場合はエラー
  // TODO: 補間するようにする
- if (singingGuide.frameRate !== editFrameRate) {
+ if (phraseQuery.frameRate !== editorFrameRate) {
  throw new Error(
- "The frame rate between the singing guide and the edit data does not match.",
+ "The frame rate between the phrase query and the editor does not match.",
  );
  }
  const unvoicedPhonemes = UNVOICED_PHONEMES;
- const f0 = singingGuide.query.f0;
- const phonemes = singingGuide.query.phonemes;
+ const f0 = phraseQuery.f0;
+ const phonemes = phraseQuery.phonemes;
 
  // 各フレームの音素の配列を生成する
  const framePhonemes = convertToFramePhonemes(phonemes);
  if (f0.length !== framePhonemes.length) {
  throw new Error("f0.length and framePhonemes.length do not match.");
  }
 
- // 歌い方の開始フレームと終了フレームを計算する
- const singingGuideFrameLength = f0.length;
- const singingGuideStartFrame = Math.round(
- singingGuide.startTime * singingGuide.frameRate,
+ // フレーズのクエリの開始フレームと終了フレームを計算する
+ const phraseQueryFrameLength = f0.length;
+ const phraseQueryStartFrame = Math.round(
+ phraseStartTime * phraseQuery.frameRate,
  );
- const singingGuideEndFrame = singingGuideStartFrame + singingGuideFrameLength;
+ const phraseQueryEndFrame = phraseQueryStartFrame + phraseQueryFrameLength;
 
  // ピッチ編集をf0に適用する
- const startFrame = Math.max(0, singingGuideStartFrame);
- const endFrame = Math.min(pitchEditData.length, singingGuideEndFrame);
+ const startFrame = Math.max(0, phraseQueryStartFrame);
+ const endFrame = Math.min(pitchEditData.length, phraseQueryEndFrame);
  for (let i = startFrame; i < endFrame; i++) {
- const phoneme = framePhonemes[i - singingGuideStartFrame];
+ const phoneme = framePhonemes[i - phraseQueryStartFrame];
  const voiced = !unvoicedPhonemes.includes(phoneme);
  if (voiced && pitchEditData[i] !== VALUE_INDICATING_NO_DATA) {
- f0[i - singingGuideStartFrame] = pitchEditData[i];
+ f0[i - phraseQueryStartFrame] = pitchEditData[i];
  }
  }
 }