Improve alignment

enjoy/src/main/echogarden.ts

@@ -10,7 +10,11 @@ import {
  trimAudioEnd,
  AudioSourceParam,
 } from "echogarden/dist/audio/AudioUtilities.js";
-import { Timeline } from "echogarden/dist/utilities/Timeline.d.js";
+import { wordTimelineToSegmentSentenceTimeline } from "echogarden/dist/utilities/Timeline.js";
+import {
+ type Timeline,
+ type TimelineEntry,
+} from "echogarden/dist/utilities/Timeline.d.js";
 import path from "path";
 import log from "@main/logger";
 import url from "url";
@@ -43,6 +47,7 @@ class EchogardenWrapper {
  public getRawAudioDuration: typeof getRawAudioDuration;
  public trimAudioStart: typeof trimAudioStart;
  public trimAudioEnd: typeof trimAudioEnd;
+ public wordTimelineToSegmentSentenceTimeline: typeof wordTimelineToSegmentSentenceTimeline;
 
  constructor() {
  this.align = Echogarden.align;
@@ -54,6 +59,8 @@ class EchogardenWrapper {
  this.getRawAudioDuration = getRawAudioDuration;
  this.trimAudioStart = trimAudioStart;
  this.trimAudioEnd = trimAudioEnd;
+ this.wordTimelineToSegmentSentenceTimeline =
+ wordTimelineToSegmentSentenceTimeline;
  }
 
  async check() {
@@ -132,6 +139,37 @@ class EchogardenWrapper {
  }
  );
 
+ ipcMain.handle(
+ "echogarden-word-to-sentence-timeline",
+ async (
+ _event,
+ wordTimeline: Timeline,
+ transcript: string,
+ language: string
+ ) => {
+ logger.debug("echogarden-word-to-sentence-timeline:", transcript);
+
+ const { segmentTimeline } =
+ await this.wordTimelineToSegmentSentenceTimeline(
+ wordTimeline,
+ transcript,
+ language.split("-")[0]
+ );
+ const timeline: Timeline = [];
+ segmentTimeline.forEach((t: TimelineEntry) => {
+ if (t.type === "sentence") {
+ timeline.push(t);
+ } else {
+ t.timeline.forEach((st) => {
+ timeline.push(st);
+ });
+ }
+ });
+
+ return timeline;
+ }
+ );
+
  ipcMain.handle(
  "echogarden-transcode",
  async (_event, url: string, sampleRate?: number) => {

enjoy/src/main/whisper.ts

@@ -105,8 +105,6 @@ class Whipser {
  `--model "${model.savePath}"`,
  "--output-json",
  `--output-file "${path.join(tmpDir, "jfk")}"`,
- `--split-on-word true`,
- `--max-len 1`,
  ];
  logger.debug(`Checking whisper command: ${commands.join(" ")}`);
  exec(
@@ -205,9 +203,9 @@ class Whipser {
  "--print-progress",
  "--language",
  model.name.includes("en") ? "en" : language?.split("-")?.[0] || "auto",
- `--split-on-word`,
- `--max-len`,
- "1",
+ // `--split-on-word`,
+ // `--max-len`,
+ // "1",
  ...extra,
  ];
 

enjoy/src/preload.ts

@@ -441,7 +441,24 @@ contextBridge.exposeInMainWorld("__ENJOY_APP__", {
  return ipcRenderer.invoke("echogarden-align", input, transcript, options);
  },
  alignSegments: (input: string, timeline: Timeline, options: any) => {
- return ipcRenderer.invoke("echogarden-align-segments", input, timeline, options);
+ return ipcRenderer.invoke(
+ "echogarden-align-segments",
+ input,
+ timeline,
+ options
+ );
+ },
+ wordToSentenceTimeline: (
+ wordTimeline: Timeline,
+ transcript: string,
+ language: string
+ ) => {
+ return ipcRenderer.invoke(
+ "echogarden-word-to-sentence-timeline",
+ wordTimeline,
+ transcript,
+ language
+ );
  },
  transcode: (input: string) => {
  return ipcRenderer.invoke("echogarden-transcode", input);

enjoy/src/renderer/components/medias/media-provider.tsx

@@ -51,11 +51,13 @@ export const MediaProvider = () => {
  language: transcription.result.language,
  })
  );
+ }, [player, transcription]);
 
+ useEffect(() => {
  return () => {
  setMediaProvider(null);
  };
- }, [player, transcription]);
+ }, [media?.src]);
 
  if (!media?.src) return null;
 

enjoy/src/renderer/context/media-player-provider.tsx

@@ -559,7 +559,7 @@ export const MediaPlayerProvider = ({
  setDecoded(false);
  setDecodeError(null);
  };
- }, [media?.src, ref, mediaProvider, layout?.playerHeight]);
+ }, [media?.src, ref?.current, mediaProvider, layout?.playerHeight]);
 
  /* cache last segment index */
  useEffect(() => {