feat(transcript): youtube and bilibili transcript fetching

apps/app/src/components/context.tsx

@@ -1,16 +1,16 @@
-import type { MediaPlayerInstance, TextTrackInit } from "@vidstack/react";
+import type { MediaPlayerInstance } from "@vidstack/react";
 import type { App, TFile, Vault } from "obsidian";
 import { createContext, useContext } from "react";
 // eslint-disable-next-line import/no-deprecated -- don't use equalityFn here
 import { createStore, useStore } from "zustand";
 import { type MediaURL } from "@/info/media-url";
 import type { MediaHost } from "@/info/supported";
+import type { LoadedTextTrack, WebsiteTextTrack } from "@/info/track-info";
 import { parseHashProps, type HashProps } from "@/lib/hash/hash-prop";
 import { TimeoutError } from "@/lib/message";
 import noop from "@/lib/no-op";
 import { WebiviewMediaProvider } from "@/lib/remote-player/provider";
 import type { ScreenshotInfo } from "@/lib/screenshot";
-import { getTracksInVault } from "@/lib/subtitle";
 import type {
   Playlist,
   PlaylistItemWithMedia,
@@ -20,7 +20,7 @@ import {
   MEDIA_FILE_VIEW_TYPE,
   type MediaViewType,
 } from "@/media-view/view-type";
-import type MediaExtended from "@/mx-main";
+import type MxPlugin from "@/mx-main";
 import type { MxSettings } from "@/settings/def";
 import { type MediaInfo, mediaInfoFromFile } from "../info/media-info";
 import { applyTempFrag, handleTempFrag } from "./state/apply-tf";
@@ -39,7 +39,7 @@ export interface SourceFacet {
    */
   title?: string | true;
   viewType: MediaViewType;
-  textTracks?: TextTrackInit[];
+  textTracks?: LoadedTextTrack[];
 }
 
 export interface MediaViewState {
@@ -63,12 +63,13 @@ export interface MediaViewState {
   disableWebFullscreen?: boolean;
   toggleControls: (showCustom: boolean) => void;
   toggleWebFullscreen: (enableWebFs: boolean) => void;
-  textTracks: TextTrackInit[];
+  textTracks: { local: LoadedTextTrack[]; remote: WebsiteTextTrack[] };
   webHost?: Exclude<MediaHost, MediaHost.Generic>;
   updateWebHost: (webHost: MediaHost) => void;
+  updateWebsiteTracks: (tracks: WebsiteTextTrack[]) => void;
 }
 
-export function createMediaViewStore() {
+export function createMediaViewStore(plugin: MxPlugin) {
   const store = createStore<MediaViewState>((set, get, store) => ({
     player: null,
     playerRef: (inst) => set({ player: inst }),
@@ -104,7 +105,10 @@ export function createMediaViewStore() {
           viewType,
           url,
         },
-        textTracks: textTracks ?? og.textTracks,
+        textTracks: {
+          local: textTracks ?? og.textTracks.local,
+          remote: og.textTracks.remote,
+        },
         hash: { ...og.hash, ...parseHashProps(hash || url.hash) },
         title:
           (title === true ? titleFromUrl(url.source.href) : title) ?? og.title,
@@ -115,15 +119,15 @@ export function createMediaViewStore() {
       set((og) => ({ hash: { ...og.hash, ...parseHashProps(hash) } }));
       applyTempFrag(get());
     },
-    async loadFile(file, { vault, subpath }) {
-      const textTracks = await getTracksInVault(file, vault);
+    async loadFile(file, { subpath }) {
       const url = mediaInfoFromFile(file, subpath ?? "");
       if (!url) {
         throw new Error("Invalid media file: " + file.path);
       }
-      set(({ source, hash }) => ({
+      const textTracks = await plugin.transcript.getTracks(url);
+      set(({ source, hash, textTracks: { remote } }) => ({
         source: { ...source, url, viewType: MEDIA_FILE_VIEW_TYPE[url.type] },
-        textTracks,
+        textTracks: { local: textTracks, remote },
         title: file.name,
         hash: subpath ? { ...hash, ...parseHashProps(subpath) } : hash,
       }));
@@ -168,7 +172,11 @@ export function createMediaViewStore() {
         player.provider.media.send("mx-toggle-webfs", enableWebFs);
       }
     },
-    textTracks: [],
+    textTracks: { local: [], remote: [] },
+    updateWebsiteTracks: (tracks) =>
+      set(({ textTracks }) => ({
+        textTracks: { ...textTracks, remote: tracks },
+      })),
     updateWebHost: (webHost) =>
       set({ webHost: webHost === "generic" ? undefined : webHost }),
   }));
@@ -181,7 +189,7 @@ export type MediaViewStoreApi = ReturnType<typeof createMediaViewStore>;
 
 export const MediaViewContext = createContext<{
   store: MediaViewStoreApi;
-  plugin: MediaExtended;
+  plugin: MxPlugin;
   embed: boolean;
   reload: () => void;
   onScreenshot?: (info: ScreenshotInfo) => any;

apps/app/src/components/hook/use-playlist.tsx

@@ -17,9 +17,9 @@ export function usePlaylist(): PlaylistWithActive | undefined {
       setPlaylist(plugin.playlist.get(media));
     }
     onPlaylistChange();
-    plugin.app.metadataCache.on("mx-playlist-change", onPlaylistChange);
+    plugin.app.metadataCache.on("mx:playlist-change", onPlaylistChange);
     return () => {
-      plugin.app.metadataCache.off("mx-playlist-change", onPlaylistChange);
+      plugin.app.metadataCache.off("mx:playlist-change", onPlaylistChange);
     };
   }, [media, plugin.playlist, plugin.app.metadataCache]);
   // get latest updated playlist

apps/app/src/components/player.tsx

@@ -6,8 +6,8 @@ import { MediaPlayer, useMediaState } from "@vidstack/react";
 import { Notice } from "obsidian";
 import { useState } from "react";
 import { useTempFragHandler } from "@/components/hook/use-temporal-frag";
+import { isFileMediaInfo } from "@/info/media-info";
 import { cn } from "@/lib/utils";
-import { isFileMediaInfo } from "../info/media-info";
 import { useIsEmbed, useMediaViewStore, useSettings } from "./context";
 import { useViewTypeDetect } from "./hook/fix-webm-audio";
 import { useControls, useDefaultVolume, useHashProps } from "./hook/use-hash";

apps/app/src/components/player/caption-menu.tsx

@@ -1,54 +1,17 @@
-import type { CaptionOption } from "@vidstack/react";
 import { useCaptionOptions, useMediaState } from "@vidstack/react";
 import { SubtitlesIcon } from "@/components/icon";
 import { dataLpPassthrough } from "./buttons";
 import { useMenu } from "./menus";
 
-function sortCaption(a: CaptionOption, b: CaptionOption) {
-  if (a.track && b.track) {
-    // if with item.track.language, sort by item.track.language
-    if (a.track.language && b.track.language) {
-      return (a.track as { language: string }).language.localeCompare(
-        (b.track as { language: string }).language,
-      );
-    }
-
-    // if not with item.track.language, sort by item.label
-    if (!a.track.language && !b.track.language) {
-      return a.label.localeCompare(b.label);
-    }
-
-    // if one has language and the other doesn't, the one with language comes first
-    if (a.track.language && !b.track.language) {
-      return -1;
-    }
-    if (!a.track.language && b.track.language) {
-      return 1;
-    }
-  }
-  // if item.track is null, put it at the end
-  if (a.track === null && b.track !== null) {
-    return 1;
-  }
-  if (a.track !== null && b.track === null) {
-    return -1;
-  }
-  return 0;
-}
 export function Captions() {
   const options = useCaptionOptions();
   const tracks = useMediaState("textTracks");
   const onClick = useMenu((menu) => {
-    options
-      .sort(sortCaption)
-      .forEach(({ label, select, selected }, idx, options) => {
-        menu.addItem((item) => {
-          if (options.length === 2 && label === "Unknown") {
-            label = "On";
-          }
-          item.setTitle(label).setChecked(selected).onClick(select);
-        });
+    options.forEach(({ label, select, selected }) => {
+      menu.addItem((item) => {
+        item.setTitle(label).setChecked(selected).onClick(select);
       });
+    });
     return true;
   });
 

apps/app/src/components/player/menus.tsx

@@ -20,6 +20,7 @@ import {
   useReload,
 } from "../context";
 import { usePlaylist } from "../hook/use-playlist";
+import { dedupeWebsiteTrack } from "../use-tracks";
 import { dataLpPassthrough } from "./buttons";
 import { addItemsToMenu } from "./playlist-menu";
 
@@ -138,9 +139,10 @@ export function MoreOptions() {
       transform,
       disableWebFullscreen,
       toggleWebFullscreen,
+      textTracks: tracks,
     } = store.getState();
     workspace.trigger(
-      "mx-media-menu",
+      "mx:media-menu",
       menu,
       {
         player,
@@ -150,6 +152,10 @@ export function MoreOptions() {
         toggleControls,
         controls,
         setTransform,
+        tracks: {
+          local: tracks.local,
+          remote: dedupeWebsiteTrack(tracks.remote, tracks.local),
+        },
         transform,
         plugin,
         disableWebFullscreen,

apps/app/src/components/provider.tsx

@@ -7,7 +7,7 @@ import {
   type MediaProviderProps,
   Track,
 } from "@vidstack/react";
-import { useCallback, useEffect, useState } from "react";
+import { useCallback } from "react";
 import { getPartition } from "@/lib/remote-player/const";
 import { WebviewProviderLoader } from "@/lib/remote-player/loader";
 import { cn } from "@/lib/utils";

apps/app/src/components/state/handle-cue-update.ts

@@ -0,0 +1,52 @@
+import type { TextTrackCueChangeEvent } from "@vidstack/react";
+import type { EventCallback } from "maverick.js/std";
+import type { Workspace } from "obsidian";
+import type { MediaInfo } from "@/info/media-info";
+import { onPlayerMounted, type MediaViewStoreApi } from "../context";
+
+declare module "obsidian" {
+  interface App {
+    commands: {
+      commands: Record<string, Command>;
+    };
+  }
+  interface Workspace {
+    on(
+      name: "mx:cue-change",
+      callback: (
+        source: MediaInfo,
+        trackId: string,
+        cueIds: string[],
+      ) => EventRef,
+    ): any;
+    trigger(
+      name: "mx:cue-change",
+      source: MediaInfo,
+      trackId: string,
+      cueIds: string[],
+    ): void;
+  }
+}
+
+export function handleTempFrag(store: MediaViewStoreApi, workspace: Workspace) {
+  return onPlayerMounted(store, (player) =>
+    player.subscribe(({ textTrack }) => {
+      if (!textTrack) return;
+      const onCueChange: EventCallback<TextTrackCueChangeEvent & Event> = (
+        evt,
+      ) => {
+        const source = store.getState().source?.url;
+        if (!source) return;
+        const track = evt.target;
+        workspace.trigger(
+          "mx:cue-change",
+          source,
+          track.id,
+          track.activeCues.map((c) => c.id),
+        );
+      };
+      textTrack.addEventListener("cue-change", onCueChange);
+      return textTrack.removeEventListener("cue-change", onCueChange);
+    }),
+  );
+}

apps/app/src/components/transcript/cue-line-list.tsx

@@ -1,10 +1,9 @@
 import { useVirtualizer } from "@tanstack/react-virtual";
-import type { VTTCueInit } from "@vidstack/react";
 import { forwardRef, useImperativeHandle, useMemo, useRef } from "react";
 import { ScrollArea } from "@/components/ui/scroll-area";
 import { cn } from "@/lib/utils";
-import type { CueSearchResult } from "@/transcript/context";
-import type { VTTCueWithId } from "@/transcript/store";
+import type { VTTCueWithId } from "@/transcript/handle/type";
+import type { CueSearchResult } from "@/transcript/view/context";
 import { CueActions, CueLine } from "./cue-line";
 
 export interface CueLineListRef {

apps/app/src/components/transcript/cue-line.tsx

@@ -2,7 +2,7 @@ import { Notice, htmlToMarkdown } from "obsidian";
 import { forwardRef, useMemo } from "react";
 import { formatDuration } from "@/lib/hash/format";
 import { cn } from "@/lib/utils";
-import type { VTTCueWithId } from "@/transcript/store";
+import type { VTTCueWithId } from "@/transcript/handle/type";
 import { CopyIcon, PlayIcon } from "../icon";
 
 export interface CueLineProps extends React.HTMLProps<HTMLDivElement> {
@@ -61,7 +61,11 @@ export const CueLine = forwardRef<HTMLDivElement, CueLineProps>(
   ) {
     const highlightedText = matches
       ? splitByKeywords(content, matches)
-      : [content];
+      : ((content: string) => {
+          const segments: React.ReactNode[] = [];
+          insertLineBreaks(content, segments, 0);
+          return segments;
+        })(content);
     return (
       <div
         {...props}
@@ -106,7 +110,8 @@ function splitByKeywords(content: string, keywords: string[]) {
       const idx = match.index!,
         matchedWord = match[0];
       const after = content.slice(idx + matchedWord.length);
-      segments.push(after, <mark key={idx}>{matchedWord}</mark>);
+      insertLineBreaks(after, segments, idx);
+      segments.push(<mark key={idx}>{matchedWord}</mark>);
       content = content.slice(0, idx);
       return segments;
     },
@@ -121,3 +126,18 @@ function Timestamp({ children: time }: { children: number }) {
   const formattedTime = useMemo(() => formatDuration(time), [time]);
   return <span>{formattedTime}</span>;
 }
+
+function insertLineBreaks(
+  content: string,
+  segments: React.ReactNode[],
+  idx: number,
+): void {
+  if (content.includes("\n")) {
+    content.split("\n").forEach((line, i) => {
+      if (i === 0) segments.push(line);
+      else segments.push(<br key={`${idx}seg-${i}`} />, line);
+    });
+  } else {
+    segments.push(content);
+  }
+}

apps/app/src/components/transcript/line.tsx

@@ -1,17 +1,17 @@
 import { assertNever } from "assert-never";
 import { useMemo, useRef, useState } from "react";
-import type { CueSearchResult } from "@/transcript/context";
+import type { CueSearchResult } from "@/transcript/view/context";
 import {
   useSearch,
   useSearchBox,
   useTranscriptViewStore,
-} from "@/transcript/context";
+} from "@/transcript/view/context";
 import type { CueLineListRef } from "./cue-line-list";
 import { CueLineList } from "./cue-line-list";
 import { SearchInput } from "./search-input";
 
 export default function Lines() {
-  const cues = useTranscriptViewStore((s) => s.captions?.result.cues || []);
+  const cues = useTranscriptViewStore((s) => s.captions?.content.cues || []);
   const cueHash = useMemo(
     () => new Map(cues.map((cue, index) => [cue.id, { cue, index }])),
     [cues],

apps/app/src/components/use-source.tsx

@@ -1,14 +1,13 @@
 import type { PlayerSrc } from "@vidstack/react";
 import { useMemo } from "react";
+import { getMediaInfoID, isFileMediaInfo } from "@/info/media-info";
 import { encodeWebpageUrl } from "@/lib/remote-player/encode";
-import { toInfoKey } from "@/media-note/note-index/def";
-import { isFileMediaInfo } from "../info/media-info";
 import { useApp, useMediaViewStore } from "./context";
 
 export function useSource() {
   const mediaInfo = useMediaViewStore((s) => s.source?.url);
   const { vault } = useApp();
-  const mediaInfoKey = mediaInfo ? toInfoKey(mediaInfo) : null;
+  const mediaInfoKey = mediaInfo ? getMediaInfoID(mediaInfo) : null;
   const src = useMemo(() => {
     if (!mediaInfo) return;
     if (isFileMediaInfo(mediaInfo)) {