feat(webview): bilibili caption support

apps/app/src/components/context.tsx

@@ -55,10 +55,7 @@ export interface MediaViewState {
   hash: HashProps;
   setHash: (hash: string) => void;
   getPlayer(): Promise<MediaPlayerInstance>;
-  loadFile(
-    file: TFile,
-    ctx: { vault: Vault; subpath?: string; defaultLang: string },
-  ): Promise<void>;
+  loadFile(file: TFile, ctx: { vault: Vault; subpath?: string }): Promise<void>;
   setSource(url: MediaURL, other: SourceFacet): void;
   transform: Partial<TransformConfig> | null;
   setTransform: (transform: Partial<TransformConfig> | null) => void;
@@ -118,8 +115,8 @@ export function createMediaViewStore() {
       set((og) => ({ hash: { ...og.hash, ...parseHashProps(hash) } }));
       applyTempFrag(get());
     },
-    async loadFile(file, { vault, subpath, defaultLang }) {
-      const textTracks = await getTracksInVault(file, vault, defaultLang);
+    async loadFile(file, { vault, subpath }) {
+      const textTracks = await getTracksInVault(file, vault);
       const url = mediaInfoFromFile(file, subpath ?? "");
       if (!url) {
         throw new Error("Invalid media file: " + file.path);

apps/app/src/components/player.tsx

@@ -16,16 +16,17 @@ import { AudioLayout } from "./player/layouts/audio-layout";
 import { VideoLayout } from "./player/layouts/video-layout";
 import { MediaProviderEnhanced } from "./provider";
 import { useSource } from "./use-source";
-import { useRemoteTracks } from "./use-tracks";
+import { useBilibiliTextTracks, useRemoteTextTracks } from "./use-tracks";
 
 function HookLoader({
   onViewTypeChange,
 }: {
   onViewTypeChange: (viewType: "audio" | "unknown") => any;
 }) {
   useViewTypeDetect(onViewTypeChange);
-  useRemoteTracks();
+  useRemoteTextTracks();
   useTempFragHandler();
+  useBilibiliTextTracks();
   useDefaultVolume();
   return <></>;
 }

apps/app/src/components/provider.tsx

@@ -13,7 +13,7 @@ import { WebviewProviderLoader } from "@/lib/remote-player/loader";
 import { cn } from "@/lib/utils";
 import { useApp, useMediaViewStore } from "./context";
 import { useControls } from "./hook/use-hash";
-import { useTracks } from "./use-tracks";
+import { useTextTracks } from "./use-tracks";
 import { WebView } from "./webview";
 
 export function MediaProviderEnhanced({
@@ -40,7 +40,7 @@ export function MediaProviderEnhanced({
       }
     });
   const controls = useControls();
-  const tracks = useTracks();
+  const tracks = useTextTracks();
   return (
     <MediaProvider
       className={cn(

apps/app/src/components/use-tracks.tsx

@@ -10,11 +10,13 @@ import type {
 } from "@vidstack/react";
 import { Notice } from "obsidian";
 import { useEffect, useMemo, useState } from "react";
+import { MediaURL } from "@/info/media-url";
+import { MediaHost } from "@/info/supported";
 import { setDefaultLang } from "@/lib/lang/default-lang";
 import { WebiviewMediaProvider } from "@/lib/remote-player/provider";
-import { useMediaViewStore, useSettings } from "./context";
+import { useMediaViewStore, usePlugin, useSettings } from "./context";
 
-export function useRemoteTracks() {
+export function useRemoteTextTracks() {
   // const externalTextTracks = useMediaViewStore(({ textTracks }) => textTracks);
   const player = useMediaPlayer();
   const loaded = useMediaState("canPlay");
@@ -61,29 +63,55 @@ export function useRemoteTracks() {
   }, [player, loaded]);
 }
 
-export function useTracks() {
+export function useBilibiliTextTracks() {
+  const isBilibili = useMediaViewStore(
+    ({ source }) =>
+      source?.url instanceof MediaURL && source.url.type === MediaHost.Bilibili,
+  );
+  const plugin = usePlugin();
+  const provider = useMediaProvider();
+  const canPlay = useMediaState("canPlay");
+  useEffect(() => {
+    if (!isBilibili || !(provider instanceof WebiviewMediaProvider) || !canPlay)
+      return;
+    provider.media.methods.bili_getManifest().then(async (manifest) => {
+      try {
+        const reqUrl = await plugin.biliReq.getPlayerV2Request(manifest);
+        provider.media.send("mx-bili-player-v2-url", reqUrl);
+      } catch (e) {
+        console.error("Failed to get player V2 API for bilibili", e);
+      }
+    });
+  }, [plugin.biliReq, provider, isBilibili, canPlay]);
+}
+
+export function useTextTracks() {
   const localTextTracks = useMediaViewStore(({ textTracks }) => textTracks);
   const [remoteTextTracks, setRemoteTracks] = useState<
     (TextTrackInit & {
       id: string;
     })[]
   >([]);
-  const loaded = useMediaState("canPlay");
   const defaultLang = useSettings((s) => s.defaultLanguage);
+  const getDefaultLang = useSettings((s) => s.getDefaultLang);
 
   const provider = useMediaProvider();
   useEffect(() => {
-    if (!(provider instanceof WebiviewMediaProvider) || !loaded) return;
-    provider.media.methods.getTracks().then((tracks): void => {
+    if (!(provider instanceof WebiviewMediaProvider)) return;
+    return provider.media.on("mx-text-tracks", ({ payload: { tracks } }) => {
       setRemoteTracks(
         tracks.map(({ src, ...t }) => ({ ...t, content: dummyVTTContent })),
       );
       if (tracks.length !== 0) console.debug("Remote tracks loaded", tracks);
     });
-  }, [provider, loaded]);
+  }, [provider]);
   return useMemo(
     () =>
-      setDefaultLang([...localTextTracks, ...remoteTextTracks], defaultLang),
+      setDefaultLang(
+        [...localTextTracks, ...remoteTextTracks],
+        getDefaultLang(),
+      ),
+    // eslint-disable-next-line react-hooks/exhaustive-deps
     [localTextTracks, remoteTextTracks, defaultLang],
   );
 }

apps/app/src/lib/message/index.ts

@@ -137,9 +137,14 @@ export class MessageController<
     ) =>
       | Promise<Awaited<ReturnType<InvokeHandlers[A]>>>
       | Awaited<ReturnType<InvokeHandlers[A]>>,
-  ): void;
-  handle(action: string, callback: (...args: any[]) => any): void {
+  ): () => void;
+  handle(action: string, callback: (...args: any[]) => any): () => void {
     this.actions[action] = callback;
+    return () => {
+      if (this.actions[action] === callback) {
+        delete this.actions[action];
+      }
+    };
   }
 
   send<A extends keyof Sends>(

apps/app/src/lib/remote-player/hook/handler-register.ts

@@ -1,13 +1,13 @@
 import pLimit from "p-limit";
 import { captureScreenshot } from "@/lib/screenshot";
-import type MediaPlugin from "../lib/plugin";
 import {
   capitalize,
   mediaActionProps,
   mediaReadonlyStateProps,
   mediaWritableStateProps,
   serializeMediaStatePropValue,
-} from "../type";
+} from "../interface";
+import type MediaPlugin from "../lib/plugin";
 
 export interface MediaStateRef {
   prevSeek: {
@@ -21,78 +21,99 @@ export function registerHandlers(this: MediaPlugin) {
   const port = this.controller;
   const ref = this.stateRef;
   mediaReadonlyStateProps.forEach((prop) => {
-    port.handle(`get${capitalize(prop)}`, () => ({
-      value: serializeMediaStatePropValue(player[prop]),
-    }));
-  });
-  port.handle("getTracks", async () => ({ value: await this.getTracks() }));
-  port.handle("getTrack", async (id) => ({ value: await this.getTrack(id) }));
-  port.handle("pictureInPictureEnabled", () => {
-    return { value: document.pictureInPictureElement === player };
-  });
-  port.handle("requestPictureInPicture", () => {
-    if (player instanceof HTMLVideoElement) player.requestPictureInPicture();
-  });
-  port.handle("exitPictureInPicture", () => {
-    document.exitPictureInPicture();
+    this.register(
+      port.handle(`get${capitalize(prop)}`, () => ({
+        value: serializeMediaStatePropValue(player[prop]),
+      })),
+    );
   });
+  this.register(
+    port.handle("getTrack", async (id) => ({ value: await this.getTrack(id) })),
+  );
+  this.register(
+    port.handle("pictureInPictureEnabled", () => {
+      return { value: document.pictureInPictureElement === player };
+    }),
+  );
+  this.register(
+    port.handle("requestPictureInPicture", () => {
+      if (player instanceof HTMLVideoElement) player.requestPictureInPicture();
+    }),
+  );
+  this.register(
+    port.handle("exitPictureInPicture", () => {
+      document.exitPictureInPicture();
+    }),
+  );
   mediaWritableStateProps.forEach((prop) => {
-    port.handle(`get${capitalize(prop)}`, () => ({
-      value: serializeMediaStatePropValue(player[prop]),
-    }));
+    this.register(
+      port.handle(`get${capitalize(prop)}`, () => ({
+        value: serializeMediaStatePropValue(player[prop]),
+      })),
+    );
     if (prop === "currentTime") {
-      port.handle(`set${capitalize(prop)}`, (val) => {
-        ref.prevSeek = {
-          value: player.currentTime,
-          time: Date.now(),
-        };
-        (player as any)[prop] = val;
-      });
+      this.register(
+        port.handle(`set${capitalize(prop)}`, (val) => {
+          ref.prevSeek = {
+            value: player.currentTime,
+            time: Date.now(),
+          };
+          (player as any)[prop] = val;
+        }),
+      );
     } else {
-      port.handle(`set${capitalize(prop)}`, (val) => {
-        (player as any)[prop] = val;
-      });
+      this.register(
+        port.handle(`set${capitalize(prop)}`, (val) => {
+          (player as any)[prop] = val;
+        }),
+      );
     }
   });
   mediaActionProps.forEach((prop) => {
-    port.handle(prop, async (...args) => ({
-      value: await (player as any)[prop](...args),
-    }));
+    this.register(
+      port.handle(prop, async (...args) => ({
+        value: await (player as any)[prop](...args),
+      })),
+    );
   });
-  port.handle("screenshot", async (type, quality) => {
-    if (!(player instanceof HTMLVideoElement))
-      throw new Error("Cannot take screenshot of non-video element");
+  this.register(
+    port.handle("screenshot", async (type, quality) => {
+      if (!(player instanceof HTMLVideoElement))
+        throw new Error("Cannot take screenshot of non-video element");
 
-    const value = await captureScreenshot(player, type, quality);
-    return {
-      value,
-      transfer: [value.blob.arrayBuffer],
-    };
-  });
+      const value = await captureScreenshot(player, type, quality);
+      return {
+        value,
+        transfer: [value.blob.arrayBuffer],
+      };
+    }),
+  );
   const reqLimit = pLimit(4);
-  port.handle("fetch", async (url, { gzip = false, ...init } = {}) => {
-    const resp = await reqLimit(() => window.fetch(url, init));
-    const blob = await resp.blob();
-    const head = {
-      type: blob.type,
-      respHeaders: Object.fromEntries(resp.headers),
-    };
-    if (!gzip) {
-      const ab = await blob.arrayBuffer();
+  this.register(
+    port.handle("fetch", async (url, { gzip = false, ...init } = {}) => {
+      const resp = await reqLimit(() => window.fetch(url, init));
+      const blob = await resp.blob();
+      const head = {
+        type: blob.type,
+        respHeaders: Object.fromEntries(resp.headers),
+      };
+      if (!gzip) {
+        const ab = await blob.arrayBuffer();
+        return {
+          value: { ab, gzip: false, ...head },
+          transfer: [ab],
+        };
+      }
+      const stream = blob.stream() as unknown as ReadableStream<Uint8Array>;
+      const ab = await streamToArrayBuffer(
+        stream.pipeThrough(new CompressionStream("gzip")),
+      );
       return {
-        value: { ab, gzip: false, ...head },
+        value: { ab, gzip: true, ...head },
         transfer: [ab],
       };
-    }
-    const stream = blob.stream() as unknown as ReadableStream<Uint8Array>;
-    const ab = await streamToArrayBuffer(
-      stream.pipeThrough(new CompressionStream("gzip")),
-    );
-    return {
-      value: { ab, gzip: true, ...head },
-      transfer: [ab],
-    };
-  });
+    }),
+  );
   return ref;
 }
 

apps/app/src/lib/remote-player/html–media-events.ts

@@ -7,8 +7,11 @@ import { useDisposalBin } from "maverick.js/std";
 // import { RAFLoop } from "../../foundation/observers/raf-loop";
 // import { getNumberOfDecimalPlaces } from "../../utils/number";
 import type { EventPayload } from "../message";
+import {
+  deserializeMediaStatePropValue,
+  type MediaEventMap,
+} from "./interface";
 import type { WebiviewMediaProvider } from "./provider";
-import { deserializeMediaStatePropValue, type MediaEventMap } from "./type";
 
 declare global {
   // eslint-disable-next-line no-var

apps/app/src/lib/remote-player/type.ts → apps/app/src/lib/remote-player/interface.ts

@@ -203,20 +203,21 @@ export type MsgCtrlRemote = MessageController<
     bili_getManifest(): {
       value: BilibiliPlayerManifest;
     };
-    getTracks(): { value: (TextTrackInit & { id: string })[] };
     getTrack(id: string): { value: VTTContent | null };
   },
   Nil,
   {
     // eslint-disable-next-line @typescript-eslint/naming-convention
     "mx-toggle-controls": boolean;
     "mx-toggle-webfs": boolean;
+    "mx-bili-player-v2-url": string;
   },
   Record<CustomEvent, void> & MediaEventPayloadMap & CustomEventWithPayload
 >;
 
 type CustomEventWithPayload = {
   "mx-open-browser": { url: string; message?: string };
+  "mx-text-tracks": { tracks: (TextTrackInit & { id: string })[] };
 };
 
 export type MsgCtrlLocal = MessageController<
@@ -243,14 +244,14 @@ export type MsgCtrlLocal = MessageController<
     requestPictureInPicture(): void;
     exitPictureInPicture(): void;
     bili_getManifest(): Promise<BilibiliPlayerManifest>;
-    getTracks(): Promise<(TextTrackInit & { id: string })[]>;
     getTrack(id: string): Promise<VTTContent | null>;
   },
   Record<CustomEvent, void> & MediaEventPayloadMap & CustomEventWithPayload,
   {
     // eslint-disable-next-line @typescript-eslint/naming-convention
     "mx-toggle-controls": boolean;
     "mx-toggle-webfs": boolean;
+    "mx-bili-player-v2-url": string;
   }
 >;
 

apps/app/src/lib/remote-player/lib/init-port.ts

@@ -1,4 +1,4 @@
-import type { MsgCtrlRemote } from "@/lib/remote-player/type";
+import type { MsgCtrlRemote } from "@/lib/remote-player/interface";
 import { MessageController } from "../../message";
 import { GET_PORT_TIMEOUT, PORT_MESSAGE } from "../const";
 

apps/app/src/lib/remote-player/lib/load-plugin.ts

@@ -1,4 +1,4 @@
-import type { MsgCtrlRemote } from "../type";
+import type { MsgCtrlRemote } from "../interface";
 import Plugin from "./plugin";
 import { require } from "./require";