feat: add tts (#215)

* feat: add tts

* fix: pnpm-lock

* fix: types

* feat: play local audio file without duration

package.json

@@ -70,6 +70,7 @@
     "@tanstack/react-query-persist-client": "5.52.0",
     "@use-gesture/react": "10.3.1",
     "@yornaath/batshit": "0.10.1",
+    "bufferutil": "4.0.8",
     "builder-util-runtime": "9.2.5-alpha.3",
     "class-variance-authority": "0.7.0",
     "click-to-react-component": "1.1.0",
@@ -88,13 +89,15 @@
     "franc-min": "6.2.0",
     "fuse.js": "7.0.0",
     "hast-util-to-jsx-runtime": "2.3.0",
+    "hast-util-to-text": "4.0.2",
     "idb-keyval": "6.2.1",
     "immer": "10.1.1",
     "jotai": "2.9.3",
     "lethargy": "1.0.9",
     "linkedom": "^0.18.4",
     "lodash-es": "4.17.21",
     "lowdb": "7.0.1",
+    "msedge-tts": "1.3.4",
     "nanoid": "5.0.7",
     "ofetch": "1.3.4",
     "path-to-regexp": "7.1.0",
@@ -127,6 +130,7 @@
     "unified": "11.0.5",
     "use-context-selector": "2.0.0",
     "usehooks-ts": "3.1.0",
+    "utf-8-validate": "6.0.4",
     "vfile": "6.0.2",
     "vscode-languagedetection": "npm:@vscode/vscode-languagedetection@^1.0.22",
     "zod": "3.23.8",

src/main/tipc/reader.ts

@@ -1,9 +1,16 @@
+import fs from "node:fs"
 import { createRequire } from "node:module"
+import path from "node:path"
+
+import { app } from "electron"
+import { MsEdgeTTS, OUTPUT_FORMAT } from "msedge-tts"
 
 import { readability } from "../lib/readability"
 import { t } from "./_instance"
 
 const require = createRequire(import.meta.url)
+const tts = new MsEdgeTTS()
+
 export const readerRoute = {
   readability: t.procedure
     .input<{ url: string }>()
@@ -17,6 +24,40 @@ export const readerRoute = {
 
       return result
     }),
+
+  tts: t.procedure
+    .input<{
+      id: string
+      text: string
+    }>()
+    .action(async ({ input }) => {
+      const { id, text } = input
+
+      if (!text) {
+        return null
+      }
+
+      const filePath = path.join(app.getPath("userData"), `${id}.webm`)
+      if (fs.existsSync(filePath)) {
+        return filePath
+      } else {
+        await tts.toFile(filePath, text)
+        return filePath
+      }
+    }),
+
+  getVoices: t.procedure
+    .action(async () => {
+      const voices = await tts.getVoices()
+      return voices
+    }),
+
+  setVoice: t.procedure
+    .input<string>()
+    .action(async ({ input }) => {
+      await tts.setMetadata(input, OUTPUT_FORMAT.WEBM_24KHZ_16BIT_MONO_OPUS)
+    }),
+
   detectCodeStringLanguage: t.procedure
     .input<{ codeString: string }>()
     .action(async ({ input }) => {

src/main/window.ts

@@ -7,7 +7,7 @@ import { imageRefererMatches } from "@shared/image"
 import type { BrowserWindowConstructorOptions } from "electron"
 import { BrowserWindow, Menu, shell } from "electron"
 
-import { isMacOS, isWindows11 } from "./env"
+import { isDev, isMacOS, isWindows11 } from "./env"
 import { getIconPath } from "./helper"
 import { store } from "./lib/store"
 import { logger } from "./logger"
@@ -42,6 +42,7 @@ export function createWindow(
       preload: path.join(__dirname, "../preload/index.mjs"),
       sandbox: false,
       webviewTag: true,
+      webSecurity: !isDev,
     },
   }
 

src/renderer/src/atoms/player.ts

@@ -92,14 +92,17 @@ export const AudioPlayer = {
         currentTime: this.audio.currentTime,
       })
     }, 1000)
+    if (Number.isNaN(this.audio.duration) || this.audio.duration === Infinity) {
+      this.audio.currentTime = 0
+    }
 
     const currentActionId = this.__currentActionId
     return this.audio.play().then(() => {
       if (currentActionId !== this.__currentActionId) return
       setAudioPlayerAtomValue({
         ...getAudioPlayerAtomValue(),
         status: "playing",
-        duration: this.audio.duration,
+        duration: this.audio.duration === Infinity ? 0 : this.audio.duration,
       })
     })
   },

src/renderer/src/atoms/settings/ui.ts

@@ -2,7 +2,7 @@ import type { UISettings } from "@shared/interface/settings"
 
 import { createSettingAtom } from "./helper"
 
-const createDefaultSettings = (): UISettings => ({
+export const createDefaultSettings = (): UISettings => ({
   // Sidebar
   entryColWidth: 356,
   feedColWidth: 256,
@@ -30,6 +30,9 @@ const createDefaultSettings = (): UISettings => ({
 
   // View
   pictureViewMasonry: true,
+
+  // TTS
+  voice: "en-US-AndrewMultilingualNeural",
 })
 
 export const {

src/renderer/src/components/ui/markdown/Markdown.tsx

@@ -38,7 +38,7 @@ export const HTML: Component<
   const stableRemarkOptions = useState({ renderInlineStyle })[0]
 
   const markdownElement = useMemo(
-    () => children && parseHtml(children, { ...stableRemarkOptions }).content,
+    () => children && parseHtml(children, { ...stableRemarkOptions }).toContent(),
     [children, stableRemarkOptions],
   )
 

src/renderer/src/constants/shortcuts.ts

@@ -69,6 +69,10 @@ export const shortcuts = {
       key: "B",
       extra: "Double Click",
     },
+    tts: {
+      name: "Play TTS",
+      key: "Shift+Meta+V",
+    },
     copyLink: {
       name: "Copy Link",
       key: "Shift+Meta+C",

src/renderer/src/hooks/biz/useEntryActions.tsx

@@ -1,3 +1,4 @@
+import { AudioPlayer, getAudioPlayerAtomValue } from "@renderer/atoms/player"
 import {
   getReadabilityStatus,
   isInReadability,
@@ -12,6 +13,7 @@ import { COPY_MAP, views } from "@renderer/constants"
 import { shortcuts } from "@renderer/constants/shortcuts"
 import { tipcClient } from "@renderer/lib/client"
 import { nextFrame } from "@renderer/lib/dom"
+import { parseHtml } from "@renderer/lib/parse-html"
 import { cn, getOS } from "@renderer/lib/utils"
 import type { CombinedEntryModel } from "@renderer/models"
 import { useTipModal } from "@renderer/modules/wallet/hooks"
@@ -22,7 +24,7 @@ import { useMutation, useQuery } from "@tanstack/react-query"
 import type { FetchError } from "ofetch"
 import { ofetch } from "ofetch"
 import type { ReactNode } from "react"
-import { useCallback, useMemo } from "react"
+import { useCallback, useMemo, useState } from "react"
 import { toast } from "sonner"
 
 export const useEntryReadabilityToggle = ({
@@ -145,6 +147,9 @@ export const useEntryActions = ({
     id: populatedEntry?.entries.id ?? "",
     url: populatedEntry?.entries.url ?? "",
   })
+
+  const [ttsLoading, setTtsLoading] = useState(false)
+
   const items = useMemo(() => {
     if (!populatedEntry || view === undefined) return []
     const items: {
@@ -212,6 +217,35 @@ export const useEntryActions = ({
           window.open(populatedEntry.entries.url, "_blank")
         },
       },
+      {
+        key: "tts",
+        name: "Play TTS",
+        shortcut: shortcuts.entry.tts.key,
+        className: ttsLoading ? "i-mgc-loading-3-cute-re animate-spin" : "i-mgc-voice-cute-re",
+        hide: !populatedEntry.entries.content,
+        onClick: async () => {
+          if (ttsLoading) return
+          if (!populatedEntry.entries.content) return
+          setTtsLoading(true)
+          if (getAudioPlayerAtomValue().entryId === populatedEntry.entries.id) {
+            AudioPlayer.togglePlayAndPause()
+          } else {
+            const filePath = await tipcClient?.tts({
+              id: populatedEntry.entries.id,
+              text: (await parseHtml(populatedEntry.entries.content)).toText(),
+            })
+            if (filePath) {
+              AudioPlayer.mount({
+                type: "audio",
+                entryId: populatedEntry.entries.id,
+                src: `file://${filePath}`,
+                currentTime: 0,
+              })
+            }
+          }
+          setTtsLoading(false)
+        },
+      },
       {
         name: "Readability",
         className: cn(

src/renderer/src/lib/parse-html.ts

@@ -9,6 +9,7 @@ import { BlockError } from "@renderer/components/ui/markdown/renderers/BlockErro
 import { Media } from "@renderer/components/ui/media"
 import type { Components } from "hast-util-to-jsx-runtime"
 import { toJsxRuntime } from "hast-util-to-jsx-runtime"
+import { toText } from "hast-util-to-text"
 import { createElement } from "react"
 import { Fragment, jsx, jsxs } from "react/jsx-runtime"
 import { renderToString } from "react-dom/server"
@@ -54,7 +55,7 @@ export const parseHtml = (
   const hastTree = pipeline.runSync(tree, file)
 
   return {
-    content: toJsxRuntime(hastTree, {
+    toContent: () => toJsxRuntime(hastTree, {
       Fragment,
       ignoreInvalidStyle: true,
       jsx: (type, props, key) => jsx(type as any, props, key),
@@ -149,6 +150,7 @@ export const parseHtml = (
           ),
       },
     }),
+    toText: () => toText(hastTree),
   }
 }
 

src/renderer/src/modules/feed-column/corner-player.tsx

@@ -111,7 +111,7 @@ const CornerPlayerImpl = () => {
     <>
       <div className="relative flex border-y bg-white transition-all duration-200 ease-in-out dark:bg-neutral-800">
         {/* play cover */}
-        <div className="relative h-full">
+        <div className="relative h-full shrink-0">
           <FeedIcon
             feed={feed}
             entry={entry.entries}
@@ -227,47 +227,53 @@ const PlayerProgress = () => {
     .startOf("y")
     .second(controlledCurrentTime)
     .format(controlledCurrentTime > ONE_HOUR_IN_SECONDS ? "H:mm:ss" : "m:ss")
-  const remainingTimeIndicator = dayjs()
-    .startOf("y")
-    .second(duration - controlledCurrentTime)
-    .format(
-      duration - controlledCurrentTime > ONE_HOUR_IN_SECONDS ?
-        "H:mm:ss" :
-        "m:ss",
-    )
+  const remainingTimeIndicator = duration ?
+    dayjs()
+      .startOf("y")
+      .second(duration - controlledCurrentTime)
+      .format(
+        duration - controlledCurrentTime > ONE_HOUR_IN_SECONDS ?
+          "H:mm:ss" :
+          "m:ss",
+      ) :
+    null
 
   return (
     <div className="relative mt-2">
       <div className="absolute bottom-1 flex w-full items-center justify-between text-theme-disabled opacity-0 duration-200 ease-in-out group-hover:opacity-100">
         <div className="text-xs">{currentTimeIndicator}</div>
-        <div className="text-xs">
-          -
-          {remainingTimeIndicator}
-        </div>
+        {!!remainingTimeIndicator && (
+          <div className="text-xs">
+            -
+            {remainingTimeIndicator}
+          </div>
+        )}
       </div>
 
       {/* slider */}
-      <Slider.Root
-        className="relative flex h-1 w-full items-center transition-all duration-200 ease-in-out"
-        min={0}
-        max={duration}
-        step={1}
-        value={[controlledCurrentTime]}
-        onPointerDown={() => setIsDraggingProgress(true)}
-        onPointerUp={() => setIsDraggingProgress(false)}
-        onValueChange={(value) => setControlledCurrentTime(value[0])}
-        onValueCommit={(value) => AudioPlayer.seek(value[0])}
-      >
-        <Slider.Track className="relative h-1 w-full grow rounded bg-gray-200 duration-200 group-hover:bg-gray-300 dark:bg-neutral-700 group-hover:dark:bg-neutral-600">
-          <Slider.Range className="absolute h-1 rounded bg-theme-accent-400 dark:bg-theme-accent-700" />
-        </Slider.Track>
+      {!!duration && (
+        <Slider.Root
+          className="relative flex h-1 w-full items-center transition-all duration-200 ease-in-out"
+          min={0}
+          max={duration}
+          step={1}
+          value={[controlledCurrentTime]}
+          onPointerDown={() => setIsDraggingProgress(true)}
+          onPointerUp={() => setIsDraggingProgress(false)}
+          onValueChange={(value) => setControlledCurrentTime(value[0])}
+          onValueCommit={(value) => AudioPlayer.seek(value[0])}
+        >
+          <Slider.Track className="relative h-1 w-full grow rounded bg-gray-200 duration-200 group-hover:bg-gray-300 dark:bg-neutral-700 group-hover:dark:bg-neutral-600">
+            <Slider.Range className="absolute h-1 rounded bg-theme-accent-400 dark:bg-theme-accent-700" />
+          </Slider.Track>
 
-        {/* indicator */}
-        <Slider.Thumb
-          className="block h-2 w-[3px] rounded-[1px] bg-accent"
-          aria-label="Progress"
-        />
-      </Slider.Root>
+          {/* indicator */}
+          <Slider.Thumb
+            className="block h-2 w-[3px] rounded-[1px] bg-accent"
+            aria-label="Progress"
+          />
+        </Slider.Root>
+      )}
     </div>
   )
 }