Feat auto download transcription (#1067)

* cal md5 for transcription result

* fix style

* auto download transcription from cloud

* refactor

* update style

enjoy/src/i18n/en.json

@@ -791,5 +791,6 @@
  "exportRecordingsConfirmation": "Select the highest score of the recordings of each segment to export as a single file.",
  "exportRecordingsSuccess": "Export recordings successfully",
  "upgrade": "Upgrade",
- "upgradeNotice": "Enjoy App v{{version}} is available. Please upgrade to the latest version."
+ "upgradeNotice": "Enjoy App v{{version}} is available. Please upgrade to the latest version.",
+ "downloadedTranscriptionFromCloud": "Downloaded transcription from cloud"
 }

enjoy/src/i18n/zh-CN.json

@@ -791,5 +791,6 @@
  "exportRecordingsConfirmation": "选择每个段落最高分的录音，导出为单个文件。",
  "exportRecordingsSuccess": "导出录音成功",
  "upgrade": "升级",
- "upgradeNotice": "Enjoy App v{{version}} 已发布。请升级到最新版本。"
+ "upgradeNotice": "Enjoy App v{{version}} 已发布。请升级到最新版本。",
+ "downloadedTranscriptionFromCloud": "从云端下载了语音文本"
 }

enjoy/src/main/db/models/transcription.ts

@@ -18,6 +18,7 @@ import { Client } from "@/api";
 import { PROCESS_TIMEOUT } from "@/constants";
 import settings from "@main/settings";
 import { AlignmentResult } from "echogarden/dist/api/Alignment";
+import { createHash } from "crypto";
 
 const logger = log.scope("db/models/transcription");
 @Table({
@@ -70,6 +71,13 @@ export class Transcription extends Model<Transcription> {
  @BelongsTo(() => Video, { foreignKey: "targetId", constraints: false })
  video: Video;
 
+ @Column(DataType.VIRTUAL)
+ get md5(): string {
+ // Calculate md5 of result
+ if (!this.result) return null;
+ return createHash("md5").update(JSON.stringify(this.result)).digest("hex");
+ }
+
  @Column(DataType.VIRTUAL)
  get isSynced(): boolean {
  return Boolean(this.syncedAt) && this.syncedAt >= this.updatedAt;

enjoy/src/renderer/components/messages/assistant-message.tsx

@@ -302,7 +302,7 @@ export const AssistantMessageComponent = (props: {
  onPointerDownOutside={(event) => event.preventDefault()}
  onInteractOutside={(event) => event.preventDefault()}
  >
- <SheetHeader className="flex items-center justify-center h-12">
+ <SheetHeader className="flex items-center justify-center">
  <SheetTitle className="sr-only">{t("shadow")}</SheetTitle>
  <SheetClose>
  <ChevronDownIcon />

enjoy/src/renderer/components/transcriptions/transcriptions-list.tsx

@@ -12,7 +12,6 @@ import {
  toast,
 } from "@renderer/components/ui";
 import { t } from "i18next";
-import { formatDateTime } from "@renderer/lib/utils";
 import {
  CheckCircleIcon,
  ChevronLeftIcon,
@@ -93,10 +92,10 @@ export const TranscriptionsList = (props: {
  <TableHeader>
  <TableRow>
  <TableHead>ID</TableHead>
- <TableHead>{t("model")}</TableHead>
- <TableHead>{t("language")}</TableHead>
- <TableHead>{t("date")}</TableHead>
- <TableHead>{t("actions")}</TableHead>
+ <TableHead className="capitalize">{t("model")}</TableHead>
+ <TableHead className="capitalize">{t("language")}</TableHead>
+ <TableHead className="capitalize">{t("downloads")}</TableHead>
+ <TableHead className="capitalize">{t("actions")}</TableHead>
  </TableRow>
  </TableHeader>
  <TableBody>
@@ -113,7 +112,7 @@ export const TranscriptionsList = (props: {
  <span className="text-sm">{tr.language || "-"}</span>
  </TableCell>
  <TableCell>
- <span className="text-xs">{formatDateTime(tr.createdAt)}</span>
+ <span className="text-xs">{tr.downloadsCount}</span>
  </TableCell>
  <TableCell>
  {transcription?.id === tr.id ? (

enjoy/src/renderer/context/chat-session-provider.tsx

@@ -389,7 +389,7 @@ export const ChatSessionProvider = ({
  onPointerDownOutside={(event) => event.preventDefault()}
  onInteractOutside={(event) => event.preventDefault()}
  >
- <SheetHeader className="flex items-center justify-center h-12">
+ <SheetHeader className="flex items-center justify-center">
  <SheetTitle className="sr-only">Shadow</SheetTitle>
  <SheetDescription className="sr-only"></SheetDescription>
  <SheetClose>

enjoy/src/renderer/context/course-provider.tsx

@@ -77,7 +77,7 @@ export const CourseProvider = ({
  onPointerDownOutside={(event) => event.preventDefault()}
  onInteractOutside={(event) => event.preventDefault()}
  >
- <SheetHeader className="flex items-center justify-center h-12">
+ <SheetHeader className="flex items-center justify-center">
  <SheetTitle className="sr-only">Shadow</SheetTitle>
  <SheetDescription className="sr-only"></SheetDescription>
  <SheetClose>

enjoy/src/renderer/hooks/use-transcriptions.tsx

@@ -9,17 +9,22 @@ import { toast } from "@renderer/components/ui";
 import { TimelineEntry } from "echogarden/dist/utilities/Timeline.d.js";
 import { MAGIC_TOKEN_REGEX, END_OF_SENTENCE_REGEX } from "@/constants";
 import { SttEngineOptionEnum } from "@/types/enums";
+import { t } from "i18next";
 
 export const useTranscriptions = (media: AudioType | VideoType) => {
  const { sttEngine } = useContext(AISettingsProviderContext);
- const { EnjoyApp, learningLanguage } = useContext(AppSettingsProviderContext);
+ const { EnjoyApp, learningLanguage, webApi } = useContext(
+ AppSettingsProviderContext
+ );
  const { addDblistener, removeDbListener } = useContext(DbProviderContext);
  const [transcription, setTranscription] = useState<TranscriptionType>(null);
  const { transcribe, output } = useTranscribe();
  const [transcribingProgress, setTranscribingProgress] = useState<number>(0);
  const [transcribing, setTranscribing] = useState<boolean>(false);
  const [transcribingOutput, setTranscribingOutput] = useState<string>("");
- const [service, setService] = useState<SttEngineOptionEnum | "upload">(sttEngine);
+ const [service, setService] = useState<SttEngineOptionEnum | "upload">(
+ sttEngine
+ );
 
  const onTransactionUpdate = (event: CustomEvent) => {
  if (!transcription) return;
@@ -38,25 +43,61 @@ export const useTranscriptions = (media: AudioType | VideoType) => {
  if (!media) return;
  if (transcription?.targetId === media.id) return;
 
- return EnjoyApp.transcriptions
- .findOrCreate({
- targetId: media.id,
- targetType: media.mediaType,
- })
- .then((t) => {
- if (t.result && !t.result["timeline"]) {
- t.result = {
- originalText: t.result?.originalText,
- };
- }
- setTranscription(t);
- return t;
- })
- .catch((err) => {
- toast.error(err.message);
- });
+ const tr = await EnjoyApp.transcriptions.findOrCreate({
+ targetId: media.id,
+ targetType: media.mediaType,
+ });
+
+ if (tr.result && !tr.result["timeline"]) {
+ tr.result = {
+ originalText: tr.result?.originalText,
+ };
+ }
+
+ const transcriptionOnline = await findTranscriptionOnline();
+ if (transcriptionOnline && !tr.result["timeline"]) {
+ return EnjoyApp.transcriptions
+ .update(tr.id, {
+ state: "finished",
+ result: transcriptionOnline.result,
+ engine: transcriptionOnline.engine,
+ model: transcriptionOnline.model,
+ language: transcriptionOnline.language || media.language,
+ })
+ .then(() => {
+ toast.success(t("downloadedTranscriptionFromCloud"));
+ setTranscription(transcriptionOnline);
+ return transcriptionOnline;
+ })
+ .catch((err) => {
+ console.error(err);
+ return tr;
+ });
+ } else {
+ setTranscription(tr);
+ return tr;
+ }
  };
 
+ const findTranscriptionOnline = async () => {
+ if (!media) return;
+
+ try {
+ const result = await webApi.transcriptions({
+ targetMd5: media.md5,
+ items: 10,
+ });
+ if (result.transcriptions.length) {
+ return result.transcriptions[0];
+ } else {
+ return null;
+ }
+ } catch (err) {
+ console.error(err);
+ return null;
+ }
+ };
+
  const generateTranscription = async (params?: {
  originalText?: string;
  language?: string;

enjoy/src/types/transcription.d.ts

@@ -8,6 +8,8 @@ type TranscriptionType = {
  model: string;
  language?: string;
  result: AlignmentResult & { original?: string };
+ md5?: string;
+ downloadsCount?: number;
  createdAt: string;
  updatedAt: string;
 };