diff --git a/web/package-lock.json b/web/package-lock.json
index 510a8c46883..28dd39f178d 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -28,6 +28,7 @@
"jsencrypt": "^3.3.2",
"lodash": "^4.17.21",
"mammoth": "^1.7.2",
+ "openai-speech-stream-player": "^1.0.8",
"rc-tween-one": "^3.0.6",
"react-copy-to-clipboard": "^5.1.0",
"react-force-graph": "^1.44.4",
@@ -20565,6 +20566,11 @@
"node": ">=12"
}
},
+ "node_modules/openai-speech-stream-player": {
+ "version": "1.0.8",
+ "resolved": "https://registry.npmmirror.com/openai-speech-stream-player/-/openai-speech-stream-player-1.0.8.tgz",
+ "integrity": "sha512-0SUybbhStl65s66ezh2QaoZE5k1kNb2t5M8tDOqJFILdHpwHaBqnYy4uHl3Hk/8F5VFWxxHaLamjKOnfNDKgbw=="
+ },
"node_modules/option": {
"version": "0.2.4",
"resolved": "https://registry.npmmirror.com/option/-/option-0.2.4.tgz",
diff --git a/web/package.json b/web/package.json
index eae40b54824..326b2f7b1ae 100644
--- a/web/package.json
+++ b/web/package.json
@@ -39,6 +39,7 @@
"jsencrypt": "^3.3.2",
"lodash": "^4.17.21",
"mammoth": "^1.7.2",
+ "openai-speech-stream-player": "^1.0.8",
"rc-tween-one": "^3.0.6",
"react-copy-to-clipboard": "^5.1.0",
"react-force-graph": "^1.44.4",
diff --git a/web/src/components/message-item/group-button.tsx b/web/src/components/message-item/group-button.tsx
index 6b433bafa71..cbbb8abc27b 100644
--- a/web/src/components/message-item/group-button.tsx
+++ b/web/src/components/message-item/group-button.tsx
@@ -5,6 +5,7 @@ import {
DeleteOutlined,
DislikeOutlined,
LikeOutlined,
+ PauseCircleOutlined,
SoundOutlined,
SyncOutlined,
} from '@ant-design/icons';
@@ -13,7 +14,7 @@ import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import SvgIcon from '../svg-icon';
import FeedbackModal from './feedback-modal';
-import { useRemoveMessage, useSendFeedback } from './hooks';
+import { useRemoveMessage, useSendFeedback, useSpeech } from './hooks';
import PromptModal from './prompt-modal';
interface IProps {
@@ -37,6 +38,7 @@ export const AssistantGroupButton = ({
showModal: showPromptModal,
} = useSetModalState();
const { t } = useTranslation();
+ const { handleRead, ref, isPlaying } = useSpeech(content);
const handleLike = useCallback(() => {
onFeedbackOk({ thumbup: true });
@@ -48,10 +50,11 @@ export const AssistantGroupButton = ({
-
+
-
+ {isPlaying ? : }
+
{showLikeButton && (
<>
diff --git a/web/src/components/message-item/hooks.ts b/web/src/components/message-item/hooks.ts
index 9e9a70f5cb7..1142509f4a3 100644
--- a/web/src/components/message-item/hooks.ts
+++ b/web/src/components/message-item/hooks.ts
@@ -1,9 +1,10 @@
import { useDeleteMessage, useFeedback } from '@/hooks/chat-hooks';
import { useSetModalState } from '@/hooks/common-hooks';
-import { IRemoveMessageById } from '@/hooks/logic-hooks';
+import { IRemoveMessageById, useSpeechWithSse } from '@/hooks/logic-hooks';
import { IFeedbackRequestBody } from '@/interfaces/request/chat';
import { getMessagePureId } from '@/utils/chat';
-import { useCallback } from 'react';
+import { SpeechPlayer } from 'openai-speech-stream-player';
+import { useCallback, useEffect, useRef, useState } from 'react';
export const useSendFeedback = (messageId: string) => {
const { visible, hideModal, showModal } = useSetModalState();
@@ -50,3 +51,52 @@ export const useRemoveMessage = (
return { onRemoveMessage, loading };
};
+
+export const useSpeech = (content: string) => {
+ const ref = useRef(null);
+ const { read } = useSpeechWithSse();
+ const player = useRef();
+ const [isPlaying, setIsPlaying] = useState(false);
+
+ const initialize = useCallback(async () => {
+ player.current = new SpeechPlayer({
+ audio: ref.current!,
+ onPlaying: () => {
+ setIsPlaying(true);
+ },
+ onPause: () => {
+ setIsPlaying(false);
+ },
+ onChunkEnd: () => {},
+ mimeType: 'audio/mpeg',
+ });
+ await player.current.init();
+ }, []);
+
+ const pause = useCallback(() => {
+ player.current?.pause();
+ }, []);
+
+ const speech = useCallback(async () => {
+ const response = await read({ text: content });
+ if (response) {
+ player?.current?.feedWithResponse(response);
+ }
+ }, [read, content]);
+
+ const handleRead = useCallback(async () => {
+ if (isPlaying) {
+ setIsPlaying(false);
+ pause();
+ } else {
+ setIsPlaying(true);
+ speech();
+ }
+ }, [setIsPlaying, speech, isPlaying, pause]);
+
+ useEffect(() => {
+ initialize();
+ }, [initialize]);
+
+ return { ref, handleRead, isPlaying };
+};
diff --git a/web/src/hooks/logic-hooks.ts b/web/src/hooks/logic-hooks.ts
index 6a283855907..80adf5188bb 100644
--- a/web/src/hooks/logic-hooks.ts
+++ b/web/src/hooks/logic-hooks.ts
@@ -278,6 +278,88 @@ export const useSendMessageWithSse = (
return { send, answer, done, setDone };
};
+export const useSpeechWithSse = (url: string = api.tts) => {
+ const read = useCallback(
+ (body: any) => {
+ const response = fetch(url, {
+ method: 'POST',
+ headers: {
+ [Authorization]: getAuthorization(),
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify(body),
+ });
+ return response;
+ },
+ [url],
+ );
+
+ return { read };
+};
+
+export const useFetchAudioWithSse = (url: string = api.tts) => {
+ // const [answer, setAnswer] = useState({} as IAnswer);
+ const [done, setDone] = useState(true);
+
+ const read = useCallback(
+ async (
+ body: any,
+ ): Promise<{ response: Response; data: ResponseType } | undefined> => {
+ try {
+ setDone(false);
+ const response = await fetch(url, {
+ method: 'POST',
+ headers: {
+ [Authorization]: getAuthorization(),
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify(body),
+ });
+
+ const res = response.clone().json();
+
+ const reader = response?.body?.getReader();
+
+ while (true) {
+ const x = await reader?.read();
+ if (x) {
+ const { done, value } = x;
+ try {
+ // const val = JSON.parse(value || '');
+ const val = value;
+ // const d = val?.data;
+ // if (typeof d !== 'boolean') {
+ // console.info('data:', d);
+ // setAnswer({
+ // ...d,
+ // conversationId: body?.conversation_id,
+ // });
+ // }
+ } catch (e) {
+ console.warn(e);
+ }
+ if (done) {
+ console.info('done');
+ break;
+ }
+ }
+ }
+ console.info('done?');
+ setDone(true);
+ // setAnswer({} as IAnswer);
+ return { data: await res, response };
+ } catch (e) {
+ setDone(true);
+ // setAnswer({} as IAnswer);
+ console.warn(e);
+ }
+ },
+ [url],
+ );
+
+ return { read, done, setDone };
+};
+
//#region chat hooks
export const useScrollToBottom = (messages?: unknown) => {