feat: 消息回复语音朗读

src/components/chat-message/MessageRobot.tsx

@@ -14,7 +14,7 @@ import { TAgentDetail } from "@/types/agent";
 
 import Taro from "@tarojs/taro";
 import ThinkAnimation from "../think-animation/index";
-import { dislikeMessage, likeMessage } from "@/service/bot";
+import { dislikeMessage, likeMessage } from "@/service/chat";
 import { EContentType, TMessage } from "@/types/bot";
 import { getLoginId, isSuccess } from "@/utils";
 import { useState } from "react";

src/images/svgs/chat/IconVolumeUp.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M1.66774 8.00074C1.66588 8.82051 1.6293 9.93865 2.13646 10.3566C2.60953 10.7465 2.94247 10.646 3.80613 10.7094C4.67042 10.7734 6.49447 13.314 7.90064 12.5104C8.62605 11.9399 8.67999 10.7441 8.67999 8.00074C8.67999 5.25742 8.62605 4.06155 7.90064 3.49112C6.49447 2.6869 4.67042 5.22812 3.80613 5.2921C2.94247 5.35548 2.60953 5.25503 2.13646 5.64488C1.6293 6.06284 1.66588 7.18097 1.66774 8.00074Z" stroke="#000000" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M13.0562 3.93652C14.7562 6.38387 14.7618 9.61212 13.0562 12.0642" stroke="#000000" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M11.3872 5.54395C12.2614 7.07107 12.2614 8.93603 11.3872 10.4584" stroke="#000000" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

src/images/svgs/chat/IconVolumeUpColor.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M1.66774 8.00074C1.66588 8.82051 1.6293 9.93865 2.13646 10.3566C2.60953 10.7465 2.94247 10.646 3.80613 10.7094C4.67042 10.7734 6.49447 13.314 7.90064 12.5104C8.62605 11.9399 8.67999 10.7441 8.67999 8.00074C8.67999 5.25742 8.62605 4.06155 7.90064 3.49112C6.49447 2.6869 4.67042 5.22812 3.80613 5.2921C2.94247 5.35548 2.60953 5.25503 2.13646 5.64488C1.6293 6.06284 1.66588 7.18097 1.66774 8.00074Z" stroke="#317CFA" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M13.0562 3.93652C14.7562 6.38387 14.7618 9.61212 13.0562 12.0642" stroke="#317CFA" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M11.3872 5.54395C12.2614 7.07107 12.2614 8.93603 11.3872 10.4584" stroke="#317CFA" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

src/pages/chat/components/ChatGreeting/index.tsx

@@ -1,17 +1,17 @@
 import { View } from "@tarojs/components";
 import style from "./index.module.less";
 import { TAgentDetail } from "@/types/agent";
-import { useChatInput } from "../input-bar/chatInput";
+import { useChatInput } from "../input-bar/chat.ts";
 interface IProps {
   agent: TAgentDetail | null;
 }
 export default ({ agent }: IProps) => {
-  const { handleOnSend } = useChatInput({
+  const { sendChatMessage } = useChatInput({
     agent,
   });
 
   const handleClick = (q: string) => {
-    handleOnSend(q);
+    sendChatMessage(q);
   };
 
   if (!agent) {

src/pages/chat/components/OptionButtons/ButtonEnableStreamVoice.tsx

@@ -0,0 +1,21 @@
+import { View, Image } from "@tarojs/components";
+import IconVolumeUp from '@/images/svgs/chat/IconVolumeUp.svg'
+import IconVolumeUpColor from '@/images/svgs/chat/IconVolumeUpColor.svg'
+import style from './index.module.less'
+
+interface IProps {
+  setEnable: (enable: boolean)=> void
+  enable: boolean
+}
+export default function ButtonEnableStreamVoice({
+  setEnable,
+  enable
+}:IProps) {
+  const handleClick = ()=> {
+    setEnable(!enable)
+  }
+  return  <View className={`${enable ? style.buttonRoundedActived: style.buttonRounded}`} onClick={handleClick}>
+      <Image className="w-16 h-16" src={enable ? IconVolumeUpColor : IconVolumeUp} mode='widthFix'></Image>
+      <View className={enable ? 'text-primary':'text-black'}>语⾳朗读</View>
+  </View>
+}

src/pages/chat/components/OptionButtons/index.module.less

@@ -0,0 +1,14 @@
+.buttonRounded{
+  display: inline-flex;
+  align-items: center;
+  gap: 2px;
+  padding: 6px 12px;
+  border-radius: 20px;
+  background-color: white;
+  border: 1px solid white;
+}
+.buttonRoundedActived{
+  .buttonRounded();
+  border: 1px solid var(--color-primary-light);
+  background-color: rgba(225, 236, 255, 1);
+}

src/pages/chat/components/RecommendQuestions/index.tsx

@@ -1,19 +1,21 @@
 import { View, ScrollView } from "@tarojs/components";
 
-import { useChatInput } from "../input-bar/chatInput";
+import { useChatInput } from "../input-bar/chat.ts";
 import { TAgentDetail } from "@/types/agent";
 
 interface IProps {
   agent: TAgentDetail;
+  enableOutputAudioStream: boolean
 }
 
-export default function Index({ agent }: IProps) {
-  const { handleOnSend, setQuestions, questions } = useChatInput({
+export default function Index({ agent, enableOutputAudioStream }: IProps) {
+  const { sendChatMessage, setQuestions, questions } = useChatInput({
     agent,
+    enableOutputAudioStream,
   });
 
   const handleClick = (q: string) => {
-    handleOnSend(q);
+    sendChatMessage(q);
     setQuestions([]);
   };
 

src/pages/chat/components/input-bar/VoiceInputBar.tsx

@@ -4,7 +4,7 @@ import { useVoiceRecord } from "@/components/voice-recorder";
 import { useState } from "react";
 import { getChatSession } from "@/store/chat";
 import style from './index.module.less'
-import { speechToText } from "@/service/bot";
+import { speechToText } from "@/service/chat";
 import { isSuccess } from "@/utils";
 interface Props {
   disabled: boolean,
@@ -46,7 +46,7 @@ export default ({agentId,disabled, onIconClick, onSend, beforeSend, onError}:Pro
     
     // 以二进制方式读取文件
     const response = await speechToText(agentId, res.tempFilePath)
-    console.log(response,111)
+    // console.log(response,111)
     if(isSuccess(response.status)){
       console.log(response.data)
       const msg = response.data?.text ?? ''

src/pages/chat/components/input-bar/chat.ts

@@ -0,0 +1,273 @@
+import { useEffect, useState } from "react";
+import TextInputBar from "./TextInputBar";
+import VoiceInputBar from "./VoiceInputBar";
+import { requestTextToChat } from "@/service/chat";
+import { useTextChat } from "@/store/textChat";
+import { TAgentDetail } from "@/types/agent";
+import { delay, getLoginId, isSuccess } from "@/utils";
+import { useAudioStreamPlayer } from "@/utils/audioStreamPlayer";
+
+import { EChatRole, EContentType } from "@/types/bot";
+
+import { usePostMessage, saveAgentChatContentToServer } from "./message";
+
+import { getRecommendPrompt } from "@/service/chat";
+import { useDidHide, useUnload } from "@tarojs/taro";
+
+interface Props {
+  agent: TAgentDetail | null;
+  enableOutputAudioStream: boolean;
+  setShowWelcome?: (b: boolean) => void;
+  setIsVoice?: (b: boolean) => void;
+  setDisabled?: (b: boolean) => void;
+}
+
+let stopReceiveChunk: (() => void) | undefined;
+export const useChatInput = ({
+  agent,
+  enableOutputAudioStream,
+  setShowWelcome,
+  setDisabled,
+}: Props) => {
+  let myMsgUk = "";
+  let mySessionId = "";
+
+  // 聊天框 store
+  const {
+    pushRobotMessage,
+    updateRobotMessage,
+    getCurrentRobotMessage,
+    pushMessage,
+    updateMessage,
+    deleteMessage,
+    setQuestions,
+    questions,
+  } = useTextChat();
+
+  // 聊天框内消息定时上报
+  const { startTimedMessage, stopTimedMessage, saveMessageToServer } =
+    usePostMessage(getCurrentRobotMessage);
+
+  // 聊天消息流式音频播报
+  const { setFistChunk, pushBase64ToQuene, playChunk, stopPlayChunk } =
+    useAudioStreamPlayer();
+
+  // 聊天
+  const chatWithGpt = async (
+    message: string,
+    sessionId: string,
+    msgUk: string
+  ) => {
+    setShowWelcome?.(false);
+    setQuestions([]);
+    stopPlayChunk();
+    let currentRobotMsgUk = "";
+    await delay(300);
+    setDisabled?.(true);
+    if (!agent?.agentId) {
+      return;
+    }
+    const loginId = getLoginId();
+    if (!loginId) {
+      return;
+    }
+
+    const newMsg = {
+      content: message,
+      contentType: EContentType.TextPlain,
+      role: EChatRole.User,
+    };
+    // 将发送的消息上报
+    const myMsgResponse = await saveMessageToServer({
+      loginId,
+      messages: [
+        {
+          ...newMsg,
+          saveStatus: 2,
+          isStreaming: false,
+          msgUk,
+        },
+      ],
+      agentId: agent.agentId,
+      sessionId,
+    });
+
+    if (!isSuccess(myMsgResponse.status)) {
+      return setDisabled?.(false);
+    }
+
+    let isFirstChunk = true;
+    // 发起文本聊天
+    const {reqTask, stopChunk} = requestTextToChat({
+      params: {
+        agentId: agent.agentId,
+        isEnableOutputAudioStream: enableOutputAudioStream,
+        isEnableSearch: false,
+        isEnableThinking: false,
+        loginId,
+        messages: [newMsg],
+        sessionId,
+      },
+      onStart: () => {
+        // 推一个空回复，用于展示 ui
+        const blankMessage = {
+          role: EChatRole.Assistant,
+          contentType: EContentType.TextPlain,
+          saveStatus: 0,
+          content: "",
+          reasoningContent: "",
+          isStreaming: false,
+          msgUk: currentRobotMsgUk,
+          dislikeReason: "",
+          robot: {
+            avatar: agent?.avatarUrl ?? "",
+            name: agent?.name ?? "",
+            agentId: agent?.agentId ?? "",
+          },
+        };
+        currentRobotMsgUk = pushRobotMessage(blankMessage);
+        // 先暂停
+        stopTimedMessage();
+        // 开始定时发送，每5秒发送一次
+        startTimedMessage(
+          {
+            loginId,
+            messages: [
+              {
+                ...blankMessage,
+              },
+            ],
+            agentId: agent.agentId ?? "",
+            sessionId,
+          },
+          5000
+        );
+
+        isFirstChunk = true;
+      },
+      // 音频输出单独处理
+      onAudioParsed: (m) => {
+        const audioStr = m?.body?.content?.audio ?? ''
+        // if(!audioStr.length){
+        //   return;
+        // }
+        if (isFirstChunk) {
+          isFirstChunk = false;
+          setFistChunk(audioStr, reqTask);
+        } else {
+          pushBase64ToQuene(audioStr);
+        }
+        playChunk();
+      },
+      onReceived: (m) => {
+        updateRobotMessage(m.content, m.body ?? {});
+      },
+      onFinished: async (m) => {
+        const currentRobotMessage = getCurrentRobotMessage();
+        console.log(
+          "回复完毕 ok, 当前robotmessage: ",
+          currentRobotMessage,
+          m?.body?.content
+        );
+
+        stopTimedMessage();
+        if (!agent.agentId) {
+          return;
+        }
+        setDisabled?.(false);
+        // 如果没有任何回答，则显示
+        if (!currentRobotMessage?.content?.length) {
+          updateRobotMessage("服务器繁忙...");
+          return;
+        }
+
+        // 将智能体的回答保存至服务器
+        // currentRobotMessage.content 保存的是当前完整的智能体回复信息文本
+        const content = currentRobotMessage.content as string;
+        updateRobotMessage(content, currentRobotMessage?.body, 2, true);
+
+        // 获取相关提示问题
+        const response = await getRecommendPrompt({
+          agentId: agent.agentId,
+          sessionId,
+        });
+        // todo: 如果用户快速输入需要将前面的问题答案丢弃，根据 currentRobotMessage.msgUk 来设置 questions
+        if (isSuccess(response.status)) {
+          setQuestions(response.data.questions);
+        }
+      },
+      onComplete: async () => {
+        stopTimedMessage();
+        console.log("回复 onComplete");
+        // 为防止服务端没有终止消息，当接口请求结束时强制再保存一次消息体，以防定时保存的消息体漏掉最后一部分
+        const currentRobotMessage = getCurrentRobotMessage();
+        if (currentRobotMessage && agent.agentId) {
+          saveAgentChatContentToServer(
+            currentRobotMessage,
+            loginId,
+            agent.agentId,
+            sessionId
+          );
+        }
+
+        setDisabled?.(false);
+
+        isFirstChunk = true;
+      },
+      onError: () => {
+        setDisabled?.(false);
+        deleteMessage(currentRobotMsgUk);
+      },
+    });
+
+    stopReceiveChunk = stopChunk
+  };
+  // 聊天录制语音转文本后发送
+  const sendVoiceChatMessage = (message: string) => {
+    updateMessage(message, myMsgUk);
+    chatWithGpt(message, mySessionId, myMsgUk);
+  };
+  // 聊天文本发送
+  const sendChatMessage = async (message: string) => {
+    const { sessionId, msgUk } = pushMessage(message);
+    chatWithGpt(message, sessionId, msgUk);
+  };
+
+  // 推一个自己的空气泡框
+  const handleBeforeSend = () => {
+    const { sessionId, msgUk } = pushMessage("");
+    myMsgUk = msgUk;
+    mySessionId = sessionId;
+  };
+
+  // 发生识别错误时，删除当前自己发出的气泡框
+  const handleVoiceError = () => {
+    deleteMessage(myMsgUk);
+  };
+
+  // 如果停止语音输出，则停止播放
+  useEffect(() => {
+    if (!enableOutputAudioStream) {
+      stopPlayChunk();
+    }
+  }, [enableOutputAudioStream]);
+
+  // 清理
+  useEffect(() => {
+    return () => {
+      stopReceiveChunk?.();
+      stopTimedMessage();
+      stopPlayChunk();
+      console.log('clear')
+    };
+  }, []);
+
+  return {
+    setQuestions,
+    sendVoiceChatMessage,
+    sendChatMessage,
+    questions,
+    handleBeforeSend,
+    handleVoiceError,
+  };
+};

src/pages/chat/components/input-bar/chatInput.ts

@@ -1,256 +0,0 @@
-import { useEffect, useState } from "react";
-import TextInputBar from "./TextInputBar";
-import VoiceInputBar from "./VoiceInputBar";
-import { textChat } from "@/service/bot";
-import { useTextChat } from "@/store/textChat";
-import { TAgentDetail } from "@/types/agent";
-import { delay, getLoginId, isSuccess } from "@/utils";
-import { 
-  useAudioPlayer
- } from "@/utils/audio";
-
-import Taro, { useUnload } from "@tarojs/taro";
-import { EChatRole, EContentType, TRobotMessage } from "@/types/bot";
-
-import { usePostMessage, saveRobotContentToServer } from './message'
-
-import { getRecommendPrompt } from "@/service/bot"
-
-
-interface Props {
-  agent: TAgentDetail | null;
-  setShowWelcome?: (b: boolean) => void;
-  setIsVoice?: (b: boolean) => void;
-  setDisabled?: (b: boolean) => void;
-}
-
-let stopReceiveChunk: (() => void) | undefined;
-export const useChatInput = ({ agent, setShowWelcome, setDisabled, }: Props) => {
-  const {
-    pushRobotMessage,
-    updateRobotMessage,
-    getCurrentRobotMessage,
-    updateRobotReasoningMessage,
-    pushMessage,
-    updateMessage,
-    deleteMessage,
-    setQuestions,
-    questions,
-  } = useTextChat();
-  const { startTimedMessage, stopTimedMessage, saveMessageToServer } = usePostMessage(getCurrentRobotMessage);
-
-  const {
-    setFistChunk,
-    pushBase64ToQuene,
-    playChunk,
-  } = useAudioPlayer()
-
-  let myMsgUk = '';
-  let mySessionId = '';
-
-
-  const chatWithGpt = async (message: string, sessionId: string, msgUk: string) => {
-    setShowWelcome?.(false)
-    setQuestions([])
-    let currentRobotMsgUk = "";
-    await delay(300);
-    setDisabled?.(true);
-    if (!agent?.agentId) {
-      return;
-    }
-    const loginId = getLoginId();
-    if (!loginId) {
-      return;
-    }
-
-    // const greeting = "欢迎光临我的智能体，你想问什么？";
-    // {
-    //   content: greeting,
-    //   contentType: EContentType.TextPlain,
-    //   role: EChatRole.System,
-    // },
-    
-    const newMsg = {
-      content: message,
-      contentType: EContentType.TextPlain,
-      role: EChatRole.User,
-    }
-    // 等发送的消息上报完成
-    const myMsgResponse = await saveMessageToServer({
-      loginId,
-      messages: [{
-        ...newMsg,
-        saveStatus: 2,
-        isStreaming: false,
-        msgUk,
-      }],
-      agentId: agent.agentId,
-      sessionId,
-    })
-    
-    if(!isSuccess(myMsgResponse.status)){
-      return setDisabled?.(false);
-    }
-    let isFirstChunk = true
-    
-    // 发起文本聊天
-    stopReceiveChunk = textChat({
-      params: {
-        agentId: agent.agentId,
-        isEnableOutputAudioStream: true,
-        isEnableSearch: false,
-        isEnableThinking: false,
-        loginId,
-        messages: [newMsg],
-        sessionId,
-      },
-      onStart: () => {
-        // 推一个空回复，用于展示 ui
-        const blankMessage = {
-          role: EChatRole.Assistant,
-          contentType: EContentType.TextPlain,
-          saveStatus: 0,
-          content: "",
-          reasoningContent: "",
-          isStreaming: false,
-          msgUk: currentRobotMsgUk,
-          dislikeReason: '',
-          robot: {
-            avatar: agent?.avatarUrl ?? "",
-            name: agent?.name ?? "",
-            agentId: agent?.agentId ?? "",
-          },
-        }
-        currentRobotMsgUk = pushRobotMessage(blankMessage);
-        // 先暂停
-        stopTimedMessage()
-        // 开始定时发送，每5秒发送一次
-        startTimedMessage({
-          loginId,
-          messages: [{
-            ...blankMessage,
-          }],
-          agentId: agent.agentId ?? '',
-          sessionId,
-        }, 5000);
-        isFirstChunk = true
-      },
-      onReceived: (m) => {
-        // console.log("received:", m);
-        if (m.reasoningContent) {
-          updateRobotReasoningMessage(
-            currentRobotMsgUk,
-            m.reasoningContent,
-            m.body,
-          );
-        } else {
-          updateRobotMessage(m.content, m.body);
-          // pushBase64ToQuene(m.body.content.audio)
-          if(isFirstChunk){
-            isFirstChunk = false
-            setFistChunk(m.body.content.audio)
-          }else{
-            pushBase64ToQuene(m.body.content.audio)
-          }
-
-          playChunk();
-        }
-      },
-      onFinished: async () => {
-        const currentRobotMessage = getCurrentRobotMessage();
-        console.log("回复完毕 ok, 当前robotmessage: ", currentRobotMessage);
-        
-        stopTimedMessage()
-        if(!agent.agentId){
-          return
-        }
-        setDisabled?.(false);
-        // 如果没有任何回答，则显示
-        if (!currentRobotMessage?.content?.length) {
-          updateRobotMessage("服务器繁忙...");
-          return 
-        }
-        
-        // 将智能体的回答保存至服务器
-        // currentRobotMessage.content 保存的是当前完整的智能体回复信息文本
-        const content = currentRobotMessage.content as string
-        updateRobotMessage(content, currentRobotMessage?.body, 2, true)
-        
-        saveRobotContentToServer(currentRobotMessage, loginId, agent.agentId, sessionId)
-        
-        
-        const response = await getRecommendPrompt({
-          agentId: agent.agentId,
-          sessionId,
-        })
-        // todo: 如果用户快速输入需要将前面的问题答案丢弃，根据 currentRobotMessage.msgUk 来设置 questions
-        if(isSuccess(response.status)){
-          setQuestions(response.data.questions)
-        }
-
-      },
-      onComplete: async ()=> {
-        stopTimedMessage()
-        console.log('回复 onComplete')
-        // 为防止服务端没有终止消息，当接口请求结束时强制再保存一次消息体，以防定时保存的消息体漏掉最后一部分
-        const currentRobotMessage = getCurrentRobotMessage();
-        if(currentRobotMessage && agent.agentId){
-          saveRobotContentToServer(currentRobotMessage, loginId, agent.agentId, sessionId) 
-        }
-        
-        setDisabled?.(false);
-        
-        isFirstChunk = true
-      },
-      onError: () => {
-        setDisabled?.(false);
-        deleteMessage(currentRobotMsgUk);
-      },
-    });
-  };
-  const handleVoiceSend = (message: string) => {
-    updateMessage(message, myMsgUk);
-    chatWithGpt(message, mySessionId, myMsgUk);
-  };
-  const handleOnSend = async (message: string) => {
-    if(!agent?.agentId){
-      return
-    }
-    const {sessionId, msgUk} = pushMessage(message);
-    chatWithGpt(message, sessionId, msgUk);
-  };
-
-  // 推一个自己的空气泡框
-  const handleBeforeSend = () => {
-    if(!agent?.agentId){
-      return
-    }
-    const {sessionId, msgUk} = pushMessage("");
-    myMsgUk = msgUk
-    mySessionId = sessionId
-  };
-
-  // 发生主意识别错误时，删除当前自己发出的气泡框
-  const handleVoiceError = () => {
-    deleteMessage(myMsgUk);
-  };
-  useEffect(()=> {
-    return ()=> {
-      stopTimedMessage();
-    }
-  }, [])
-  useUnload(() => {
-    if (stopReceiveChunk) {
-      stopReceiveChunk();
-      stopTimedMessage();
-    }
-  });
-  return {
-    setQuestions,
-    handleVoiceSend,
-    handleOnSend,
-    questions,
-    handleBeforeSend,
-    handleVoiceError,
-  }
-}

src/pages/chat/components/input-bar/index.module.less

@@ -15,5 +15,6 @@
   background: url(https://cdn.wehome.cn/cmn/gif/199/META-H8UKVHWU-KIGP3BIL7M5AYC6XHNUA2-OSAK6A2M-72.gif) center center no-repeat;
   background-size: 124px 27px;
   background-color: #000;
-  background-color: var(--color-primary);
+  // background-color: var(--color-primary);
+  // background-color: var(--color-primary);
 }

src/pages/chat/components/input-bar/index.tsx

@@ -3,21 +3,23 @@ import TextInputBar from "./TextInputBar";
 import VoiceInputBar from "./VoiceInputBar";
 import { TAgentDetail } from "@/types/agent";
 import { TMessage, TRobotMessage } from "@/types/bot";
-import { useChatInput } from './chatInput'
+import { useChatInput } from './chat.ts'
 interface Props {
   agent: TAgentDetail | null;
   histories: (TMessage|TRobotMessage)[];
   setShowWelcome: (b: boolean) => void;
+  enableOutputAudioStream: boolean
 }
 
-export default ({ agent, setShowWelcome }: Props) => {
+export default ({ agent, enableOutputAudioStream, setShowWelcome }: Props) => {
   const [isVoice, setIsVoice] = useState(false);
   const [disabled, setDisabled] = useState(false);
-  const {handleBeforeSend, handleVoiceError, handleVoiceSend, handleOnSend} = useChatInput({
+  const {handleBeforeSend, handleVoiceError, sendVoiceChatMessage, sendChatMessage} = useChatInput({
     agent,
     setShowWelcome,
     setIsVoice,
     setDisabled,
+    enableOutputAudioStream,
   })
 
   const handleTextInputBarSwitch = () => {
@@ -37,7 +39,7 @@ export default ({ agent, setShowWelcome }: Props) => {
       <VoiceInputBar
         agentId={agent?.agentId}
         disabled={disabled}
-        onSend={handleVoiceSend}
+        onSend={sendVoiceChatMessage}
         beforeSend={handleBeforeSend}
         onIconClick={handleVoiceInputBarSwitch}
         onError={handleVoiceError}
@@ -49,7 +51,7 @@ export default ({ agent, setShowWelcome }: Props) => {
   return (
     <TextInputBar
       disabled={disabled}
-      onSend={handleOnSend}
+      onSend={sendChatMessage}
       onIconClick={handleTextInputBarSwitch}
     ></TextInputBar>
   );

src/pages/chat/components/input-bar/message.ts

@@ -1,5 +1,5 @@
-import { appendMessages } from "@/service/bot"
-import { type TMessageHistories, type TRequestBody, type TAppendMessages, TRobotMessage, EContentType } from "@/types/bot";
+import { appendMessages } from "@/service/chat"
+import { type TAppendMessages, TRobotMessage, EContentType } from "@/types/bot";
 import { useRef, useEffect, useState } from "react";
 
 export const saveMessageToServer = (data: TAppendMessages) => {
@@ -18,8 +18,8 @@ export const saveMessageToServer = (data: TAppendMessages) => {
 }
 
 
-
-export const saveRobotContentToServer = async (currentRobotMessage: TRobotMessage, loginId:string, agentId:string, sessionId: string) => {
+// 保存智能体消息内容至服务器
+export const saveAgentChatContentToServer = async (currentRobotMessage: TRobotMessage, loginId:string, agentId:string, sessionId: string) => {
   const currentContentType = currentRobotMessage?.body?.contentType
   let content = ''
   // audio chunk 不需要上报，在初始时已经上报
@@ -88,6 +88,7 @@ export const usePostMessage = (getCurrentRobotMessage:() => TRobotMessage | unde
         }
         const message = {
           ...msg,
+          body: undefined, // 删除保存在 msg 内的 body 信息 不需要上报至服务器
           isStreaming: msg.isStreaming ?? false,
           contentType: msg.contentType ?? EContentType.TextPlain,
           saveStatus: msg.saveStatus ?? 1

src/pages/chat/index.tsx

@@ -18,10 +18,11 @@ import ChatGreeting from "./components/ChatGreeting";
 import IconArrowLeftWhite24 from "@/components/icon/IconArrowLeftWhite24";
 import IconArrowLeft from "@/components/icon/icon-arrow-left";
 import PersonalCard from "./components/personal-card";
+import ButtonEnableStreamVoice from "./components/OptionButtons/ButtonEnableStreamVoice";
 import { useAgentStore } from "@/store/agentStore";
 
 import { useLoadMoreInfinite, createKey } from "@/utils/loadMoreInfinite";
-import { getMessageHistories } from "@/service/bot";
+import { getMessageHistories } from "@/service/chat";
 
 import RecommendQuestions from "./components/RecommendQuestions";
 import { useKeyboard } from "./components/keyboard";
@@ -36,6 +37,7 @@ export default function Index() {
   }
 
   const { fetchAgent, fetchAgentProfile } = useAgentStore();
+
   const bottomSafeHeight = useAppStore( state => state.bottomSafeHeight)
 
   const agent = useAgentStore((state) => {
@@ -47,6 +49,7 @@ export default function Index() {
   const scrollViewRef = useRef<any>(null);
   const messageList = useTextChat((state) => state.list);
   const [inputContainerHeight,setInputContainerHeight]= useState(0)
+  const [streamVoiceEnable,setStreamVoiceEnable]= useState(false)
 
   const { keyboardHeight, marginTopOffset, triggerHeightUpdate } = useKeyboard(
     scrollViewRef,
@@ -293,7 +296,7 @@ export default function Index() {
             })}
           </View>
           <View className="pb-40 pt-8">
-            {agent && <RecommendQuestions agent={agent} />}
+            {agent && <RecommendQuestions enableOutputAudioStream={streamVoiceEnable} agent={agent} />}
           </View>
         </ScrollView>
         <View className="w-full h-60" style={{
@@ -308,13 +311,19 @@ export default function Index() {
           }}
         >
           <View className="bg-[#F5FAFF]">
-            {agent && (
-              <InputBar
-                agent={agent}
-                histories={list}
-                setShowWelcome={setShowWelcome}
-              ></InputBar>
-            )}
+            {agent &&
+              <View className="flex flex-col w-full gap-8">
+                <InputBar
+                  enableOutputAudioStream={streamVoiceEnable}
+                  agent={agent}
+                  histories={list}
+                  setShowWelcome={setShowWelcome}
+                ></InputBar>
+                <View>
+                  <ButtonEnableStreamVoice setEnable={setStreamVoiceEnable} enable={streamVoiceEnable}></ButtonEnableStreamVoice>
+                </View>
+              </View>
+            }
           </View>
         </View>
       </View>

src/service/bot.ts → src/service/chat.ts

@@ -80,36 +80,39 @@ export type TTextChatParams = {
   params: TRequestBody;
   onStart: () => void;
   onReceived: (m: ICompleteCallback) => void;
-  onFinished: () => void;
+  onAudioParsed: (m: ICompleteCallback) => void;
+  onFinished: (m: ICompleteCallback) => void;
   onComplete?: () => void; // 无论失败或成功都会执行
   onError: () => void;
 };
 
-export const textChat = ({
+export const requestTextToChat = ({
   params,
   onStart,
   onReceived,
+  onAudioParsed,
   onFinished,
   onComplete,
   onError,
 }: TTextChatParams) => {
   onStart();
 
-  let reqTask: Taro.RequestTask<any>;
+  let reqTask: Taro.RequestTask<any>|null = null;
   const jsonParser = new JsonChunkParser();
   jsonParser.onParseComplete((m) => {
-    onFinished();
+    onFinished(m);
   });
+  
   const onChunkReceived = (chunk: any) => {
     // console.log('chunkReceived: ', chunk);
     const uint8Array = new Uint8Array(chunk.data);
     // console.log('uint8Array: ', uint8Array);
     var string = new TextDecoder("utf-8").decode(uint8Array);
-    // console.log(string);
+    // console.log('chunked:', string);
     jsonParser.parseChunk(string, (m) => {
-      console.log('parseChunk', m);
+      // console.log('parseChunk', m);
       onReceived(m);
-    });
+    }, onAudioParsed);
   };
   const header = getSimpleHeader()
   
@@ -141,8 +144,8 @@ export const textChat = ({
   }
 
   const stopChunk = () => {
-    reqTask.offChunkReceived(onChunkReceived);
+    reqTask?.offChunkReceived(onChunkReceived);
   };
 
-  return stopChunk;
+  return {reqTask: reqTask, stopChunk};
 };

src/store/textChat.ts

@@ -5,7 +5,7 @@
 import { create } from "zustand";
 import { generateUUID } from '@/utils/index'
 
-import { getMessageHistories, type TGetMessageHistoriesParams,  } from '@/service/bot'
+import { getMessageHistories, type TGetMessageHistoriesParams,  } from '@/service/chat'
 import { EChatRole, TAnyMessage, TRobotMessage, TMessage } from "@/types/bot";
 import {getFormatNow} from '@/utils/timeUtils'
 

src/utils/audio.ts → src/utils/audioStreamPlayer.ts

@@ -1,8 +1,11 @@
 import Taro from "@tarojs/taro";
-import { decode } from '@/utils'
+import { decode } from "@/utils";
 
-
-function combineArrayBuffers(arrays: ArrayBuffer[], totalLength: number): ArrayBuffer {
+// 将多个 chunk 合并
+function combineArrayBuffers(
+  arrays: ArrayBuffer[],
+  totalLength: number
+): ArrayBuffer {
   const result = new Uint8Array(totalLength);
   let offset = 0;
 
@@ -14,9 +17,8 @@ function combineArrayBuffers(arrays: ArrayBuffer[], totalLength: number): ArrayB
 
   return result.buffer;
 }
-
-
-function combineHeaderAndChunk(header:ArrayBuffer, chunk:ArrayBuffer) {
+// 播放片断时需要与 wav 头组合才能正常解析
+function combineHeaderAndChunk(header: ArrayBuffer, chunk: ArrayBuffer) {
   // Create a new ArrayBuffer to hold both the header and the chunk
   const combinedBuffer = new ArrayBuffer(header.byteLength + chunk.byteLength);
 
@@ -32,62 +34,60 @@ function combineHeaderAndChunk(header:ArrayBuffer, chunk:ArrayBuffer) {
   return combinedBuffer;
 }
 
-
-
-let audioCtx = Taro.createWebAudioContext()
+let audioCtx = Taro.createWebAudioContext();
 let source: AudioBufferSourceNode;
 let enablePlay = true; // 用于中断流式播放
 let chunks: ArrayBuffer[] = []; // 流式播放　chunks
 let requestTask = null;
-let audioBase64 = ''
-const WAV_HEADER_LENGTH = 44
+// let audioBase64 = ''
+const WAV_HEADER_LENGTH = 44; // wav 头长度
 
-export const useAudioPlayer = () => {
-  
-  let totalLength = 0
-  let playing = false
+export const useAudioStreamPlayer = () => {
+  let totalLength = WAV_HEADER_LENGTH;
+  let playing = false;
   let wavHeader: ArrayBuffer;
-  
+
   const setFistChunk = (base64Str: string, _requestTask?: any) => {
-    if(_requestTask){
-      requestTask = _requestTask  
+    if (_requestTask) {
+      requestTask = _requestTask;
     }
     enablePlay = true;
-    audioBase64 = base64Str;
-    let chunk = decode(base64Str)
+    // audioBase64 = base64Str;
+    let chunk = decode(base64Str);
     emptyQuene();
     // 第一个 chunk 内包含了头信息
     wavHeader = chunk.slice(0, WAV_HEADER_LENGTH);
     const firstChunkData = chunk.slice(WAV_HEADER_LENGTH);
-    pushChunk2Quene(firstChunkData)
-  }
+    pushChunk2Quene(firstChunkData);
+  };
 
-  const pushBase64ToQuene = (base64Str: string)=> {
-    audioBase64 += base64Str 
+  const pushBase64ToQuene = (base64Str: string) => {
+    // audioBase64 += base64Str
     let buf = decode(base64Str);
     pushChunk2Quene(buf);
-  }
+  };
 
   const pushChunk2Quene = (chunk: ArrayBuffer) => {
     chunks.push(chunk);
     totalLength += chunk.byteLength;
-  }
+  };
 
   const emptyQuene = () => {
-    chunks = []
+    chunks = [];
+    // audioBase64 = ''
     totalLength = WAV_HEADER_LENGTH;
-  }
+  };
 
   const playChunk = () => {
-    if(!enablePlay){
+    if (!enablePlay) {
       emptyQuene();
-      return ;
+      return;
     }
-    if(playing){
+    if (playing) {
       return;
     }
     if (!chunks.length) {
-      playing = false
+      playing = false;
       return;
     }
     playing = true;
@@ -95,43 +95,44 @@ export const useAudioPlayer = () => {
     let tmp = [...chunks];
     const _chunk = combineArrayBuffers(tmp, totalLength);
     const partChunks = combineHeaderAndChunk(wavHeader, _chunk);
-    
+
     emptyQuene();
-    //@ts-ignore
-    audioCtx.decodeAudioData(partChunks, (decodedBuffer: AudioBuffer) => {
-      source = audioCtx.createBufferSource();
-      //@ts-ignore
-      source.connect(audioCtx.destination);
-      source.buffer = decodedBuffer;
-      console.log('play start')
-      source.onended = () => {
-        console.log('play end')
-        playing = false
-        playChunk()
-        console.log('finally', audioBase64)
-      };
-      source.start(0);
-    }, (err:any) => {
-      console.log(err)
-      playing = false;
-    })
-  }
+    // @ts-ignore
+    audioCtx.decodeAudioData( partChunks,
+      (decodedBuffer: AudioBuffer) => {
+        source = audioCtx.createBufferSource();
+        //@ts-ignore
+        source.connect(audioCtx.destination);
+        source.buffer = decodedBuffer;
+        console.info("play start");
+        source.onended = () => {
+          console.info("play end");
+          playing = false;
+          playChunk();
+          // console.log('finally', audioBase64)
+        };
+        source.start(0);
+      },
+      (err: any) => {
+        console.warn(err);
+        playing = false;
+      }
+    );
+  };
 
   const stopPlayChunk = () => {
     // 如果有请求任务取消
-    if(requestTask){
+    if (requestTask) {
       //@ts-ignore
-      requestTask?.abort?.()
+      requestTask?.abort?.();
       //@ts-ignore
-      requestTask?.offChunkReceived?.()
+      requestTask?.offChunkReceived?.();
     }
+    source?.stop();
     emptyQuene();
-    
-    enablePlay = false;
 
-  }
-
-  
+    enablePlay = false;
+  };
 
   return {
     pushChunk2Quene,
@@ -139,6 +140,5 @@ export const useAudioPlayer = () => {
     playChunk,
     stopPlayChunk,
     setFistChunk,
-  }
-}
-
+  };
+};

src/utils/jsonChunkParser.ts

@@ -33,7 +33,10 @@ data: [DONE]
 export interface ICompleteCallback {
   content: string;
   reasoningContent?: string;
-  body: Record<string,any>
+  body: null| (TJsonMessage & Record<string,any>)
+}
+export interface IAudioPared {
+  body: null| (TJsonMessage & Record<string,any>)
 }
 
 type TContentType = "text/plain" | "aiseek/qa" | 'application/json' | 'aiseek/audio_chunk' | 'aiseek/thinking'  |  'aiseek/function_call' | 'aiseek/multimodal'
@@ -49,83 +52,94 @@ export default class JsonChunkParser {
   public onParseComplete(completeCallback: (data: ICompleteCallback) => void) {
     this.complete = completeCallback;
   }
+  
   // 解析接收到的 chunk
-  public parseChunk(chunk: string, onParsed: (json: ICompleteCallback) => void): void {
+  public parseChunk(
+      chunk: string, 
+      onParsed: (json: ICompleteCallback) => void,
+      onAudioParsed: (json: IAudioPared) => void
+    ): void {
     // 将新接收到的 chunk 添加到 buffer
     this.buffer += chunk;
 
     // 按换行符分割字符串
     const lines = this.buffer.split("\n");
-    let receivedJsonBody = {}
-    let audio = ''
-    let combinedContent: string[] = []; // 用于合并 content 字段
-    let combinedReasoningContent: string[] = []; // 用于合并 reasoner 字段
+    let receivedJsonBody:any = {}
+    let audio = '' // todo: audio 是否需要拼接保留？？
+    let combinedContentText: string[] = []; // 用于合并 content 字段
     
     // 遍历每一行
     for (let i = 0; i < lines.length; i++) {
       const line = lines[i].trim(); // 去除前后空格
-
+      // 如果行不为空
       if (line) {
-        // 如果行不为空
-        if (line === "data:[DONE]") {
-          // 如果遇到 DONE，合并并调用 onParsed
-          if (combinedContent.length > 0) {
-            onParsed({ content: combinedContent.join("") , body: receivedJsonBody}); // 合并 content
+        // 如果遇到 DONE，合并并调用 onParsed
+        // 同时兼容 "data:[DONE]" 和 "data: [DONE]" 两种格式
+        if (line.startsWith("data:")) {
+          const dataPayload = line.substring(5).trim();
+          if (dataPayload === "[DONE]") {
+            if (combinedContentText.length > 0 || audio.length > 0) {
+              onParsed({ content: combinedContentText.join("") , body: receivedJsonBody});
+            }
+            this.complete({ content: combinedContentText.join(""), body: receivedJsonBody });
+            this.buffer = "";
+            return;
           }
-          this.complete({ content: combinedContent.join(""), body: receivedJsonBody });
-          this.buffer = ""; // 清空 buffer
-          return; // 结束解析
         }
-
+        // 尝试解析为 json 对象
         try {
           // 处理 data: data: 前缀
           if (line.startsWith("data:")) {
-            const jsonStr = line.substring(5); // 移除 "data:" 前缀
+            const jsonStr = line.substring(5).trim(); // 移除 "data:" 前缀并去除空格
             const json: TJsonMessage = JSON.parse(jsonStr);
             receivedJsonBody = json
-            console.log(555555, receivedJsonBody)
+            // console.log(11111, receivedJsonBody)
             // 文本回复
             if(json.contentType === "text/plain") {
               if (json.content) {
-                combinedContent.push(json.content as string); // 收集 content 字段
+                combinedContentText.push(json.content as string); // 收集 content 字段
               }
-              if (json.reasoningContent) {
-                combinedReasoningContent.push(json.reasoningContent); // 收集 content 字段
-              }  
             }
             // QA 回复
             else if(json.contentType === "aiseek/qa") {
               if (json.content.answer?.text) {
-                combinedContent.push(json.content.answer.text); // 收集 QA 的 answer 文本
+                combinedContentText.push(json.content.answer.text); // 收集 QA 的 answer 文本
               }
-            }else if(json.contentType === "aiseek/audio_chunk"){
-              if(json.content.sentenceBegin){
-                combinedContent.push(json.content.sentenceBegin); // 收集 sentenceBegin 文本
+            }
+            // 语音回复消息
+            else if(json.contentType === "aiseek/audio_chunk"){
+              if(json.content?.sentenceBegin?.length){
+                combinedContentText.push(json.content.sentenceBegin); // 收集 sentenceBegin 文本
               }
+              // 收集 audio base64 格式文本
               if(json.content.audio){
-                audio+=audio;
+                audio += json.content.audio;
+              }
+              // 音频流单独处理回调
+              // 由于 语音和文本长度是不同步的，只要有语音就要调用一次 onAudioParsed
+              if (audio.length >= 0) {
+                onAudioParsed({body: json  }); // 合并并输出
               }
             }
-            
           }
         } catch (error) {
-          // 如果解析失败，说明当前行不是完整的 JSON
+          // 如果解析失败，说明当前行不是完整的 JSON（通常是最后一行）
+          // 在返回前，先把已成功解析并累计的文本吐出去，避免丢失
+          if (combinedContentText.length > 0) {
+            onParsed({ content: combinedContentText.join(''),  body: receivedJsonBody  });
+          }
           // 将当前行保留在 buffer 中，等待下一个 chunk
-          this.buffer = lines.slice(i).join("\n"); // 更新 buffer
+          this.buffer = lines.slice(i).join("\n");
           return; // 直接返回，等待下一个 chunk
         }
       }
     }
 
-    // 如果当前 chunk 解析完毕且有合并的内容，调用 onParsed
-    if (combinedContent.length > 0 || audio.length >= 0) {
-      onParsed({ content: combinedContent.join(""),  body: receivedJsonBody  }); // 合并并输出
+    // 如果当前 combinedContent有内容合并后 调用  onParsed 
+    if (combinedContentText.length > 0) {
+      onParsed({ content: combinedContentText.join(''),  body: receivedJsonBody  }); // 合并并输出
     }
 
-    // if (combinedReasoningContent.length > 0) {
-    //   onParsed({ reasoningContent: combinedReasoningContent.join(""), body: receivedJsonBody ,  body: receivedJsonBody   }); // 合并并输出
-    // }
-
     // 清空 buffer
     this.buffer = "";
   }