add banbanmini backend

talkingq-url/implementations/aliyun_asr.py

@@ -0,0 +1,251 @@
+import asyncio
+from dashscope.audio.asr import TranslationRecognizerChat
+from dashscope.audio.asr import TranscriptionResult
+from dashscope.audio.asr import TranslationRecognizerCallback
+from config import settings
+from interfaces.asr import ASR
+from implementations.base_service import BaseService
+from services.interrupt_handler import interrupt_handler
+from utils.logger import session_logger
+
+
+class GummyCallback(TranslationRecognizerCallback):
+    def __init__(self, device_id, session_id):
+        self.device_id = device_id
+        self.session_id = session_id
+        self.transcript = ""
+        self.transcript_event = asyncio.Event()
+        self.received_result = False
+        self.error_message = None
+        self.is_closed = False
+        self.session_end = False
+
+    def on_open(self) -> None:
+        session_logger.info(self.device_id, self.session_id, "Gummy ASR连接建立成功")
+
+    def on_event(
+        self,
+        request_id,
+        transcription_result: TranscriptionResult,
+        translation_result,
+        usage,
+    ) -> None:
+        if transcription_result:
+            if transcription_result.is_sentence_end:
+                self.transcript = transcription_result.text
+                self.received_result = True
+                session_logger.info(
+                    self.device_id,
+                    self.session_id,
+                    f"Gummy ASR完整句子: {self.transcript}",
+                )
+                self.transcript_event.set()
+            else:
+                self.transcript = transcription_result.text
+
+    def on_complete(self) -> None:
+        session_logger.info(self.device_id, self.session_id, "Gummy ASR识别完成")
+        self.session_end = True
+        self.transcript_event.set()
+
+    def on_error(self, result) -> None:
+        self.error_message = str(result)
+        session_logger.error(
+            self.device_id, self.session_id, f"Gummy ASR错误: {self.error_message}"
+        )
+        self.transcript_event.set()
+
+    def on_close(self) -> None:
+        session_logger.info(self.device_id, self.session_id, "Gummy ASR连接已关闭")
+        self.is_closed = True
+        self.transcript_event.set()
+
+
+class AliyunASR(ASR, BaseService):
+    def __init__(self, selected_role=None):
+        BaseService.__init__(self, "asr", selected_role)
+        self.api_key = settings.aliyun_api_key
+        self.vocabulary_id = settings.aliyun_vocabulary_id
+        self.connected = False
+        self.transcript = ""
+        self.device_id = "unknown"
+        self.session_id = "unknown"
+        self.sample_rate = 16000
+        self.audio_buffer = bytearray()
+        self.translator = None
+        self.callback = None
+        self.buffer_size = 3200  # 约100ms的音频，按16000采样率计算
+        self.max_end_silence = 2000  # 设为2000ms，允许更长的停顿
+
+    async def connect(self):
+        try:
+            if self.connected and self.translator:
+                session_logger.info(
+                    self.device_id, self.session_id, "Gummy ASR连接已存在，复用当前连接"
+                )
+                return
+
+            self.transcript = ""
+            self.audio_buffer = bytearray()
+
+            self.callback = GummyCallback(self.device_id, self.session_id)
+
+            self.translator = TranslationRecognizerChat(
+                model="gummy-chat-v1",
+                format="mp3",
+                sample_rate=self.sample_rate,
+                callback=self.callback,
+                transcription_enabled=True,
+                source_language="auto",
+                semantic_punctuation_enabled=False,
+                max_end_silence=self.max_end_silence,
+                api_key=self.api_key,
+                vocabulary_id=self.vocabulary_id,
+            )
+
+            self.translator.start()
+            self.callback.transcript_event.clear()
+            self.connected = True
+
+        except Exception as e:
+            session_logger.error(
+                self.device_id, self.session_id, f"建立Gummy ASR连接失败: {str(e)}"
+            )
+            self.connected = False
+            if self.translator:
+                try:
+                    self.translator.stop()
+                except:
+                    pass
+                self.translator = None
+
+    async def send_audio(self, audio_data: bytes):
+        if not self.connected or not self.translator:
+            session_logger.error(self.device_id, self.session_id, "Gummy ASR连接未建立")
+            return
+
+        try:
+            self.audio_buffer.extend(audio_data)
+
+            while len(self.audio_buffer) >= self.buffer_size:
+                chunk = bytes(self.audio_buffer[: self.buffer_size])
+                self.audio_buffer = self.audio_buffer[self.buffer_size :]
+
+                if self.device_id and self.session_id:
+                    session_key = (self.device_id, self.session_id)
+                    if interrupt_handler.is_interrupted(session_key):
+                        session_logger.info(
+                            self.device_id,
+                            self.session_id,
+                            "检测到中断，停止发送音频数据",
+                        )
+                        return
+
+                try:
+                    self.translator.send_audio_frame(chunk)
+                except Exception as e:
+                    session_logger.error(
+                        self.device_id,
+                        self.session_id,
+                        f"发送音频帧到Gummy ASR出错: {str(e)}",
+                    )
+
+        except Exception as e:
+            session_logger.error(
+                self.device_id, self.session_id, f"发送音频到Gummy ASR出错: {str(e)}"
+            )
+
+    async def send_end(self):
+        if not self.connected or not self.translator:
+            session_logger.error(
+                self.device_id, self.session_id, "Gummy ASR连接未建立，无法发送结束信号"
+            )
+            return
+
+        try:
+            if len(self.audio_buffer) > 0:
+                chunk = bytes(self.audio_buffer)
+                self.audio_buffer = bytearray()
+                self.translator.send_audio_frame(chunk)
+
+            session_logger.info(
+                self.device_id,
+                self.session_id,
+                "客户端发送结束包，等待Gummy ASR处理完毕",
+            )
+
+            loop = asyncio.get_running_loop()
+            await loop.run_in_executor(None, lambda: self._safe_stop_translator())
+
+        except Exception as e:
+            session_logger.error(
+                self.device_id,
+                self.session_id,
+                f"发送结束信号到Gummy ASR出错: {str(e)}",
+            )
+
+    async def receive_results(self):
+        if not self.connected:
+            session_logger.error(
+                self.device_id, self.session_id, "Gummy ASR连接未建立，无法接收结果"
+            )
+            return
+
+        try:
+            try:
+                await asyncio.wait_for(
+                    self.callback.transcript_event.wait(), timeout=0.5
+                )
+            except asyncio.TimeoutError:
+                session_logger.warning(
+                    self.device_id,
+                    self.session_id,
+                    "接收Gummy ASR结果超时，使用最后的中间结果",
+                )
+
+            if self.callback.received_result or self.callback.transcript:
+                self.transcript = self.callback.transcript
+
+        except Exception as e:
+            session_logger.error(
+                self.device_id, self.session_id, f"接收Gummy ASR结果出错: {str(e)}"
+            )
+
+        return self.transcript
+
+    async def close(self):
+        if self.connected:
+            try:
+                if self.translator is not None:
+                    try:
+                        self.translator.stop()
+                    except Exception as e:
+                        if "has stopped" not in str(e):
+                            session_logger.warning(
+                                self.device_id, 
+                                self.session_id, 
+                                f"关闭Gummy translator出错: {str(e)}"
+                            )
+                    self.translator = None
+                self.connected = False
+                session_logger.info(
+                    self.device_id, self.session_id, "Gummy ASR连接已关闭"
+                )
+            except Exception as e:
+                session_logger.error(
+                    self.device_id, self.session_id, f"关闭Gummy ASR连接出错: {str(e)}"
+                )
+
+    def _safe_stop_translator(self):
+        """安全停止translator，忽略已停止的异常"""
+        try:
+            if self.translator:
+                self.translator.stop()
+        except Exception as e:
+            if "has stopped" not in str(e):
+                session_logger.warning(
+                    self.device_id,
+                    self.session_id,
+                    f"停止translator时出现非标准错误: {str(e)}",
+                )
+        return None

talkingq-url/implementations/base_llm.py

@@ -0,0 +1,212 @@
+import json
+import aiohttp
+import asyncio
+import time
+from typing import List, Dict, AsyncGenerator
+from interfaces.llm import LLM
+from utils.logger import session_logger
+from config import settings
+from services.interrupt_handler import interrupt_handler
+from implementations.base_service import BaseService
+
+class BaseLLM(LLM, BaseService):
+    def __init__(self, api_key: str, base_url: str, service_name: str, **kwargs):
+        BaseService.__init__(self, "llm", kwargs.get("selected_role"))
+        self.session = None
+        self.api_key = api_key
+        self.base_url = base_url
+        self.connected = False
+        self.closed = False
+        self.response = None
+        self.device_id = None
+        self.session_id = None
+        self.service_name = service_name
+
+    async def connect(self):
+        try:
+            if self.session:
+                await self.session.close()
+            self.session = aiohttp.ClientSession()
+            self.connected = True
+        except Exception as e:
+            session_logger.error(
+                self.device_id,
+                self.session_id,
+                f"[{self.service_name}] 连接建立失败: {str(e)}",
+            )
+            self.connected = False
+
+    async def prepare_request(
+        self, transcript: str, history: List[Dict[str, str]], selected_role: dict
+    ) -> Dict:
+        start_time = time.perf_counter()
+        if (
+            hasattr(self, "_system_message")
+            and self._system_message.get("content") == selected_role["content"]
+        ):
+            messages = [self._system_message]  # 复用缓存的系统消息
+        else:
+            messages = [{"role": "system", "content": selected_role["content"]}]
+            self._system_message = messages[0]  # 更新缓存
+        recent_history = history[-settings.max_conversation_history :]
+        for exchange in recent_history:
+            messages.append({"role": "user", "content": exchange["user"]})
+            messages.append({"role": "assistant", "content": exchange["assistant"]})
+        messages.append({"role": "user", "content": transcript})
+        data = {
+            "model": self.model_id,
+            "messages": messages,
+            "stream": True,
+        }
+        end_time = time.perf_counter()
+        session_logger.info(
+            self.device_id,
+            self.session_id,
+            f"[{self.service_name}] 请求准备完成: {len(messages)}条消息, 耗时: {end_time - start_time:.4f}秒",
+        )
+        return data
+
+    async def generate_response_stream(
+        self, transcript: str, history: List[Dict[str, str]], selected_role: dict
+    ) -> AsyncGenerator[str, None]:
+        if not self.connected or not self.session:
+            await self.connect()
+            if not self.connected:
+                yield None
+                return
+        self.device_id = getattr(self, "device_id", "unknown")
+        self.session_id = getattr(self, "session_id", "unknown")
+        session_key = (
+            (self.device_id, self.session_id)
+            if self.device_id != "unknown" and self.session_id != "unknown"
+            else None
+        )
+        try:
+            if session_key:
+                self.closed = interrupt_handler.is_interrupted(session_key)
+            data = await self.prepare_request(transcript, history, selected_role)
+            if session_key:
+                await interrupt_handler.set_interrupt_state(session_key, False)
+                self.closed = False
+            session_logger.info(
+                self.device_id, self.session_id, f"[{self.service_name}] 开始生成回复"
+            )
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            }
+            try:
+                if not self.session or self.session.closed:
+                    self.session = aiohttp.ClientSession()
+                    self.connected = True
+                response = await self.session.post(
+                    f"{self.base_url}/chat/completions", headers=headers, json=data
+                )
+                self.response = response
+                if response.status == 200:
+                    async for line in response.content:
+                        if session_key and interrupt_handler.is_interrupted(
+                            session_key
+                        ):
+                            self.closed = True
+                        if self.closed:
+                            session_logger.info(
+                                self.device_id,
+                                self.session_id,
+                                f"[{self.service_name}] 生成被中断",
+                            )
+                            break
+                        decoded_line = line.decode("utf-8").strip()
+                        if not decoded_line:
+                            continue
+                        if decoded_line.startswith("data:"):
+                            data_str = decoded_line[len("data:") :].strip()
+                            if data_str == "[DONE]":
+                                break
+                            try:
+                                data_json = json.loads(data_str)
+                            except json.JSONDecodeError:
+                                continue
+                            choices = data_json.get("choices", [])
+                            if choices:
+                                delta = choices[0].get("delta", {})
+                                content = delta.get("content")
+                                if content:
+                                    if self.closed:
+                                        session_logger.info(
+                                            self.device_id,
+                                            self.session_id,
+                                            f"[{self.service_name}] 生成被中断",
+                                        )
+                                        break
+                                    yield content
+                else:
+                    error_text = await response.text()
+                    session_logger.error(
+                        self.device_id,
+                        self.session_id,
+                        f"[{self.service_name}] 生成回复失败: {error_text}",
+                    )
+                    yield None
+            except asyncio.CancelledError:
+                session_logger.info(
+                    self.device_id, self.session_id, f"[{self.service_name}] 请求被取消"
+                )
+                yield None
+            except Exception as e:
+                session_logger.error(
+                    self.device_id,
+                    self.session_id,
+                    f"[{self.service_name}] API请求错误: {str(e)}",
+                )
+                yield None
+        except Exception as e:
+            session_logger.error(
+                self.device_id,
+                self.session_id,
+                f"[{self.service_name}] 生成过程发生错误: {str(e)}",
+            )
+            yield None
+        finally:
+            if self.response:
+                await self.response.release()
+                self.response = None
+
+    async def close(self):
+        if (
+            hasattr(self, "device_id")
+            and hasattr(self, "session_id")
+            and self.device_id != "unknown"
+            and self.session_id != "unknown"
+        ):
+            session_key = (self.device_id, self.session_id)
+            await interrupt_handler.set_interrupt_state(session_key, True)
+        self.closed = True  # 本地标志也同步设置
+        if self.response:
+            try:
+                self.response.close()  # 立即关闭响应，不等待release
+                await self.response.release()
+            except Exception as e:
+                if hasattr(self, "device_id") and hasattr(self, "session_id"):
+                    session_logger.warning(
+                        self.device_id,
+                        self.session_id,
+                        f"[{self.service_name}] 关闭响应时出错: {e}",
+                    )
+            self.response = None
+        if self.session:
+            try:
+                await self.session.close()
+            except Exception as e:
+                if hasattr(self, "device_id") and hasattr(self, "session_id"):
+                    session_logger.warning(
+                        self.device_id,
+                        self.session_id,
+                        f"[{self.service_name}] 关闭会话时出错: {e}",
+                    )
+            self.session = None
+        self.connected = False
+        if hasattr(self, "device_id") and hasattr(self, "session_id"):
+            session_logger.info(
+                self.device_id, self.session_id, f"[{self.service_name}] 连接已关闭"
+            )

talkingq-url/implementations/base_service.py

@@ -0,0 +1,21 @@
+from typing import Any, Dict, Optional
+from services.config_manager import config_manager
+
+class BaseService:
+    """所有服务实现的基类，提供统一的配置处理功能"""
+    
+    def __init__(self, service_type: str, selected_role: Optional[Dict[str, Any]] = None):
+        self.selected_role = selected_role
+        self.service_type = service_type
+        self.config = config_manager.get_service_config(service_type, selected_role)
+    
+    def get_config(self, key: str, default_value: Any = None) -> Any:
+        """获取配置值，按照优先级规则"""
+        if key in self.config:
+            return self.config[key]
+        return config_manager.get_config_value(key, self.selected_role, default_value)
+    
+    def update_config(self, new_role_config: Dict[str, Any] = None) -> None:
+        """更新服务配置，用于动态切换角色时"""
+        self.selected_role = new_role_config if new_role_config else self.selected_role
+        self.config = config_manager.get_service_config(self.service_type, self.selected_role)

talkingq-url/implementations/minimax_tts.py

@@ -0,0 +1,202 @@
+import os
+import uuid
+import aiohttp
+import aiofiles
+import binascii
+import json
+from typing import Optional, List, Any, AsyncGenerator
+from config import settings
+from interfaces.tts import TTS
+from implementations.base_service import BaseService
+from utils.logger import session_logger
+from services.tts_audio_cleaner import TTSAudioCleaner
+
+class MiniMaxTTS(TTS, BaseService):
+    def __init__(self, selected_role=None):
+        BaseService.__init__(self, "tts", selected_role)
+        self.api_key = settings.minimax_api_key
+        self.group_id = settings.minimax_group_id
+        self.base_url = settings.minimax_base_url
+        self.connected = False
+        self.device_id = "unknown"
+        self.session_id = "unknown"
+        self.voice_id = "cartoon-boy-01"  # 默认音色
+        if selected_role and "minimax_voice_id" in selected_role:
+            self.voice_id = selected_role["minimax_voice_id"]
+        self.model = "speech-02-turbo"
+        self.selected_role = selected_role
+
+    async def connect(self):
+        """连接到MiniMax服务"""
+        self.connected = True
+        session_logger.info(
+            self.device_id, 
+            self.session_id,
+            f"MiniMax TTS服务已连接，使用音色: {self.voice_id}"
+        )
+
+    async def close(self):
+        """关闭MiniMax服务连接"""
+        self.connected = False
+
+    async def tts(
+        self,
+        text: str,
+        output_file_prefix: Optional[str] = None,
+        tts_format: str = "mp3",
+        selected_role: dict = None,
+        session: Optional[Any] = None,
+        language: str = None,  
+    ) -> Optional[List[str]]:
+        """
+        将文本转换为语音
+        返回值为包含音频URL的列表
+        """
+        if selected_role:
+            self.selected_role = selected_role
+            if "minimax_voice_id" in selected_role:
+                self.voice_id = selected_role["minimax_voice_id"]
+
+        if session and hasattr(session, "device_id") and session.device_id:
+            self.device_id = session.device_id
+        if session and hasattr(session, "session_id") and session.session_id:
+            self.session_id = session.session_id
+
+        unique_id = f"{self.session_id}_{uuid.uuid4().hex[:8]}"
+        
+        if output_file_prefix:
+            output_file = f"{output_file_prefix}.{tts_format}"
+        else:
+            await TTSAudioCleaner.prepare_output_directory()
+            tts_dir = os.path.join(settings.assets_dir, "tts_audio")
+            output_file = os.path.join(tts_dir, f"{unique_id}.{tts_format}")
+
+        try:
+            session_logger.info(
+                self.device_id,
+                self.session_id,
+                f"开始MiniMax流式语音合成，文本长度: {len(text)}，输出文件: {output_file}"
+            )
+            
+            urls = await self._process_stream(text, output_file, language)
+            return urls if urls else None
+        except Exception as e:
+            session_logger.error(
+                self.device_id,
+                self.session_id,
+                f"MiniMax TTS合成失败: {str(e)}",
+                exc_info=True
+            )
+            return None
+
+    async def _process_stream(self, text: str, output_file: str, language: str = None) -> Optional[List[str]]:
+        """使用流式方式处理文本并返回URL"""
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        
+        payload = {
+            "model": self.model,
+            "text": text,
+            "stream": True,
+            "voice_setting": {
+                "voice_id": self.voice_id,
+                "speed": 1.0,
+                "vol": 2.0,
+                "pitch": 0
+            },
+            "audio_setting": {
+                "sample_rate": 16000,
+                "bitrate": 32000,
+                "format": "mp3",
+                "channel": 1
+            }
+        }
+        
+        if language:
+            lang_mapping = {
+                "zh": "Chinese",
+                "en": "English",
+                "fr": "French",
+                "de": "German",
+                "es": "Spanish",
+                "yue": "Chinese,Yue"
+            }
+            payload["language_boost"] = lang_mapping.get(language, "auto")
+        else:
+            payload["language_boost"] = "auto"
+            
+        url = f"{self.base_url}?GroupId={self.group_id}"
+        
+        try:
+            session_logger.info(
+                self.device_id, 
+                self.session_id,
+                f"发送MiniMax TTS流式请求: {url}"
+            )
+            
+            audio_buffer = bytearray()
+            
+            async with aiohttp.ClientSession() as client_session:
+                async with client_session.post(url, json=payload, headers=headers) as response:
+                    if response.status != 200:
+                        error_text = await response.text()
+                        session_logger.error(
+                            self.device_id,
+                            self.session_id,
+                            f"MiniMax TTS请求失败: {response.status}, {error_text}"
+                        )
+                        return None
+                    
+                    session_logger.info(
+                        self.device_id,
+                        self.session_id,
+                        "开始接收MiniMax TTS流式响应"
+                    )
+                    
+                    async for line in response.content:
+                        if line.startswith(b'data:'):
+                            try:
+                                data_json = json.loads(line[5:])
+                                if "data" in data_json and "audio" in data_json["data"]:
+                                    status = data_json["data"].get("status", 1)
+                                                                     
+                                    if status == 1: # 只处理status=1(合成中)的音频数据，忽略status=2(合成结束)的汇总数据
+                                        audio_hex = data_json["data"]["audio"]
+                                        audio_binary = binascii.unhexlify(audio_hex)
+                                        audio_buffer.extend(audio_binary)
+                                    
+                                    if status == 2:  # 合成结束
+                                        session_logger.info(
+                                            self.device_id,
+                                            self.session_id,
+                                            "MiniMax TTS流式合成完成"
+                                        )
+                            except Exception as e:
+                                session_logger.error(
+                                    self.device_id,
+                                    self.session_id,
+                                    f"处理MiniMax TTS流式响应出错: {str(e)}",
+                                    exc_info=True
+                                )
+            
+            async with aiofiles.open(output_file, 'wb') as f:
+                await f.write(audio_buffer)
+            
+            relative_path = os.path.relpath(output_file, settings.assets_dir)
+            url = f"assets/{relative_path}"
+            session_logger.info(
+                self.device_id,
+                self.session_id,
+                f"MiniMax TTS流式合成完成，生成URL: {url}"
+            )
+            return [url]
+        except Exception as e:
+            session_logger.error(
+                self.device_id,
+                self.session_id,
+                f"MiniMax TTS流式处理失败: {str(e)}",
+                exc_info=True
+            )
+            return None

talkingq-url/implementations/volcano_llm.py

@@ -0,0 +1,41 @@
+from implementations.base_llm import BaseLLM
+from config import settings
+from typing import List, Dict
+
+class VolcanoLLM(BaseLLM):
+    def __init__(self, selected_role=None):
+        self.model_id = settings.volcano_model_id
+        if selected_role and "volcano_model_id" in selected_role:
+            self.model_id = selected_role["volcano_model_id"]
+            session_source = "角色配置"
+        else:
+            session_source = "全局配置"
+            
+        if hasattr(self, 'logger'):
+            self.logger.info(
+                "unknown", 
+                f"Volcano模型ID: {self.model_id} (来源: {session_source})"
+            )
+            
+        if self.model_id.startswith("bot-"):
+            base_url = f"{settings.volcano_base_url}/bots"
+        else:
+            base_url = settings.volcano_base_url
+            
+        super().__init__(
+            api_key=settings.volcano_api_key,
+            base_url=base_url,
+            service_name="Volcano LLM",
+            selected_role=selected_role,
+        )
+
+    async def prepare_request(
+        self, transcript: str, history: List[Dict[str, str]], selected_role: dict
+    ) -> Dict:
+        if selected_role and "volcano_model_id" in selected_role:
+            self.model_id = selected_role["volcano_model_id"]
+            if self.model_id.startswith("bot-"):
+                self.base_url = f"{settings.volcano_base_url}/bots"
+            else:
+                self.base_url = settings.volcano_base_url
+        return await super().prepare_request(transcript, history, selected_role)