feat(asr): switch to Qwen streaming and close interrupted providers
This commit is contained in:
stu2not
@@ -1,8 +1,11 @@
|
||||
"""Async Qwen-Audio/Fun-ASR adapter using the DashScope recognition protocol."""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from dashscope.audio.asr import TranslationRecognizerChat
|
||||
from dashscope.audio.asr import TranscriptionResult
|
||||
from dashscope.audio.asr import TranslationRecognizerCallback
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import aiohttp
|
||||
|
||||
from config import settings
|
||||
from interfaces.asr import ASR
|
||||
from implementations.base_service import BaseService
|
||||
@@ -10,333 +13,248 @@ from services.interrupt_handler import interrupt_handler
|
||||
from utils.logger import session_logger
|
||||
|
||||
|
||||
class GummyCallback(TranslationRecognizerCallback):
|
||||
def __init__(self, device_id, session_id, asr_instance=None):
|
||||
self.device_id = device_id
|
||||
self.session_id = session_id
|
||||
self.asr_instance = asr_instance
|
||||
self.transcript = ""
|
||||
self.transcript_event = asyncio.Event()
|
||||
self.received_result = False
|
||||
self.error_message = None
|
||||
self.is_closed = False
|
||||
self.session_end = False
|
||||
self.reconnect_event = threading.Event()
|
||||
self.is_rate_limit_error = False
|
||||
|
||||
def on_open(self) -> None:
|
||||
session_logger.info(self.device_id, self.session_id, "Gummy ASR连接建立成功")
|
||||
|
||||
def on_event(
|
||||
self,
|
||||
request_id,
|
||||
transcription_result: TranscriptionResult,
|
||||
translation_result,
|
||||
usage,
|
||||
) -> None:
|
||||
if transcription_result:
|
||||
if transcription_result.is_sentence_end:
|
||||
self.transcript = transcription_result.text
|
||||
self.received_result = True
|
||||
session_logger.info(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"Gummy ASR完整句子: {self.transcript}",
|
||||
)
|
||||
self.transcript_event.set()
|
||||
else:
|
||||
self.transcript = transcription_result.text
|
||||
|
||||
def on_complete(self) -> None:
|
||||
session_logger.info(self.device_id, self.session_id, "Gummy ASR识别完成")
|
||||
self.session_end = True
|
||||
self.transcript_event.set()
|
||||
|
||||
def on_error(self, result) -> None:
|
||||
self.error_message = str(result)
|
||||
error_str = str(result).lower()
|
||||
self.is_rate_limit_error = "throttling" in error_str or "rate limit" in error_str
|
||||
|
||||
if self.is_rate_limit_error:
|
||||
session_logger.warning(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"Gummy ASR限流错误，触发重连信号: {self.error_message}"
|
||||
)
|
||||
self.reconnect_event.set()
|
||||
if self.asr_instance:
|
||||
self.asr_instance.need_reconnect = True
|
||||
else:
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id, f"Gummy ASR错误: {self.error_message}"
|
||||
)
|
||||
self.transcript_event.set()
|
||||
|
||||
def on_close(self) -> None:
|
||||
session_logger.info(self.device_id, self.session_id, "Gummy ASR连接已关闭")
|
||||
self.is_closed = True
|
||||
self.transcript_event.set()
|
||||
class ASRError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class AliyunASR(ASR, BaseService):
|
||||
def __init__(self, selected_role=None):
|
||||
BaseService.__init__(self, "asr", selected_role)
|
||||
self.api_key = settings.aliyun_api_key
|
||||
self.vocabulary_id = settings.aliyun_vocabulary_id
|
||||
self.connected = False
|
||||
self.transcript = ""
|
||||
self.model = settings.aliyun_asr_model
|
||||
self.url = settings.aliyun_asr_url
|
||||
# Gummy vocabulary IDs are model-specific; never reuse them implicitly.
|
||||
self.vocabulary_id = settings.aliyun_asr_vocabulary_id
|
||||
self.sample_rate = 16000
|
||||
self.device_id = "unknown"
|
||||
self.session_id = "unknown"
|
||||
self.sample_rate = 16000
|
||||
self.connected = False
|
||||
self.transcript = ""
|
||||
self.error = None
|
||||
self.task_id = None
|
||||
self.audio_buffer = bytearray()
|
||||
self.translator = None
|
||||
self.callback = None
|
||||
self.buffer_size = 3200
|
||||
self.max_end_silence = 2000
|
||||
self.max_retries = 3
|
||||
self.retry_delay = 1.0
|
||||
self.need_reconnect = False
|
||||
self._client = None
|
||||
self._websocket = None
|
||||
self._receiver = None
|
||||
self._ready = asyncio.Event()
|
||||
self._finished = asyncio.Event()
|
||||
self._sentences = {}
|
||||
self._pending = set()
|
||||
self._complete = False
|
||||
self._ending = False
|
||||
self._closing = False
|
||||
self._end_lock = asyncio.Lock()
|
||||
self._close_lock = asyncio.Lock()
|
||||
|
||||
def _fail(self, message):
|
||||
if self.error is None:
|
||||
self.error = ASRError(message)
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id,
|
||||
f"ASR失败 model={self.model} task_id={self.task_id}: {message}",
|
||||
)
|
||||
self.connected = False
|
||||
self.transcript = ""
|
||||
self._ready.set()
|
||||
self._finished.set()
|
||||
|
||||
def _check_error(self):
|
||||
if self.error:
|
||||
raise self.error
|
||||
|
||||
def _command(self, action, payload):
|
||||
return {
|
||||
"header": {"action": action, "task_id": self.task_id, "streaming": "duplex"},
|
||||
"payload": payload,
|
||||
}
|
||||
|
||||
async def connect(self):
|
||||
if self.connected:
|
||||
return
|
||||
await self.close()
|
||||
self._closing = False
|
||||
self._ending = False
|
||||
self._complete = False
|
||||
self.error = None
|
||||
self.transcript = ""
|
||||
self.audio_buffer.clear()
|
||||
self._sentences.clear()
|
||||
self._pending.clear()
|
||||
self._ready.clear()
|
||||
self._finished.clear()
|
||||
self.task_id = uuid.uuid4().hex
|
||||
parameters = {
|
||||
"format": "mp3",
|
||||
"sample_rate": self.sample_rate,
|
||||
"semantic_punctuation_enabled": False,
|
||||
"max_sentence_silence": 2000,
|
||||
"heartbeat": True,
|
||||
}
|
||||
if self.vocabulary_id:
|
||||
parameters["vocabulary_id"] = self.vocabulary_id
|
||||
try:
|
||||
if self.connected and self.translator:
|
||||
session_logger.info(
|
||||
self.device_id, self.session_id, "Gummy ASR连接已存在，复用当前连接"
|
||||
)
|
||||
return
|
||||
|
||||
self.transcript = ""
|
||||
|
||||
self.callback = GummyCallback(self.device_id, self.session_id, self)
|
||||
|
||||
self.translator = TranslationRecognizerChat(
|
||||
model="gummy-chat-v1",
|
||||
format="mp3",
|
||||
sample_rate=self.sample_rate,
|
||||
callback=self.callback,
|
||||
transcription_enabled=True,
|
||||
source_language="auto",
|
||||
semantic_punctuation_enabled=False,
|
||||
max_end_silence=self.max_end_silence,
|
||||
api_key=self.api_key,
|
||||
vocabulary_id=self.vocabulary_id,
|
||||
if not self.api_key:
|
||||
raise ASRError("未配置 ALIYUN_API_KEY")
|
||||
# Native async I/O allows cancellation and bounded cleanup; the old
|
||||
# SDK's stop() joins a worker thread without a timeout.
|
||||
self._client = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=None, connect=settings.asr_connect_timeout_seconds),
|
||||
)
|
||||
|
||||
self.translator.start()
|
||||
self.callback.transcript_event.clear()
|
||||
self.connected = True
|
||||
self.need_reconnect = False
|
||||
|
||||
except Exception as e:
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id, f"建立Gummy ASR连接失败: {str(e)}"
|
||||
self._websocket = await asyncio.wait_for(
|
||||
self._client.ws_connect(
|
||||
self.url, headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
heartbeat=20,
|
||||
),
|
||||
timeout=settings.asr_connect_timeout_seconds,
|
||||
)
|
||||
await self._cleanup_connection()
|
||||
|
||||
async def _cleanup_connection(self):
|
||||
if self.translator:
|
||||
try:
|
||||
self.translator.stop()
|
||||
except:
|
||||
pass
|
||||
self.translator = None
|
||||
self.callback = None
|
||||
self.connected = False
|
||||
|
||||
async def _handle_reconnect(self):
|
||||
if not self.need_reconnect:
|
||||
return True
|
||||
|
||||
session_logger.warning(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
"检测到限流错误，准备重连..."
|
||||
)
|
||||
|
||||
await self._cleanup_connection()
|
||||
|
||||
current_delay = self.retry_delay
|
||||
for attempt in range(self.max_retries):
|
||||
self._receiver = asyncio.create_task(self._receive(), name=f"asr:{self.task_id}")
|
||||
await asyncio.wait_for(self._websocket.send_json(self._command("run-task", {
|
||||
"task_group": "audio", "task": "asr", "function": "recognition",
|
||||
"model": self.model, "parameters": parameters, "input": {},
|
||||
})), timeout=settings.asr_send_timeout_seconds)
|
||||
await asyncio.wait_for(self._ready.wait(), timeout=settings.asr_connect_timeout_seconds)
|
||||
self._check_error()
|
||||
session_logger.info(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"重连尝试 {attempt + 1}/{self.max_retries}，等待{current_delay}秒"
|
||||
self.device_id, self.session_id,
|
||||
f"ASR任务已就绪 model={self.model} task_id={self.task_id}",
|
||||
)
|
||||
await asyncio.sleep(current_delay)
|
||||
|
||||
try:
|
||||
await self.connect()
|
||||
if self.connected:
|
||||
session_logger.info(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
"重连成功"
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
session_logger.error(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"重连失败: {str(e)}"
|
||||
)
|
||||
|
||||
current_delay *= 2
|
||||
self.retry_delay = current_delay
|
||||
|
||||
session_logger.error(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
"重连失败，已达到最大重试次数"
|
||||
)
|
||||
self.need_reconnect = False
|
||||
return False
|
||||
except asyncio.CancelledError:
|
||||
await self.close()
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._fail(f"启动失败: {type(exc).__name__}: {exc}")
|
||||
await self.close()
|
||||
raise self.error from exc
|
||||
|
||||
async def _receive(self):
|
||||
try:
|
||||
async for message in self._websocket:
|
||||
if message.type == aiohttp.WSMsgType.ERROR:
|
||||
raise ASRError("识别 WebSocket 传输错误")
|
||||
if message.type != aiohttp.WSMsgType.TEXT:
|
||||
continue
|
||||
event = json.loads(message.data)
|
||||
header = event.get("header", {})
|
||||
if header.get("task_id") != self.task_id:
|
||||
continue
|
||||
kind = header.get("event")
|
||||
if kind == "task-started":
|
||||
self.connected = True
|
||||
self._ready.set()
|
||||
elif kind == "result-generated":
|
||||
sentence = event.get("payload", {}).get("output", {}).get("sentence", {})
|
||||
if not sentence or sentence.get("heartbeat"):
|
||||
continue
|
||||
key = sentence.get("sentence_id", sentence.get("begin_time"))
|
||||
if key is None:
|
||||
raise ASRError("识别结果缺少句子编号和起始时间")
|
||||
if sentence.get("sentence_end"):
|
||||
self._sentences[key] = sentence.get("text", "")
|
||||
self._pending.discard(key)
|
||||
elif sentence.get("text") and key not in self._sentences:
|
||||
self._pending.add(key)
|
||||
elif kind == "task-finished":
|
||||
if not self._ending:
|
||||
self._fail("识别任务提前结束，设备音频尚未发送完毕")
|
||||
return
|
||||
self._complete = True
|
||||
self.connected = False
|
||||
self._finished.set()
|
||||
return
|
||||
elif kind == "task-failed":
|
||||
self._fail(f"{header.get('error_code', 'Unknown')}: {header.get('error_message', '')}")
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._fail(f"接收失败: {type(exc).__name__}: {exc}")
|
||||
finally:
|
||||
if not self._complete and not self._closing and not self.error:
|
||||
self._fail("识别连接提前关闭，未收到 task-finished")
|
||||
|
||||
async def _send_bytes(self, data):
|
||||
self._check_error()
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._websocket.send_bytes(data), timeout=settings.asr_send_timeout_seconds,
|
||||
)
|
||||
except Exception as exc:
|
||||
self._fail(f"发送音频失败: {type(exc).__name__}: {exc}")
|
||||
raise self.error from exc
|
||||
|
||||
async def send_audio(self, audio_data: bytes):
|
||||
if self.need_reconnect:
|
||||
reconnected = await self._handle_reconnect()
|
||||
if not reconnected:
|
||||
session_logger.error(self.device_id, self.session_id, "Gummy ASR重连失败，无法发送音频")
|
||||
return
|
||||
|
||||
if not self.connected or not self.translator:
|
||||
session_logger.error(self.device_id, self.session_id, "Gummy ASR连接未建立")
|
||||
return
|
||||
|
||||
try:
|
||||
self.audio_buffer.extend(audio_data)
|
||||
|
||||
while len(self.audio_buffer) >= self.buffer_size:
|
||||
if self.need_reconnect:
|
||||
session_logger.info(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
"检测到限流错误，暂停发送音频，准备重连"
|
||||
)
|
||||
reconnected = await self._handle_reconnect()
|
||||
if not reconnected:
|
||||
return
|
||||
if not self.connected or not self.translator:
|
||||
return
|
||||
continue
|
||||
|
||||
chunk = bytes(self.audio_buffer[: self.buffer_size])
|
||||
self.audio_buffer = self.audio_buffer[self.buffer_size :]
|
||||
|
||||
if self.device_id and self.session_id:
|
||||
session_key = (self.device_id, self.session_id)
|
||||
if interrupt_handler.is_interrupted(session_key):
|
||||
session_logger.info(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
"检测到中断，停止发送音频数据",
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
self.translator.send_audio_frame(chunk)
|
||||
except Exception as e:
|
||||
session_logger.error(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"发送音频帧到Gummy ASR出错: {str(e)}",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id, f"发送音频到Gummy ASR出错: {str(e)}"
|
||||
)
|
||||
self._check_error()
|
||||
if interrupt_handler.is_interrupted((self.device_id, self.session_id)):
|
||||
raise ASRError("会话已中断")
|
||||
if not self.connected or self._ending:
|
||||
raise ASRError("识别任务未就绪或已结束")
|
||||
self.audio_buffer.extend(audio_data)
|
||||
while len(self.audio_buffer) >= self.buffer_size:
|
||||
chunk = bytes(self.audio_buffer[:self.buffer_size])
|
||||
await self._send_bytes(chunk)
|
||||
del self.audio_buffer[:self.buffer_size]
|
||||
|
||||
async def send_end(self):
|
||||
if not self.connected or not self.translator:
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id, "Gummy ASR连接未建立，无法发送结束信号"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
if len(self.audio_buffer) > 0:
|
||||
chunk = bytes(self.audio_buffer)
|
||||
self.audio_buffer = bytearray()
|
||||
self.translator.send_audio_frame(chunk)
|
||||
|
||||
session_logger.info(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
"客户端发送结束包，等待Gummy ASR处理完毕",
|
||||
)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, lambda: self._safe_stop_translator())
|
||||
|
||||
except Exception as e:
|
||||
session_logger.error(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"发送结束信号到Gummy ASR出错: {str(e)}",
|
||||
)
|
||||
async with self._end_lock:
|
||||
self._check_error()
|
||||
if self._complete:
|
||||
return
|
||||
if not self.connected:
|
||||
raise ASRError("识别任务未就绪")
|
||||
self._ending = True
|
||||
try:
|
||||
if self.audio_buffer:
|
||||
await self._send_bytes(bytes(self.audio_buffer))
|
||||
self.audio_buffer.clear()
|
||||
await asyncio.wait_for(
|
||||
self._websocket.send_json(self._command("finish-task", {"input": {}})),
|
||||
timeout=settings.asr_send_timeout_seconds,
|
||||
)
|
||||
await asyncio.wait_for(self._finished.wait(), timeout=settings.asr_final_timeout_seconds)
|
||||
self._check_error()
|
||||
except asyncio.CancelledError:
|
||||
await self.close()
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._fail(f"结束识别失败: {type(exc).__name__}: {exc}")
|
||||
raise self.error from exc
|
||||
|
||||
async def receive_results(self):
|
||||
if not self.connected:
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id, "Gummy ASR连接未建立，无法接收结果"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self.callback.transcript_event.wait(), timeout=0.5
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
session_logger.warning(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
"接收Gummy ASR结果超时，使用最后的中间结果",
|
||||
)
|
||||
|
||||
if self.callback.received_result or self.callback.transcript:
|
||||
self.transcript = self.callback.transcript
|
||||
|
||||
except Exception as e:
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id, f"接收Gummy ASR结果出错: {str(e)}"
|
||||
)
|
||||
|
||||
self._check_error()
|
||||
if not self._complete or self._pending:
|
||||
raise ASRError("识别未完整结束，拒绝使用不完整转录")
|
||||
self.transcript = "".join(self._sentences[key] for key in sorted(self._sentences))
|
||||
session_logger.info(
|
||||
self.device_id, self.session_id,
|
||||
f"ASR最终结果 model={self.model} task_id={self.task_id}: {self.transcript}",
|
||||
)
|
||||
return self.transcript
|
||||
|
||||
async def close(self):
|
||||
if self.connected:
|
||||
try:
|
||||
if self.translator is not None:
|
||||
try:
|
||||
self.translator.stop()
|
||||
except Exception as e:
|
||||
if "has stopped" not in str(e):
|
||||
session_logger.warning(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"关闭Gummy translator出错: {str(e)}"
|
||||
)
|
||||
self.translator = None
|
||||
self.connected = False
|
||||
session_logger.info(
|
||||
self.device_id, self.session_id, "Gummy ASR连接已关闭"
|
||||
)
|
||||
except Exception as e:
|
||||
session_logger.error(
|
||||
self.device_id, self.session_id, f"关闭Gummy ASR连接出错: {str(e)}"
|
||||
)
|
||||
async with self._close_lock:
|
||||
await self._close_connection()
|
||||
|
||||
def _safe_stop_translator(self):
|
||||
"""安全停止translator，忽略已停止的异常"""
|
||||
async def _close_connection(self):
|
||||
self._closing = True
|
||||
self.connected = False
|
||||
# Wake waiters if the session is interrupted during start or finish.
|
||||
if self.task_id and not self._complete and not self.error:
|
||||
self.error = ASRError("识别会话已关闭")
|
||||
self._ready.set()
|
||||
self._finished.set()
|
||||
if self._receiver:
|
||||
self._receiver.cancel()
|
||||
await asyncio.gather(self._receiver, return_exceptions=True)
|
||||
self._receiver = None
|
||||
try:
|
||||
if self.translator:
|
||||
self.translator.stop()
|
||||
except Exception as e:
|
||||
if "has stopped" not in str(e):
|
||||
session_logger.warning(
|
||||
self.device_id,
|
||||
self.session_id,
|
||||
f"停止translator时出现非标准错误: {str(e)}",
|
||||
)
|
||||
return None
|
||||
if self._websocket and not self._websocket.closed:
|
||||
await asyncio.wait_for(self._websocket.close(), timeout=1.0)
|
||||
except (asyncio.TimeoutError, aiohttp.ClientError):
|
||||
pass
|
||||
finally:
|
||||
if self._client:
|
||||
await self._client.close()
|
||||
self._client = None
|
||||
self._websocket = None
|
||||
self.audio_buffer.clear()
|
||||
|
||||
Reference in New Issue
Block a user