fix(voice): recover dialogue after abandoned message recordings
This commit is contained in:
stu2not
@@ -71,6 +71,7 @@ class Settings(BaseSettings):
|
||||
|
||||
assets_dir: str = Field(default="assets", validation_alias="ASSETS_DIR")
|
||||
session_timeout: int = Field(default=600, validation_alias="SESSION_TIMEOUT")
|
||||
voice_target_timeout_seconds: float = Field(default=120, gt=0, le=600, validation_alias="VOICE_TARGET_TIMEOUT_SECONDS")
|
||||
conversation_history_timeout: int = Field(default=1800, validation_alias="CONVERSATION_HISTORY_TIMEOUT")
|
||||
max_conversation_history: int = Field(default=5, validation_alias="MAX_CONVERSATION_HISTORY")
|
||||
cleanup_interval: int = Field(default=300, validation_alias="CLEANUP_INTERVAL")
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# 留言状态与后续 AI 对话恢复
|
||||
|
||||
留言目标只属于一次录音和一个 WebSocket 连接。目标设置后默认空闲有效期为
|
||||
120 秒，可用 `VOICE_TARGET_TIMEOUT_SECONDS` 调整；录音开始及有效留言音频会刷新
|
||||
空闲期限。每次访问均检查过期状态，后台会定期回收过期目标及缓存。
|
||||
|
||||
开始包统一返回一次 `START_ACK`。存在留言目标时，根据首个音频包选择流程：
|
||||
类型 `0` 清除待处理留言目标并完成 AI 初始化，原始首包随后正常送入 ASR；
|
||||
类型 `4` 写入本次留言的独立缓存。录音开始传输音频后不允许切换类型。
|
||||
|
||||
结束、取消、异常、空音频、超时和连接断开均会释放对应目标及音频缓存。
|
||||
结束时先消费本次目标，再处理上传和通知，避免旧处理完成时清除新设置的目标。
|
||||
旧连接断开只清理该连接的会话和任务，不影响替换它的新连接。
|
||||
音频缓存键按发送设备及本次目标独立生成，多个发送方对同一接收方留言时互不覆盖。
|
||||
|
||||
本次修复保留既有数据包类型、鉴权及设备协议。家长留言的上传、会话保存和通知流程
|
||||
沿用原实现。
|
||||
|
||||
## 本地验证（2026-10-28）
|
||||
|
||||
- 全量后端回归 243/243，包括新增的 28 项留言状态与恢复测试。
|
||||
- 本机隔离 MySQL、原有 `/ws` 路由和真实 ASR/LLM/TTS：正常对话及静音 10/10，
|
||||
中断及恢复 4/4；包含两个角色在待处理、过期、取消家长留言后的 AI 对话。
|
||||
- 实际 WebSocket 家长留言取消 2 次，均收到取消确认及处理完成通知，随后 AI 对话成功。
|
||||
- 共 52 段音频通过 HTTP 200、Range 206、完整 MP3 解码及非零 PCM 检查。
|
||||
- 正常留言的上传、保存、通知和失败分支使用本地替身测试，未向真实家长发送通知。
|
||||
- 没有访问或部署生产服务，未验证实物设备的麦克风、扬声器及播放中打断。
|
||||
|
||||
覆盖边界：留言取消、空音频、处理异常、过期及未结束录音后的 AI 恢复已有回归；
|
||||
AI 中断后再次 AI 对话已有真实服务验证。留言取消后再次留言验证了录音隔离，
|
||||
AI 中断后留言验证了启动、收音及取消，尚未串联验证这两种情况下第二条留言完整发送。
|
||||
正常家长留言处理的成功、空音频和上传异常分支已用模拟外部接口验证。
|
||||
|
||||
状态测试：`.venv/bin/python -m pytest tests/test_voice_target_lifecycle.py -q`
|
||||
|
||||
完整结果：[本地测试汇总](../output/voice-target-recovery-20261028/summary.json)。
|
||||
@@ -22,8 +22,10 @@ async def handle_websocket_data(
|
||||
|
||||
if packet_type == 1: # 开始包
|
||||
try:
|
||||
await websocket.send_text("START_ACK")
|
||||
session_logger.info(device_id, session_id, "已发送开始包确认")
|
||||
if not session.start_ack_sent:
|
||||
await websocket.send_text("START_ACK")
|
||||
session.start_ack_sent = True
|
||||
session_logger.info(device_id, session_id, "已发送开始包确认")
|
||||
except Exception as e:
|
||||
session_logger.error(device_id, session_id, f"发送开始包确认失败: {str(e)}")
|
||||
session.sample_rate = sample_rate
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from services.interrupt_handler import interrupt_handler
|
||||
from services.audio_session import audio_session_manager
|
||||
from utils.logger import session_logger
|
||||
from services.device_target_cache import device_target_cache
|
||||
from services.task_manager import task_manager
|
||||
|
||||
async def handle_old_session_cleanup(old_session, session_key):
|
||||
"""
|
||||
@@ -21,19 +23,22 @@ async def handle_old_session_cleanup(old_session, session_key):
|
||||
session_logger.error("unknown", "unknown", f"清理旧会话资源时出错: {e}")
|
||||
|
||||
|
||||
async def cleanup_device_sessions(device_id):
|
||||
async def cleanup_device_sessions(device_id, websocket=None):
|
||||
"""
|
||||
清理设备所有会话
|
||||
|
||||
Args:
|
||||
device_id: 设备ID
|
||||
"""
|
||||
await device_target_cache.remove_target(device_id, websocket=websocket, reason="disconnected")
|
||||
sessions_to_clean = []
|
||||
for session_key, session in await audio_session_manager.get_all_sessions():
|
||||
if session_key[0] == device_id:
|
||||
if session_key[0] == device_id and (websocket is None or session.websocket is websocket):
|
||||
sessions_to_clean.append((session_key, session))
|
||||
|
||||
for session_key, session in sessions_to_clean:
|
||||
if websocket is not None and session.websocket is not websocket:
|
||||
continue
|
||||
if hasattr(session, "set_interrupted"):
|
||||
session_logger.info(
|
||||
device_id, session_key[1], "WebSocket连接断开，中断会话"
|
||||
@@ -44,8 +49,11 @@ async def cleanup_device_sessions(device_id):
|
||||
await interrupt_handler.set_interrupt_state(session_key, True)
|
||||
if session.llm_service and hasattr(session.llm_service, "closed"):
|
||||
session.llm_service.closed = True
|
||||
await task_manager.cancel_session_tasks(session_key)
|
||||
|
||||
for session_key, _ in sessions_to_clean:
|
||||
for session_key, session in sessions_to_clean:
|
||||
if websocket is not None and session.websocket is not websocket:
|
||||
continue
|
||||
await audio_session_manager.remove_session(session_key)
|
||||
await interrupt_handler.remove_session(session_key)
|
||||
session_logger.info(
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from fastapi import WebSocket, WebSocketDisconnect
|
||||
from utils.logger import session_logger
|
||||
from services.connection_manager import connection_manager
|
||||
from services.task_manager import task_manager
|
||||
from services.offline_audio_cache import offline_audio_cache
|
||||
from handlers.websocket_auth_handler import authenticate_websocket
|
||||
from handlers.websocket_message_handler import handle_websocket_messages
|
||||
@@ -59,9 +58,7 @@ async def websocket_endpoint(websocket: WebSocket):
|
||||
finally:
|
||||
if device_id:
|
||||
await connection_manager.remove_connection(device_id, websocket)
|
||||
await cleanup_device_sessions(device_id)
|
||||
# 清理设备相关的所有异步任务
|
||||
await task_manager.cancel_device_tasks(device_id)
|
||||
await cleanup_device_sessions(device_id, websocket)
|
||||
|
||||
session_logger.info(
|
||||
device_id or "unknown",
|
||||
|
||||
@@ -44,6 +44,7 @@ async def handle_websocket_messages(websocket: WebSocket, device_id: str, serial
|
||||
websocket: WebSocket连接
|
||||
device_id: 设备ID
|
||||
"""
|
||||
await device_target_cache.bind_connection(device_id, websocket)
|
||||
current_active_session = None
|
||||
first_audio_received_time = None
|
||||
|
||||
@@ -156,54 +157,111 @@ async def handle_text_message(websocket: WebSocket, device_id: str, text_data: s
|
||||
|
||||
async def handle_binary_message(websocket: WebSocket, device_id: str, serial_number: str, binary_data, current_active_session, first_audio_received_time):
|
||||
"""处理二进制音频消息"""
|
||||
active_connection = await connection_manager.get_connection(device_id)
|
||||
if active_connection is not None and active_connection is not websocket:
|
||||
session_logger.warning(device_id, "connection", "忽略已被替换连接的录音包")
|
||||
return current_active_session, first_audio_received_time
|
||||
session_key, session, packet_type, audio_data, sample_rate = await parse_packet(binary_data, websocket)
|
||||
|
||||
if session:
|
||||
session.websocket = websocket
|
||||
session_id = session_key[1]
|
||||
|
||||
if packet_type == 1: # 开始包
|
||||
voice_target = await device_target_cache.get_voice_target(device_id)
|
||||
if voice_target:
|
||||
await target_audio_cache.clear_audio_data(voice_target.audio_cache_key)
|
||||
return None, None
|
||||
current_active_session = await handle_start_packet(
|
||||
device_id, session_id, session_key, session, current_active_session
|
||||
if not session:
|
||||
return current_active_session, first_audio_received_time
|
||||
session_id = session_key[1]
|
||||
if session.websocket is not None and session.websocket is not websocket:
|
||||
session_logger.warning(device_id, session_id, "忽略其他连接的录音包")
|
||||
return current_active_session, first_audio_received_time
|
||||
session.websocket = websocket
|
||||
session.last_activity_time = asyncio.get_running_loop().time()
|
||||
|
||||
if packet_type == 1:
|
||||
if session.start_received:
|
||||
return current_active_session, first_audio_received_time
|
||||
session.start_received = True
|
||||
session.voice_target = await device_target_cache.claim_target(device_id, websocket, session_id)
|
||||
if session.voice_target:
|
||||
# Acknowledge START now; the first audio packet selects AI (0) or
|
||||
# message (4). A pending message must not swallow an AI recording.
|
||||
await websocket.send_text("START_ACK")
|
||||
session.start_ack_sent = True
|
||||
return session_key, None
|
||||
current_active_session = await start_ai_recording(
|
||||
websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
|
||||
)
|
||||
return current_active_session, None
|
||||
|
||||
if not session.start_received or session.recording_finished:
|
||||
session_logger.warning(device_id, session_id, "忽略未开始或已结束的录音包")
|
||||
return current_active_session, first_audio_received_time
|
||||
|
||||
if packet_type == 0:
|
||||
if session.message_audio_received:
|
||||
session_logger.warning(device_id, session_id, "忽略留言录音中混入的普通对话音频")
|
||||
return current_active_session, first_audio_received_time
|
||||
if not session.ai_started:
|
||||
target = session.voice_target
|
||||
if target:
|
||||
await device_target_cache.remove_target(device_id, expected=target, reason="ai_audio")
|
||||
session.voice_target = None
|
||||
current_active_session = await start_ai_recording(
|
||||
websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
|
||||
)
|
||||
elif packet_type == 3: # 中断包
|
||||
voice_target = await device_target_cache.get_voice_target(device_id)
|
||||
if voice_target:
|
||||
return None, None
|
||||
current_active_session = await handle_interrupt_packet(
|
||||
device_id, session_id, session_key, session, current_active_session
|
||||
)
|
||||
elif packet_type == 2: # 结束包
|
||||
voice_target = await device_target_cache.get_voice_target(device_id)
|
||||
if voice_target:
|
||||
session_logger.info(device_id, session_id, f"收到结束包，开始处理缓存音频数据")
|
||||
if voice_target.kind == VOICE_TARGET_KIND_PARENT:
|
||||
await process_parent_leave_message(device_id, voice_target.audio_cache_key)
|
||||
else:
|
||||
await process_cached_audio(device_id, voice_target.target_device_id, serial_number)
|
||||
await device_target_cache.remove_target(device_id)
|
||||
return None, None
|
||||
if current_active_session == session_key:
|
||||
current_active_session = None
|
||||
elif packet_type == 4: # 发送给目标设备的音频包
|
||||
await handle_target_audio_packet(device_id, audio_data)
|
||||
return None, None
|
||||
elif packet_type == 0 and first_audio_received_time is None:
|
||||
if first_audio_received_time is None:
|
||||
first_audio_received_time = time.perf_counter()
|
||||
session.start_time = first_audio_received_time
|
||||
session_logger.info(
|
||||
device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}"
|
||||
)
|
||||
|
||||
await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate)
|
||||
|
||||
session_logger.info(device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}")
|
||||
elif packet_type == 4:
|
||||
target = session.voice_target
|
||||
if target and not session.ai_started:
|
||||
if await handle_target_audio_packet(device_id, audio_data, target, websocket, session_id):
|
||||
session.message_audio_received = True
|
||||
else:
|
||||
session_logger.warning(device_id, session_id, "忽略没有对应留言目标的留言音频包")
|
||||
return current_active_session, first_audio_received_time
|
||||
elif packet_type == 3:
|
||||
if session.voice_target:
|
||||
await device_target_cache.remove_target(device_id, expected=session.voice_target, reason="interrupted")
|
||||
session.voice_target = None
|
||||
session.recording_finished = True
|
||||
await websocket.send_text("INTERRUPT_ACK")
|
||||
await websocket.send_text("INTERRUPT_PROCESSED")
|
||||
return None if current_active_session == session_key else current_active_session, None
|
||||
current_active_session = await handle_interrupt_packet(
|
||||
device_id, session_id, session_key, session, current_active_session
|
||||
)
|
||||
elif packet_type == 2:
|
||||
if session.voice_target:
|
||||
target = await device_target_cache.take_target(device_id, session.voice_target, websocket, session_id)
|
||||
session.voice_target = None
|
||||
session.recording_finished = True
|
||||
try:
|
||||
await websocket.send_text("END_ACK")
|
||||
if target:
|
||||
session_logger.info(device_id, session_id, "收到结束包，开始处理缓存音频数据")
|
||||
if target.kind == VOICE_TARGET_KIND_PARENT:
|
||||
await process_parent_leave_message(device_id, target.audio_cache_key)
|
||||
else:
|
||||
await process_cached_audio(device_id, target.target_device_id, serial_number,
|
||||
audio_cache_key=target.audio_cache_key)
|
||||
finally:
|
||||
if target:
|
||||
await target_audio_cache.clear_audio_data(target.audio_cache_key)
|
||||
return None if current_active_session == session_key else current_active_session, None
|
||||
if current_active_session == session_key:
|
||||
current_active_session = None
|
||||
|
||||
await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate)
|
||||
if packet_type in (2, 3):
|
||||
session.recording_finished = True
|
||||
return current_active_session, first_audio_received_time
|
||||
|
||||
|
||||
async def start_ai_recording(websocket, device_id, session_id, session_key, session, current_active_session, sample_rate):
|
||||
current_active_session = await handle_start_packet(
|
||||
device_id, session_id, session_key, session, current_active_session
|
||||
)
|
||||
await handle_websocket_data(websocket, session_key, session, 1, b"", sample_rate)
|
||||
session.ai_started = True
|
||||
return current_active_session
|
||||
|
||||
|
||||
async def handle_start_packet(device_id, session_id, session_key, session, current_active_session):
|
||||
"""处理会话开始包"""
|
||||
if current_active_session and current_active_session[1] != session_id:
|
||||
@@ -255,20 +313,17 @@ async def handle_interrupt_packet(device_id, session_id, session_key, session, c
|
||||
return current_active_session
|
||||
|
||||
|
||||
async def handle_target_audio_packet(device_id: str, audio_data: bytes):
|
||||
async def handle_target_audio_packet(device_id: str, audio_data: bytes, voice_target, websocket, session_id):
|
||||
"""处理发送给目标设备的音频包"""
|
||||
try:
|
||||
voice_target = await device_target_cache.get_voice_target(device_id)
|
||||
|
||||
if not voice_target:
|
||||
session_logger.warning(device_id, "target", "未设置目标设备，无法发送音频")
|
||||
return
|
||||
|
||||
await target_audio_cache.add_audio_data(voice_target.audio_cache_key, audio_data)
|
||||
if not await device_target_cache.append_audio(device_id, voice_target, websocket, session_id, audio_data):
|
||||
session_logger.warning(device_id, session_id, "忽略已过期或不属于本次录音的留言音频")
|
||||
return
|
||||
if voice_target.kind == VOICE_TARGET_KIND_PARENT:
|
||||
session_logger.info(device_id, "target", "已缓存发给家长的留言音频数据")
|
||||
else:
|
||||
session_logger.info(device_id, "target", f"已缓存音频数据到目标设备 {voice_target.target_device_id}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
session_logger.error(device_id, "target", f"处理目标音频包时出错: {e}", exc_info=True)
|
||||
@@ -337,10 +392,11 @@ async def process_parent_leave_message(device_id: str, audio_cache_key: str):
|
||||
await target_audio_cache.clear_audio_data(audio_cache_key)
|
||||
|
||||
|
||||
async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str):
|
||||
async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str, *, audio_cache_key=None):
|
||||
"""处理缓存的音频数据并发送音频URL"""
|
||||
try:
|
||||
cached_audio = await target_audio_cache.get_audio_data(target_device_id)
|
||||
cache_key = audio_cache_key or target_device_id
|
||||
cached_audio = await target_audio_cache.get_audio_data(cache_key)
|
||||
if not cached_audio:
|
||||
session_logger.info(device_id, "target", f"目标设备 {target_device_id} 没有缓存的音频数据")
|
||||
return
|
||||
@@ -398,4 +454,4 @@ async def process_cached_audio(device_id: str, target_device_id: str, serial_num
|
||||
session_logger.error(device_id, "target", f"处理缓存音频时出错: {e}", exc_info=True)
|
||||
finally:
|
||||
# 清除缓存
|
||||
await target_audio_cache.clear_audio_data(target_device_id)
|
||||
await target_audio_cache.clear_audio_data(cache_key)
|
||||
|
||||
@@ -30,6 +30,12 @@ class AudioSession:
|
||||
self.session_id = None
|
||||
self.sample_rate = sample_rate
|
||||
self.audio_queue = None
|
||||
self.voice_target = None
|
||||
self.start_received = False
|
||||
self.start_ack_sent = False
|
||||
self.ai_started = False
|
||||
self.recording_finished = False
|
||||
self.message_audio_received = False
|
||||
|
||||
async def set_interrupted(self, interrupted=True):
|
||||
|
||||
|
||||
@@ -1,59 +1,143 @@
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from config import settings
|
||||
from services.connection_manager import connection_manager
|
||||
from services.target_audio_cache import target_audio_cache
|
||||
from utils.logger import session_logger
|
||||
|
||||
|
||||
VOICE_TARGET_KIND_DEVICE = "device"
|
||||
VOICE_TARGET_KIND_PARENT = "parent"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@dataclass
|
||||
class VoiceTarget:
|
||||
kind: str
|
||||
target_device_id: Optional[str] = None
|
||||
uuid: Optional[str] = None
|
||||
audio_cache_key: Optional[str] = None
|
||||
websocket: object = None
|
||||
session_id: Optional[str] = None
|
||||
expires_at: float = 0
|
||||
|
||||
|
||||
class DeviceTargetCache:
|
||||
def __init__(self):
|
||||
def __init__(self, *, timeout_seconds=None, clock=None):
|
||||
self.targets: Dict[str, VoiceTarget] = {}
|
||||
self.lock = asyncio.Lock()
|
||||
self.timeout_seconds = timeout_seconds if timeout_seconds is not None else settings.voice_target_timeout_seconds
|
||||
self.clock = clock or time.monotonic
|
||||
|
||||
async def _remove_locked(self, device_id, target, reason, *, clear_audio=True):
|
||||
del self.targets[device_id]
|
||||
if clear_audio:
|
||||
await target_audio_cache.clear_audio_data(target.audio_cache_key)
|
||||
session_logger.info(device_id, target.session_id or "target",
|
||||
f"留言目标已清除: kind={target.kind}, reason={reason}")
|
||||
|
||||
async def _get_locked(self, device_id):
|
||||
target = self.targets.get(device_id)
|
||||
if target and self.clock() >= target.expires_at:
|
||||
await self._remove_locked(device_id, target, "timeout")
|
||||
return None
|
||||
return target
|
||||
|
||||
async def get_target(self, device_id: str) -> Optional[str]:
|
||||
async with self.lock:
|
||||
target = self.targets.get(device_id)
|
||||
if not target or target.kind != VOICE_TARGET_KIND_DEVICE:
|
||||
return None
|
||||
return target.target_device_id
|
||||
target = await self.get_voice_target(device_id)
|
||||
return target.target_device_id if target and target.kind == VOICE_TARGET_KIND_DEVICE else None
|
||||
|
||||
async def get_voice_target(self, device_id: str) -> Optional[VoiceTarget]:
|
||||
async with self.lock:
|
||||
return self.targets.get(device_id)
|
||||
return await self._get_locked(device_id)
|
||||
|
||||
async def _set_target(self, device_id, *, kind, target_device_id=None, uuid=None):
|
||||
async with self.lock:
|
||||
old = self.targets.get(device_id)
|
||||
if old:
|
||||
await self._remove_locked(device_id, old, "replaced")
|
||||
self.targets[device_id] = VoiceTarget(
|
||||
kind=kind, target_device_id=target_device_id, uuid=uuid,
|
||||
audio_cache_key=f"voice:{device_id}:{uuid4().hex}",
|
||||
websocket=await connection_manager.get_connection(device_id),
|
||||
expires_at=self.clock() + self.timeout_seconds,
|
||||
)
|
||||
session_logger.info(device_id, "target", f"留言目标已设置: kind={kind}, timeout={self.timeout_seconds}s")
|
||||
|
||||
async def set_target(self, device_id: str, target_device_id: str):
|
||||
async with self.lock:
|
||||
self.targets[device_id] = VoiceTarget(
|
||||
kind=VOICE_TARGET_KIND_DEVICE,
|
||||
target_device_id=target_device_id,
|
||||
audio_cache_key=target_device_id,
|
||||
)
|
||||
await self._set_target(device_id, kind=VOICE_TARGET_KIND_DEVICE, target_device_id=target_device_id)
|
||||
|
||||
async def set_parent_target(self, device_id: str, uuid: str):
|
||||
async with self.lock:
|
||||
self.targets[device_id] = VoiceTarget(
|
||||
kind=VOICE_TARGET_KIND_PARENT,
|
||||
uuid=uuid,
|
||||
audio_cache_key=f"parent:{device_id}",
|
||||
)
|
||||
await self._set_target(device_id, kind=VOICE_TARGET_KIND_PARENT, uuid=uuid)
|
||||
|
||||
async def remove_target(self, device_id: str):
|
||||
async def claim_target(self, device_id, websocket, session_id):
|
||||
"""Bind one pending intent to one recording on one connection."""
|
||||
async with self.lock:
|
||||
if device_id in self.targets:
|
||||
del self.targets[device_id]
|
||||
active = await connection_manager.get_connection(device_id)
|
||||
if active is not None and active is not websocket:
|
||||
return None
|
||||
target = await self._get_locked(device_id)
|
||||
if target is None:
|
||||
return None
|
||||
if target.websocket is not None and target.websocket is not websocket:
|
||||
await self._remove_locked(device_id, target, "connection_changed")
|
||||
return None
|
||||
if target.session_id is not None and target.session_id != session_id:
|
||||
await self._remove_locked(device_id, target, "new_recording")
|
||||
return None
|
||||
target.websocket, target.session_id = websocket, session_id
|
||||
target.expires_at = self.clock() + self.timeout_seconds
|
||||
session_logger.info(device_id, session_id, f"留言目标已关联录音: kind={target.kind}")
|
||||
return target
|
||||
|
||||
async def bind_connection(self, device_id, websocket):
|
||||
"""Adopt commands received before connect; discard older socket intents."""
|
||||
async with self.lock:
|
||||
if await connection_manager.get_connection(device_id) is not websocket:
|
||||
return
|
||||
target = await self._get_locked(device_id)
|
||||
if target is None:
|
||||
return
|
||||
if target.websocket is not None and target.websocket is not websocket:
|
||||
await self._remove_locked(device_id, target, "connection_changed")
|
||||
else:
|
||||
target.websocket = websocket
|
||||
|
||||
async def append_audio(self, device_id, target, websocket, session_id, audio):
|
||||
async with self.lock:
|
||||
current = await self._get_locked(device_id)
|
||||
if current is not target or target.websocket is not websocket or target.session_id != session_id:
|
||||
return False
|
||||
target.expires_at = self.clock() + self.timeout_seconds
|
||||
await target_audio_cache.add_audio_data(target.audio_cache_key, audio)
|
||||
return True
|
||||
|
||||
async def take_target(self, device_id, target, websocket, session_id):
|
||||
"""Consume the exact recording before slow upload/notification work."""
|
||||
async with self.lock:
|
||||
current = await self._get_locked(device_id)
|
||||
if current is not target or target.websocket is not websocket or target.session_id != session_id:
|
||||
return None
|
||||
await self._remove_locked(device_id, target, "recording_ended", clear_audio=False)
|
||||
return target
|
||||
|
||||
async def remove_target(self, device_id: str, *, expected=None, websocket=None, reason="cancelled"):
|
||||
async with self.lock:
|
||||
target = self.targets.get(device_id)
|
||||
if target is None or (expected is not None and target is not expected):
|
||||
return False
|
||||
if websocket is not None and target.websocket is not websocket:
|
||||
return False
|
||||
await self._remove_locked(device_id, target, reason)
|
||||
return True
|
||||
|
||||
async def get_all_targets(self) -> list:
|
||||
async with self.lock:
|
||||
for device_id in list(self.targets):
|
||||
await self._get_locked(device_id)
|
||||
return list(self.targets.items())
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from config import settings
|
||||
from services.audio_session import audio_session_manager
|
||||
from utils.logger import session_logger
|
||||
from services.conversation_history import conversation_history_manager
|
||||
from services.device_target_cache import device_target_cache
|
||||
|
||||
async def run_in_executor(func, *args):
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -12,6 +13,7 @@ async def background_cleanup_task():
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(settings.cleanup_interval)
|
||||
await device_target_cache.get_all_targets()
|
||||
|
||||
current_time = asyncio.get_running_loop().time()
|
||||
audio_sessions_to_clean = []
|
||||
@@ -54,4 +56,4 @@ async def background_cleanup_task():
|
||||
except asyncio.CancelledError:
|
||||
session_logger.info("system", "cleanup", "后台清理任务已取消")
|
||||
except Exception as e:
|
||||
session_logger.error("system", "cleanup", f"后台清理任务出错: {str(e)}")
|
||||
session_logger.error("system", "cleanup", f"后台清理任务出错: {str(e)}")
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
"""Regression for stale message intents swallowing the next AI recording."""
|
||||
import asyncio
|
||||
import struct
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from handlers import audio_packet_parser as parser
|
||||
from handlers import audio_session_handler as audio_handler
|
||||
from handlers import session_cleanup_handler as cleanup
|
||||
from handlers import websocket_message_handler as messages
|
||||
from handlers import websocket_handler as endpoint
|
||||
from services import device_target_cache as targets_module
|
||||
from services.audio_session import AudioSessionManager
|
||||
from services.connection_manager import ConnectionManager
|
||||
from services.device_target_cache import DeviceTargetCache
|
||||
from services.target_audio_cache import TargetAudioCache
|
||||
|
||||
DEVICE = 'TalkingQ_local_voice'
|
||||
PROCESS_PARENT_MESSAGE = messages.process_parent_leave_message
|
||||
|
||||
|
||||
def packet(sid, kind, audio=b''):
|
||||
return DEVICE.encode()+b'\0'+sid.encode()+b'\0'+struct.pack('<IBI', 1, kind, len(audio))+audio
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def state(monkeypatch):
|
||||
now = [100.0]
|
||||
connections, sessions, buffers = ConnectionManager(), AudioSessionManager(), TargetAudioCache()
|
||||
targets = DeviceTargetCache(timeout_seconds=10, clock=lambda: now[0])
|
||||
started, ended = [], []
|
||||
ws = SimpleNamespace(send_text=AsyncMock(), accept=AsyncMock(), client='local',
|
||||
client_state=SimpleNamespace(name='CONNECTED'))
|
||||
for module in [messages, endpoint, targets_module]:
|
||||
monkeypatch.setattr(module, 'connection_manager', connections)
|
||||
for module in [messages, cleanup]:
|
||||
monkeypatch.setattr(module, 'device_target_cache', targets)
|
||||
for module in [messages, targets_module]:
|
||||
monkeypatch.setattr(module, 'target_audio_cache', buffers)
|
||||
for module in [messages, cleanup, parser]:
|
||||
monkeypatch.setattr(module, 'audio_session_manager', sessions)
|
||||
monkeypatch.setattr(cleanup, 'task_manager', SimpleNamespace(cancel_session_tasks=AsyncMock()))
|
||||
monkeypatch.setattr(parser, 'get_asr', AsyncMock(side_effect=lambda _: SimpleNamespace(connected=False)))
|
||||
monkeypatch.setattr(parser, 'get_llm', AsyncMock(side_effect=lambda _: SimpleNamespace(close=AsyncMock())))
|
||||
monkeypatch.setattr(parser, 'get_tts', AsyncMock(side_effect=lambda _: SimpleNamespace(close=AsyncMock())))
|
||||
|
||||
async def prepare(session, device_id, sid):
|
||||
started.append(sid)
|
||||
session.asr_service.connected = True
|
||||
session.send_queue = asyncio.Queue()
|
||||
|
||||
async def finish(session, key):
|
||||
chunks = []
|
||||
while not session.send_queue.empty():
|
||||
chunks.append(session.send_queue.get_nowait())
|
||||
ended.append((key[1], b''.join(chunks)))
|
||||
|
||||
monkeypatch.setattr(audio_handler, 'handle_start_packet', prepare)
|
||||
monkeypatch.setattr(audio_handler, 'handle_end_packet', finish)
|
||||
monkeypatch.setattr(messages, 'process_parent_leave_message', AsyncMock())
|
||||
monkeypatch.setattr(messages, 'process_cached_audio', AsyncMock())
|
||||
return SimpleNamespace(now=now, connections=connections, sessions=sessions, buffers=buffers,
|
||||
targets=targets, ws=ws, started=started, ended=ended)
|
||||
|
||||
|
||||
async def send(state, sid, kind, audio=b'', *, ws=None, active=None):
|
||||
return await messages.handle_binary_message(ws or state.ws, DEVICE, 'LOCAL-SERIAL',
|
||||
packet(sid, kind, audio), active, None)
|
||||
|
||||
|
||||
async def set_target(state, kind):
|
||||
if kind == 'parent':
|
||||
await state.targets.set_parent_target(DEVICE, DEVICE)
|
||||
else:
|
||||
await state.targets.set_target(DEVICE, 'TalkingQ_receiver')
|
||||
return await state.targets.get_voice_target(DEVICE)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('kind', ['parent', 'device'])
|
||||
async def test_ai_audio_recovers_from_pending_message_without_losing_first_chunk(state, kind):
|
||||
target = await set_target(state, kind)
|
||||
await send(state, 'ai', 1)
|
||||
assert state.started == []
|
||||
await send(state, 'ai', 0, b'first-audio')
|
||||
await send(state, 'ai', 0, b'-tail')
|
||||
await send(state, 'ai', 2)
|
||||
assert state.started == ['ai']
|
||||
assert state.ended == [('ai', b'first-audio-tail')]
|
||||
assert [c.args[0] for c in state.ws.send_text.call_args_list] == ['START_ACK', 'END_ACK']
|
||||
assert await state.targets.get_voice_target(DEVICE) is None
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
|
||||
messages.process_parent_leave_message.assert_not_awaited()
|
||||
messages.process_cached_audio.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('kind', ['parent', 'device'])
|
||||
@pytest.mark.parametrize('outcome', ['success', 'empty', 'error', 'cancelled'])
|
||||
async def test_message_terminal_paths_release_only_their_recording(state, kind, outcome):
|
||||
target = await set_target(state, kind)
|
||||
await send(state, 'message', 1)
|
||||
if outcome != 'empty':
|
||||
await send(state, 'message', 4, b'message-audio')
|
||||
processor = messages.process_parent_leave_message if kind == 'parent' else messages.process_cached_audio
|
||||
if outcome == 'error':
|
||||
processor.side_effect = RuntimeError('upload failed')
|
||||
with pytest.raises(RuntimeError, match='upload failed'):
|
||||
await send(state, 'message', 2)
|
||||
else:
|
||||
await send(state, 'message', 3 if outcome == 'cancelled' else 2)
|
||||
assert await state.targets.get_voice_target(DEVICE) is None
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
|
||||
assert state.started == []
|
||||
if outcome == 'cancelled':
|
||||
processor.assert_not_awaited()
|
||||
assert [c.args[0] for c in state.ws.send_text.call_args_list][-2:] == ['INTERRUPT_ACK', 'INTERRUPT_PROCESSED']
|
||||
else:
|
||||
processor.assert_awaited_once()
|
||||
await send(state, 'next-ai', 1)
|
||||
await send(state, 'next-ai', 0, b'next-question')
|
||||
await send(state, 'next-ai', 2)
|
||||
assert state.ended == [('next-ai', b'next-question')]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_message_cannot_receive_late_audio_or_end(state):
|
||||
await set_target(state, 'parent')
|
||||
await send(state, 'cancelled', 1)
|
||||
await send(state, 'cancelled', 4, b'old')
|
||||
await send(state, 'cancelled', 3)
|
||||
await set_target(state, 'parent')
|
||||
fresh = await state.targets.get_voice_target(DEVICE)
|
||||
await send(state, 'fresh', 1)
|
||||
await send(state, 'fresh', 4, b'new')
|
||||
await send(state, 'cancelled', 4, b'late')
|
||||
await send(state, 'cancelled', 2)
|
||||
assert await state.targets.get_voice_target(DEVICE) is fresh
|
||||
assert await state.buffers.get_audio_data(fresh.audio_cache_key) == b'new'
|
||||
messages.process_parent_leave_message.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('claimed', [False, True])
|
||||
async def test_timeout_clears_pending_or_partial_message_and_allows_ai(state, claimed):
|
||||
target = await set_target(state, 'parent')
|
||||
if claimed:
|
||||
await send(state, 'expired', 1)
|
||||
await send(state, 'expired', 4, b'partial')
|
||||
state.now[0] += 11
|
||||
assert await state.targets.get_voice_target(DEVICE) is None
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
|
||||
await send(state, 'ai', 1)
|
||||
await send(state, 'ai', 0, b'question')
|
||||
await send(state, 'ai', 2)
|
||||
assert state.ended == [('ai', b'question')]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_active_message_refreshes_idle_timeout(state):
|
||||
target = await set_target(state, 'parent')
|
||||
await send(state, 'message', 1)
|
||||
for _ in range(4):
|
||||
state.now[0] += 9
|
||||
await send(state, 'message', 4, b'a')
|
||||
assert await state.targets.get_voice_target(DEVICE) is target
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) == b'aaaa'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replacement_during_upload_survives_old_recording_cleanup(state):
|
||||
old = await set_target(state, 'parent')
|
||||
await send(state, 'old', 1)
|
||||
await send(state, 'old', 4, b'old-audio')
|
||||
replacement = []
|
||||
|
||||
async def upload(*args):
|
||||
await set_target(state, 'parent')
|
||||
replacement.append(await state.targets.get_voice_target(DEVICE))
|
||||
|
||||
messages.process_parent_leave_message.side_effect = upload
|
||||
await send(state, 'old', 2)
|
||||
assert await state.targets.get_voice_target(DEVICE) is replacement[0]
|
||||
assert replacement[0].audio_cache_key != old.audio_cache_key
|
||||
assert await state.buffers.get_audio_data(old.audio_cache_key) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_message_command_during_ai_does_not_hijack_its_end(state):
|
||||
await send(state, 'ai', 1)
|
||||
await send(state, 'ai', 0, b'question')
|
||||
target = await set_target(state, 'parent')
|
||||
await send(state, 'ai', 2)
|
||||
assert state.ended == [('ai', b'question')]
|
||||
assert await state.targets.get_voice_target(DEVICE) is target
|
||||
messages.process_parent_leave_message.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_senders_to_same_receiver_use_separate_buffers(state):
|
||||
await state.targets.set_target(DEVICE, 'receiver')
|
||||
await state.targets.set_target('other-sender', 'receiver')
|
||||
a = await state.targets.claim_target(DEVICE, state.ws, 'a')
|
||||
b = await state.targets.claim_target('other-sender', state.ws, 'b')
|
||||
await state.targets.append_audio(DEVICE, a, state.ws, 'a', b'one')
|
||||
await state.targets.append_audio('other-sender', b, state.ws, 'b', b'two')
|
||||
await state.targets.remove_target(DEVICE, expected=a)
|
||||
assert await state.buffers.get_audio_data(b.audio_cache_key) == b'two'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('claimed', [False, True])
|
||||
async def test_disconnect_clears_pending_or_partial_target(state, claimed):
|
||||
await state.connections.add_connection(DEVICE, state.ws)
|
||||
target = await set_target(state, 'parent')
|
||||
if claimed:
|
||||
await send(state, 'message', 1)
|
||||
await send(state, 'message', 4, b'partial')
|
||||
session = await state.sessions.get_session((DEVICE, 'message'))
|
||||
session.set_interrupted = AsyncMock()
|
||||
await cleanup.cleanup_device_sessions(DEVICE, state.ws)
|
||||
assert await state.targets.get_voice_target(DEVICE) is None
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
|
||||
assert await state.sessions.get_all_sessions() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_socket_cleanup_and_late_packets_preserve_new_connection(state, monkeypatch):
|
||||
old = state.ws
|
||||
new = SimpleNamespace(send_text=AsyncMock())
|
||||
|
||||
async def authenticate(ws):
|
||||
await state.connections.add_connection(DEVICE, old)
|
||||
return True, DEVICE, 'LOCAL'
|
||||
|
||||
async def replacement_connects(*args):
|
||||
await state.connections.add_connection(DEVICE, new)
|
||||
await set_target(state, 'parent')
|
||||
await send(state, 'new', 1, ws=new)
|
||||
await send(state, 'new', 4, b'new-audio', ws=new)
|
||||
await send(state, 'old-late-start', 1, ws=old)
|
||||
await send(state, 'old-late-end', 2, ws=old)
|
||||
|
||||
monkeypatch.setattr(endpoint, 'authenticate_websocket', authenticate)
|
||||
monkeypatch.setattr(endpoint, 'handle_websocket_messages', replacement_connects)
|
||||
await endpoint.websocket_endpoint(old)
|
||||
target = await state.targets.get_voice_target(DEVICE)
|
||||
assert target.websocket is new
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) == b'new-audio'
|
||||
assert await state.connections.get_connection(DEVICE) is new
|
||||
assert await state.sessions.get_session((DEVICE, 'new')) is not None
|
||||
cleanup.task_manager.cancel_session_tasks.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_command_before_connect_is_adopted_and_cleared_on_early_disconnect(state):
|
||||
target = await set_target(state, 'parent')
|
||||
assert target.websocket is None
|
||||
await state.connections.add_connection(DEVICE, state.ws)
|
||||
await state.targets.bind_connection(DEVICE, state.ws)
|
||||
await cleanup.cleanup_device_sessions(DEVICE, state.ws)
|
||||
assert await state.targets.get_voice_target(DEVICE) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconnect_discards_old_partial_message(state):
|
||||
await state.connections.add_connection(DEVICE, state.ws)
|
||||
old = await set_target(state, 'parent')
|
||||
await send(state, 'old', 1)
|
||||
await send(state, 'old', 4, b'old')
|
||||
new = SimpleNamespace(send_text=AsyncMock())
|
||||
await state.connections.add_connection(DEVICE, new)
|
||||
await state.targets.bind_connection(DEVICE, new)
|
||||
assert await state.targets.get_voice_target(DEVICE) is None
|
||||
assert await state.buffers.get_audio_data(old.audio_cache_key) is None
|
||||
await send(state, 'new-ai', 1, ws=new)
|
||||
await send(state, 'new-ai', 0, b'reconnected-question', ws=new)
|
||||
await send(state, 'new-ai', 2, ws=new)
|
||||
assert state.ended == [('new-ai', b'reconnected-question')]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('outcome', ['success', 'empty', 'upload_error'])
|
||||
async def test_real_parent_message_processing_keeps_notifications_and_clears_state(state, monkeypatch, outcome):
|
||||
monkeypatch.setattr(messages, 'process_parent_leave_message', PROCESS_PARENT_MESSAGE)
|
||||
upload = AsyncMock(return_value=SimpleNamespace(file_key='local-test/audio.wav'))
|
||||
create = AsyncMock(return_value=(SimpleNamespace(child_name='local-test'), None, []))
|
||||
notify = Mock()
|
||||
if outcome == 'upload_error':
|
||||
upload.side_effect = RuntimeError('simulated upload failure')
|
||||
monkeypatch.setattr(messages, 'upload_message_audio', upload)
|
||||
monkeypatch.setattr(messages.im_conversation_service, 'create_device_parent_leave_message', create)
|
||||
monkeypatch.setattr(messages.wechat_mp_notification_service, 'schedule_leave_message_notification', notify)
|
||||
await state.connections.add_connection(DEVICE, state.ws)
|
||||
target = await set_target(state, 'parent')
|
||||
await send(state, 'parent', 1)
|
||||
if outcome != 'empty':
|
||||
await send(state, 'parent', 4, b'\x01\x00'*400)
|
||||
await send(state, 'parent', 2)
|
||||
assert await state.targets.get_voice_target(DEVICE) is None
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
|
||||
if outcome == 'success':
|
||||
upload.assert_awaited_once()
|
||||
create.assert_awaited_once()
|
||||
assert create.call_args.kwargs['audio_content'].startswith(b'RIFF')
|
||||
notify.assert_called_once()
|
||||
assert state.ws.send_text.call_args.args[0].startswith('PROMPT_SOUND_URL:')
|
||||
else:
|
||||
create.assert_not_awaited()
|
||||
notify.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('first_kind', [0, 4])
|
||||
async def test_recording_cannot_switch_between_ai_and_message_after_audio(state, first_kind):
|
||||
target = await set_target(state, 'parent')
|
||||
await send(state, 'mixed', 1)
|
||||
await send(state, 'mixed', first_kind, b'first')
|
||||
await send(state, 'mixed', 4 if first_kind == 0 else 0, b'wrong-mode')
|
||||
if first_kind == 0:
|
||||
await send(state, 'mixed', 2)
|
||||
assert state.ended == [('mixed', b'first')]
|
||||
messages.process_parent_leave_message.assert_not_awaited()
|
||||
else:
|
||||
assert await state.buffers.get_audio_data(target.audio_cache_key) == b'first'
|
||||
assert state.started == []
|
||||
await send(state, 'mixed', 2)
|
||||
messages.process_parent_leave_message.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_recording_recovers_abandoned_message_without_reusing_audio(state):
|
||||
abandoned = await set_target(state, 'parent')
|
||||
await send(state, 'abandoned', 1)
|
||||
await send(state, 'abandoned', 4, b'abandoned-audio')
|
||||
await send(state, 'new-ai', 1)
|
||||
await send(state, 'new-ai', 0, b'question')
|
||||
await send(state, 'abandoned', 2, active=(DEVICE, 'new-ai'))
|
||||
await send(state, 'new-ai', 2)
|
||||
assert state.ended == [('new-ai', b'question')]
|
||||
assert await state.buffers.get_audio_data(abandoned.audio_cache_key) is None
|
||||
messages.process_parent_leave_message.assert_not_awaited()
|
||||
Reference in New Issue
Block a user