fix(voice): recover dialogue after abandoned message recordings

This commit is contained in:
stu2not
2026-10-28 12:22:58 +08:00
parent a02e0d5f13
commit 9ebb1a78ec
10 changed files with 622 additions and 85 deletions
+1
View File
@@ -71,6 +71,7 @@ class Settings(BaseSettings):
assets_dir: str = Field(default="assets", validation_alias="ASSETS_DIR") assets_dir: str = Field(default="assets", validation_alias="ASSETS_DIR")
session_timeout: int = Field(default=600, validation_alias="SESSION_TIMEOUT") session_timeout: int = Field(default=600, validation_alias="SESSION_TIMEOUT")
voice_target_timeout_seconds: float = Field(default=120, gt=0, le=600, validation_alias="VOICE_TARGET_TIMEOUT_SECONDS")
conversation_history_timeout: int = Field(default=1800, validation_alias="CONVERSATION_HISTORY_TIMEOUT") conversation_history_timeout: int = Field(default=1800, validation_alias="CONVERSATION_HISTORY_TIMEOUT")
max_conversation_history: int = Field(default=5, validation_alias="MAX_CONVERSATION_HISTORY") max_conversation_history: int = Field(default=5, validation_alias="MAX_CONVERSATION_HISTORY")
cleanup_interval: int = Field(default=300, validation_alias="CLEANUP_INTERVAL") cleanup_interval: int = Field(default=300, validation_alias="CLEANUP_INTERVAL")
@@ -0,0 +1,36 @@
# 留言状态与后续 AI 对话恢复
留言目标只属于一次录音和一个 WebSocket 连接。目标设置后默认空闲有效期为
120 秒,可用 `VOICE_TARGET_TIMEOUT_SECONDS` 调整;录音开始及有效留言音频会刷新
空闲期限。每次访问均检查过期状态,后台会定期回收过期目标及缓存。
开始包统一返回一次 `START_ACK`。存在留言目标时,根据首个音频包选择流程:
类型 `0` 清除待处理留言目标并完成 AI 初始化,原始首包随后正常送入 ASR;
类型 `4` 写入本次留言的独立缓存。录音开始传输音频后不允许切换类型。
结束、取消、异常、空音频、超时和连接断开均会释放对应目标及音频缓存。
结束时先消费本次目标,再处理上传和通知,避免旧处理完成时清除新设置的目标。
旧连接断开只清理该连接的会话和任务,不影响替换它的新连接。
音频缓存键按发送设备及本次目标独立生成,多个发送方对同一接收方留言时互不覆盖。
本次修复保留既有数据包类型、鉴权及设备协议。家长留言的上传、会话保存和通知流程
沿用原实现。
## 本地验证(2026-10-28)
- 全量后端回归 243/243,包括新增的 28 项留言状态与恢复测试。
- 本机隔离 MySQL、原有 `/ws` 路由和真实 ASR/LLM/TTS:正常对话及静音 10/10,
中断及恢复 4/4;包含两个角色在待处理、过期、取消家长留言后的 AI 对话。
- 实际 WebSocket 家长留言取消 2 次,均收到取消确认及处理完成通知,随后 AI 对话成功。
- 共 52 段音频通过 HTTP 200、Range 206、完整 MP3 解码及非零 PCM 检查。
- 正常留言的上传、保存、通知和失败分支使用本地替身测试,未向真实家长发送通知。
- 没有访问或部署生产服务,未验证实物设备的麦克风、扬声器及播放中打断。
覆盖边界:留言取消、空音频、处理异常、过期及未结束录音后的 AI 恢复已有回归;
AI 中断后再次 AI 对话已有真实服务验证。留言取消后再次留言验证了录音隔离,
AI 中断后留言验证了启动、收音及取消,尚未串联验证这两种情况下第二条留言完整发送。
正常家长留言处理的成功、空音频和上传异常分支已用模拟外部接口验证。
状态测试:`.venv/bin/python -m pytest tests/test_voice_target_lifecycle.py -q`
完整结果:[本地测试汇总](../output/voice-target-recovery-20261028/summary.json)。
@@ -22,8 +22,10 @@ async def handle_websocket_data(
if packet_type == 1: # 开始包 if packet_type == 1: # 开始包
try: try:
await websocket.send_text("START_ACK") if not session.start_ack_sent:
session_logger.info(device_id, session_id, "已发送开始包确认") await websocket.send_text("START_ACK")
session.start_ack_sent = True
session_logger.info(device_id, session_id, "已发送开始包确认")
except Exception as e: except Exception as e:
session_logger.error(device_id, session_id, f"发送开始包确认失败: {str(e)}") session_logger.error(device_id, session_id, f"发送开始包确认失败: {str(e)}")
session.sample_rate = sample_rate session.sample_rate = sample_rate
@@ -1,6 +1,8 @@
from services.interrupt_handler import interrupt_handler from services.interrupt_handler import interrupt_handler
from services.audio_session import audio_session_manager from services.audio_session import audio_session_manager
from utils.logger import session_logger from utils.logger import session_logger
from services.device_target_cache import device_target_cache
from services.task_manager import task_manager
async def handle_old_session_cleanup(old_session, session_key): async def handle_old_session_cleanup(old_session, session_key):
""" """
@@ -21,19 +23,22 @@ async def handle_old_session_cleanup(old_session, session_key):
session_logger.error("unknown", "unknown", f"清理旧会话资源时出错: {e}") session_logger.error("unknown", "unknown", f"清理旧会话资源时出错: {e}")
async def cleanup_device_sessions(device_id): async def cleanup_device_sessions(device_id, websocket=None):
""" """
清理设备所有会话 清理设备所有会话
Args: Args:
device_id: 设备ID device_id: 设备ID
""" """
await device_target_cache.remove_target(device_id, websocket=websocket, reason="disconnected")
sessions_to_clean = [] sessions_to_clean = []
for session_key, session in await audio_session_manager.get_all_sessions(): for session_key, session in await audio_session_manager.get_all_sessions():
if session_key[0] == device_id: if session_key[0] == device_id and (websocket is None or session.websocket is websocket):
sessions_to_clean.append((session_key, session)) sessions_to_clean.append((session_key, session))
for session_key, session in sessions_to_clean: for session_key, session in sessions_to_clean:
if websocket is not None and session.websocket is not websocket:
continue
if hasattr(session, "set_interrupted"): if hasattr(session, "set_interrupted"):
session_logger.info( session_logger.info(
device_id, session_key[1], "WebSocket连接断开,中断会话" device_id, session_key[1], "WebSocket连接断开,中断会话"
@@ -44,8 +49,11 @@ async def cleanup_device_sessions(device_id):
await interrupt_handler.set_interrupt_state(session_key, True) await interrupt_handler.set_interrupt_state(session_key, True)
if session.llm_service and hasattr(session.llm_service, "closed"): if session.llm_service and hasattr(session.llm_service, "closed"):
session.llm_service.closed = True session.llm_service.closed = True
await task_manager.cancel_session_tasks(session_key)
for session_key, _ in sessions_to_clean: for session_key, session in sessions_to_clean:
if websocket is not None and session.websocket is not websocket:
continue
await audio_session_manager.remove_session(session_key) await audio_session_manager.remove_session(session_key)
await interrupt_handler.remove_session(session_key) await interrupt_handler.remove_session(session_key)
session_logger.info( session_logger.info(
+1 -4
View File
@@ -1,7 +1,6 @@
from fastapi import WebSocket, WebSocketDisconnect from fastapi import WebSocket, WebSocketDisconnect
from utils.logger import session_logger from utils.logger import session_logger
from services.connection_manager import connection_manager from services.connection_manager import connection_manager
from services.task_manager import task_manager
from services.offline_audio_cache import offline_audio_cache from services.offline_audio_cache import offline_audio_cache
from handlers.websocket_auth_handler import authenticate_websocket from handlers.websocket_auth_handler import authenticate_websocket
from handlers.websocket_message_handler import handle_websocket_messages from handlers.websocket_message_handler import handle_websocket_messages
@@ -59,9 +58,7 @@ async def websocket_endpoint(websocket: WebSocket):
finally: finally:
if device_id: if device_id:
await connection_manager.remove_connection(device_id, websocket) await connection_manager.remove_connection(device_id, websocket)
await cleanup_device_sessions(device_id) await cleanup_device_sessions(device_id, websocket)
# 清理设备相关的所有异步任务
await task_manager.cancel_device_tasks(device_id)
session_logger.info( session_logger.info(
device_id or "unknown", device_id or "unknown",
@@ -44,6 +44,7 @@ async def handle_websocket_messages(websocket: WebSocket, device_id: str, serial
websocket: WebSocket连接 websocket: WebSocket连接
device_id: 设备ID device_id: 设备ID
""" """
await device_target_cache.bind_connection(device_id, websocket)
current_active_session = None current_active_session = None
first_audio_received_time = None first_audio_received_time = None
@@ -156,54 +157,111 @@ async def handle_text_message(websocket: WebSocket, device_id: str, text_data: s
async def handle_binary_message(websocket: WebSocket, device_id: str, serial_number: str, binary_data, current_active_session, first_audio_received_time): async def handle_binary_message(websocket: WebSocket, device_id: str, serial_number: str, binary_data, current_active_session, first_audio_received_time):
"""处理二进制音频消息""" """处理二进制音频消息"""
active_connection = await connection_manager.get_connection(device_id)
if active_connection is not None and active_connection is not websocket:
session_logger.warning(device_id, "connection", "忽略已被替换连接的录音包")
return current_active_session, first_audio_received_time
session_key, session, packet_type, audio_data, sample_rate = await parse_packet(binary_data, websocket) session_key, session, packet_type, audio_data, sample_rate = await parse_packet(binary_data, websocket)
if not session:
if session: return current_active_session, first_audio_received_time
session.websocket = websocket session_id = session_key[1]
session_id = session_key[1] if session.websocket is not None and session.websocket is not websocket:
session_logger.warning(device_id, session_id, "忽略其他连接的录音包")
if packet_type == 1: # 开始包 return current_active_session, first_audio_received_time
voice_target = await device_target_cache.get_voice_target(device_id) session.websocket = websocket
if voice_target: session.last_activity_time = asyncio.get_running_loop().time()
await target_audio_cache.clear_audio_data(voice_target.audio_cache_key)
return None, None if packet_type == 1:
current_active_session = await handle_start_packet( if session.start_received:
device_id, session_id, session_key, session, current_active_session return current_active_session, first_audio_received_time
session.start_received = True
session.voice_target = await device_target_cache.claim_target(device_id, websocket, session_id)
if session.voice_target:
# Acknowledge START now; the first audio packet selects AI (0) or
# message (4). A pending message must not swallow an AI recording.
await websocket.send_text("START_ACK")
session.start_ack_sent = True
return session_key, None
current_active_session = await start_ai_recording(
websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
)
return current_active_session, None
if not session.start_received or session.recording_finished:
session_logger.warning(device_id, session_id, "忽略未开始或已结束的录音包")
return current_active_session, first_audio_received_time
if packet_type == 0:
if session.message_audio_received:
session_logger.warning(device_id, session_id, "忽略留言录音中混入的普通对话音频")
return current_active_session, first_audio_received_time
if not session.ai_started:
target = session.voice_target
if target:
await device_target_cache.remove_target(device_id, expected=target, reason="ai_audio")
session.voice_target = None
current_active_session = await start_ai_recording(
websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
) )
elif packet_type == 3: # 中断包 if first_audio_received_time is None:
voice_target = await device_target_cache.get_voice_target(device_id)
if voice_target:
return None, None
current_active_session = await handle_interrupt_packet(
device_id, session_id, session_key, session, current_active_session
)
elif packet_type == 2: # 结束包
voice_target = await device_target_cache.get_voice_target(device_id)
if voice_target:
session_logger.info(device_id, session_id, f"收到结束包,开始处理缓存音频数据")
if voice_target.kind == VOICE_TARGET_KIND_PARENT:
await process_parent_leave_message(device_id, voice_target.audio_cache_key)
else:
await process_cached_audio(device_id, voice_target.target_device_id, serial_number)
await device_target_cache.remove_target(device_id)
return None, None
if current_active_session == session_key:
current_active_session = None
elif packet_type == 4: # 发送给目标设备的音频包
await handle_target_audio_packet(device_id, audio_data)
return None, None
elif packet_type == 0 and first_audio_received_time is None:
first_audio_received_time = time.perf_counter() first_audio_received_time = time.perf_counter()
session.start_time = first_audio_received_time session.start_time = first_audio_received_time
session_logger.info( session_logger.info(device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}")
device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}" elif packet_type == 4:
) target = session.voice_target
if target and not session.ai_started:
await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate) if await handle_target_audio_packet(device_id, audio_data, target, websocket, session_id):
session.message_audio_received = True
else:
session_logger.warning(device_id, session_id, "忽略没有对应留言目标的留言音频包")
return current_active_session, first_audio_received_time
elif packet_type == 3:
if session.voice_target:
await device_target_cache.remove_target(device_id, expected=session.voice_target, reason="interrupted")
session.voice_target = None
session.recording_finished = True
await websocket.send_text("INTERRUPT_ACK")
await websocket.send_text("INTERRUPT_PROCESSED")
return None if current_active_session == session_key else current_active_session, None
current_active_session = await handle_interrupt_packet(
device_id, session_id, session_key, session, current_active_session
)
elif packet_type == 2:
if session.voice_target:
target = await device_target_cache.take_target(device_id, session.voice_target, websocket, session_id)
session.voice_target = None
session.recording_finished = True
try:
await websocket.send_text("END_ACK")
if target:
session_logger.info(device_id, session_id, "收到结束包,开始处理缓存音频数据")
if target.kind == VOICE_TARGET_KIND_PARENT:
await process_parent_leave_message(device_id, target.audio_cache_key)
else:
await process_cached_audio(device_id, target.target_device_id, serial_number,
audio_cache_key=target.audio_cache_key)
finally:
if target:
await target_audio_cache.clear_audio_data(target.audio_cache_key)
return None if current_active_session == session_key else current_active_session, None
if current_active_session == session_key:
current_active_session = None
await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate)
if packet_type in (2, 3):
session.recording_finished = True
return current_active_session, first_audio_received_time return current_active_session, first_audio_received_time
async def start_ai_recording(websocket, device_id, session_id, session_key, session, current_active_session, sample_rate):
current_active_session = await handle_start_packet(
device_id, session_id, session_key, session, current_active_session
)
await handle_websocket_data(websocket, session_key, session, 1, b"", sample_rate)
session.ai_started = True
return current_active_session
async def handle_start_packet(device_id, session_id, session_key, session, current_active_session): async def handle_start_packet(device_id, session_id, session_key, session, current_active_session):
"""处理会话开始包""" """处理会话开始包"""
if current_active_session and current_active_session[1] != session_id: if current_active_session and current_active_session[1] != session_id:
@@ -255,20 +313,17 @@ async def handle_interrupt_packet(device_id, session_id, session_key, session, c
return current_active_session return current_active_session
async def handle_target_audio_packet(device_id: str, audio_data: bytes): async def handle_target_audio_packet(device_id: str, audio_data: bytes, voice_target, websocket, session_id):
"""处理发送给目标设备的音频包""" """处理发送给目标设备的音频包"""
try: try:
voice_target = await device_target_cache.get_voice_target(device_id) if not await device_target_cache.append_audio(device_id, voice_target, websocket, session_id, audio_data):
session_logger.warning(device_id, session_id, "忽略已过期或不属于本次录音的留言音频")
if not voice_target: return
session_logger.warning(device_id, "target", "未设置目标设备,无法发送音频")
return
await target_audio_cache.add_audio_data(voice_target.audio_cache_key, audio_data)
if voice_target.kind == VOICE_TARGET_KIND_PARENT: if voice_target.kind == VOICE_TARGET_KIND_PARENT:
session_logger.info(device_id, "target", "已缓存发给家长的留言音频数据") session_logger.info(device_id, "target", "已缓存发给家长的留言音频数据")
else: else:
session_logger.info(device_id, "target", f"已缓存音频数据到目标设备 {voice_target.target_device_id}") session_logger.info(device_id, "target", f"已缓存音频数据到目标设备 {voice_target.target_device_id}")
return True
except Exception as e: except Exception as e:
session_logger.error(device_id, "target", f"处理目标音频包时出错: {e}", exc_info=True) session_logger.error(device_id, "target", f"处理目标音频包时出错: {e}", exc_info=True)
@@ -337,10 +392,11 @@ async def process_parent_leave_message(device_id: str, audio_cache_key: str):
await target_audio_cache.clear_audio_data(audio_cache_key) await target_audio_cache.clear_audio_data(audio_cache_key)
async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str): async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str, *, audio_cache_key=None):
"""处理缓存的音频数据并发送音频URL""" """处理缓存的音频数据并发送音频URL"""
try: try:
cached_audio = await target_audio_cache.get_audio_data(target_device_id) cache_key = audio_cache_key or target_device_id
cached_audio = await target_audio_cache.get_audio_data(cache_key)
if not cached_audio: if not cached_audio:
session_logger.info(device_id, "target", f"目标设备 {target_device_id} 没有缓存的音频数据") session_logger.info(device_id, "target", f"目标设备 {target_device_id} 没有缓存的音频数据")
return return
@@ -398,4 +454,4 @@ async def process_cached_audio(device_id: str, target_device_id: str, serial_num
session_logger.error(device_id, "target", f"处理缓存音频时出错: {e}", exc_info=True) session_logger.error(device_id, "target", f"处理缓存音频时出错: {e}", exc_info=True)
finally: finally:
# 清除缓存 # 清除缓存
await target_audio_cache.clear_audio_data(target_device_id) await target_audio_cache.clear_audio_data(cache_key)
+6
View File
@@ -30,6 +30,12 @@ class AudioSession:
self.session_id = None self.session_id = None
self.sample_rate = sample_rate self.sample_rate = sample_rate
self.audio_queue = None self.audio_queue = None
self.voice_target = None
self.start_received = False
self.start_ack_sent = False
self.ai_started = False
self.recording_finished = False
self.message_audio_received = False
async def set_interrupted(self, interrupted=True): async def set_interrupted(self, interrupted=True):
+107 -23
View File
@@ -1,59 +1,143 @@
import asyncio import asyncio
import time
from dataclasses import dataclass from dataclasses import dataclass
from typing import Dict, Optional from typing import Dict, Optional
from uuid import uuid4
from config import settings
from services.connection_manager import connection_manager
from services.target_audio_cache import target_audio_cache
from utils.logger import session_logger
VOICE_TARGET_KIND_DEVICE = "device" VOICE_TARGET_KIND_DEVICE = "device"
VOICE_TARGET_KIND_PARENT = "parent" VOICE_TARGET_KIND_PARENT = "parent"
@dataclass(frozen=True) @dataclass
class VoiceTarget: class VoiceTarget:
kind: str kind: str
target_device_id: Optional[str] = None target_device_id: Optional[str] = None
uuid: Optional[str] = None uuid: Optional[str] = None
audio_cache_key: Optional[str] = None audio_cache_key: Optional[str] = None
websocket: object = None
session_id: Optional[str] = None
expires_at: float = 0
class DeviceTargetCache: class DeviceTargetCache:
def __init__(self): def __init__(self, *, timeout_seconds=None, clock=None):
self.targets: Dict[str, VoiceTarget] = {} self.targets: Dict[str, VoiceTarget] = {}
self.lock = asyncio.Lock() self.lock = asyncio.Lock()
self.timeout_seconds = timeout_seconds if timeout_seconds is not None else settings.voice_target_timeout_seconds
self.clock = clock or time.monotonic
async def _remove_locked(self, device_id, target, reason, *, clear_audio=True):
del self.targets[device_id]
if clear_audio:
await target_audio_cache.clear_audio_data(target.audio_cache_key)
session_logger.info(device_id, target.session_id or "target",
f"留言目标已清除: kind={target.kind}, reason={reason}")
async def _get_locked(self, device_id):
target = self.targets.get(device_id)
if target and self.clock() >= target.expires_at:
await self._remove_locked(device_id, target, "timeout")
return None
return target
async def get_target(self, device_id: str) -> Optional[str]: async def get_target(self, device_id: str) -> Optional[str]:
async with self.lock: target = await self.get_voice_target(device_id)
target = self.targets.get(device_id) return target.target_device_id if target and target.kind == VOICE_TARGET_KIND_DEVICE else None
if not target or target.kind != VOICE_TARGET_KIND_DEVICE:
return None
return target.target_device_id
async def get_voice_target(self, device_id: str) -> Optional[VoiceTarget]: async def get_voice_target(self, device_id: str) -> Optional[VoiceTarget]:
async with self.lock: async with self.lock:
return self.targets.get(device_id) return await self._get_locked(device_id)
async def _set_target(self, device_id, *, kind, target_device_id=None, uuid=None):
async with self.lock:
old = self.targets.get(device_id)
if old:
await self._remove_locked(device_id, old, "replaced")
self.targets[device_id] = VoiceTarget(
kind=kind, target_device_id=target_device_id, uuid=uuid,
audio_cache_key=f"voice:{device_id}:{uuid4().hex}",
websocket=await connection_manager.get_connection(device_id),
expires_at=self.clock() + self.timeout_seconds,
)
session_logger.info(device_id, "target", f"留言目标已设置: kind={kind}, timeout={self.timeout_seconds}s")
async def set_target(self, device_id: str, target_device_id: str): async def set_target(self, device_id: str, target_device_id: str):
async with self.lock: await self._set_target(device_id, kind=VOICE_TARGET_KIND_DEVICE, target_device_id=target_device_id)
self.targets[device_id] = VoiceTarget(
kind=VOICE_TARGET_KIND_DEVICE,
target_device_id=target_device_id,
audio_cache_key=target_device_id,
)
async def set_parent_target(self, device_id: str, uuid: str): async def set_parent_target(self, device_id: str, uuid: str):
async with self.lock: await self._set_target(device_id, kind=VOICE_TARGET_KIND_PARENT, uuid=uuid)
self.targets[device_id] = VoiceTarget(
kind=VOICE_TARGET_KIND_PARENT,
uuid=uuid,
audio_cache_key=f"parent:{device_id}",
)
async def remove_target(self, device_id: str): async def claim_target(self, device_id, websocket, session_id):
"""Bind one pending intent to one recording on one connection."""
async with self.lock: async with self.lock:
if device_id in self.targets: active = await connection_manager.get_connection(device_id)
del self.targets[device_id] if active is not None and active is not websocket:
return None
target = await self._get_locked(device_id)
if target is None:
return None
if target.websocket is not None and target.websocket is not websocket:
await self._remove_locked(device_id, target, "connection_changed")
return None
if target.session_id is not None and target.session_id != session_id:
await self._remove_locked(device_id, target, "new_recording")
return None
target.websocket, target.session_id = websocket, session_id
target.expires_at = self.clock() + self.timeout_seconds
session_logger.info(device_id, session_id, f"留言目标已关联录音: kind={target.kind}")
return target
async def bind_connection(self, device_id, websocket):
"""Adopt commands received before connect; discard older socket intents."""
async with self.lock:
if await connection_manager.get_connection(device_id) is not websocket:
return
target = await self._get_locked(device_id)
if target is None:
return
if target.websocket is not None and target.websocket is not websocket:
await self._remove_locked(device_id, target, "connection_changed")
else:
target.websocket = websocket
async def append_audio(self, device_id, target, websocket, session_id, audio):
async with self.lock:
current = await self._get_locked(device_id)
if current is not target or target.websocket is not websocket or target.session_id != session_id:
return False
target.expires_at = self.clock() + self.timeout_seconds
await target_audio_cache.add_audio_data(target.audio_cache_key, audio)
return True
async def take_target(self, device_id, target, websocket, session_id):
"""Consume the exact recording before slow upload/notification work."""
async with self.lock:
current = await self._get_locked(device_id)
if current is not target or target.websocket is not websocket or target.session_id != session_id:
return None
await self._remove_locked(device_id, target, "recording_ended", clear_audio=False)
return target
async def remove_target(self, device_id: str, *, expected=None, websocket=None, reason="cancelled"):
async with self.lock:
target = self.targets.get(device_id)
if target is None or (expected is not None and target is not expected):
return False
if websocket is not None and target.websocket is not websocket:
return False
await self._remove_locked(device_id, target, reason)
return True
async def get_all_targets(self) -> list: async def get_all_targets(self) -> list:
async with self.lock: async with self.lock:
for device_id in list(self.targets):
await self._get_locked(device_id)
return list(self.targets.items()) return list(self.targets.items())
@@ -3,6 +3,7 @@ from config import settings
from services.audio_session import audio_session_manager from services.audio_session import audio_session_manager
from utils.logger import session_logger from utils.logger import session_logger
from services.conversation_history import conversation_history_manager from services.conversation_history import conversation_history_manager
from services.device_target_cache import device_target_cache
async def run_in_executor(func, *args): async def run_in_executor(func, *args):
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
@@ -12,6 +13,7 @@ async def background_cleanup_task():
try: try:
while True: while True:
await asyncio.sleep(settings.cleanup_interval) await asyncio.sleep(settings.cleanup_interval)
await device_target_cache.get_all_targets()
current_time = asyncio.get_running_loop().time() current_time = asyncio.get_running_loop().time()
audio_sessions_to_clean = [] audio_sessions_to_clean = []
@@ -54,4 +56,4 @@ async def background_cleanup_task():
except asyncio.CancelledError: except asyncio.CancelledError:
session_logger.info("system", "cleanup", "后台清理任务已取消") session_logger.info("system", "cleanup", "后台清理任务已取消")
except Exception as e: except Exception as e:
session_logger.error("system", "cleanup", f"后台清理任务出错: {str(e)}") session_logger.error("system", "cleanup", f"后台清理任务出错: {str(e)}")
@@ -0,0 +1,345 @@
"""Regression for stale message intents swallowing the next AI recording."""
import asyncio
import struct
from types import SimpleNamespace
from unittest.mock import AsyncMock
from unittest.mock import Mock
import pytest
from handlers import audio_packet_parser as parser
from handlers import audio_session_handler as audio_handler
from handlers import session_cleanup_handler as cleanup
from handlers import websocket_message_handler as messages
from handlers import websocket_handler as endpoint
from services import device_target_cache as targets_module
from services.audio_session import AudioSessionManager
from services.connection_manager import ConnectionManager
from services.device_target_cache import DeviceTargetCache
from services.target_audio_cache import TargetAudioCache
DEVICE = 'TalkingQ_local_voice'
PROCESS_PARENT_MESSAGE = messages.process_parent_leave_message
def packet(sid, kind, audio=b''):
return DEVICE.encode()+b'\0'+sid.encode()+b'\0'+struct.pack('<IBI', 1, kind, len(audio))+audio
@pytest.fixture
def state(monkeypatch):
now = [100.0]
connections, sessions, buffers = ConnectionManager(), AudioSessionManager(), TargetAudioCache()
targets = DeviceTargetCache(timeout_seconds=10, clock=lambda: now[0])
started, ended = [], []
ws = SimpleNamespace(send_text=AsyncMock(), accept=AsyncMock(), client='local',
client_state=SimpleNamespace(name='CONNECTED'))
for module in [messages, endpoint, targets_module]:
monkeypatch.setattr(module, 'connection_manager', connections)
for module in [messages, cleanup]:
monkeypatch.setattr(module, 'device_target_cache', targets)
for module in [messages, targets_module]:
monkeypatch.setattr(module, 'target_audio_cache', buffers)
for module in [messages, cleanup, parser]:
monkeypatch.setattr(module, 'audio_session_manager', sessions)
monkeypatch.setattr(cleanup, 'task_manager', SimpleNamespace(cancel_session_tasks=AsyncMock()))
monkeypatch.setattr(parser, 'get_asr', AsyncMock(side_effect=lambda _: SimpleNamespace(connected=False)))
monkeypatch.setattr(parser, 'get_llm', AsyncMock(side_effect=lambda _: SimpleNamespace(close=AsyncMock())))
monkeypatch.setattr(parser, 'get_tts', AsyncMock(side_effect=lambda _: SimpleNamespace(close=AsyncMock())))
async def prepare(session, device_id, sid):
started.append(sid)
session.asr_service.connected = True
session.send_queue = asyncio.Queue()
async def finish(session, key):
chunks = []
while not session.send_queue.empty():
chunks.append(session.send_queue.get_nowait())
ended.append((key[1], b''.join(chunks)))
monkeypatch.setattr(audio_handler, 'handle_start_packet', prepare)
monkeypatch.setattr(audio_handler, 'handle_end_packet', finish)
monkeypatch.setattr(messages, 'process_parent_leave_message', AsyncMock())
monkeypatch.setattr(messages, 'process_cached_audio', AsyncMock())
return SimpleNamespace(now=now, connections=connections, sessions=sessions, buffers=buffers,
targets=targets, ws=ws, started=started, ended=ended)
async def send(state, sid, kind, audio=b'', *, ws=None, active=None):
return await messages.handle_binary_message(ws or state.ws, DEVICE, 'LOCAL-SERIAL',
packet(sid, kind, audio), active, None)
async def set_target(state, kind):
if kind == 'parent':
await state.targets.set_parent_target(DEVICE, DEVICE)
else:
await state.targets.set_target(DEVICE, 'TalkingQ_receiver')
return await state.targets.get_voice_target(DEVICE)
@pytest.mark.asyncio
@pytest.mark.parametrize('kind', ['parent', 'device'])
async def test_ai_audio_recovers_from_pending_message_without_losing_first_chunk(state, kind):
target = await set_target(state, kind)
await send(state, 'ai', 1)
assert state.started == []
await send(state, 'ai', 0, b'first-audio')
await send(state, 'ai', 0, b'-tail')
await send(state, 'ai', 2)
assert state.started == ['ai']
assert state.ended == [('ai', b'first-audio-tail')]
assert [c.args[0] for c in state.ws.send_text.call_args_list] == ['START_ACK', 'END_ACK']
assert await state.targets.get_voice_target(DEVICE) is None
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
messages.process_parent_leave_message.assert_not_awaited()
messages.process_cached_audio.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize('kind', ['parent', 'device'])
@pytest.mark.parametrize('outcome', ['success', 'empty', 'error', 'cancelled'])
async def test_message_terminal_paths_release_only_their_recording(state, kind, outcome):
target = await set_target(state, kind)
await send(state, 'message', 1)
if outcome != 'empty':
await send(state, 'message', 4, b'message-audio')
processor = messages.process_parent_leave_message if kind == 'parent' else messages.process_cached_audio
if outcome == 'error':
processor.side_effect = RuntimeError('upload failed')
with pytest.raises(RuntimeError, match='upload failed'):
await send(state, 'message', 2)
else:
await send(state, 'message', 3 if outcome == 'cancelled' else 2)
assert await state.targets.get_voice_target(DEVICE) is None
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
assert state.started == []
if outcome == 'cancelled':
processor.assert_not_awaited()
assert [c.args[0] for c in state.ws.send_text.call_args_list][-2:] == ['INTERRUPT_ACK', 'INTERRUPT_PROCESSED']
else:
processor.assert_awaited_once()
await send(state, 'next-ai', 1)
await send(state, 'next-ai', 0, b'next-question')
await send(state, 'next-ai', 2)
assert state.ended == [('next-ai', b'next-question')]
@pytest.mark.asyncio
async def test_cancelled_message_cannot_receive_late_audio_or_end(state):
await set_target(state, 'parent')
await send(state, 'cancelled', 1)
await send(state, 'cancelled', 4, b'old')
await send(state, 'cancelled', 3)
await set_target(state, 'parent')
fresh = await state.targets.get_voice_target(DEVICE)
await send(state, 'fresh', 1)
await send(state, 'fresh', 4, b'new')
await send(state, 'cancelled', 4, b'late')
await send(state, 'cancelled', 2)
assert await state.targets.get_voice_target(DEVICE) is fresh
assert await state.buffers.get_audio_data(fresh.audio_cache_key) == b'new'
messages.process_parent_leave_message.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize('claimed', [False, True])
async def test_timeout_clears_pending_or_partial_message_and_allows_ai(state, claimed):
target = await set_target(state, 'parent')
if claimed:
await send(state, 'expired', 1)
await send(state, 'expired', 4, b'partial')
state.now[0] += 11
assert await state.targets.get_voice_target(DEVICE) is None
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
await send(state, 'ai', 1)
await send(state, 'ai', 0, b'question')
await send(state, 'ai', 2)
assert state.ended == [('ai', b'question')]
@pytest.mark.asyncio
async def test_active_message_refreshes_idle_timeout(state):
target = await set_target(state, 'parent')
await send(state, 'message', 1)
for _ in range(4):
state.now[0] += 9
await send(state, 'message', 4, b'a')
assert await state.targets.get_voice_target(DEVICE) is target
assert await state.buffers.get_audio_data(target.audio_cache_key) == b'aaaa'
@pytest.mark.asyncio
async def test_replacement_during_upload_survives_old_recording_cleanup(state):
old = await set_target(state, 'parent')
await send(state, 'old', 1)
await send(state, 'old', 4, b'old-audio')
replacement = []
async def upload(*args):
await set_target(state, 'parent')
replacement.append(await state.targets.get_voice_target(DEVICE))
messages.process_parent_leave_message.side_effect = upload
await send(state, 'old', 2)
assert await state.targets.get_voice_target(DEVICE) is replacement[0]
assert replacement[0].audio_cache_key != old.audio_cache_key
assert await state.buffers.get_audio_data(old.audio_cache_key) is None
@pytest.mark.asyncio
async def test_new_message_command_during_ai_does_not_hijack_its_end(state):
await send(state, 'ai', 1)
await send(state, 'ai', 0, b'question')
target = await set_target(state, 'parent')
await send(state, 'ai', 2)
assert state.ended == [('ai', b'question')]
assert await state.targets.get_voice_target(DEVICE) is target
messages.process_parent_leave_message.assert_not_awaited()
@pytest.mark.asyncio
async def test_two_senders_to_same_receiver_use_separate_buffers(state):
await state.targets.set_target(DEVICE, 'receiver')
await state.targets.set_target('other-sender', 'receiver')
a = await state.targets.claim_target(DEVICE, state.ws, 'a')
b = await state.targets.claim_target('other-sender', state.ws, 'b')
await state.targets.append_audio(DEVICE, a, state.ws, 'a', b'one')
await state.targets.append_audio('other-sender', b, state.ws, 'b', b'two')
await state.targets.remove_target(DEVICE, expected=a)
assert await state.buffers.get_audio_data(b.audio_cache_key) == b'two'
@pytest.mark.asyncio
@pytest.mark.parametrize('claimed', [False, True])
async def test_disconnect_clears_pending_or_partial_target(state, claimed):
await state.connections.add_connection(DEVICE, state.ws)
target = await set_target(state, 'parent')
if claimed:
await send(state, 'message', 1)
await send(state, 'message', 4, b'partial')
session = await state.sessions.get_session((DEVICE, 'message'))
session.set_interrupted = AsyncMock()
await cleanup.cleanup_device_sessions(DEVICE, state.ws)
assert await state.targets.get_voice_target(DEVICE) is None
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
assert await state.sessions.get_all_sessions() == []
@pytest.mark.asyncio
async def test_old_socket_cleanup_and_late_packets_preserve_new_connection(state, monkeypatch):
old = state.ws
new = SimpleNamespace(send_text=AsyncMock())
async def authenticate(ws):
await state.connections.add_connection(DEVICE, old)
return True, DEVICE, 'LOCAL'
async def replacement_connects(*args):
await state.connections.add_connection(DEVICE, new)
await set_target(state, 'parent')
await send(state, 'new', 1, ws=new)
await send(state, 'new', 4, b'new-audio', ws=new)
await send(state, 'old-late-start', 1, ws=old)
await send(state, 'old-late-end', 2, ws=old)
monkeypatch.setattr(endpoint, 'authenticate_websocket', authenticate)
monkeypatch.setattr(endpoint, 'handle_websocket_messages', replacement_connects)
await endpoint.websocket_endpoint(old)
target = await state.targets.get_voice_target(DEVICE)
assert target.websocket is new
assert await state.buffers.get_audio_data(target.audio_cache_key) == b'new-audio'
assert await state.connections.get_connection(DEVICE) is new
assert await state.sessions.get_session((DEVICE, 'new')) is not None
cleanup.task_manager.cancel_session_tasks.assert_not_awaited()
@pytest.mark.asyncio
async def test_command_before_connect_is_adopted_and_cleared_on_early_disconnect(state):
target = await set_target(state, 'parent')
assert target.websocket is None
await state.connections.add_connection(DEVICE, state.ws)
await state.targets.bind_connection(DEVICE, state.ws)
await cleanup.cleanup_device_sessions(DEVICE, state.ws)
assert await state.targets.get_voice_target(DEVICE) is None
@pytest.mark.asyncio
async def test_reconnect_discards_old_partial_message(state):
await state.connections.add_connection(DEVICE, state.ws)
old = await set_target(state, 'parent')
await send(state, 'old', 1)
await send(state, 'old', 4, b'old')
new = SimpleNamespace(send_text=AsyncMock())
await state.connections.add_connection(DEVICE, new)
await state.targets.bind_connection(DEVICE, new)
assert await state.targets.get_voice_target(DEVICE) is None
assert await state.buffers.get_audio_data(old.audio_cache_key) is None
await send(state, 'new-ai', 1, ws=new)
await send(state, 'new-ai', 0, b'reconnected-question', ws=new)
await send(state, 'new-ai', 2, ws=new)
assert state.ended == [('new-ai', b'reconnected-question')]
@pytest.mark.asyncio
@pytest.mark.parametrize('outcome', ['success', 'empty', 'upload_error'])
async def test_real_parent_message_processing_keeps_notifications_and_clears_state(state, monkeypatch, outcome):
monkeypatch.setattr(messages, 'process_parent_leave_message', PROCESS_PARENT_MESSAGE)
upload = AsyncMock(return_value=SimpleNamespace(file_key='local-test/audio.wav'))
create = AsyncMock(return_value=(SimpleNamespace(child_name='local-test'), None, []))
notify = Mock()
if outcome == 'upload_error':
upload.side_effect = RuntimeError('simulated upload failure')
monkeypatch.setattr(messages, 'upload_message_audio', upload)
monkeypatch.setattr(messages.im_conversation_service, 'create_device_parent_leave_message', create)
monkeypatch.setattr(messages.wechat_mp_notification_service, 'schedule_leave_message_notification', notify)
await state.connections.add_connection(DEVICE, state.ws)
target = await set_target(state, 'parent')
await send(state, 'parent', 1)
if outcome != 'empty':
await send(state, 'parent', 4, b'\x01\x00'*400)
await send(state, 'parent', 2)
assert await state.targets.get_voice_target(DEVICE) is None
assert await state.buffers.get_audio_data(target.audio_cache_key) is None
if outcome == 'success':
upload.assert_awaited_once()
create.assert_awaited_once()
assert create.call_args.kwargs['audio_content'].startswith(b'RIFF')
notify.assert_called_once()
assert state.ws.send_text.call_args.args[0].startswith('PROMPT_SOUND_URL:')
else:
create.assert_not_awaited()
notify.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize('first_kind', [0, 4])
async def test_recording_cannot_switch_between_ai_and_message_after_audio(state, first_kind):
target = await set_target(state, 'parent')
await send(state, 'mixed', 1)
await send(state, 'mixed', first_kind, b'first')
await send(state, 'mixed', 4 if first_kind == 0 else 0, b'wrong-mode')
if first_kind == 0:
await send(state, 'mixed', 2)
assert state.ended == [('mixed', b'first')]
messages.process_parent_leave_message.assert_not_awaited()
else:
assert await state.buffers.get_audio_data(target.audio_cache_key) == b'first'
assert state.started == []
await send(state, 'mixed', 2)
messages.process_parent_leave_message.assert_awaited_once()
@pytest.mark.asyncio
async def test_new_recording_recovers_abandoned_message_without_reusing_audio(state):
abandoned = await set_target(state, 'parent')
await send(state, 'abandoned', 1)
await send(state, 'abandoned', 4, b'abandoned-audio')
await send(state, 'new-ai', 1)
await send(state, 'new-ai', 0, b'question')
await send(state, 'abandoned', 2, active=(DEVICE, 'new-ai'))
await send(state, 'new-ai', 2)
assert state.ended == [('new-ai', b'question')]
assert await state.buffers.get_audio_data(abandoned.audio_cache_key) is None
messages.process_parent_leave_message.assert_not_awaited()