fix(voice): recover dialogue after abandoned message recordings

2026-10-28 12:22:58 +08:00
parent a02e0d5f13
commit 9ebb1a78ec
10 changed files with 622 additions and 85 deletions
@@ -22,8 +22,10 @@ async def handle_websocket_data(

    if packet_type == 1:  # 开始包
        try:
-            await websocket.send_text("START_ACK")
-            session_logger.info(device_id, session_id, "已发送开始包确认")
+            if not session.start_ack_sent:
+                await websocket.send_text("START_ACK")
+                session.start_ack_sent = True
+                session_logger.info(device_id, session_id, "已发送开始包确认")
        except Exception as e:
            session_logger.error(device_id, session_id, f"发送开始包确认失败: {str(e)}")
        session.sample_rate = sample_rate
@@ -1,6 +1,8 @@
 from services.interrupt_handler import interrupt_handler
 from services.audio_session import audio_session_manager
 from utils.logger import session_logger
+from services.device_target_cache import device_target_cache
+from services.task_manager import task_manager

 async def handle_old_session_cleanup(old_session, session_key):
    """
@@ -21,19 +23,22 @@ async def handle_old_session_cleanup(old_session, session_key):
        session_logger.error("unknown", "unknown", f"清理旧会话资源时出错: {e}")


-async def cleanup_device_sessions(device_id):
+async def cleanup_device_sessions(device_id, websocket=None):
    """
    清理设备所有会话
    
    Args:
        device_id: 设备ID
    """
+    await device_target_cache.remove_target(device_id, websocket=websocket, reason="disconnected")
    sessions_to_clean = []
    for session_key, session in await audio_session_manager.get_all_sessions():
-        if session_key[0] == device_id:
+        if session_key[0] == device_id and (websocket is None or session.websocket is websocket):
            sessions_to_clean.append((session_key, session))
    
    for session_key, session in sessions_to_clean:
+        if websocket is not None and session.websocket is not websocket:
+            continue
        if hasattr(session, "set_interrupted"):
            session_logger.info(
                device_id, session_key[1], "WebSocket连接断开，中断会话"
@@ -44,8 +49,11 @@ async def cleanup_device_sessions(device_id):
            await interrupt_handler.set_interrupt_state(session_key, True)
            if session.llm_service and hasattr(session.llm_service, "closed"):
                session.llm_service.closed = True
+        await task_manager.cancel_session_tasks(session_key)
    
-    for session_key, _ in sessions_to_clean:
+    for session_key, session in sessions_to_clean:
+        if websocket is not None and session.websocket is not websocket:
+            continue
        await audio_session_manager.remove_session(session_key)
        await interrupt_handler.remove_session(session_key)
        session_logger.info(
@@ -1,7 +1,6 @@
 from fastapi import WebSocket, WebSocketDisconnect
 from utils.logger import session_logger
 from services.connection_manager import connection_manager
-from services.task_manager import task_manager
 from services.offline_audio_cache import offline_audio_cache
 from handlers.websocket_auth_handler import authenticate_websocket
 from handlers.websocket_message_handler import handle_websocket_messages
@@ -59,9 +58,7 @@ async def websocket_endpoint(websocket: WebSocket):
    finally:
        if device_id:
            await connection_manager.remove_connection(device_id, websocket)
-            await cleanup_device_sessions(device_id)
-            # 清理设备相关的所有异步任务
-            await task_manager.cancel_device_tasks(device_id)
+            await cleanup_device_sessions(device_id, websocket)
        
        session_logger.info(
            device_id or "unknown",
@@ -44,6 +44,7 @@ async def handle_websocket_messages(websocket: WebSocket, device_id: str, serial
        websocket: WebSocket连接
        device_id: 设备ID
    """
+    await device_target_cache.bind_connection(device_id, websocket)
    current_active_session = None
    first_audio_received_time = None
    
@@ -156,54 +157,111 @@ async def handle_text_message(websocket: WebSocket, device_id: str, text_data: s

 async def handle_binary_message(websocket: WebSocket, device_id: str, serial_number: str, binary_data, current_active_session, first_audio_received_time):
    """处理二进制音频消息"""
+    active_connection = await connection_manager.get_connection(device_id)
+    if active_connection is not None and active_connection is not websocket:
+        session_logger.warning(device_id, "connection", "忽略已被替换连接的录音包")
+        return current_active_session, first_audio_received_time
    session_key, session, packet_type, audio_data, sample_rate = await parse_packet(binary_data, websocket)
-    
-    if session:
-        session.websocket = websocket
-        session_id = session_key[1]
-        
-        if packet_type == 1:  # 开始包
-            voice_target = await device_target_cache.get_voice_target(device_id)
-            if voice_target:
-                await target_audio_cache.clear_audio_data(voice_target.audio_cache_key)
-                return None, None
-            current_active_session = await handle_start_packet(
-                device_id, session_id, session_key, session, current_active_session
+    if not session:
+        return current_active_session, first_audio_received_time
+    session_id = session_key[1]
+    if session.websocket is not None and session.websocket is not websocket:
+        session_logger.warning(device_id, session_id, "忽略其他连接的录音包")
+        return current_active_session, first_audio_received_time
+    session.websocket = websocket
+    session.last_activity_time = asyncio.get_running_loop().time()
+
+    if packet_type == 1:
+        if session.start_received:
+            return current_active_session, first_audio_received_time
+        session.start_received = True
+        session.voice_target = await device_target_cache.claim_target(device_id, websocket, session_id)
+        if session.voice_target:
+            # Acknowledge START now; the first audio packet selects AI (0) or
+            # message (4). A pending message must not swallow an AI recording.
+            await websocket.send_text("START_ACK")
+            session.start_ack_sent = True
+            return session_key, None
+        current_active_session = await start_ai_recording(
+            websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
+        )
+        return current_active_session, None
+
+    if not session.start_received or session.recording_finished:
+        session_logger.warning(device_id, session_id, "忽略未开始或已结束的录音包")
+        return current_active_session, first_audio_received_time
+
+    if packet_type == 0:
+        if session.message_audio_received:
+            session_logger.warning(device_id, session_id, "忽略留言录音中混入的普通对话音频")
+            return current_active_session, first_audio_received_time
+        if not session.ai_started:
+            target = session.voice_target
+            if target:
+                await device_target_cache.remove_target(device_id, expected=target, reason="ai_audio")
+                session.voice_target = None
+            current_active_session = await start_ai_recording(
+                websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
            )
-        elif packet_type == 3:  # 中断包
-            voice_target = await device_target_cache.get_voice_target(device_id)
-            if voice_target:
-                return None, None
-            current_active_session = await handle_interrupt_packet(
-                device_id, session_id, session_key, session, current_active_session
-            )
-        elif packet_type == 2:  # 结束包
-            voice_target = await device_target_cache.get_voice_target(device_id)
-            if voice_target:
-                session_logger.info(device_id, session_id, f"收到结束包，开始处理缓存音频数据")
-                if voice_target.kind == VOICE_TARGET_KIND_PARENT:
-                    await process_parent_leave_message(device_id, voice_target.audio_cache_key)
-                else:
-                    await process_cached_audio(device_id, voice_target.target_device_id, serial_number)
-                await device_target_cache.remove_target(device_id)
-                return None, None
-            if current_active_session == session_key:
-                current_active_session = None
-        elif packet_type == 4:  # 发送给目标设备的音频包
-            await handle_target_audio_packet(device_id, audio_data)
-            return None, None
-        elif packet_type == 0 and first_audio_received_time is None:
+        if first_audio_received_time is None:
            first_audio_received_time = time.perf_counter()
            session.start_time = first_audio_received_time
-            session_logger.info(
-                device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}"
-            )
-        
-        await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate)
-    
+            session_logger.info(device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}")
+    elif packet_type == 4:
+        target = session.voice_target
+        if target and not session.ai_started:
+            if await handle_target_audio_packet(device_id, audio_data, target, websocket, session_id):
+                session.message_audio_received = True
+        else:
+            session_logger.warning(device_id, session_id, "忽略没有对应留言目标的留言音频包")
+        return current_active_session, first_audio_received_time
+    elif packet_type == 3:
+        if session.voice_target:
+            await device_target_cache.remove_target(device_id, expected=session.voice_target, reason="interrupted")
+            session.voice_target = None
+            session.recording_finished = True
+            await websocket.send_text("INTERRUPT_ACK")
+            await websocket.send_text("INTERRUPT_PROCESSED")
+            return None if current_active_session == session_key else current_active_session, None
+        current_active_session = await handle_interrupt_packet(
+            device_id, session_id, session_key, session, current_active_session
+        )
+    elif packet_type == 2:
+        if session.voice_target:
+            target = await device_target_cache.take_target(device_id, session.voice_target, websocket, session_id)
+            session.voice_target = None
+            session.recording_finished = True
+            try:
+                await websocket.send_text("END_ACK")
+                if target:
+                    session_logger.info(device_id, session_id, "收到结束包，开始处理缓存音频数据")
+                    if target.kind == VOICE_TARGET_KIND_PARENT:
+                        await process_parent_leave_message(device_id, target.audio_cache_key)
+                    else:
+                        await process_cached_audio(device_id, target.target_device_id, serial_number,
+                                                   audio_cache_key=target.audio_cache_key)
+            finally:
+                if target:
+                    await target_audio_cache.clear_audio_data(target.audio_cache_key)
+            return None if current_active_session == session_key else current_active_session, None
+        if current_active_session == session_key:
+            current_active_session = None
+
+    await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate)
+    if packet_type in (2, 3):
+        session.recording_finished = True
    return current_active_session, first_audio_received_time


+async def start_ai_recording(websocket, device_id, session_id, session_key, session, current_active_session, sample_rate):
+    current_active_session = await handle_start_packet(
+        device_id, session_id, session_key, session, current_active_session
+    )
+    await handle_websocket_data(websocket, session_key, session, 1, b"", sample_rate)
+    session.ai_started = True
+    return current_active_session
+
+
 async def handle_start_packet(device_id, session_id, session_key, session, current_active_session):
    """处理会话开始包"""
    if current_active_session and current_active_session[1] != session_id:
@@ -255,20 +313,17 @@ async def handle_interrupt_packet(device_id, session_id, session_key, session, c
    return current_active_session


-async def handle_target_audio_packet(device_id: str, audio_data: bytes):
+async def handle_target_audio_packet(device_id: str, audio_data: bytes, voice_target, websocket, session_id):
    """处理发送给目标设备的音频包"""
    try:
-        voice_target = await device_target_cache.get_voice_target(device_id)
-        
-        if not voice_target:
-            session_logger.warning(device_id, "target", "未设置目标设备，无法发送音频")
-            return 
-        
-        await target_audio_cache.add_audio_data(voice_target.audio_cache_key, audio_data)
+        if not await device_target_cache.append_audio(device_id, voice_target, websocket, session_id, audio_data):
+            session_logger.warning(device_id, session_id, "忽略已过期或不属于本次录音的留言音频")
+            return
        if voice_target.kind == VOICE_TARGET_KIND_PARENT:
            session_logger.info(device_id, "target", "已缓存发给家长的留言音频数据")
        else:
            session_logger.info(device_id, "target", f"已缓存音频数据到目标设备 {voice_target.target_device_id}")
+        return True
        
    except Exception as e:
        session_logger.error(device_id, "target", f"处理目标音频包时出错: {e}", exc_info=True)
@@ -337,10 +392,11 @@ async def process_parent_leave_message(device_id: str, audio_cache_key: str):
        await target_audio_cache.clear_audio_data(audio_cache_key)


-async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str):
+async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str, *, audio_cache_key=None):
    """处理缓存的音频数据并发送音频URL"""
    try:
-        cached_audio = await target_audio_cache.get_audio_data(target_device_id)
+        cache_key = audio_cache_key or target_device_id
+        cached_audio = await target_audio_cache.get_audio_data(cache_key)
        if not cached_audio:
            session_logger.info(device_id, "target", f"目标设备 {target_device_id} 没有缓存的音频数据")
            return
@@ -398,4 +454,4 @@ async def process_cached_audio(device_id: str, target_device_id: str, serial_num
            session_logger.error(device_id, "target", f"处理缓存音频时出错: {e}", exc_info=True)
    finally:
        # 清除缓存
-        await target_audio_cache.clear_audio_data(target_device_id)
+        await target_audio_cache.clear_audio_data(cache_key)