huangyicao/banban
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(voice): recover dialogue after abandoned message recordings

Browse Source
This commit is contained in:
stu2not
2026-10-28 12:22:58 +08:00
parent a02e0d5f13
commit 9ebb1a78ec
10 changed files with 622 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
talkingq-url/handlers/websocket_message_handler.py
+108 -52
View File
@@ -44,6 +44,7 @@ async def handle_websocket_messages(websocket: WebSocket, device_id: str, serial
websocket: WebSocket连接
device_id: 设备ID
"""
await device_target_cache.bind_connection(device_id, websocket)
current_active_session = None
first_audio_received_time = None
@@ -156,54 +157,111 @@ async def handle_text_message(websocket: WebSocket, device_id: str, text_data: s
async def handle_binary_message(websocket: WebSocket, device_id: str, serial_number: str, binary_data, current_active_session, first_audio_received_time):
"""处理二进制音频消息"""
active_connection = await connection_manager.get_connection(device_id)
if active_connection is not None and active_connection is not websocket:
session_logger.warning(device_id, "connection", "忽略已被替换连接的录音包")
return current_active_session, first_audio_received_time
session_key, session, packet_type, audio_data, sample_rate = await parse_packet(binary_data, websocket)
if session:
session.websocket = websocket
session_id = session_key[1]
if packet_type == 1: # 开始包
voice_target = await device_target_cache.get_voice_target(device_id)
if voice_target:
await target_audio_cache.clear_audio_data(voice_target.audio_cache_key)
return None, None
current_active_session = await handle_start_packet(
device_id, session_id, session_key, session, current_active_session
if not session:
return current_active_session, first_audio_received_time
session_id = session_key[1]
if session.websocket is not None and session.websocket is not websocket:
session_logger.warning(device_id, session_id, "忽略其他连接的录音包")
return current_active_session, first_audio_received_time
session.websocket = websocket
session.last_activity_time = asyncio.get_running_loop().time()
if packet_type == 1:
if session.start_received:
return current_active_session, first_audio_received_time
session.start_received = True
session.voice_target = await device_target_cache.claim_target(device_id, websocket, session_id)
if session.voice_target:
# Acknowledge START now; the first audio packet selects AI (0) or
# message (4). A pending message must not swallow an AI recording.
await websocket.send_text("START_ACK")
session.start_ack_sent = True
return session_key, None
current_active_session = await start_ai_recording(
websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
)
return current_active_session, None
if not session.start_received or session.recording_finished:
session_logger.warning(device_id, session_id, "忽略未开始或已结束的录音包")
return current_active_session, first_audio_received_time
if packet_type == 0:
if session.message_audio_received:
session_logger.warning(device_id, session_id, "忽略留言录音中混入的普通对话音频")
return current_active_session, first_audio_received_time
if not session.ai_started:
target = session.voice_target
if target:
await device_target_cache.remove_target(device_id, expected=target, reason="ai_audio")
session.voice_target = None
current_active_session = await start_ai_recording(
websocket, device_id, session_id, session_key, session, current_active_session, sample_rate
)
elif packet_type == 3: # 中断包
voice_target = await device_target_cache.get_voice_target(device_id)
if voice_target:
return None, None
current_active_session = await handle_interrupt_packet(
device_id, session_id, session_key, session, current_active_session
)
elif packet_type == 2: # 结束包
voice_target = await device_target_cache.get_voice_target(device_id)
if voice_target:
session_logger.info(device_id, session_id, f"收到结束包，开始处理缓存音频数据")
if voice_target.kind == VOICE_TARGET_KIND_PARENT:
await process_parent_leave_message(device_id, voice_target.audio_cache_key)
else:
await process_cached_audio(device_id, voice_target.target_device_id, serial_number)
await device_target_cache.remove_target(device_id)
return None, None
if current_active_session == session_key:
current_active_session = None
elif packet_type == 4: # 发送给目标设备的音频包
await handle_target_audio_packet(device_id, audio_data)
return None, None
elif packet_type == 0 and first_audio_received_time is None:
if first_audio_received_time is None:
first_audio_received_time = time.perf_counter()
session.start_time = first_audio_received_time
session_logger.info(
device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}"
)
await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate)
session_logger.info(device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}")
elif packet_type == 4:
target = session.voice_target
if target and not session.ai_started:
if await handle_target_audio_packet(device_id, audio_data, target, websocket, session_id):
session.message_audio_received = True
else:
session_logger.warning(device_id, session_id, "忽略没有对应留言目标的留言音频包")
return current_active_session, first_audio_received_time
elif packet_type == 3:
if session.voice_target:
await device_target_cache.remove_target(device_id, expected=session.voice_target, reason="interrupted")
session.voice_target = None
session.recording_finished = True
await websocket.send_text("INTERRUPT_ACK")
await websocket.send_text("INTERRUPT_PROCESSED")
return None if current_active_session == session_key else current_active_session, None
current_active_session = await handle_interrupt_packet(
device_id, session_id, session_key, session, current_active_session
)
elif packet_type == 2:
if session.voice_target:
target = await device_target_cache.take_target(device_id, session.voice_target, websocket, session_id)
session.voice_target = None
session.recording_finished = True
try:
await websocket.send_text("END_ACK")
if target:
session_logger.info(device_id, session_id, "收到结束包，开始处理缓存音频数据")
if target.kind == VOICE_TARGET_KIND_PARENT:
await process_parent_leave_message(device_id, target.audio_cache_key)
else:
await process_cached_audio(device_id, target.target_device_id, serial_number,
audio_cache_key=target.audio_cache_key)
finally:
if target:
await target_audio_cache.clear_audio_data(target.audio_cache_key)
return None if current_active_session == session_key else current_active_session, None
if current_active_session == session_key:
current_active_session = None
await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate)
if packet_type in (2, 3):
session.recording_finished = True
return current_active_session, first_audio_received_time
async def start_ai_recording(websocket, device_id, session_id, session_key, session, current_active_session, sample_rate):
current_active_session = await handle_start_packet(
device_id, session_id, session_key, session, current_active_session
)
await handle_websocket_data(websocket, session_key, session, 1, b"", sample_rate)
session.ai_started = True
return current_active_session
async def handle_start_packet(device_id, session_id, session_key, session, current_active_session):
"""处理会话开始包"""
if current_active_session and current_active_session[1] != session_id:
@@ -255,20 +313,17 @@ async def handle_interrupt_packet(device_id, session_id, session_key, session, c
return current_active_session
async def handle_target_audio_packet(device_id: str, audio_data: bytes):
async def handle_target_audio_packet(device_id: str, audio_data: bytes, voice_target, websocket, session_id):
"""处理发送给目标设备的音频包"""
try:
voice_target = await device_target_cache.get_voice_target(device_id)
if not voice_target:
session_logger.warning(device_id, "target", "未设置目标设备，无法发送音频")
return
await target_audio_cache.add_audio_data(voice_target.audio_cache_key, audio_data)
if not await device_target_cache.append_audio(device_id, voice_target, websocket, session_id, audio_data):
session_logger.warning(device_id, session_id, "忽略已过期或不属于本次录音的留言音频")
return
if voice_target.kind == VOICE_TARGET_KIND_PARENT:
session_logger.info(device_id, "target", "已缓存发给家长的留言音频数据")
else:
session_logger.info(device_id, "target", f"已缓存音频数据到目标设备 {voice_target.target_device_id}")
return True
except Exception as e:
session_logger.error(device_id, "target", f"处理目标音频包时出错: {e}", exc_info=True)
@@ -337,10 +392,11 @@ async def process_parent_leave_message(device_id: str, audio_cache_key: str):
await target_audio_cache.clear_audio_data(audio_cache_key)
async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str):
async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str, *, audio_cache_key=None):
"""处理缓存的音频数据并发送音频URL"""
try:
cached_audio = await target_audio_cache.get_audio_data(target_device_id)
cache_key = audio_cache_key or target_device_id
cached_audio = await target_audio_cache.get_audio_data(cache_key)
if not cached_audio:
session_logger.info(device_id, "target", f"目标设备 {target_device_id} 没有缓存的音频数据")
return
@@ -398,4 +454,4 @@ async def process_cached_audio(device_id: str, target_device_id: str, serial_num
session_logger.error(device_id, "target", f"处理缓存音频时出错: {e}", exc_info=True)
finally:
# 清除缓存
await target_audio_cache.clear_audio_data(target_device_id)
await target_audio_cache.clear_audio_data(cache_key)