import json import time import asyncio from fastapi import WebSocket from handlers.audio_packet_parser import parse_packet from handlers.audio_session_handler import handle_websocket_data from handlers.audio_file_handler import upload_message_audio from banban.service.device_audio_cache import device_audio_cache_service from banban.service.device_voice_archive import device_voice_archive_service from banban.service.pending_voice_message import pending_voice_message_service from services.audio_session import audio_session_manager from services.interrupt_handler import interrupt_handler from services.task_manager import task_manager from services.connection_manager import connection_manager from services.device_target_cache import device_target_cache from services.offline_audio_cache import offline_audio_cache from services.target_audio_cache import target_audio_cache from services.card_service import card_service from services.device_target_cache import VOICE_TARGET_KIND_PARENT from utils.logger import session_logger from handlers.prompt_sound_handler import handle_prompt_sound_request from handlers.session_cleanup_handler import handle_old_session_cleanup from config import settings from banban.service.im import im_service as im_conversation_service from utils.audio_format import detect_audio_format, wrap_pcm_as_wav from fastapi import HTTPException def prepare_message_audio(audio_data: bytes) -> tuple[bytes, str, str, str]: source_format = detect_audio_format(audio_data) if source_format == "wav": return audio_data, "audio/wav", "wav", source_format if source_format == "mp3": return audio_data, "audio/mpeg", "mp3", source_format return wrap_pcm_as_wav(audio_data), "audio/wav", "wav", source_format async def handle_websocket_messages(websocket: WebSocket, device_id: str, serial_number: str): """ 处理WebSocket连接中的所有消息 Args: websocket: WebSocket连接 device_id: 设备ID """ current_active_session = None first_audio_received_time = None while True: message = await websocket.receive() if not websocket.client_state.name == "CONNECTED": session_logger.info( device_id, "connection", "WebSocket连接已关闭，退出消息循环", ) break if message["type"] == "websocket.receive": data = message["text"] if "text" in message else message["bytes"] if isinstance(data, str): await handle_text_message(websocket, device_id, data) else: # 处理二进制音频数据 current_active_session, first_audio_received_time = await handle_binary_message( websocket, device_id, serial_number, data, current_active_session, first_audio_received_time ) elif message["type"] == "websocket.disconnect": break async def handle_text_message(websocket: WebSocket, device_id: str, text_data: str): """处理文本消息""" if text_data.startswith("REGISTER_TARGET_DEVICE:"): card_uuid = text_data.split(":", 1)[1].strip() # 检查卡片是否存在 existing_card = await card_service.get_card_by_uuid(card_uuid) if existing_card: # 卡片已存在，使用卡片绑定的设备ID作为目标设备ID target_device_id = existing_card.device_id session_logger.info(device_id, "card", f"卡片已存在，绑定的设备ID: {target_device_id}") else: # 卡片不存在，创建新卡片并绑定到当前设备 new_card = await card_service.activate_card(card_uuid, device_id) session_logger.info(device_id, "card", f"新卡片{card_uuid}已创建并激活，绑定到设备: {device_id}") return # 设置目标设备 await device_target_cache.set_target(device_id, target_device_id) # 检查目标设备是否在线 target_websocket = await connection_manager.get_connection(target_device_id) if target_websocket and target_websocket.client_state.name == "CONNECTED": sound_url = f"http://{settings.server_host}:{settings.server_port}/assets/audio/welcome.mp3" await websocket.send_text(f"TARGET_DEVICE_REGISTERED_URL:{sound_url}") session_logger.info(device_id, "target", f"成功注册目标设备: {target_device_id}") else: sound_url = f"http://{settings.server_host}:{settings.server_port}/assets/audio/offline.mp3" await websocket.send_text(f"TARGET_DEVICE_REGISTERED_URL:{sound_url}") session_logger.warning(device_id, "target", f"目标设备 {target_device_id} 不在线") elif text_data.startswith("REQUEST_PROMPT_SOUND:"): prompt_type = text_data.split(":", 1)[1] if device_id: await handle_prompt_sound_request(device_id, prompt_type) elif text_data == "NETWORK_RESET_ACKNOWLEDGED": session_logger.info(device_id, "network", f"设备 {device_id} 确认网络重置") elif text_data.startswith("FIRMWARE_UPDATE_STATUS:"): status_info = text_data.split(":", 1)[1] parts = status_info.split(",") status_dict = {} for part in parts: if "=" in part: key, value = part.split("=", 1) status_dict[key.strip()] = value.strip() from services.device_update_manager import device_firmware_update_manager update_status = status_dict.get("status", "updating") progress = float(status_dict.get("progress", "0")) if "progress" in status_dict else 0.0 await device_firmware_update_manager.update_firmware_progress(device_id, progress) if update_status in ["success", "failed", "completed"]: version = status_dict.get("version", "unknown") await device_firmware_update_manager.update_firmware_update( device_id, firmware_version=version, update_status=update_status ) session_logger.info(device_id, "update", f"固件更新状态已保存: {update_status}, 进度: {progress}") elif text_data.startswith("FIRMWARE_VERSION:"): version_info = text_data.split(":", 1)[1].strip() session_logger.info(device_id, "update", f"收到设备固件版本: {version_info}") from services.device_update_manager import device_firmware_update_manager await device_firmware_update_manager.update_device_firmware_version( device_id, version_info ) await device_firmware_update_manager.update_firmware_update( device_id, version_info, "success" # 收到版本信息说明设备当前状态正常 ) async def handle_binary_message(websocket: WebSocket, device_id: str, serial_number: str, binary_data, current_active_session, first_audio_received_time): """处理二进制音频消息""" session_key, session, packet_type, audio_data, sample_rate = await parse_packet(binary_data, websocket) if session: session.websocket = websocket session_id = session_key[1] if packet_type == 1: # 开始包 voice_target = await device_target_cache.get_voice_target(device_id) if voice_target: await target_audio_cache.clear_audio_data(voice_target.audio_cache_key) return None, None current_active_session = await handle_start_packet( device_id, session_id, session_key, session, current_active_session ) elif packet_type == 3: # 中断包 voice_target = await device_target_cache.get_voice_target(device_id) if voice_target: return None, None current_active_session = await handle_interrupt_packet( device_id, session_id, session_key, session, current_active_session ) elif packet_type == 2: # 结束包 voice_target = await device_target_cache.get_voice_target(device_id) if voice_target: session_logger.info(device_id, session_id, f"收到结束包，开始处理缓存音频数据") if voice_target.kind == VOICE_TARGET_KIND_PARENT: await process_parent_leave_message(device_id, voice_target.audio_cache_key) else: await process_cached_audio(device_id, voice_target.target_device_id, serial_number) await device_target_cache.remove_target(device_id) return None, None if current_active_session == session_key: current_active_session = None elif packet_type == 4: # 发送给目标设备的音频包 await handle_target_audio_packet(device_id, audio_data) return None, None elif packet_type == 0 and first_audio_received_time is None: first_audio_received_time = time.perf_counter() session.start_time = first_audio_received_time session_logger.info( device_id, session_id, f"收到第一个音频数据包时间: {first_audio_received_time}" ) await handle_websocket_data(websocket, session_key, session, packet_type, audio_data, sample_rate) return current_active_session, first_audio_received_time async def handle_start_packet(device_id, session_id, session_key, session, current_active_session): """处理会话开始包""" if current_active_session and current_active_session[1] != session_id: old_session = await audio_session_manager.get_session(current_active_session) if old_session: session_logger.info( device_id, current_active_session[1], f"检测到新会话 {session_id}，优先中断旧会话 {current_active_session[1]}", ) await interrupt_handler.set_interrupt_state(current_active_session, True) if hasattr(old_session.llm_service, "close"): await old_session.llm_service.close() if hasattr(old_session.tts_service, "close"): await old_session.tts_service.close() await task_manager.create_task( handle_old_session_cleanup(old_session, current_active_session), device_id=device_id, session_key=current_active_session, task_type="cleanup" ) current_active_session = session_key await interrupt_handler.set_interrupt_state(session_key, False) if hasattr(session.llm_service, "closed"): session.llm_service.closed = False session_logger.info(device_id, session_id, f"设置当前活跃会话: {session_id}") await interrupt_handler.register_session(session_key) return current_active_session async def handle_interrupt_packet(device_id, session_id, session_key, session, current_active_session): """处理中断包""" if current_active_session == session_key: current_active_session = None session_logger.info(device_id, session_id, "收到中断包，立即设置中断状态") await interrupt_handler.set_interrupt_state(session_key, True) if hasattr(session.llm_service, "closed"): session.llm_service.closed = True if interrupt_handler.is_processing_interrupt(session_key): session_logger.info(device_id, session_id, "检测到连续中断，中断请求将被排队") await task_manager.create_task( interrupt_handler.handle_interrupt(session_key), device_id=device_id, session_key=session_key, task_type="interrupt" ) return current_active_session async def handle_target_audio_packet(device_id: str, audio_data: bytes): """处理发送给目标设备的音频包""" try: voice_target = await device_target_cache.get_voice_target(device_id) if not voice_target: session_logger.warning(device_id, "target", "未设置目标设备，无法发送音频") return await target_audio_cache.add_audio_data(voice_target.audio_cache_key, audio_data) if voice_target.kind == VOICE_TARGET_KIND_PARENT: session_logger.info(device_id, "target", "已缓存发给家长的留言音频数据") else: session_logger.info(device_id, "target", f"已缓存音频数据到目标设备 {voice_target.target_device_id}") except Exception as e: session_logger.error(device_id, "target", f"处理目标音频包时出错: {e}", exc_info=True) async def process_parent_leave_message(device_id: str, audio_cache_key: str): """处理设备发给家长的留言音频并写入家长会话。""" try: cached_audio = await target_audio_cache.get_audio_data(audio_cache_key) if not cached_audio: session_logger.info(device_id, "parent", "发给家长的留言没有缓存音频数据") return archive_audio, media_mime_type, extension, source_format = prepare_message_audio(cached_audio) stored_audio = await upload_message_audio( archive_audio, device_id, content_type=media_mime_type, extension=extension, ) await im_conversation_service.create_device_parent_leave_message( device_id=device_id, media_file_key=stored_audio.file_key, media_mime_type=media_mime_type, media_size_bytes=len(archive_audio), ext_json={ "source": "device_ws_parent_leave_message", "storage": "cos", "source_format": source_format, "archive_format": extension, }, ) session_logger.info( device_id, "parent", f"发给家长的留言已上传COS并写入家长会话: {stored_audio.file_key}", ) websocket = await connection_manager.get_connection(device_id) if websocket and websocket.client_state.name == "CONNECTED": success_audio_url = f"http://{settings.server_host}:{settings.server_port}/assets/audio/message_ok.mp3" await websocket.send_text(f"PROMPT_SOUND_URL:{success_audio_url}") session_logger.info(device_id, "device", f"留言成功提示音已发送给设备 {device_id}") else: session_logger.warning(device_id, "device", f"设备 {device_id} 不在线，暂不发送留言成功提示音") except Exception as e: if isinstance(e, HTTPException): session_logger.error(device_id, "parent", f"发给家长的留言保存失败，返回HTTPException: {e}") websocket = await connection_manager.get_connection(device_id) if e.status_code == 404 or e.status_code == 400: if websocket and websocket.client_state.name == "CONNECTED": fail_audio_url = f"http://{settings.server_host}:{settings.server_port}/assets/audio/save_audio_fail.mp3" await websocket.send_text(f"PROMPT_SOUND_URL:{fail_audio_url}") session_logger.info(device_id, "device", f"留言失败提示音已发送给设备 {device_id}") else: session_logger.warning(device_id, "device", f"设备 {device_id} 不在线，暂不发送留言失败提示音") else: session_logger.error(device_id, "parent", f"处理家长留言音频时出错: {e}", exc_info=True) finally: await target_audio_cache.clear_audio_data(audio_cache_key) async def process_cached_audio(device_id: str, target_device_id: str, serial_number: str): """处理缓存的音频数据并发送音频URL""" try: cached_audio = await target_audio_cache.get_audio_data(target_device_id) if not cached_audio: session_logger.info(device_id, "target", f"目标设备 {target_device_id} 没有缓存的音频数据") return audio_url, local_audio_path = await device_audio_cache_service.save_device_audio( cached_audio, device_id=target_device_id, ) archive_result = await device_voice_archive_service.archive_peer_voice_message( sender_device_id=device_id, receiver_device_id=target_device_id, local_audio_path=str(local_audio_path), ) if archive_result is not None: await pending_voice_message_service.add_pending_message( target_device_id=target_device_id, sender_device_id=device_id, im_message_id=archive_result.message_id, media_file_key=archive_result.media_file_key, audio_url=audio_url, source="device_peer_voice", ) session_logger.info( device_id, "target", f"设备留言已加入待收听队列: target_device_id={target_device_id}, audio_url={audio_url}", ) else: await offline_audio_cache.add_audio_url(target_device_id, audio_url) session_logger.warning( device_id, "target", "设备留言已写入内存待收听队列，但归档失败，重启后无法恢复这条留言", ) websocket = await connection_manager.get_connection(device_id) if websocket and websocket.client_state.name == "CONNECTED": success_audio_url = f"http://{settings.server_host}:{settings.server_port}/assets/audio/message_ok.mp3" await websocket.send_text(f"PROMPT_SOUND_URL:{success_audio_url}") session_logger.info(device_id, "device", f"留言已收到音频URL给设备 {device_id}") else: session_logger.warning(device_id, "device", f"设备 {device_id} 不在线，暂不发送留言已收到音频URL") except Exception as e: if isinstance(e, HTTPException): session_logger.error(device_id, "target", f"保存音频文件时出错，返回HTTPException: {e}") websocket = await connection_manager.get_connection(device_id) if e.status_code == 404 or e.status_code == 400: if websocket and websocket.client_state.name == "CONNECTED": success_audio_url = f"http://{settings.server_host}:{settings.server_port}/assets/audio/save_audio_fail.mp3" await websocket.send_text(f"PROMPT_SOUND_URL:{success_audio_url}") session_logger.info(device_id, "device", f"留言已收到音频URL给设备 {device_id}") else: session_logger.warning(device_id, "device", f"设备 {device_id} 不在线，暂不发送留言已收到音频URL") else: session_logger.error(device_id, "target", f"处理缓存音频时出错: {e}", exc_info=True) finally: # 清除缓存 await target_audio_cache.clear_audio_data(target_device_id)