huangyicao/banban
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): upload device audio to cos and enable playback

Browse Source
This commit is contained in:
ChengCan
2026-04-30 01:03:34 +08:00
parent 282d0874d8
commit efb13de8ed
17 changed files with 640 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
talkingq-url/banban/dao/__init__.py
View File

@@ -7,7 +7,8 @@ class BaseDAO:
self.db = db
async def execute(self, query, params: dict = None):
return await self.db.execute(text(query), params or {})
statement = query if hasattr(query, "_execute_on_connection") else text(query)
return await self.db.execute(statement, params or {})
async def commit(self):
await self.db.commit()

30
talkingq-url/banban/dao/im.py
View File

@@ -25,6 +25,14 @@ class ConversationMessageCreateResult:
conversation_type: int
message: dict
@property
def conversation_type_name(self) -> str:
if self.conversation_type == 1:
return "child_peer"
if self.conversation_type == 2:
return "parent_child"
return f"unknown_{self.conversation_type}"
class ImDAO(BaseDAO):
async def assert_parent_child_access(self, *, user_id: int, child_id: int) -> Mapping[str, Any]:
@@ -179,6 +187,22 @@ class ImDAO(BaseDAO):
return "[image]"
return "[json]"
async def ensure_parent_child_conversation(self, *, parent_user_id: int, child_id: int) -> int:
child_row = await self.assert_parent_child_access(user_id=parent_user_id, child_id=child_id)
parent_row = await self._get_parent_row(parent_user_id)
if not parent_row:
from fastapi import HTTPException
raise HTTPException(status_code=404, detail="parent not found")
return await self._get_or_create_conversation(
conversation_type=2,
participant_a_type=2,
participant_a_id=str(child_id),
participant_b_type=1,
participant_b_id=str(parent_user_id),
pair_key=f"{child_id}:{parent_user_id}",
)
async def create_message(
self,
*,
@@ -428,8 +452,8 @@ class ImDAO(BaseDAO):
FROM im_conversations
WHERE conversation_type = :conversation_type
AND pair_key = :pair_key
{lock_clause}
LIMIT 1
{lock_clause}
"""
),
{"conversation_type": conversation_type, "pair_key": pair_key},
@@ -449,8 +473,8 @@ class ImDAO(BaseDAO):
SELECT id, conversation_type, last_seq, status
FROM im_conversations
WHERE id = :conversation_id
{lock_clause}
LIMIT 1
{lock_clause}
"""
),
{"conversation_id": conversation_id},
@@ -484,4 +508,4 @@ class ImDAO(BaseDAO):
async def _next_primary_key(self, table_name: str) -> int | None:
result = await self.execute(text(f"SELECT 1"))
return None
return None

6
talkingq-url/banban/routers/im.py
View File

@@ -16,7 +16,7 @@ try:
ConversationMessageCreateResponse,
ParentChildMessageCreateRequest,
)
from banban.service.im import ImService, im_service
from banban.service.im import ImService, im_service, present_message_item
except ModuleNotFoundError:
from banban.security import get_current_user_id
from banban.schemas.im import (
@@ -27,7 +27,7 @@ except ModuleNotFoundError:
ConversationMessageCreateResponse,
ParentChildMessageCreateRequest,
)
from banban.service.im import ImService, im_service
from banban.service.im import ImService, im_service, present_message_item
router = APIRouter(prefix="/children", tags=["im"])
@@ -506,7 +506,7 @@ async def list_child_conversation_messages(
rows = rows[:limit]
rows = list(rows)
rows.reverse()
items = [_row_to_message_item(row) for row in rows]
items = [await present_message_item(row, audio_storage=im_service.audio_storage) for row in rows]
next_cursor_seq = items[0].seq if has_more and items else None
logger.info(

274
talkingq-url/banban/service/im.py
View File

@@ -1,10 +1,12 @@
from dataclasses import dataclass
import hashlib
import json
from collections.abc import Mapping
from typing import Any
from fastapi import HTTPException, status
from fastapi import HTTPException
from services.database_service_base import DatabaseServiceBase
from banban.service.message_audio_storage import MessageAudioStorageService, MessageAudioStorageError
try:
from banban.dao.im import ImDAO, DeviceIdentity, ConversationMessageCreateResult
@@ -43,6 +45,12 @@ def participant_type_name(participant_type: int) -> str:
return PARTICIPANT_TYPE_NAMES.get(participant_type, f"unknown_{participant_type}")
def build_device_audio_client_msg_id(*, device_id: str, target_device_id: str, audio_url: str) -> str:
raw = f"{device_id}|{target_device_id}|{audio_url}"
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()
return f"device-audio-{digest[:32]}"
def normalize_content_json(value: Any) -> dict[str, Any] | None:
if value is None:
return None
@@ -85,9 +93,24 @@ def row_to_message_item(row: Mapping[str, Any]) -> ChildConversationMessageItem:
)
async def present_message_item(
row: Mapping[str, Any],
*,
audio_storage: MessageAudioStorageService,
) -> ChildConversationMessageItem:
item = row_to_message_item(row)
if item.content_type == 2 and item.media_file_key:
try:
item.media_file_key = await audio_storage.get_audio_url(item.media_file_key)
except MessageAudioStorageError:
pass
return item
class ImService(DatabaseServiceBase):
def __init__(self):
super().__init__(service_name="im_service")
self.audio_storage = MessageAudioStorageService()
async def assert_parent_child_access(self, *, user_id: int, child_id: int) -> Mapping[str, Any]:
db_session = await self.get_session()
@@ -105,6 +128,34 @@ class ImService(DatabaseServiceBase):
finally:
await db_session.close()
async def _build_message_create_result(
self,
*,
dao: ImDAO,
idempotent: bool,
conversation_id: int,
conversation_type: int,
client_msg_id: str,
) -> ConversationMessageCreateResult:
message_row = await dao._get_message_by_conversation_client_id(
conversation_id=conversation_id,
client_msg_id=client_msg_id,
)
if not message_row:
raise RuntimeError("message was not found after insert")
presented_message = await present_message_item(
message_row,
audio_storage=self.audio_storage,
)
return ConversationMessageCreateResult(
idempotent=idempotent,
conversation_id=conversation_id,
conversation_type=conversation_type,
message=presented_message,
)
async def create_parent_child_message(
self,
*,
@@ -137,18 +188,12 @@ class ImService(DatabaseServiceBase):
receiver_avatar_snapshot=None,
payload=payload,
)
message_row = await dao._get_message_by_conversation_client_id(
conversation_id=conversation_id,
client_msg_id=payload.client_msg_id,
)
if not message_row:
raise RuntimeError("message was not found after insert")
return ConversationMessageCreateResult(
return await self._build_message_create_result(
dao=dao,
idempotent=idempotent,
conversation_id=conversation_id,
conversation_type=PARENT_CHILD_CONVERSATION_TYPE,
message=row_to_message_item(message_row),
client_msg_id=payload.client_msg_id,
)
except Exception:
await db_session.rollback()
@@ -156,13 +201,87 @@ class ImService(DatabaseServiceBase):
finally:
await db_session.close()
async def _create_device_message_with_payload(
self,
*,
dao: ImDAO,
device_identity: DeviceIdentity,
payload: DeviceMessageCreateRequest,
) -> ConversationMessageCreateResult:
if payload.conversation_type == CHILD_PEER_CONVERSATION_TYPE:
if payload.peer_child_id == device_identity.child_id:
raise HTTPException(status_code=400, detail="peer_child_id must be different from current child")
sender_child_row = await dao.assert_child_exists(child_id=device_identity.child_id)
receiver_child_row = await dao.assert_child_exists(child_id=payload.peer_child_id)
participant_a_id, participant_b_id, pair_key = dao._build_child_peer_pair(
device_identity.child_id,
payload.peer_child_id,
)
conversation_id, idempotent = await dao.create_message(
conversation_type=CHILD_PEER_CONVERSATION_TYPE,
participant_a_type=CHILD_PARTICIPANT_TYPE,
participant_a_id=participant_a_id,
participant_b_type=CHILD_PARTICIPANT_TYPE,
participant_b_id=participant_b_id,
pair_key=pair_key,
sender_type=CHILD_PARTICIPANT_TYPE,
sender_id=str(device_identity.child_id),
receiver_type=CHILD_PARTICIPANT_TYPE,
receiver_id=str(payload.peer_child_id),
sender_name_snapshot=sender_child_row["child_name"],
sender_avatar_snapshot=None,
receiver_name_snapshot=receiver_child_row["child_name"],
receiver_avatar_snapshot=None,
payload=payload,
)
else:
sender_child_row = await dao.assert_child_exists(child_id=device_identity.child_id)
parent_row = await dao._get_parent_row(payload.parent_user_id)
if not parent_row:
raise HTTPException(status_code=404, detail="parent not found")
await dao.assert_parent_child_access(user_id=payload.parent_user_id, child_id=device_identity.child_id)
conversation_id, idempotent = await dao.create_message(
conversation_type=PARENT_CHILD_CONVERSATION_TYPE,
participant_a_type=CHILD_PARTICIPANT_TYPE,
participant_a_id=str(device_identity.child_id),
participant_b_type=PARENT_PARTICIPANT_TYPE,
participant_b_id=str(payload.parent_user_id),
pair_key=f"{device_identity.child_id}:{payload.parent_user_id}",
sender_type=CHILD_PARTICIPANT_TYPE,
sender_id=str(device_identity.child_id),
receiver_type=PARENT_PARTICIPANT_TYPE,
receiver_id=str(payload.parent_user_id),
sender_name_snapshot=sender_child_row["child_name"],
sender_avatar_snapshot=None,
receiver_name_snapshot=parent_row["nickname"],
receiver_avatar_snapshot=parent_row["avatar_url"],
payload=payload,
)
return await self._build_message_create_result(
dao=dao,
idempotent=idempotent,
conversation_id=conversation_id,
conversation_type=payload.conversation_type,
client_msg_id=payload.client_msg_id,
)
async def create_device_message(
self,
*,
device_id: str,
serial_number: str,
payload: DeviceMessageCreateRequest,
payload: DeviceMessageCreateRequest | None = None,
target_device_id: str | None = None,
audio_url: str | None = None,
) -> tuple[DeviceIdentity, ConversationMessageCreateResult]:
if payload is None and (not target_device_id or not audio_url):
raise ValueError("payload or target_device_id/audio_url is required")
if payload is not None and (target_device_id is not None or audio_url is not None):
raise ValueError("payload and target_device_id/audio_url cannot be used together")
db_session = await self.get_session()
try:
dao = ImDAO(db_session)
@@ -171,70 +290,31 @@ class ImService(DatabaseServiceBase):
serial_number=serial_number,
)
if payload.conversation_type == CHILD_PEER_CONVERSATION_TYPE:
if payload.peer_child_id == device_identity.child_id:
raise HTTPException(status_code=400, detail="peer_child_id must be different from current child")
sender_child_row = await dao.assert_child_exists(child_id=device_identity.child_id)
receiver_child_row = await dao.assert_child_exists(child_id=payload.peer_child_id)
participant_a_id, participant_b_id, pair_key = dao._build_child_peer_pair(
device_identity.child_id,
payload.peer_child_id,
)
conversation_id, idempotent = await dao.create_message(
resolved_payload = payload
if resolved_payload is None:
target_device_identity = await dao.get_device_by_id(device_id=target_device_id)
resolved_payload = DeviceMessageCreateRequest(
conversation_type=CHILD_PEER_CONVERSATION_TYPE,
participant_a_type=CHILD_PARTICIPANT_TYPE,
participant_a_id=participant_a_id,
participant_b_type=CHILD_PARTICIPANT_TYPE,
participant_b_id=participant_b_id,
pair_key=pair_key,
sender_type=CHILD_PARTICIPANT_TYPE,
sender_id=str(device_identity.child_id),
receiver_type=CHILD_PARTICIPANT_TYPE,
receiver_id=str(payload.peer_child_id),
sender_name_snapshot=sender_child_row["child_name"],
sender_avatar_snapshot=None,
receiver_name_snapshot=receiver_child_row["child_name"],
receiver_avatar_snapshot=None,
payload=payload,
)
else:
sender_child_row = await dao.assert_child_exists(child_id=device_identity.child_id)
parent_row = await dao._get_parent_row(payload.parent_user_id)
if not parent_row:
raise HTTPException(status_code=404, detail="parent not found")
await dao.assert_parent_child_access(user_id=payload.parent_user_id, child_id=device_identity.child_id)
conversation_id, idempotent = await dao.create_message(
conversation_type=PARENT_CHILD_CONVERSATION_TYPE,
participant_a_type=CHILD_PARTICIPANT_TYPE,
participant_a_id=str(device_identity.child_id),
participant_b_type=PARENT_PARTICIPANT_TYPE,
participant_b_id=str(payload.parent_user_id),
pair_key=f"{device_identity.child_id}:{payload.parent_user_id}",
sender_type=CHILD_PARTICIPANT_TYPE,
sender_id=str(device_identity.child_id),
receiver_type=PARENT_PARTICIPANT_TYPE,
receiver_id=str(payload.parent_user_id),
sender_name_snapshot=sender_child_row["child_name"],
sender_avatar_snapshot=None,
receiver_name_snapshot=parent_row["nickname"],
receiver_avatar_snapshot=parent_row["avatar_url"],
payload=payload,
peer_child_id=target_device_identity.child_id,
content_type=2,
media_file_key=audio_url,
media_mime_type="audio/mpeg",
client_msg_id=build_device_audio_client_msg_id(
device_id=device_id,
target_device_id=target_device_id,
audio_url=audio_url,
),
ext_json={
"source": "device_audio_message",
"source_device_id": device_id,
"target_device_id": target_device_id,
},
)
message_row = await dao._get_message_by_conversation_client_id(
conversation_id=conversation_id,
client_msg_id=payload.client_msg_id,
)
if not message_row:
raise RuntimeError("message was not found after insert")
result = ConversationMessageCreateResult(
idempotent=idempotent,
conversation_id=conversation_id,
conversation_type=payload.conversation_type,
message=row_to_message_item(message_row),
result = await self._create_device_message_with_payload(
dao=dao,
device_identity=device_identity,
payload=resolved_payload,
)
return device_identity, result
except Exception:
@@ -251,55 +331,5 @@ class ImService(DatabaseServiceBase):
finally:
await db_session.close()
'''
Todo 创建设备消息, 还不完善
'''
async def create_device_message(
self,
*,
device_id: str,
serial_number: str,
target_device_id: str,
audio_url: str,
):
db_session = await self.get_session()
try:
dao = ImDAO(db_session)
device_identity = await dao.authenticate_device_identity(
device_id=device_id,
serial_number=serial_number,
)
target_device_identity = await dao.get_device_by_id(device_id=target_device_id)
sender_child_row = await dao.assert_child_exists(child_id=device_identity.child_id)
receiver_child_row = await dao.assert_child_exists(child_id=target_device_identity.child_id)
conversation_id, idempotent = await dao.create_message(
conversation_type=PARENT_CHILD_CONVERSATION_TYPE,
participant_a_type=CHILD_PARTICIPANT_TYPE,
participant_a_id=str(device_identity.child_id),
participant_b_type=PARENT_PARTICIPANT_TYPE,
participant_b_id=str(receiver_child_row.child_id),
pair_key=f"{device_identity.child_id}:{target_device_identity.child_id}",
sender_type=CHILD_PARTICIPANT_TYPE,
sender_id=str(device_identity.child_id),
receiver_type=PARENT_PARTICIPANT_TYPE,
receiver_id=str(receiver_child_row.child_id),
sender_name_snapshot=sender_child_row["child_name"],
sender_avatar_snapshot=None,
receiver_name_snapshot=receiver_child_row["child_name"],
receiver_avatar_snapshot=None,
payload=None,
)
return device_identity
except Exception:
await db_session.rollback()
raise
finally:
await db_session.close()
# 创建全局 ImService 实例
im_service = ImService()

129
talkingq-url/banban/service/message_audio_storage.py Normal file
View File

@@ -0,0 +1,129 @@
import asyncio
from dataclasses import dataclass
from datetime import UTC, datetime
from urllib.parse import urlparse
from uuid import uuid4
try:
from qcloud_cos import CosConfig, CosS3Client
except ModuleNotFoundError: # pragma: no cover - exercised in runtime env
CosConfig = None
CosS3Client = None
from config import settings
class MessageAudioStorageError(Exception):
pass
@dataclass(frozen=True)
class StoredMessageAudio:
file_key: str
public_url: str
class MessageAudioStorageService:
def __init__(self) -> None:
self._client = None
def _assert_ready(self) -> None:
if CosConfig is None or CosS3Client is None:
raise MessageAudioStorageError("COS SDK is not installed")
required_pairs = {
"COS_SECRET_ID": settings.cos_secret_id,
"COS_SECRET_KEY": settings.cos_secret_key,
"COS_REGION": settings.cos_region,
"COS_BUCKET_MESSAGE": settings.cos_bucket_message,
}
missing = [key for key, value in required_pairs.items() if not value]
if missing:
raise MessageAudioStorageError(f"missing COS message config: {', '.join(missing)}")
def _get_client(self):
if self._client is None:
config = CosConfig(
Region=settings.cos_region,
SecretId=settings.cos_secret_id,
SecretKey=settings.cos_secret_key,
Scheme="https",
)
self._client = CosS3Client(config)
return self._client
def _build_key(self, *, device_id: str, extension: str) -> str:
prefix = settings.cos_message_prefix.strip("/") or "messages/audio"
now = datetime.now(UTC)
return (
f"{prefix}/{device_id}/{now.strftime('%Y/%m/%d')}/"
f"{uuid4().hex}.{extension}"
)
def _build_public_url(self, *, file_key: str) -> str:
base_url = settings.cos_public_base_url.strip().rstrip("/")
if not base_url:
base_url = f"https://{settings.cos_bucket_message}.cos.{settings.cos_region}.myqcloud.com"
return f"{base_url}/{file_key.lstrip('/')}"
def _normalize_file_key(self, file_key_or_url: str) -> str:
value = (file_key_or_url or "").strip()
if not value:
raise MessageAudioStorageError("audio file key is required")
if value.startswith("http://") or value.startswith("https://"):
parsed = urlparse(value)
path = parsed.path.lstrip("/")
if not path:
raise MessageAudioStorageError("audio file key is invalid")
return path
return value.lstrip("/")
async def upload_audio(
self,
*,
device_id: str,
content: bytes,
content_type: str = "audio/mpeg",
extension: str = "mp3",
) -> StoredMessageAudio:
self._assert_ready()
if not content:
raise MessageAudioStorageError("audio content is empty")
file_key = self._build_key(device_id=device_id, extension=extension)
await asyncio.to_thread(
self._upload_audio_sync,
file_key=file_key,
content=content,
content_type=content_type,
)
return StoredMessageAudio(
file_key=file_key,
public_url=self._build_public_url(file_key=file_key),
)
async def get_audio_url(self, file_key_or_url: str) -> str:
self._assert_ready()
normalized_key = self._normalize_file_key(file_key_or_url)
return await asyncio.to_thread(
self._get_client().get_presigned_url,
Bucket=settings.cos_bucket_message,
Key=normalized_key,
Method="GET",
Expired=settings.cos_avatar_url_expire_seconds,
)
def _upload_audio_sync(
self,
*,
file_key: str,
content: bytes,
content_type: str,
) -> None:
self._get_client().put_object(
Bucket=settings.cos_bucket_message,
Body=content,
Key=file_key,
ContentType=content_type,
EnableMD5=False,
)

1
talkingq-url/config.py
View File

@@ -89,6 +89,7 @@ class Settings(BaseSettings):
cos_bucket_message: str = Field(default="", validation_alias="COS_BUCKET_MESSAGE")
cos_bucket_ava: str = Field(default="", validation_alias="COS_BUCKET_AVA")
cos_public_base_url: str = Field(default="", validation_alias="COS_PUBLIC_BASE_URL")
cos_message_prefix: str = Field(default="messages/audio/", validation_alias="COS_MESSAGE_PREFIX")
cos_avatar_prefix: str = Field(default="avatars/", validation_alias="COS_AVATAR_PREFIX")
cos_avatar_url_expire_seconds: int = Field(
default=86400,

43
talkingq-url/handlers/audio_file_handler.py
View File

@@ -1,36 +1,19 @@
import os
import uuid
from config import settings
from banban.service.message_audio_storage import MessageAudioStorageService
from utils.logger import session_logger
# from utils.audio_denoiser import reduce_background_noise
message_audio_storage_service = MessageAudioStorageService()
async def save_audio_file(audio_data: bytes, device_id: str) -> str:
"""
保存音频数据到 assets/audio 目录
Args:
audio_data: 音频二进制数据
device_id: 设备ID
Returns:
音频文件的相对路径
"""
"""Upload device audio to COS and return its object key."""
try:
audio_dir = os.path.join(settings.assets_dir, "audio")
os.makedirs(audio_dir, exist_ok=True)
filename = f"{device_id}_{uuid.uuid4().hex[:8]}.mp3"
filepath = os.path.join(audio_dir, filename)
with open(filepath, 'wb') as f:
f.write(audio_data)
# relative_path = f"assets/audio/{filename}"
session_logger.info(device_id, "audio", f"音频文件已保存: {filepath}")
# reduce_background_noise(filepath, relative_path,noise_path='assets/audio/noise_sample.wav',normalize_volume=True)
return filepath
stored = await message_audio_storage_service.upload_audio(
device_id=device_id,
content=audio_data,
)
session_logger.info(device_id, "audio", f"audio uploaded to COS: {stored.file_key}")
return stored.file_key
except Exception as e:
session_logger.error(device_id, "audio", f"保存音频文件时出错: {e}", exc_info=True)
raise
session_logger.error(device_id, "audio", f"failed to store audio: {e}", exc_info=True)
raise

13
talkingq-url/handlers/websocket_message_handler.py
View File

@@ -4,7 +4,7 @@ import asyncio
from fastapi import WebSocket
from handlers.audio_packet_parser import parse_packet
from handlers.audio_session_handler import handle_websocket_data
from handlers.audio_file_handler import save_audio_file
from handlers.audio_file_handler import message_audio_storage_service, save_audio_file
from services.audio_session import audio_session_manager
from services.interrupt_handler import interrupt_handler
from services.task_manager import task_manager
@@ -265,11 +265,14 @@ async def process_cached_audio(device_id: str, target_device_id: str, serial_num
return
# 保存音频文件
audio_path = await save_audio_file(cached_audio, device_id)
audio_url = f"http://{settings.server_host}:{settings.server_port}/{audio_path}"
audio_file_key = await save_audio_file(cached_audio, device_id)
# 将音频URL保存到数据库 im_conversation和im_message
await im_conversation_service.create_device_message(device_id=device_id, serial_number=serial_number, target_device_id=target_device_id, audio_url=audio_url)
await im_conversation_service.create_device_message(device_id=device_id, serial_number=serial_number, target_device_id=target_device_id, audio_url=audio_file_key)
try:
audio_url = await message_audio_storage_service.get_audio_url(audio_file_key)
except Exception:
audio_url = audio_file_key
# 发送URL给目标设备
# target_websocket = await connection_manager.get_connection(target_device_id)
# if target_websocket and target_websocket.client_state.name == "CONNECTED":