add banbanmini backend
This commit is contained in:
HycJack
0
talkingq-url/utils/__init__.py
Normal file
0
talkingq-url/utils/__init__.py
Normal file
210
talkingq-url/utils/language_detector.py
Normal file
210
talkingq-url/utils/language_detector.py
Normal file
@@ -0,0 +1,210 @@
|
||||
import pycld2 as cld2
|
||||
from utils.logger import session_logger
|
||||
import re
|
||||
import unicodedata
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
|
||||
class LanguageDetector:
|
||||
LANGUAGE_UNICODE_RANGES = {
|
||||
'zh': [ # 中文
|
||||
(0x4E00, 0x9FFF), # CJK统一汉字
|
||||
(0x3400, 0x4DBF), # CJK扩展A
|
||||
(0xF900, 0xFAFF), # CJK兼容汉字
|
||||
]
|
||||
}
|
||||
|
||||
LANGUAGE_FEATURES = {
|
||||
'zh': ['的', '了', '是', '我', '你', '他', '们', '在', '有', '和', '这', '那', '什么', '怎么', '为什么',
|
||||
'好的', '嗯', '啊', '哦', '吧', '呢', '吗', '哈', '嘿', '喂', '您好', '谢谢', '对不起', '没关系',
|
||||
'好啊', '行', '可以', '不行', '不可以', '是的', '不是', '要', '不要', '能', '能不能', '请问',
|
||||
'请', '帮我', '告诉我', '知道', '不知道', '明白', '不明白', '对', '不对', '听见', '听不见'],
|
||||
|
||||
'en': ['i', 'you', 'he', 'she', 'we', 'they', 'this', 'that', 'what', 'how', 'why', 'where', 'when',
|
||||
'okay', 'yes', 'no', 'yeah', 'oh', 'hey', 'hi', 'hello', 'thanks', 'thank', 'sorry', 'please',
|
||||
'can', 'could', 'would', 'will', 'want', 'need', 'like', 'tell', 'know', 'think', 'mean',
|
||||
'say', 'said', 'help', 'hear', 'listen', 'understand', 'right', 'wrong', 'good', 'bad',
|
||||
'ok', 'alright', 'excuse', 'pardon', 'well', 'sure', 'maybe', 'perhaps'],
|
||||
|
||||
'de': ['ich', 'du', 'er', 'sie', 'es', 'wir', 'ihr', 'sie', 'das', 'was', 'wie', 'warum', 'wo', 'wann',
|
||||
'ja', 'nein', 'okay', 'hallo', 'danke', 'bitte', 'entschuldigung', 'tschüss', 'guten', 'tag',
|
||||
'morgen', 'abend', 'kann', 'könnte', 'würde', 'möchte', 'wollen', 'brauchen', 'verstehe',
|
||||
'weiß', 'denke', 'meine', 'helfen', 'hören', 'zuhören', 'richtig', 'falsch', 'gut', 'schlecht',
|
||||
'alles', 'klar', 'natürlich', 'vielleicht'],
|
||||
|
||||
'fr': ['je', 'tu', 'il', 'elle', 'nous', 'vous', 'ils', 'elles', 'ce', 'ça', 'quoi', 'comment', 'pourquoi',
|
||||
'où', 'quand', 'oui', 'non', 'salut', 'bonjour', 'merci', 'pardon', 'excusez', 'au revoir',
|
||||
'peux', 'pouvez', 'voulez', 'veux', 'besoin', 'comprends', 'sais', 'pense', 'dire', 'aider',
|
||||
'entendre', 'écouter', 'correct', 'incorrect', 'bon', 'bien', 'mauvais', 'mal',
|
||||
'd\'accord', 'bien sûr', 'peut-être'],
|
||||
|
||||
'es': ['yo', 'tú', 'él', 'ella', 'nosotros', 'vosotros', 'ellos', 'ellas', 'esto', 'eso', 'qué', 'cómo',
|
||||
'por qué', 'dónde', 'cuándo', 'sí', 'no', 'hola', 'gracias', 'perdón', 'disculpe', 'adiós',
|
||||
'puede', 'puedes', 'quiere', 'quiero', 'necesito', 'entiendo', 'sé', 'pienso', 'digo', 'ayudar',
|
||||
'oír', 'escuchar', 'correcto', 'incorrecto', 'bueno', 'malo', 'vale', 'claro', 'quizás', 'tal vez']
|
||||
}
|
||||
|
||||
LANGUAGE_PUNCTUATION = {
|
||||
'zh': ['。', '，', '、', '：', '；', '！', '？', '"', '"', '（', '）', '《', '》', '【', '】', '——', '……', '～'],
|
||||
'en': [], # 英语标点大多通用，不作为特征
|
||||
'de': ['„', '"'], # 德语特有引号
|
||||
'fr': ['«', '»', '…'], # 法语特有引号和省略号
|
||||
'es': ['¿', '¡'] # 西班牙语特有的倒置问号和感叹号
|
||||
}
|
||||
|
||||
SPEECH_FILLERS = {
|
||||
'zh': ['嗯', '呃', '啊', '哦', '唉', '那个', '这个', '就是', '其实', '然后', '所以', '但是'],
|
||||
'en': ['um', 'uh', 'er', 'ah', 'like', 'you know', 'i mean', 'well', 'so', 'anyway', 'actually'],
|
||||
'de': ['äh', 'ähm', 'hmm', 'na ja', 'also', 'sozusagen', 'quasi', 'naja', 'eigentlich', 'tja'],
|
||||
'fr': ['euh', 'ben', 'bah', 'eh bien', 'bon', 'alors', 'donc', 'voilà', 'en fait', 'quoi'],
|
||||
'es': ['eh', 'este', 'em', 'pues', 'bueno', 'o sea', 'vale', 'vamos', 'mira', 'entonces']
|
||||
}
|
||||
|
||||
SPEECH_PATTERNS = {
|
||||
'zh': [r'^(你好|您好|喂|在吗|请问|那个|这个|帮我|麻烦|请|要|需要)'],
|
||||
'en': [r'^(hello|hi|hey|excuse me|sorry|please|can you|could you|would you|I want|I need)'],
|
||||
'de': [r'^(hallo|guten tag|entschuldigung|bitte|können sie|könntest du|ich möchte|ich brauche)'],
|
||||
'fr': [r'^(bonjour|salut|excusez-moi|pardon|s\'il vous plaît|pouvez-vous|pourriez-vous|je voudrais|j\'ai besoin)'],
|
||||
'es': [r'^(hola|buenos días|perdón|disculpe|por favor|puede usted|puedes|quiero|necesito)']
|
||||
}
|
||||
|
||||
_executor = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=4,
|
||||
thread_name_prefix="lang_detector"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def detect_language(text):
|
||||
try:
|
||||
SUPPORTED_LANGUAGES = ["zh", "en", "fr", "de", "es"]
|
||||
if not text:
|
||||
return "en" # 默认返回英文
|
||||
cleaned_text = ' '.join(text.lower().split())
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
speech_pattern_lang = await loop.run_in_executor(
|
||||
LanguageDetector._executor,
|
||||
LanguageDetector._detect_by_speech_patterns,
|
||||
cleaned_text
|
||||
)
|
||||
|
||||
if speech_pattern_lang != "unknown":
|
||||
return speech_pattern_lang
|
||||
|
||||
char_based_lang = await loop.run_in_executor(
|
||||
LanguageDetector._executor,
|
||||
LanguageDetector._detect_by_characters,
|
||||
cleaned_text
|
||||
)
|
||||
|
||||
if char_based_lang != "unknown":
|
||||
return char_based_lang
|
||||
|
||||
feature_based_lang = await loop.run_in_executor(
|
||||
LanguageDetector._executor,
|
||||
LanguageDetector._detect_by_features,
|
||||
cleaned_text
|
||||
)
|
||||
|
||||
if feature_based_lang != "unknown":
|
||||
return feature_based_lang
|
||||
|
||||
try:
|
||||
is_reliable, _, details = cld2.detect(cleaned_text)
|
||||
if is_reliable and details[0][1] in SUPPORTED_LANGUAGES:
|
||||
return details[0][1]
|
||||
except:
|
||||
pass
|
||||
|
||||
for ch in cleaned_text:
|
||||
if '\u4e00' <= ch <= '\u9fff': # 检测中文字符
|
||||
return "zh"
|
||||
|
||||
return "en" # 默认返回英语
|
||||
except Exception as e:
|
||||
session_logger.error("system", "language_detector", f"语言检测错误: {str(e)}")
|
||||
return "en" # 出错时默认使用英文
|
||||
|
||||
@staticmethod
|
||||
def _detect_by_speech_patterns(text: str) -> str:
|
||||
"""检测常见的语音对话开头模式"""
|
||||
for lang, patterns in LanguageDetector.SPEECH_PATTERNS.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, text):
|
||||
return lang
|
||||
return "unknown"
|
||||
|
||||
@staticmethod
|
||||
def _detect_by_characters(text: str) -> str:
|
||||
"""基于字符分布的语言检测，特别适合短文本"""
|
||||
text = text.strip()
|
||||
text = text.lower()
|
||||
text_len = max(1, len(text))
|
||||
char_counts = {lang: 0 for lang in LanguageDetector.LANGUAGE_UNICODE_RANGES}
|
||||
|
||||
for char in text:
|
||||
code_point = ord(char)
|
||||
for lang, ranges in LanguageDetector.LANGUAGE_UNICODE_RANGES.items():
|
||||
for start, end in ranges:
|
||||
if start <= code_point <= end:
|
||||
char_counts[lang] += 1
|
||||
|
||||
for lang, count in char_counts.items():
|
||||
ratio = count / text_len
|
||||
if lang == 'zh' and ratio > 0.12:
|
||||
return 'zh'
|
||||
|
||||
latin_chars = sum(1 for c in text if unicodedata.category(c).startswith('L'))
|
||||
latin_ratio = latin_chars / text_len
|
||||
|
||||
if latin_ratio > 0.4: # 降低拉丁字符检测阈值，更灵敏地捕捉语音转文本
|
||||
if any(c in text for c in "äöüß"):
|
||||
return "de" # 德语特有字符
|
||||
elif any(c in text for c in "éèêëàâçùûüÿôœæ"):
|
||||
return "fr" # 法语特有字符
|
||||
elif any(c in text for c in "áéíóúñ¿¡"):
|
||||
return "es" # 西班牙语特有字符
|
||||
|
||||
if "¿" in text or "¡" in text:
|
||||
return "es" # 西班牙语特有标点
|
||||
elif "«" in text or "»" in text:
|
||||
return "fr" # 法语引号
|
||||
|
||||
return "en" # 默认英语
|
||||
|
||||
return "unknown"
|
||||
|
||||
@staticmethod
|
||||
def _detect_by_features(text: str) -> str:
|
||||
"""基于特征词和语气词的语言检测，针对口语场景优化"""
|
||||
text_lower = text.lower()
|
||||
matches = {lang: 0 for lang in LanguageDetector.LANGUAGE_FEATURES}
|
||||
|
||||
for lang, features in LanguageDetector.LANGUAGE_FEATURES.items():
|
||||
for word in features:
|
||||
if lang == 'zh': # 中文不需要空格分隔
|
||||
if word in text_lower:
|
||||
matches[lang] += 2
|
||||
elif f" {word} " in f" {text_lower} ":
|
||||
matches[lang] += 1
|
||||
|
||||
for lang, fillers in LanguageDetector.SPEECH_FILLERS.items():
|
||||
for filler in fillers:
|
||||
if lang == 'zh':
|
||||
count = text_lower.count(filler)
|
||||
else:
|
||||
count = text_lower.count(f" {filler} ")
|
||||
|
||||
matches[lang] += count * 3
|
||||
|
||||
for lang, puncts in LanguageDetector.LANGUAGE_PUNCTUATION.items():
|
||||
for punct in puncts:
|
||||
matches[lang] += text.count(punct) * 5
|
||||
|
||||
nonzero_matches = {lang: score for lang, score in matches.items() if score > 0}
|
||||
if nonzero_matches:
|
||||
best_lang = max(nonzero_matches.items(), key=lambda x: x[1])
|
||||
if best_lang[1] > 0:
|
||||
return best_lang[0]
|
||||
|
||||
return "unknown"
|
||||
92
talkingq-url/utils/logger.py
Normal file
92
talkingq-url/utils/logger.py
Normal file
@@ -0,0 +1,92 @@
|
||||
import os
|
||||
from typing import Optional
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class SessionLogger:
|
||||
def __init__(self):
|
||||
self.is_main_process = os.environ.get("UVICORN_WID", "0") == "0"
|
||||
|
||||
# 创建日志目录
|
||||
os.makedirs("logs", exist_ok=True)
|
||||
|
||||
# 配置loguru
|
||||
logger.remove() # 移除默认handler
|
||||
|
||||
# 控制台输出
|
||||
logger.add(
|
||||
lambda msg: print(msg, end=""),
|
||||
format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> - <level>{level}</level> - [设备: {extra[device_id]}] [会话: {extra[session_id]}] {message}",
|
||||
level="INFO"
|
||||
)
|
||||
|
||||
# 文件输出 - 持久化保存
|
||||
logger.add(
|
||||
"logs/talkingq_{time:YYYY-MM-DD}.log",
|
||||
format="{time:YYYY-MM-DD HH:mm:ss} - {level} - [设备: {extra[device_id]}] [会话: {extra[session_id]}] {message}",
|
||||
rotation="00:00", # 每天轮转
|
||||
retention="30 days", # 保留30天
|
||||
compression="zip", # 压缩旧日志
|
||||
level="INFO"
|
||||
)
|
||||
|
||||
self.logger = logger
|
||||
|
||||
def log(
|
||||
self,
|
||||
level: str,
|
||||
device_id: Optional[str],
|
||||
session_id: Optional[str],
|
||||
message: str,
|
||||
**kwargs
|
||||
):
|
||||
if isinstance(device_id, str) and device_id.startswith("{"):
|
||||
try:
|
||||
import json
|
||||
device_info = json.loads(device_id)
|
||||
device_id = device_info.get("device_id", device_id)
|
||||
except:
|
||||
pass # 如果解析失败，保持原样
|
||||
|
||||
# 绑定额外信息到logger
|
||||
bound_logger = self.logger.bind(
|
||||
device_id=device_id or "unknown",
|
||||
session_id=session_id or "unknown"
|
||||
)
|
||||
|
||||
# 根据level调用对应方法
|
||||
if level.upper() == "INFO":
|
||||
bound_logger.info(message)
|
||||
elif level.upper() == "ERROR":
|
||||
bound_logger.error(message)
|
||||
elif level.upper() == "WARNING":
|
||||
bound_logger.warning(message)
|
||||
else:
|
||||
bound_logger.log(level, message)
|
||||
|
||||
def system_log(self, level: str, session_id: str, message: str):
|
||||
if self.is_main_process:
|
||||
self.log(level, "system", session_id, message)
|
||||
|
||||
def info(self, device_id: Optional[str], session_id: Optional[str], message: str):
|
||||
self.log("INFO", device_id, session_id, message)
|
||||
|
||||
def system_info(self, session_id: str, message: str):
|
||||
self.system_log("INFO", session_id, message)
|
||||
|
||||
def error(
|
||||
self,
|
||||
device_id: Optional[str],
|
||||
session_id: Optional[str],
|
||||
message: str,
|
||||
**kwargs
|
||||
):
|
||||
self.log("ERROR", device_id, session_id, message, **kwargs)
|
||||
|
||||
def warning(
|
||||
self, device_id: Optional[str], session_id: Optional[str], message: str
|
||||
):
|
||||
self.log("WARNING", device_id, session_id, message)
|
||||
|
||||
|
||||
session_logger = SessionLogger()
|
||||
145
talkingq-url/utils/text_splitter.py
Normal file
145
talkingq-url/utils/text_splitter.py
Normal file
@@ -0,0 +1,145 @@
|
||||
import re
|
||||
|
||||
SENTENCE_ENDINGS = set("。！？!?；;：:.。\n")
|
||||
PUNCTUATION_MARKS = set('。！？!?.,，；;：:~…—－_\'"″""》《）》《<>()[]{}【】\'、~～¡¿äöüßáéíóúüñç\n')
|
||||
APOSTROPHE_SUFFIXES = {'s', 't', 've', 'll', 're', 'd', 'm'}
|
||||
LANGUAGE_CONTRACTIONS = {
|
||||
'fr': ["l'", "d'", "s'", "qu'", "c'", "j'", "n'", "m'", "t'", "jusqu'"],
|
||||
'de': ["geht's", "gibt's"],
|
||||
'es': ["el-", "del", "al"],
|
||||
'ms': ["-lah", "-kah", "-nya"]
|
||||
}
|
||||
|
||||
def is_numbered_list_item(text):
|
||||
pattern = r"^\s*\d+\.\s"
|
||||
return bool(re.search(pattern, text))
|
||||
|
||||
def split_into_sentences(text):
|
||||
return _custom_split_sentences(text)
|
||||
|
||||
def _custom_split_sentences(text):
|
||||
text = re.sub(r"([。！？!?;；])", r"\1\n", text)
|
||||
|
||||
sentences = []
|
||||
current_sentence = ""
|
||||
i = 0
|
||||
|
||||
while i < len(text):
|
||||
current_sentence += text[i]
|
||||
|
||||
if text[i] in SENTENCE_ENDINGS and not is_special_context(text, i):
|
||||
if current_sentence.strip():
|
||||
sentences.append(current_sentence)
|
||||
current_sentence = ""
|
||||
|
||||
i += 1
|
||||
|
||||
if current_sentence.strip():
|
||||
sentences.append(current_sentence)
|
||||
|
||||
merged_sentences = []
|
||||
temp = ""
|
||||
|
||||
for s in sentences:
|
||||
s_stripped = s.strip()
|
||||
|
||||
if len(s_stripped) < 5 and not any(p in s for p in "。！？!?"):
|
||||
temp += s
|
||||
else:
|
||||
if temp:
|
||||
merged_sentences.append(temp + s)
|
||||
temp = ""
|
||||
else:
|
||||
merged_sentences.append(s)
|
||||
|
||||
if temp:
|
||||
merged_sentences.append(temp)
|
||||
|
||||
final_sentences = []
|
||||
for idx, sentence in enumerate(merged_sentences):
|
||||
sentence_start = sentence.strip()
|
||||
if idx > 0:
|
||||
prev_sentence_end = merged_sentences[idx-1].strip()
|
||||
|
||||
if (sentence_start.startswith("s ") and prev_sentence_end.endswith("'")) or \
|
||||
any(sentence_start.startswith(suffix + " ") for suffix in APOSTROPHE_SUFFIXES) and prev_sentence_end.endswith("'"):
|
||||
final_sentences[-1] += sentence
|
||||
continue
|
||||
|
||||
if any(sentence_start.startswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['fr']):
|
||||
final_sentences[-1] += sentence
|
||||
continue
|
||||
|
||||
if (any(prev_sentence_end.endswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['de']) or
|
||||
any(sentence_start.startswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['es']) or
|
||||
any(prev_sentence_end.endswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['ms'])):
|
||||
final_sentences[-1] += sentence
|
||||
continue
|
||||
|
||||
if sentence_start.startswith("¿") or sentence_start.startswith("¡"):
|
||||
final_sentences.append(sentence)
|
||||
continue
|
||||
|
||||
final_sentences.append(sentence)
|
||||
|
||||
return final_sentences
|
||||
|
||||
def is_special_context(text, pos):
|
||||
"""判断是否是特殊上下文，不应该在此处断句"""
|
||||
if text[pos] == "." and pos > 0 and pos < len(text) - 1:
|
||||
if text[pos - 1].isdigit() and text[pos + 1].isdigit():
|
||||
return True
|
||||
|
||||
if text[pos] == "'" and pos > 0 and pos < len(text) - 1:
|
||||
if text[pos - 1].isalpha(): # 前面是字母
|
||||
for suffix in APOSTROPHE_SUFFIXES:
|
||||
if pos + len(suffix) <= len(text) and text[pos+1:pos+1+len(suffix)] == suffix:
|
||||
if pos + len(suffix) + 1 >= len(text) or not text[pos+1+len(suffix)].isalpha():
|
||||
return True
|
||||
|
||||
if text[pos] == "." and pos > 0 and pos < len(text) - 2:
|
||||
if text[pos - 1].isupper() and text[pos + 1].isupper():
|
||||
return True
|
||||
|
||||
if text[pos] == "." and pos > 0 and pos < len(text) - 1:
|
||||
if not text[pos - 1].isspace() and not text[pos + 1].isspace():
|
||||
left_context = text[max(0, pos-15):pos]
|
||||
if "www." in left_context or "http" in left_context or "@" in left_context:
|
||||
return True
|
||||
|
||||
if text[pos] == "." and pos > 0 and pos < len(text) - 1:
|
||||
if pos + 2 < len(text) and text[pos:pos+3] == "...":
|
||||
return True
|
||||
|
||||
if text[pos] == "'" and pos > 0 and pos < len(text) - 1:
|
||||
if pos > 1 and text[pos-2:pos+1] in ["l'", "d'", "c'", "j'", "n'", "m'", "t'"]:
|
||||
return True
|
||||
|
||||
if text[pos] == "." and pos > 0:
|
||||
if pos >= 3 and text[pos-3:pos] in ["bzw", "usw", "etc"]:
|
||||
return True
|
||||
|
||||
if text[pos] in "¿¡" and pos < len(text) - 1:
|
||||
return True
|
||||
|
||||
if text[pos] == "-" and pos > 0 and pos < len(text) - 2:
|
||||
if text[pos+1:pos+4] in ["lah", "kah", "nya"]:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def should_skip_tts(text_chunk):
|
||||
"""判断一段文本是否应该跳过TTS处理"""
|
||||
if not text_chunk.strip():
|
||||
return True
|
||||
|
||||
if is_numbered_list_item(text_chunk) and len(text_chunk.strip()) <= 5:
|
||||
return True
|
||||
|
||||
if all(c in PUNCTUATION_MARKS for c in text_chunk.strip()):
|
||||
return True
|
||||
|
||||
if len(text_chunk.strip()) <= 2 and not any(c.isalnum() for c in text_chunk):
|
||||
return True
|
||||
|
||||
return False
|
||||
Reference in New Issue
Block a user