add banbanmini backend

talkingq-url/utils/text_splitter.py

@@ -0,0 +1,145 @@
+import re
+
+SENTENCE_ENDINGS = set("。！？!?；;：:.。\n")
+PUNCTUATION_MARKS = set('。！？!?.,，；;：:~…—－_\'"″""》《）》《<>()[]{}【】\'、~～¡¿äöüßáéíóúüñç\n')
+APOSTROPHE_SUFFIXES = {'s', 't', 've', 'll', 're', 'd', 'm'}
+LANGUAGE_CONTRACTIONS = {
+    'fr': ["l'", "d'", "s'", "qu'", "c'", "j'", "n'", "m'", "t'", "jusqu'"],
+    'de': ["geht's", "gibt's"],
+    'es': ["el-", "del", "al"],
+    'ms': ["-lah", "-kah", "-nya"]
+}
+
+def is_numbered_list_item(text):
+    pattern = r"^\s*\d+\.\s"
+    return bool(re.search(pattern, text))
+
+def split_into_sentences(text):
+    return _custom_split_sentences(text)
+
+def _custom_split_sentences(text):
+    text = re.sub(r"([。！？!?;；])", r"\1\n", text)
+    
+    sentences = []
+    current_sentence = ""
+    i = 0
+    
+    while i < len(text):
+        current_sentence += text[i]
+        
+        if text[i] in SENTENCE_ENDINGS and not is_special_context(text, i):
+            if current_sentence.strip():
+                sentences.append(current_sentence)
+            current_sentence = ""
+        
+        i += 1
+        
+    if current_sentence.strip():
+        sentences.append(current_sentence)
+    
+    merged_sentences = []
+    temp = ""
+    
+    for s in sentences:
+        s_stripped = s.strip()
+        
+        if len(s_stripped) < 5 and not any(p in s for p in "。！？!?"):
+            temp += s
+        else:
+            if temp:
+                merged_sentences.append(temp + s)
+                temp = ""
+            else:
+                merged_sentences.append(s)
+    
+    if temp:
+        merged_sentences.append(temp)
+    
+    final_sentences = []
+    for idx, sentence in enumerate(merged_sentences):
+        sentence_start = sentence.strip()
+        if idx > 0:
+            prev_sentence_end = merged_sentences[idx-1].strip()
+            
+            if (sentence_start.startswith("s ") and prev_sentence_end.endswith("'")) or \
+               any(sentence_start.startswith(suffix + " ") for suffix in APOSTROPHE_SUFFIXES) and prev_sentence_end.endswith("'"):
+                final_sentences[-1] += sentence
+                continue
+                
+            if any(sentence_start.startswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['fr']):
+                final_sentences[-1] += sentence
+                continue
+                
+            if (any(prev_sentence_end.endswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['de']) or
+                any(sentence_start.startswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['es']) or
+                any(prev_sentence_end.endswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['ms'])):
+                final_sentences[-1] += sentence
+                continue
+        
+        if sentence_start.startswith("¿") or sentence_start.startswith("¡"):
+            final_sentences.append(sentence)
+            continue
+            
+        final_sentences.append(sentence)
+    
+    return final_sentences
+
+def is_special_context(text, pos):
+    """判断是否是特殊上下文，不应该在此处断句"""
+    if text[pos] == "." and pos > 0 and pos < len(text) - 1:
+        if text[pos - 1].isdigit() and text[pos + 1].isdigit():
+            return True
+    
+    if text[pos] == "'" and pos > 0 and pos < len(text) - 1:
+        if text[pos - 1].isalpha():  # 前面是字母
+            for suffix in APOSTROPHE_SUFFIXES:
+                if pos + len(suffix) <= len(text) and text[pos+1:pos+1+len(suffix)] == suffix:
+                    if pos + len(suffix) + 1 >= len(text) or not text[pos+1+len(suffix)].isalpha():
+                        return True
+    
+    if text[pos] == "." and pos > 0 and pos < len(text) - 2:
+        if text[pos - 1].isupper() and text[pos + 1].isupper():
+            return True
+    
+    if text[pos] == "." and pos > 0 and pos < len(text) - 1:
+        if not text[pos - 1].isspace() and not text[pos + 1].isspace():
+            left_context = text[max(0, pos-15):pos]
+            if "www." in left_context or "http" in left_context or "@" in left_context:
+                return True
+    
+    if text[pos] == "." and pos > 0 and pos < len(text) - 1:
+        if pos + 2 < len(text) and text[pos:pos+3] == "...":
+            return True
+            
+    if text[pos] == "'" and pos > 0 and pos < len(text) - 1:
+        if pos > 1 and text[pos-2:pos+1] in ["l'", "d'", "c'", "j'", "n'", "m'", "t'"]:
+            return True
+            
+    if text[pos] == "." and pos > 0:
+        if pos >= 3 and text[pos-3:pos] in ["bzw", "usw", "etc"]:
+            return True
+            
+    if text[pos] in "¿¡" and pos < len(text) - 1:
+        return True
+        
+    if text[pos] == "-" and pos > 0 and pos < len(text) - 2:
+        if text[pos+1:pos+4] in ["lah", "kah", "nya"]:
+            return True
+    
+    return False
+
+def should_skip_tts(text_chunk):
+    """判断一段文本是否应该跳过TTS处理"""
+    if not text_chunk.strip():
+        return True
+    
+    if is_numbered_list_item(text_chunk) and len(text_chunk.strip()) <= 5:
+        return True
+    
+    if all(c in PUNCTUATION_MARKS for c in text_chunk.strip()):
+        return True
+    
+    if len(text_chunk.strip()) <= 2 and not any(c.isalnum() for c in text_chunk):
+        return True
+    
+    return False