146 lines
5.2 KiB
Python
146 lines
5.2 KiB
Python
import re
|
||
|
||
SENTENCE_ENDINGS = set("。！？!?；;：:.。\n")
|
||
PUNCTUATION_MARKS = set('。！？!?.,，；;：:~…—－_\'"″""》《）》《<>()[]{}【】\'、~～¡¿äöüßáéíóúüñç\n')
|
||
APOSTROPHE_SUFFIXES = {'s', 't', 've', 'll', 're', 'd', 'm'}
|
||
LANGUAGE_CONTRACTIONS = {
|
||
'fr': ["l'", "d'", "s'", "qu'", "c'", "j'", "n'", "m'", "t'", "jusqu'"],
|
||
'de': ["geht's", "gibt's"],
|
||
'es': ["el-", "del", "al"],
|
||
'ms': ["-lah", "-kah", "-nya"]
|
||
}
|
||
|
||
def is_numbered_list_item(text):
|
||
pattern = r"^\s*\d+\.\s"
|
||
return bool(re.search(pattern, text))
|
||
|
||
def split_into_sentences(text):
|
||
return _custom_split_sentences(text)
|
||
|
||
def _custom_split_sentences(text):
|
||
text = re.sub(r"([。！？!?;；])", r"\1\n", text)
|
||
|
||
sentences = []
|
||
current_sentence = ""
|
||
i = 0
|
||
|
||
while i < len(text):
|
||
current_sentence += text[i]
|
||
|
||
if text[i] in SENTENCE_ENDINGS and not is_special_context(text, i):
|
||
if current_sentence.strip():
|
||
sentences.append(current_sentence)
|
||
current_sentence = ""
|
||
|
||
i += 1
|
||
|
||
if current_sentence.strip():
|
||
sentences.append(current_sentence)
|
||
|
||
merged_sentences = []
|
||
temp = ""
|
||
|
||
for s in sentences:
|
||
s_stripped = s.strip()
|
||
|
||
if len(s_stripped) < 5 and not any(p in s for p in "。！？!?"):
|
||
temp += s
|
||
else:
|
||
if temp:
|
||
merged_sentences.append(temp + s)
|
||
temp = ""
|
||
else:
|
||
merged_sentences.append(s)
|
||
|
||
if temp:
|
||
merged_sentences.append(temp)
|
||
|
||
final_sentences = []
|
||
for idx, sentence in enumerate(merged_sentences):
|
||
sentence_start = sentence.strip()
|
||
if idx > 0:
|
||
prev_sentence_end = merged_sentences[idx-1].strip()
|
||
|
||
if (sentence_start.startswith("s ") and prev_sentence_end.endswith("'")) or \
|
||
any(sentence_start.startswith(suffix + " ") for suffix in APOSTROPHE_SUFFIXES) and prev_sentence_end.endswith("'"):
|
||
final_sentences[-1] += sentence
|
||
continue
|
||
|
||
if any(sentence_start.startswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['fr']):
|
||
final_sentences[-1] += sentence
|
||
continue
|
||
|
||
if (any(prev_sentence_end.endswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['de']) or
|
||
any(sentence_start.startswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['es']) or
|
||
any(prev_sentence_end.endswith(contraction) for contraction in LANGUAGE_CONTRACTIONS['ms'])):
|
||
final_sentences[-1] += sentence
|
||
continue
|
||
|
||
if sentence_start.startswith("¿") or sentence_start.startswith("¡"):
|
||
final_sentences.append(sentence)
|
||
continue
|
||
|
||
final_sentences.append(sentence)
|
||
|
||
return final_sentences
|
||
|
||
def is_special_context(text, pos):
|
||
"""判断是否是特殊上下文，不应该在此处断句"""
|
||
if text[pos] == "." and pos > 0 and pos < len(text) - 1:
|
||
if text[pos - 1].isdigit() and text[pos + 1].isdigit():
|
||
return True
|
||
|
||
if text[pos] == "'" and pos > 0 and pos < len(text) - 1:
|
||
if text[pos - 1].isalpha(): # 前面是字母
|
||
for suffix in APOSTROPHE_SUFFIXES:
|
||
if pos + len(suffix) <= len(text) and text[pos+1:pos+1+len(suffix)] == suffix:
|
||
if pos + len(suffix) + 1 >= len(text) or not text[pos+1+len(suffix)].isalpha():
|
||
return True
|
||
|
||
if text[pos] == "." and pos > 0 and pos < len(text) - 2:
|
||
if text[pos - 1].isupper() and text[pos + 1].isupper():
|
||
return True
|
||
|
||
if text[pos] == "." and pos > 0 and pos < len(text) - 1:
|
||
if not text[pos - 1].isspace() and not text[pos + 1].isspace():
|
||
left_context = text[max(0, pos-15):pos]
|
||
if "www." in left_context or "http" in left_context or "@" in left_context:
|
||
return True
|
||
|
||
if text[pos] == "." and pos > 0 and pos < len(text) - 1:
|
||
if pos + 2 < len(text) and text[pos:pos+3] == "...":
|
||
return True
|
||
|
||
if text[pos] == "'" and pos > 0 and pos < len(text) - 1:
|
||
if pos > 1 and text[pos-2:pos+1] in ["l'", "d'", "c'", "j'", "n'", "m'", "t'"]:
|
||
return True
|
||
|
||
if text[pos] == "." and pos > 0:
|
||
if pos >= 3 and text[pos-3:pos] in ["bzw", "usw", "etc"]:
|
||
return True
|
||
|
||
if text[pos] in "¿¡" and pos < len(text) - 1:
|
||
return True
|
||
|
||
if text[pos] == "-" and pos > 0 and pos < len(text) - 2:
|
||
if text[pos+1:pos+4] in ["lah", "kah", "nya"]:
|
||
return True
|
||
|
||
return False
|
||
|
||
def should_skip_tts(text_chunk):
|
||
"""判断一段文本是否应该跳过TTS处理"""
|
||
if not text_chunk.strip():
|
||
return True
|
||
|
||
if is_numbered_list_item(text_chunk) and len(text_chunk.strip()) <= 5:
|
||
return True
|
||
|
||
if all(c in PUNCTUATION_MARKS for c in text_chunk.strip()):
|
||
return True
|
||
|
||
if len(text_chunk.strip()) <= 2 and not any(c.isalnum() for c in text_chunk):
|
||
return True
|
||
|
||
return False
|