mirror of https://github.com/hpcaitech/ColossalAI
57 lines
2.5 KiB
Python
57 lines
2.5 KiB
Python
|
"""
|
|||
|
|
Code for Chinese text splitter
|
|||
|
|
"""
|
|||
|
|
from typing import Any, List, Optional
|
|||
|
|
|
|||
|
|
from colossalqa.text_splitter.utils import get_cleaned_paragraph
|
|||
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ChineseTextSplitter(RecursiveCharacterTextSplitter):
|
|||
|
|
def __init__(self, separators: Optional[List[str]] = None, is_separator_regrx: bool = False, **kwargs: Any):
|
|||
|
|
self._separators = separators or ["\n\n", "\n", "，", "。", "！", "？", "?"]
|
|||
|
|
if "chunk_size" not in kwargs:
|
|||
|
|
kwargs["chunk_size"] = 50
|
|||
|
|
if "chunk_overlap" not in kwargs:
|
|||
|
|
kwargs["chunk_overlap"] = 10
|
|||
|
|
super().__init__(separators=separators, keep_separator=True, **kwargs)
|
|||
|
|
self._is_separator_regex = is_separator_regrx
|
|||
|
|
|
|||
|
|
def split_text(self, text: str) -> List[str]:
|
|||
|
|
"""Return the list of separated text chunks"""
|
|||
|
|
cleaned_paragraph = get_cleaned_paragraph(text)
|
|||
|
|
splitted = []
|
|||
|
|
for paragraph in cleaned_paragraph:
|
|||
|
|
segs = super().split_text(paragraph)
|
|||
|
|
for i in range(len(segs) - 1):
|
|||
|
|
if segs[i][-1] not in self._separators:
|
|||
|
|
pos = text.find(segs[i])
|
|||
|
|
pos_end = pos + len(segs[i])
|
|||
|
|
if i > 0:
|
|||
|
|
last_sentence_start = max([text.rfind(m, 0, pos) for m in ["。", "！", "？"]])
|
|||
|
|
pos = last_sentence_start + 1
|
|||
|
|
segs[i] = str(text[pos:pos_end])
|
|||
|
|
if i != len(segs) - 1:
|
|||
|
|
next_sentence_end = max([text.find(m, pos_end) for m in ["。", "！", "？"]])
|
|||
|
|
segs[i] = str(text[pos : next_sentence_end + 1])
|
|||
|
|
splitted.append(segs[i])
|
|||
|
|
if len(splitted) <= 1:
|
|||
|
|
return splitted
|
|||
|
|
splitted_text = []
|
|||
|
|
i = 1
|
|||
|
|
if splitted[0] not in splitted[1]:
|
|||
|
|
splitted_text.append([splitted[0], 0])
|
|||
|
|
if splitted[-1] not in splitted[-2]:
|
|||
|
|
splitted_text.append([splitted[-1], len(splitted) - 1])
|
|||
|
|
while i < len(splitted) - 1:
|
|||
|
|
if splitted[i] not in splitted[i + 1] and splitted[i] not in splitted[i - 1]:
|
|||
|
|
splitted_text.append([splitted[i], i])
|
|||
|
|
i += 1
|
|||
|
|
splitted_text = sorted(splitted_text, key=lambda x: x[1])
|
|||
|
|
splitted_text = [splitted_text[i][0] for i in range(len(splitted_text))]
|
|||
|
|
ret = []
|
|||
|
|
for s in splitted_text:
|
|||
|
|
if s not in ret:
|
|||
|
|
ret.append(s)
|
|||
|
|
return ret
|