ZhouYI/02_RAG/agent/rag/splitter.py

from langchain_text_splitters import RecursiveCharacterTextSplitter  
from langchain_experimental.text_splitter import SemanticChunker
from langchain_community.embeddings import DashScopeEmbeddings
from agent.config import load_config
def recursive_split(documents:list, chunk_size:int=500, chunk_overlap:int=100) -> list:
    """
    使用递归字符分割器将文本分割为较小的块。
    
    参数:
        documents (list): 要分割的文档列表。
        chunk_size (int): 每个块的最大字符数。
        chunk_overlap (int): 块之间的重叠字符数。
    """
    text_splitter = RecursiveCharacterTextSplitter(
        separators=["\n\n", "\n", "。", "！", "？", ".", " ",""],
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap,
        length_function=len,
    )
    chunks = text_splitter.split_documents(documents)
    return chunks


def embedding_split(text:str) -> list:
    """
    使用 Embedding 模型计算相邻句⼦的语义相似度，当相似度「断崖式下降」时，就在那⾥切⼀⼑
    
    """
    config=load_config()
    # 使⽤通义的 Embedding 模型做语义分块
    emmbedding_model=DashScopeEmbeddings(
        model="text-embedding-v4",
        dashscope_api_key=config.api_key,
    )
    # 创建语义分块器
    semantic_splitter=SemanticChunker(
        emmbedding_model=emmbedding_model,
        breakpoint_threshold_type="percentile",
        breakpoint_threshold_amount=85
    )
    #分块
    chunks=semantic_splitter.split_text(text)
    return chunks