ZhouYI/03_RAG优化/agent/rag/document.py

"""文档加载和文本清洗。"""

import re

from langchain_community.document_loaders import PyMuPDFLoader
from langchain_core.documents import Document


def load_pdf_document(pdf_path: str) -> list[Document]:
    """按页加载 PDF 文档。"""
    return PyMuPDFLoader(pdf_path).load()


def clean_pdf_text(text: str) -> str:
    """规整 PDF 提取文本中的空白字符。"""
    text = re.sub(r"[\t\r\f\v ]+", " ", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()