enterprise-kb-rag/2.2_LangChain基础知识/23_indexes_markdown_header_splitter.py

from pathlib import Path
import sys

sys.path.append(str(Path(__file__).resolve().parents[1]))

from _chapter2_config import print_dependency_hint


try:
    from langchain_text_splitters import MarkdownHeaderTextSplitter
except ModuleNotFoundError as exc:
    print_dependency_hint(exc)
    raise SystemExit(1)


def main() -> None:
    headers_to_split_on = [
        ("#", "Header 1"),
        ("##", "Header 2"),
        ("###", "Header 3"),
    ]

    markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
    markdown_text = "# Header 1\nSome text\n## Header 2\nMore text\n### Header 3\nEven more text"
    docs = markdown_splitter.split_text(markdown_text)
    print(docs)


if __name__ == "__main__":
    main()