enterprise-kb-rag/2.2_LangChain基础知识/19_indexes_document_loader.py

from pathlib import Path
import sys

sys.path.append(str(Path(__file__).resolve().parents[1]))

from _chapter2_config import print_dependency_hint


DATA_FILE = Path(__file__).with_name("data") / "衣服属性.txt"


def main() -> None:
    try:
        from langchain_community.document_loaders import TextLoader
    except ModuleNotFoundError as exc:
        print_dependency_hint(exc)
        raise SystemExit(1)

    try:
        from langchain_unstructured import UnstructuredLoader
    except ModuleNotFoundError:
        UnstructuredLoader = None

    if UnstructuredLoader is not None:
        loader = UnstructuredLoader(str(DATA_FILE), encoding="utf8")
        docs = loader.load()
        print(f"unstructured docs-->{docs}")
        print(f"len-->{len(docs)}")
        print(f"第一行数据-->{docs[0].page_content}")
        print("*" * 100)
    else:
        print("未安装 langchain-unstructured，跳过 UnstructuredLoader 示例。")

    loader = TextLoader(str(DATA_FILE), encoding="utf8")
    docs = loader.load()
    print(f"TextLoader docs-->{docs}")
    print(f"len-->{len(docs)}")
    print("第一行数据-->{}".format(docs[0].page_content.split("\n")[0]))


if __name__ == "__main__":
    main()