{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "0b103582", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "文档类型：\n", "PDF 共 354 页\n", "元数据：{'producer': 'PDFlib+PDI 9.0.6 (C++/Win64)', 'creator': 'PTC Arbortext Publishing Engine', 'creationdate': '2023-06-16T15:35:59+08:00', 'source': './car_info.pdf', 'file_path': './car_info.pdf', 'total_pages': 354, 'format': 'PDF 1.7', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '2023-06-16T17:41:45+08:00', 'trapped': '', 'modDate': \"D:20230616174145+08'00'\", 'creationDate': \"D:20230616153559+08'00'\", 'page': 0}\n", "内容预览：欢迎\n", "感谢您选择了具有优良安全性、舒适性、动力性和经济性的Lynk & Co领克汽车。\n", "首次使用前请仔细、完整地阅读本手册内容，将有助于您更好地了解和使用车辆。\n", "本手册中的所有资料均为出版时的最新资料，但本公司将对产品进行不断的改进和优化，您所购的车辆可能与本手册中的描述有所不同，请以实际\n", "接收的车辆为准。\n", "如您有任何问题，或需要预约服务，请拨打电话4006-010101 联系我们。您也可以开车前\n" ] } ], "source": [ "from langchain_community.document_loaders import PyMuPDFLoader\n", "# 安装依赖：uv pip install pymupdf -i https://pypi.tuna.tsinghua.edu.cn/simple\n", "\n", "# 创建加载器实例，传入 PDF 文件路径\n", "pdf_loader = PyMuPDFLoader(\"./car_info.pdf\")\n", "\n", "# 调用 load() 方法，返回一个 Document 列表（每页一个 Document）\n", "pdf_pages = pdf_loader.load()\n", "\n", "# 看看加载结果\n", "print(f\"文档类型：{type(pdf_pages)}\")\n", "print(f\"PDF 共 {len(pdf_pages)} 页\")\n", "# 查看第一页的内容和元数据\n", "first_page = pdf_pages[0]\n", "print(f\"元数据：{first_page.metadata}\")\n", "print(f\"内容预览：{first_page.page_content[:200]}\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "f1ab96e1", "metadata": {}, "outputs": [], "source": [ "import re\n", "\n", "# ---- 问题1：PDF 解析产生的多余换行符 ----\n", "# 现象：一句话被拆成多行，中间插了 \\n\n", "# 例如：\"领克汽车\\n车顶行李架\\n最大载荷\"\n", "# 解决：用正则匹配并删除非中文字符之间的换行符\n", "raw_text = \"领克汽车\\n车顶行李架\\n最大载荷。\"\n", "pattern = re.compile(r'[^一](\\n)[^一]', re.DOTALL)\n", "clean_text = re.sub(pattern, lambda m: m.group(0).replace('\\n', ''), raw_text)\n", "\n", "# ---- 问题2：特殊符号干扰 ----\n", "# 现象：PDF 中的项目符号 • 、多余空格等\n", "clean_text = clean_text.replace('•', '')\n", "clean_text = clean_text.replace(' ', ' ') # 合并多余空格\n", "\n", "# ---- 问题3：页眉页脚噪声 ----\n", "# 现象：每页都有 \"第X页\"、\"公司名称\" 等重复内容\n", "# 解决：根据元数据中的页码信息，过滤掉固定位置的噪声文本" ] }, { "cell_type": "code", "execution_count": 9, "id": "9defa53b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "原数据：领克汽车\n", "车顶行李架\n", "最大载荷。\n", "领克汽车车顶行李架最大载荷。\n" ] } ], "source": [ "text = \"领克汽车\\n车顶行李架\\n最大载荷。\"\n", "print(\"原数据：领克汽车\\n车顶行李架\\n最大载荷。\")\n", " \n", "# 删除非中文字符之间的换行符\n", "text = re.sub(r'[^一](\\n)[^一]', \n", " lambda m: m.group(0).replace('\\n', ''), text)\n", " \n", "# 删除项目符号和多余空格\n", "text = text.replace('•', '').replace(' ', ' ')\n", " \n", "# 删除连续的换行符（保留一个）\n", "text = re.sub(r'\\n{2,}', '\\n', text)\n", "print(text)" ] }, { "cell_type": "code", "execution_count": 21, "id": "4c8a3996", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- 块1 ---\n", "LangChain 是一个让你的 LLM 变得更强大的开源框架。\n", "你想开发一个基于 LLM 的应用，需要什么组件它都有，直接使用就行。\n", "甚至针对常规的应用流程\n", "\n", "--- 块2 ---\n", "需要什么组件它都有，直接使用就行。\n", "甚至针对常规的应用流程，它利用 Chain 这个概念已经内置标准化方案了。\n", "下面我们从新兴的大语言模型技术栈的角度来看看为何它的理念这么受欢迎。\n", "\n" ] } ], "source": [ "#安装依赖：uv pip install langchain-text-splitters -i https://mirrors.aliyun.com/pypi/simple\n", "# 手动实现固定字符分块\n", "from langchain_text_splitters import CharacterTextSplitter\n", "\n", "# 1. 定义分块器\n", "text_splitter = CharacterTextSplitter(\n", " separator=\"，\", # 指定分隔符，默认为 \"\\n\\n\"\n", " chunk_size=100, # 每个块的最大字符数\n", " chunk_overlap=50, # 块与块之间的重叠字符数，建议设置以保持上下文连贯[citation:4]\n", " length_function=len, # 计算长度的方法，默认按字符数\n", " is_separator_regex=False # 是否将分隔符视为正则表达式\n", ")\n", "\n", "# 2. 执行分块，返回字符串列表\n", "text = \"\"\"\n", "LangChain 是一个让你的 LLM 变得更强大的开源框架。\n", "你想开发一个基于 LLM 的应用，需要什么组件它都有，直接使用就行。\n", "甚至针对常规的应用流程，它利用 Chain 这个概念已经内置标准化方案了。\n", "下面我们从新兴的大语言模型技术栈的角度来看看为何它的理念这么受欢迎。\n", "\"\"\"\n", "chunks = text_splitter.split_text(text)\n", "for i, chunk in enumerate(chunks):\n", " print(f\"--- 块{i+1} ---\\n{chunk}\\n\")" ] }, { "cell_type": "code", "execution_count": 22, "id": "192375ec", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- 块1 ---\n", "LangChain 是一个让你的 LLM 变得更强大的开源框架。\n", "\n", "--- 块2 ---\n", "你想开发一个基于 LLM 的应用，需要什么组件它都有，直接使用就行。\n", "\n", "--- 块3 ---\n", "甚至针对常规的应用流程，它利用 Chain 这个概念已经内置标准化方案了。\n", "\n", "--- 块4 ---\n", "下面我们从新兴的大语言模型技术栈的角度来看看为何它的理念这么受欢迎。\n", "\n" ] } ], "source": [ "''' \n", "* RecursiveCharacterTextSplitter 递归字符文本分割\n", "RecursiveCharacterTextSplitter 将按不同的字符递归地分割(按照这个优先级[\"\\n\\n\", \"\\n\", \" \", \"\"])，\n", " 这样就能尽量把所有和语义相关的内容尽可能长时间地保留在同一位置\n", "RecursiveCharacterTextSplitter需要关注的是4个参数：\n", "\n", "* separators - 分隔符字符串数组\n", "* chunk_size - 每个文档的字符数量限制\n", "* chunk_overlap - 两份文档重叠区域的长度\n", "* length_function - 长度计算函数\n", "'''\n", "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", "\n", "# 创建递归字符分割器\n", "text_splitter = RecursiveCharacterTextSplitter(\n", " # 分隔符优先级：段落 → 换行 → 句号 → 空格 → 硬切\n", " separators=[\"\\n\\n\", \"\\n\", \"。\", \"！\", \"？\", \" \", \"\"],\n", " \n", " # 每个块最大 50 字符\n", " chunk_size=50,\n", " \n", " # 相邻块重叠 10 字符（chunk_size 的 20%）\n", " chunk_overlap=10,\n", " \n", " # 长度计算函数\n", " length_function=len\n", ")\n", "\n", "# 对单段文本进行分割\n", "text = \"\"\"\n", "LangChain 是一个让你的 LLM 变得更强大的开源框架。\n", "你想开发一个基于 LLM 的应用，需要什么组件它都有，直接使用就行。\n", "甚至针对常规的应用流程，它利用 Chain 这个概念已经内置标准化方案了。\n", "下面我们从新兴的大语言模型技术栈的角度来看看为何它的理念这么受欢迎。\n", "\"\"\"\n", "chunks = text_splitter.split_text(text)\n", "for i, chunk in enumerate(chunks):\n", " print(f\"--- 块{i+1} ---\\n{chunk}\\n\")" ] }, { "cell_type": "code", "execution_count": 23, "id": "45738e55", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "切分后的文件数量：4243\n", "切分后的字符数（可以用来大致评估 token 数）：148617\n" ] } ], "source": [ "split_docs = text_splitter.split_documents(pdf_pages)\n", "print(f\"切分后的文件数量：{len(split_docs)}\")\n", "print(f\"切分后的字符数（可以用来大致评估 token 数）：{sum([len(doc.page_content) for doc in split_docs])}\")" ] }, { "cell_type": "code", "execution_count": 25, "id": "b6061663", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "有效块数量：4243\n", "总字符数（可大致评估 Token 数）：148617\n" ] } ], "source": [ "# 过滤掉 page_content 为空或仅含空白的文档\n", "valid_docs = [\n", " doc for doc in split_docs \n", " if doc.page_content and doc.page_content.strip()\n", "]\n", "\n", "print(f\"有效块数量：{len(valid_docs)}\")\n", "print(f\"总字符数（可大致评估 Token 数）：{sum(len(d.page_content) for d in valid_docs)}\")" ] }, { "cell_type": "code", "execution_count": 44, "id": "a2f58237", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "向量维度：1024\n", "前5个值：[-0.013686168007552624, 0.015642698854207993, -0.052721332758665085, 0.033709585666656494, -0.08253694325685501]\n" ] } ], "source": [ "from langchain_community.embeddings import DashScopeEmbeddings\n", "\n", "# 初始化 Embedding 模型\n", "# 需要先在 https://dashscope.console.aliyun.com/ 获取 API Key\n", "embedding_model = DashScopeEmbeddings(\n", " model=\"text-embedding-v3\", # 模型名称\n", " dashscope_api_key=\"sk-ws-H.EMYERML.28mU.MEQCIBRtVOGRuQoEtyXeguqxh68NBcmxhdFiY2aj5VOUpwLGAiApVSH0WKpaWounDN24jiTo-kNa2uSZzcgvNBMl5QwVMw\" # 替换为你的真实 Key\n", ")\n", "\n", "# 单条文本向量化\n", "text = \"RAG系统搭建实战\"\n", "embedding = embedding_model.embed_query(text)\n", "print(f\"向量维度：{len(embedding)}\")\n", "print(f\"前5个值：{embedding[:5]}\")" ] }, { "cell_type": "code", "execution_count": 54, "id": "dd6b3aff", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "总块数: 48\n", "平均块大小: 15.3 字符\n", "\n", "块 #1 (14字符)\n", "小明特别喜欢吃脆甜多汁的苹果\n", "\n", "块 #2 (15字符)\n", "小红对榴莲那独特的味道情有独钟\n", "\n", "块 #3 (13字符)\n", "小明和小丽是一对甜蜜的情侣\n", "\n", "块 #4 (17字符)\n", "王老师教学认真负责，是公认的好老师\n", "\n", "块 #5 (11字符)\n", "小李每天都要吃一根香蕉\n" ] } ], "source": [ "from langchain_community.embeddings import DashScopeEmbeddings\n", "from langchain_community.vectorstores import Chroma\n", "import os\n", "from dotenv import load_dotenv\n", "\n", "# 加载环境变量\n", "load_dotenv()\n", "# 加载数据库\n", "embedding = DashScopeEmbeddings(\n", " model=\"text-embedding-v3\",\n", " dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\")\n", ")\n", "\n", "vectordb = Chroma(\n", " persist_directory=\"./chroma_db1\",\n", " embedding_function=embedding\n", ")\n", "\n", "# 查看所有块\n", "result = vectordb.get()\n", "docs = result['documents']\n", "\n", "print(f\"总块数: {len(docs)}\")\n", "print(f\"平均块大小: {sum(len(d) for d in docs) / len(docs):.1f} 字符\")\n", "\n", "# 查看前5个块\n", "for i in range(5):\n", " print(f\"\\n块 #{i+1} ({len(docs[i])}字符)\")\n", " print(docs[i][:200])\n" ] }, { "cell_type": "code", "execution_count": 32, "id": "c4ebd145", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "生成了 4 个向量\n", "每个向量维度：1024\n" ] } ], "source": [ "# 批量文本向量化\n", "texts = [\n", " \"hello world\",\n", " \"什么是大语言模型\",\n", " \"RAG的概念和原理\",\n", " \"牛顿第一定律是什么\",\n", "]\n", "\n", "embeddings = embedding_model.embed_documents(texts)\n", "print(f\"生成了 {len(embeddings)} 个向量\")\n", "print(f\"每个向量维度：{len(embeddings[0])}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "f0d7504a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "匹配内容：小明特别喜欢吃脆甜多汁的苹果\n", "内容：小明特别喜欢吃脆甜多汁的苹果 | 相似度分数：0.4177\n", "内容：小明特别喜欢吃脆甜多汁的苹果 | 相似度分数：0.4177\n", "内容：小明特别喜欢吃脆甜多汁的苹果 | 相似度分数：0.4177\n", "内容：小明特别喜欢吃脆甜多汁的苹果 | 相似度分数：0.4177\n", "内容：小明特别喜欢吃脆甜多汁的苹果 | 相似度分数：0.4177\n" ] } ], "source": [ "#安装依赖：uv pip install chromadb -i https://pypi.tuna.tsinghua.edu.cn/simple\n", "from langchain_community.vectorstores import Chroma\n", "from langchain_community.embeddings import DashScopeEmbeddings\n", "\n", "# 准备一些测试数据\n", "datas = [\n", " \"小明特别喜欢吃脆甜多汁的苹果\",\n", " \"小红对榴莲那独特的味道情有独钟\",\n", " \"小明和小丽是一对甜蜜的情侣\",\n", " \"王老师教学认真负责，是公认的好老师\",\n", " \"小李每天都要吃一根香蕉\",\n", " \"小王的男朋友长得阳光帅气，是大家公认的大帅哥\"\n", "]\n", "\n", "# 创建向量数据库并持久化(会同时把文本和对应的向量存入数据库)\n", "db = Chroma.from_texts(\n", " texts=datas,\n", " embedding=embedding_model,\n", " collection_metadata={\"hnsw:space\": \"cosine\"}, # 关键配置：指定距离算法为余弦相似度\n", " persist_directory=\"./chroma_db1\" # 数据库存储路径\n", ")\n", "# 相似度检索\n", "query = \"夏天适合吃什么水果\"\n", "results = db.similarity_search(query, k=1) # 返回最相似的 1 条\n", "\n", "for doc in results:\n", " print(f\"匹配内容：{doc.page_content}\")\n", "# 带分数的相似度检索（分数越低越相似，0 表示完全匹配）\n", "query = \"夏天适合吃什么水果\"\n", "results = db.similarity_search_with_score(query, k=3)\n", "\n", "for doc, score in results:\n", " print(f\"内容：{doc.page_content} | 相似度分数：{score:.4f}\")" ] }, { "cell_type": "code", "execution_count": 45, "id": "51a34231", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- 结果1 ---\n", "内容：说明！\n", "□当车速在0 – 40km/h的范围内，且车辆在陡坡上低速下坡行驶\n", "时，HDC才会激活。...\n", "来源：{'modDate': \"D:20230616174145+08'00'\", 'producer': 'PDFlib+PDI 9.0.6 (C++/Win64)', 'author': '', 'total_pages': 354, 'creator': 'PTC Arbortext Publishing Engine', 'creationDate': \"D:20230616153559+08'00'\", 'title': '', 'format': 'PDF 1.7', 'creationdate': '2023-06-16T15:35:59+08:00', 'file_path': './car_info.pdf', 'subject': '', 'keywords': '', 'moddate': '2023-06-16T17:41:45+08:00', 'page': 160, 'trapped': '', 'source': './car_info.pdf'}\n", "\n", "--- 结果2 ---\n", "内容：□当车速在40-60 km/h范围内无法激活HDC功能，车速超过60 km/...\n", "来源：{'author': '', 'producer': 'PDFlib+PDI 9.0.6 (C++/Win64)', 'modDate': \"D:20230616174145+08'00'\", 'file_path': './car_info.pdf', 'moddate': '2023-06-16T17:41:45+08:00', 'total_pages': 354, 'creationDate': \"D:20230616153559+08'00'\", 'source': './car_info.pdf', 'format': 'PDF 1.7', 'keywords': '', 'trapped': '', 'title': '', 'creationdate': '2023-06-16T15:35:59+08:00', 'creator': 'PTC Arbortext Publishing Engine', 'page': 160, 'subject': ''}\n", "\n", "--- 结果3 ---\n", "内容：h时，HDC自动退出。\n", "□激活HDC后，您可以通过踩下制动踏板或油门踏板调整下坡车\n", "速。...\n", "来源：{'creator': 'PTC Arbortext Publishing Engine', 'source': './car_info.pdf', 'page': 160, 'producer': 'PDFlib+PDI 9.0.6 (C++/Win64)', 'creationdate': '2023-06-16T15:35:59+08:00', 'subject': '', 'author': '', 'keywords': '', 'moddate': '2023-06-16T17:41:45+08:00', 'title': '', 'creationDate': \"D:20230616153559+08'00'\", 'format': 'PDF 1.7', 'trapped': '', 'modDate': \"D:20230616174145+08'00'\", 'file_path': './car_info.pdf', 'total_pages': 354}\n", "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\Sundear\\AppData\\Local\\Temp\\ipykernel_7968\\2664928941.py:26: LangChainDeprecationWarning: The class `Chroma` was deprecated in LangChain 0.2.9 and will be removed in 1.0. An updated version of the class exists in the `langchain-chroma package and should be used instead. To use it run `pip install -U `langchain-chroma` and import as `from `langchain_chroma import Chroma``.\n", " vectordb = Chroma(\n" ] } ], "source": [ "from langchain_community.vectorstores import Chroma\n", "from langchain_community.embeddings import DashScopeEmbeddings\n", "\n", "# 假设 split_docs 是前面分块后的文档列表\n", "\n", "# 从文档创建向量库(写数据)\n", "vectorstore = Chroma.from_documents(\n", " documents=valid_docs,\n", " embedding=embedding_model,\n", " collection_metadata={\"hnsw:space\": \"cosine\"},\n", " persist_directory=\"./my_knowledge_db\"\n", ")\n", "\n", "# 创建检索器，设置返回 Top-3 最相关文档\n", "retriever = vectorstore.as_retriever(search_kwargs={\"k\": 3})\n", "# 测试检索\n", "query = \"在多少速度范围内，HDC才会激活？\"\n", "relevant_docs = retriever.invoke(query)\n", "\n", "for i, doc in enumerate(relevant_docs):\n", " print(f\"--- 结果{i+1} ---\")\n", " print(f\"内容：{doc.page_content[:100]}...\")\n", " print(f\"来源：{doc.metadata}\")\n", " print()\n", "# 直接加载已持久化的数据库（无需再次添加文档或持久化）\n", "vectordb = Chroma(\n", " persist_directory=\"./my_knowledge_db\",\n", " embedding_function=embedding_model\n", ")" ] }, { "cell_type": "code", "execution_count": 46, "id": "7a744b66", "metadata": {}, "outputs": [], "source": [ "from langchain_core.prompts import ChatPromptTemplate\n", "\n", "# 构建 Prompt 模板\n", "prompt = ChatPromptTemplate.from_template(\"\"\"\n", "你是一个专业的知识库助手。请根据以下检索到的上下文回答用户问题。\n", "\n", "**规则：**\n", "- 只基于提供的上下文回答，不要编造\n", "- 如果上下文中没有相关信息，直接说「根据现有资料，我找不到这个问题的答案」\n", "- 回答要简洁直接，引用原文时用引号\n", "\n", "**检索到的上下文：**\n", "{context}\n", "\n", "**用户问题：**\n", "{question}\n", "\"\"\")" ] }, { "cell_type": "code", "execution_count": 51, "id": "0dff7c44", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "根据上下文，HDC激活的速度范围是“0 – 40km/h”。\n" ] } ], "source": [ "from langchain_community.chat_models import ChatTongyi\n", "from langchain_core.output_parsers import StrOutputParser\n", "from langchain_openai import ChatOpenAI\n", "\n", "# 初始化大模型（这里用通义千问，也可以换成 DeepSeek）\n", "llm = ChatOpenAI(\n", " model_name=\"deepseek-v4-flash\",\n", " api_key='sk-80a123483afb480285c6452985eea18e',\n", " base_url=\"https://api.deepseek.com/v1\"\n", ")\n", "\n", "# 拼装上下文\n", "context_text = \"\\n\\n---\\n\\n\".join([doc.page_content for doc in relevant_docs])\n", "\n", "# 构建 Chain 并调用\n", "chain = prompt | llm | StrOutputParser()\n", "\n", "response = chain.invoke({\n", " \"context\": context_text,\n", " \"question\": query\n", "})\n", "\n", "print(response)" ] }, { "cell_type": "code", "execution_count": null, "id": "24f7f23b", "metadata": {}, "outputs": [], "source": [ "from langchain_community.document_loaders import PyMuPDFLoader\n", "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", "from langchain_community.embeddings import DashScopeEmbeddings\n", "from langchain_community.vectorstores import Chroma\n", "from langchain_core.prompts import ChatPromptTemplate\n", "from langchain_community.chat_models import ChatTongyi\n", "from langchain_core.output_parsers import StrOutputParser\n", "\n", "# ========== 第一步：加载文档 ==========\n", "loader = PyMuPDFLoader(\"./your_document.pdf\")\n", "pages = loader.load()\n", "\n", "# ========== 第二步：清洗数据（可选，根据文档质量决定）==========\n", "# clean_pages = [clean_pdf_text(page) for page in pages]\n", "\n", "# ========== 第三步：分块 ==========\n", "splitter = RecursiveCharacterTextSplitter(\n", " # 分隔符优先级：段落 → 换行 → 句号 → 空格 → 硬切\n", " separators=[\"\\n\\n\", \"\\n\", \"。\", \"！\", \"？\", \" \", \"\"],\n", " # 每个块最大 50 字符\n", " chunk_size=50,\n", " # 相邻块重叠 10 字符（chunk_size 的 20%）\n", " chunk_overlap=10,\n", " # 长度计算函数\n", " length_function=len\n", ")\n", "docs = splitter.split_documents(pages)\n", "\n", "# ========== 第四步：向量化 + 存入向量库 ==========\n", "embedding_model = DashScopeEmbeddings(\n", " model=\"text-embedding-v3\",\n", " dashscope_api_key=\"your-api-key\"\n", ")\n", "vectorstore = Chroma.from_documents(\n", " documents=docs,\n", " embedding=embedding_model,\n", " persist_directory=\"./knowledge_db\"\n", ")\n", "\n", "# ========== 第五步：创建检索器 ==========\n", "retriever = vectorstore.as_retriever(search_kwargs={\"k\": 3})\n", "\n", "# ========== 第六步：提问 ==========\n", "query = \"你的问题\"\n", "relevant_docs = retriever.invoke(query)\n", "\n", "# ========== 第七步：生成回答 ==========\n", "context = \"\\n\\n---\\n\\n\".join([d.page_content for d in relevant_docs])\n", "\n", "prompt = ChatPromptTemplate.from_template(\"\"\"\n", "你是一个专业的知识库助手。请根据以下上下文回答问题。\n", "\n", "**规则：**\n", "- 只基于提供的上下文回答，不要编造\n", "- 如果上下文中没有相关信息，直接说「根据现有资料，我找不到这个问题的答案」\n", "- 回答要简洁直接，引用原文时用引号\n", "\n", "**上下文：**\n", "{context}\n", "\n", "**问题：**\n", "{question}\n", "\"\"\")\n", "\n", "llm = ChatTongyi(model=\"qwen-plus\", dashscope_api_key=\"your-api-key\")\n", "chain = prompt | llm | StrOutputParser()\n", "\n", "answer = chain.invoke({\"context\": context, \"question\": query})\n", "print(answer)" ] }, { "cell_type": "code", "execution_count": 59, "id": "5f02c3cf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "总块数: 94\n", "平均块大小: 114.9 字符\n", "\n", "块 #1 (674字符)\n", "疯狂的里海投资体系框架\n", "数据来源：《疯狂的里海》2019-2025年度文集合集（含投资周记、周直播、公众号文章）\n", "整理日期：2026年5月26日\n", "一、总体框架：\"2444\"体系\n", "疯狂的里海将自己的投资体系浓缩为 \"2-4-4-4\" 结构：\n", "层次\n", "内容\n", "说明\n", "\"2\"\n", "认知 + 修行\n", "投资分为两大部分：认知是知识的集合，修行是心性的磨练\n", "第一个\"4\"\n", "常识、概率、赔率、频率\n", "底层思维框架\n", "第二个\"4\"\n", "\n", "块 #2 (508字符)\n", "四、选股模型：隐形冠军模型\n", "核心公式：\n", "隐形冠军 + 资产结构简单 + 低估值 + 有变化 = 买入信号\n", "四大要素详解：\n", "1. 简单（最重要的前提）\n", "业务结构简单：主营业务清晰，产品线不复杂，散户看得懂\n", "资产负债结构简单：没有大量有息负债，没有复杂的表外风险，资产质量\"实\"\n", "典型特征：货币资金充裕、经营性负债为主、无暴雷风险\n", "2. 底部（安全边际）\n", "股价在历史低位区间，跌无可跌\n", "估值指标：PB 1\n", "\n", "块 #3 (591字符)\n", "维度\n", "战略股\n", "战术股\n", "持有时间\n", "1-3年甚至更长\n", "数周到数月\n", "预期收益\n", "翻倍以上（3-10倍）\n", "30-80%\n", "仓位\n", "重仓（40-60%）\n", "轻仓/中仓（10-30%）\n", "核心逻辑\n", "业绩从低点释放到高点的全过程\n", "赔率保护下的价值发现波段\n", "典型案例\n", "涪陵榨菜、太极集团\n", "秦安股份、阳谷华泰、花园生物\n", "操作要点\n", "低位重仓、不惧波动、拿住\n", "到预期目标果断走，切换到下一个\n", "关键认知：\n", "\"战略股都是战术股走出来的\"\n", "\n", "块 #4 (555字符)\n", "在市场不关注（成交低迷）时买入\n", "利用市场定价错误获取安全边际\n", "强者突破（能力圈内的深度跟踪）\n", "光选股不够，必须跟踪变化\n", "在能力圈内做\"强者\"：深入产业链、实地调研、跟踪数据\n", "本地公司优势：随时上门、通过合作机构了解、朋友在公司内\n", "\"散户做强者突破，只能是一点：能力圈内选股\"\n", "八、操作节奏与卖出原则\n", "买入时机\n", "股价在底部区间（PB低、PE低、成交萎缩）\n", "基本面出现向上的\"变化\"信号\n", "大势不需要太好，\n", "\n", "块 #5 (569字符)\n", "震荡市/结构性行情\n", "\"聚焦微观，做好当下，死磕投资\"\n", "放低预期，追求跑赢指数\n", "找结构性机会：局部需求增长的行业/产业链\n", "十、投资进阶路径\n", "里海总结的散户进阶之路：\n", "第一阶段：中巴式（买好公司长期持有）\n", " ↓\n", "第二阶段：成长股（关注业绩增长）\n", " ↓\n", "第三阶段：基本面 + 市场情绪（戴维斯双击）\n", " ↓\n", "第四阶段：基本面 + 技术/交易（择时能力）\n", " ↓\n", "第五阶段：赚时代的钱（前瞻性认知）\n", "里海自身定位\n" ] } ], "source": [ "from langchain_community.embeddings import DashScopeEmbeddings\n", "from langchain_community.vectorstores import Chroma\n", "import os\n", "from dotenv import load_dotenv\n", "from langchain_community.embeddings import DashScopeEmbeddings\n", "\n", "# 加载环境变量\n", "load_dotenv()\n", "\n", "# 加载数据库\n", "embedding = DashScopeEmbeddings(\n", " model=\"text-embedding-v3\",\n", " dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\")\n", ")\n", "\n", "vectordb = Chroma(\n", " persist_directory=\"./investment_knowledge_db\",\n", " embedding_function=embedding\n", ")\n", "\n", "# 查看所有块\n", "result = vectordb.get()\n", "docs = result['documents']\n", "\n", "print(f\"总块数: {len(docs)}\")\n", "print(f\"平均块大小: {sum(len(d) for d in docs) / len(docs):.1f} 字符\")\n", "\n", "# 查看前5个块\n", "for i in range(5):\n", " print(f\"\\n块 #{i+1} ({len(docs[i])}字符)\")\n", " print(docs[i][:200])\n" ] }, { "cell_type": "code", "execution_count": 60, "id": "1fd285f3", "metadata": {}, "outputs": [ { "ename": "ModuleNotFoundError", "evalue": "No module named 'pymilvus'", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", "Cell \u001b[1;32mIn[60], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpymilvus\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MilvusClient, DataType\n\u001b[0;32m 3\u001b[0m \u001b[38;5;66;03m# 连接 Milvus 服务（默认端口 19530）\u001b[39;00m\n\u001b[0;32m 4\u001b[0m client \u001b[38;5;241m=\u001b[39m MilvusClient(uri\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://localhost:19530\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'pymilvus'" ] } ], "source": [ "from pymilvus import MilvusClient, DataType\n", "\n", "# 连接 Milvus 服务（默认端口 19530）\n", "client = MilvusClient(uri=\"http://localhost:19530\")" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.20" } }, "nbformat": 4, "nbformat_minor": 5 }