feat: 第二次作业

02_rag/01_rag_chroma.ipynb

@@ -0,0 +1,164 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6b4b5b35",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "86fc8f30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import PyMuPDFLoader\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "from langchain_community.embeddings import DashScopeEmbeddings\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_community.chat_models import ChatTongyi\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "# ========== 第一步：加载文档 ==========\n",
+    "loader = PyMuPDFLoader(\"./profile.pdf\")\n",
+    "pages = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12a71183",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ========== 第二步：清洗数据（可选，根据文档质量决定）==========\n",
+    "# clean_pages = [clean_pdf_text(page) for page in pages]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96e8b2ac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ========== 第三步：分块 ==========\n",
+    "splitter = RecursiveCharacterTextSplitter(\n",
+    "    # 分隔符优先级：段落 → 换行 → 句号 → 空格 → 硬切\n",
+    "    separators=[\"\\n\\n\", \"\\n\", \"。\", \"！\", \"？\", \" \", \"\"],\n",
+    "    # 每个块最大 100 字符\n",
+    "    chunk_size=500,\n",
+    "    # 相邻块重叠 10 字符（chunk_size 的 20%）\n",
+    "    chunk_overlap=100,\n",
+    "    # 长度计算函数\n",
+    "    length_function=len\n",
+    ")\n",
+    "docs = splitter.split_documents(pages)\n",
+    "# ========== 第四步：向量化 + 存入向量库 ==========\n",
+    "embedding_model = DashScopeEmbeddings(\n",
+    "    model=\"text-embedding-v3\",\n",
+    "    dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\")\n",
+    ")\n",
+    "\n",
+    "vectorstore = Chroma.from_documents(\n",
+    "    documents=docs,\n",
+    "    embedding=embedding_model,\n",
+    "    persist_directory=\"./knowledge_db_500_char\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "c73b682e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "吴雅娟是**高级前端工程师**，目前为自由职业者（2024/05至今），此前就职于智库人工智能科技有限公司（2021/08–2024/04）担任Web前端高级工程师，更早前在北京路遥科技有限公司（2016/08–2021/05）任Web前端开发工程师。\n",
+      "\n",
+      "履历包括：  \n",
+      "- **教育背景**：五邑大学本科，2016年毕业；  \n",
+      "- **技术能力**：熟悉HTML5/CSS3、ES6+、Vue2/Vue3（含Vuex、Pinia、Vue Router）、React.js（含Redux/Router）、HTTP/TCP协议、浏览器渲染机制、Webpack/Vite工程化等；  \n",
+      "- **项目经验**：主导“看数需求”项目（含公式编辑器优化、Zrender高自定义图表开发、IndexedDB缓存、长列表性能优化、展板导出优化）、DMP数据管理系统（RBAC权限控制、数据处理流程优化）及“智慧‘气象+’城市赋能平台”等；  \n",
+      "- **其他**：承接向暖小程序等外包项目。\n"
+     ]
+    }
+   ],
+   "source": [
+    "# ========== 第五步：创建检索器 ==========\n",
+    "retriever = vectorstore.as_retriever(search_kwargs={\"k\": 3})\n",
+    "\n",
+    "# ========== 第六步：提问 ==========\n",
+    "query = \"吴雅娟是干什么的？有什么履历？\"\n",
+    "relevant_docs = retriever.invoke(query)\n",
+    "\n",
+    "# ========== 第七步：生成回答 ==========\n",
+    "context = \"\\n\\n---\\n\\n\".join([d.page_content for d in relevant_docs])\n",
+    "\n",
+    "prompt = ChatPromptTemplate.from_template(\"\"\"\n",
+    "你是一个专业的知识库助手。请根据以下上下文回答问题。\n",
+    "\n",
+    "**规则：**\n",
+    "- 只基于提供的上下文回答，不要编造\n",
+    "- 如果上下文中没有相关信息，直接说「根据现有资料，我找不到这个问题的答案」\n",
+    "- 回答要简洁直接，引用原文时用引号\n",
+    "\n",
+    "**上下文：**\n",
+    "{context}\n",
+    "\n",
+    "**问题：**\n",
+    "{question}\n",
+    "\"\"\")\n",
+    "\n",
+    "llm = ChatTongyi(model=\"qwen-plus\", dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\"))\n",
+    "chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "answer = chain.invoke({\"context\": context, \"question\": query})\n",
+    "print(answer)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "01_LANGCHAIN",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0rc2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}