feat：第三次作业

02_rag/02_rag.ipynb

@@ -0,0 +1,260 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6b4b5b35",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "86fc8f30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import PyMuPDFLoader\n",
+    "\n",
+    "# ========== 第一步：加载文档 ==========\n",
+    "loader = PyMuPDFLoader(\"./../../resource/人事管理流程.docx\")\n",
+    "pages = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12a71183",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ========== 第二步：清洗数据（可选，根据文档质量决定）==========\n",
+    "# clean_pages = [clean_pdf_text(page) for page in pages]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96e8b2ac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "# ========== 第三步：递归分块 ==========\n",
+    "splitter = RecursiveCharacterTextSplitter(\n",
+    "    # 分隔符优先级：段落 → 换行 → 句号 → 空格 → 硬切\n",
+    "    separators=[\"\\n\\n\", \"\\n\", \"。\", \"！\", \"？\", \" \", \"\"],\n",
+    "    # 每个块最大 100 字符\n",
+    "    chunk_size=500,\n",
+    "    # 相邻块重叠 10 字符（chunk_size 的 20%）\n",
+    "    chunk_overlap=100,\n",
+    "    # 长度计算函数\n",
+    "    length_function=len\n",
+    ")\n",
+    "\n",
+    "# (可优化点： 语义分块)\n",
+    "\n",
+    "\n",
+    "valid_docs = splitter.split_documents(pages)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "3c6d3e2c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:213: PyMilvusDeprecationWarning: `utility.has_collection` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  if utility.has_collection(self.collection_name, using=self.alias):\n",
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:401: PyMilvusDeprecationWarning: `Collection` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  self.col = Collection(\n",
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:430: PyMilvusDeprecationWarning: `Collection.indexes` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  for x in self.col.indexes:\n",
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:450: PyMilvusDeprecationWarning: `Collection.create_index` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  self.col.create_index(\n",
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:432: PyMilvusDeprecationWarning: `Index.to_dict` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  return x.to_dict()\n",
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:506: PyMilvusDeprecationWarning: `utility.load_state` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  and utility.load_state(self.collection_name, using=self.alias)\n",
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:509: PyMilvusDeprecationWarning: `Collection.load` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  self.col.load(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully stored 21 documents in Milvus collection 'car_info_collection'.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:629: PyMilvusDeprecationWarning: `Collection.insert` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  res = self.col.insert(insert_list, timeout=timeout, **kwargs)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 第四步 ：向量化（Milvus）\n",
+    "from langchain_community.vectorstores import Milvus\n",
+    "from langchain_community.embeddings import DashScopeEmbeddings\n",
+    "\n",
+    "embedding_model = DashScopeEmbeddings(\n",
+    "    model=\"text-embedding-v3\",\n",
+    "    dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\")\n",
+    ")\n",
+    "\n",
+    "vectorstore = Milvus.from_documents(\n",
+    "  valid_docs,\n",
+    "  embedding_model,\n",
+    "  connection_args={\"uri\": \"http://127.0.0.1:19530\"},\n",
+    "  collection_name=\"hr_management_instructions_collection\",\n",
+    ")\n",
+    "\n",
+    "print(f\"Successfully stored {len(valid_docs)} documents in Milvus collection 'car_info_collection'.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46744f15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.retrievers import BM25Retriever\n",
+    "from langchain_classic.retrievers import EnsembleRetriever\n",
+    "# ========== 第五步：创建检索器 ==========\n",
+    "# retriever = vectorstore.as_retriever(search_kwargs={\"k\": 1})\n",
+    "\n",
+    "# 可优化： 混合检索\n",
+    "bm25_retriever = BM25Retriever.from_documents(valid_docs)\n",
+    "retriever = EnsembleRetriever(\n",
+    "    retrievers=[bm25_retriever, vectorstore.as_retriever(search_kwargs={\"k\": 1})],\n",
+    "    weights=[0.4, 0.6],\n",
+    "    normalize_scores=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a053e4dc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Relevant docs: [Document(metadata={'producer': '', 'creator': '', 'creationdate': '', 'source': './../../resource/人事管理流程.docx', 'file_path': './../../resource/人事管理流程.docx', 'total_pages': 13, 'format': 'Office document', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'encryption': '', 'modDate': '', 'creationDate': '', 'page': 9, 'pk': 467492688208533558}, page_content='女）去世的员工请丧假，可享受2个工作日丧假；\\n3.10.3以上假期超过部分按事假处理；\\n销假管理\\n为方便行政人事部对员工假期的管理以及考勤统计，员工需于假期结\\n束后第一个工作日内，到行政人事部进行销假处理；需要续延假期的\\n应重新办理请假手续。\\n以下行为，公司有权予以辞退处理并不支付任何经济补偿金：\\n迟到/早退次数：月度合计达到5次者，或季度合计达到12次者，或半\\n年达到20次者，或年度达到36次及以上者，公司有权单方面予以辞退\\n处理。\\n员工（含试用期）月度事假累计5天，或季度累计10天，或半年累计')]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:784: PyMilvusDeprecationWarning: `Collection.search` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
+      "  res = self.col.search(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# ========== 第六步：提问 ==========\n",
+    "query = \"我这个月请了5天假，会被辞退吗？\"\n",
+    "\n",
+    "# 查询侧优化ß\n",
+    "\n",
+    "relevant_docs = retriever.invoke(query)\n",
+    "print(f\"Relevant docs: {relevant_docs}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c73b682e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "是的，会被辞退。  \n",
+      "原文明确指出：“员工（含试用期）月度事假累计5天……公司有权予以辞退处理并不支付任何经济补偿金”。\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_community.chat_models import ChatTongyi\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "\n",
+    "# ========== 第七步：生成回答 ==========\n",
+    "context = \"\\n\\n---\\n\\n\".join([d.page_content for d in relevant_docs])\n",
+    "\n",
+    "prompt = ChatPromptTemplate.from_template(\"\"\"\n",
+    "你是一个专业的知识库助手。请根据以下上下文回答问题。\n",
+    "\n",
+    "**规则：**\n",
+    "- 只基于提供的上下文回答，不要编造\n",
+    "- 如果上下文中没有相关信息，直接说「根据现有资料，我找不到这个问题的答案」\n",
+    "- 回答要简洁直接，引用原文时用引号\n",
+    "\n",
+    "**上下文：**\n",
+    "{context}\n",
+    "\n",
+    "**问题：**\n",
+    "{question}\n",
+    "\"\"\")\n",
+    "\n",
+    "llm = ChatTongyi(model=\"qwen-plus\", dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\"))\n",
+    "chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "answer = chain.invoke({\"context\": context, \"question\": query})\n",
+    "print(answer)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "01_LANGCHAIN",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0rc2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}