{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6b4b5b35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86fc8f30",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyMuPDFLoader\n",
    "\n",
    "# ========== 第一步：加载文档 ==========\n",
    "loader = PyMuPDFLoader(\"./../../resource/人事管理流程.docx\")\n",
    "pages = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12a71183",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ========== 第二步：清洗数据（可选，根据文档质量决定）==========\n",
    "# clean_pages = [clean_pdf_text(page) for page in pages]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96e8b2ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "# ========== 第三步：递归分块 ==========\n",
    "splitter = RecursiveCharacterTextSplitter(\n",
    "    # 分隔符优先级：段落 → 换行 → 句号 → 空格 → 硬切\n",
    "    separators=[\"\\n\\n\", \"\\n\", \"。\", \"！\", \"？\", \" \", \"\"],\n",
    "    # 每个块最大 100 字符\n",
    "    chunk_size=500,\n",
    "    # 相邻块重叠 10 字符（chunk_size 的 20%）\n",
    "    chunk_overlap=100,\n",
    "    # 长度计算函数\n",
    "    length_function=len\n",
    ")\n",
    "\n",
    "# (可优化点： 语义分块)\n",
    "\n",
    "\n",
    "valid_docs = splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3c6d3e2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:213: PyMilvusDeprecationWarning: `utility.has_collection` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  if utility.has_collection(self.collection_name, using=self.alias):\n",
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:401: PyMilvusDeprecationWarning: `Collection` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  self.col = Collection(\n",
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:430: PyMilvusDeprecationWarning: `Collection.indexes` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  for x in self.col.indexes:\n",
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:450: PyMilvusDeprecationWarning: `Collection.create_index` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  self.col.create_index(\n",
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:432: PyMilvusDeprecationWarning: `Index.to_dict` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  return x.to_dict()\n",
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:506: PyMilvusDeprecationWarning: `utility.load_state` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  and utility.load_state(self.collection_name, using=self.alias)\n",
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:509: PyMilvusDeprecationWarning: `Collection.load` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  self.col.load(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully stored 21 documents in Milvus collection 'car_info_collection'.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:629: PyMilvusDeprecationWarning: `Collection.insert` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  res = self.col.insert(insert_list, timeout=timeout, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "# 第四步 ：向量化（Milvus）\n",
    "from langchain_community.vectorstores import Milvus\n",
    "from langchain_community.embeddings import DashScopeEmbeddings\n",
    "\n",
    "embedding_model = DashScopeEmbeddings(\n",
    "    model=\"text-embedding-v3\",\n",
    "    dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\")\n",
    ")\n",
    "\n",
    "vectorstore = Milvus.from_documents(\n",
    "  valid_docs,\n",
    "  embedding_model,\n",
    "  connection_args={\"uri\": \"http://127.0.0.1:19530\"},\n",
    "  collection_name=\"hr_management_instructions_collection\",\n",
    ")\n",
    "\n",
    "print(f\"Successfully stored {len(valid_docs)} documents in Milvus collection 'car_info_collection'.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46744f15",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.retrievers import BM25Retriever\n",
    "from langchain_classic.retrievers import EnsembleRetriever\n",
    "# ========== 第五步：创建检索器 ==========\n",
    "# retriever = vectorstore.as_retriever(search_kwargs={\"k\": 1})\n",
    "\n",
    "# 可优化： 混合检索\n",
    "bm25_retriever = BM25Retriever.from_documents(valid_docs)\n",
    "retriever = EnsembleRetriever(\n",
    "    retrievers=[bm25_retriever, vectorstore.as_retriever(search_kwargs={\"k\": 1})],\n",
    "    weights=[0.4, 0.6],\n",
    "    normalize_scores=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a053e4dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Relevant docs: [Document(metadata={'producer': '', 'creator': '', 'creationdate': '', 'source': './../../resource/人事管理流程.docx', 'file_path': './../../resource/人事管理流程.docx', 'total_pages': 13, 'format': 'Office document', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'encryption': '', 'modDate': '', 'creationDate': '', 'page': 9, 'pk': 467492688208533558}, page_content='女）去世的员工请丧假，可享受2个工作日丧假；\\n3.10.3以上假期超过部分按事假处理；\\n销假管理\\n为方便行政人事部对员工假期的管理以及考勤统计，员工需于假期结\\n束后第一个工作日内，到行政人事部进行销假处理；需要续延假期的\\n应重新办理请假手续。\\n以下行为，公司有权予以辞退处理并不支付任何经济补偿金：\\n迟到/早退次数：月度合计达到5次者，或季度合计达到12次者，或半\\n年达到20次者，或年度达到36次及以上者，公司有权单方面予以辞退\\n处理。\\n员工（含试用期）月度事假累计5天，或季度累计10天，或半年累计')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:784: PyMilvusDeprecationWarning: `Collection.search` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n",
      "  res = self.col.search(\n"
     ]
    }
   ],
   "source": [
    "# ========== 第六步：提问 ==========\n",
    "query = \"我这个月请了5天假，会被辞退吗？\"\n",
    "\n",
    "# 查询侧优化ß\n",
    "\n",
    "relevant_docs = retriever.invoke(query)\n",
    "print(f\"Relevant docs: {relevant_docs}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c73b682e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "是的，会被辞退。  \n",
      "原文明确指出：“员工（含试用期）月度事假累计5天……公司有权予以辞退处理并不支付任何经济补偿金”。\n"
     ]
    }
   ],
   "source": [
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_community.chat_models import ChatTongyi\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "\n",
    "\n",
    "# ========== 第七步：生成回答 ==========\n",
    "context = \"\\n\\n---\\n\\n\".join([d.page_content for d in relevant_docs])\n",
    "\n",
    "prompt = ChatPromptTemplate.from_template(\"\"\"\n",
    "你是一个专业的知识库助手。请根据以下上下文回答问题。\n",
    "\n",
    "**规则：**\n",
    "- 只基于提供的上下文回答，不要编造\n",
    "- 如果上下文中没有相关信息，直接说「根据现有资料，我找不到这个问题的答案」\n",
    "- 回答要简洁直接，引用原文时用引号\n",
    "\n",
    "**上下文：**\n",
    "{context}\n",
    "\n",
    "**问题：**\n",
    "{question}\n",
    "\"\"\")\n",
    "\n",
    "llm = ChatTongyi(model=\"qwen-plus\", dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\"))\n",
    "chain = prompt | llm | StrOutputParser()\n",
    "\n",
    "answer = chain.invoke({\"context\": context, \"question\": query})\n",
    "print(answer)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "01_LANGCHAIN",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}