{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "6b4b5b35", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import os\n", "from dotenv import load_dotenv\n", "load_dotenv()" ] }, { "cell_type": "code", "execution_count": null, "id": "86fc8f30", "metadata": {}, "outputs": [], "source": [ "from langchain_community.document_loaders import PyMuPDFLoader\n", "\n", "# ========== 第一步：加载文档 ==========\n", "loader = PyMuPDFLoader(\"./../../resource/人事管理流程.docx\")\n", "pages = loader.load()" ] }, { "cell_type": "code", "execution_count": null, "id": "12a71183", "metadata": {}, "outputs": [], "source": [ "# ========== 第二步：清洗数据（可选，根据文档质量决定）==========\n", "# clean_pages = [clean_pdf_text(page) for page in pages]" ] }, { "cell_type": "code", "execution_count": null, "id": "96e8b2ac", "metadata": {}, "outputs": [], "source": [ "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", "# ========== 第三步：递归分块 ==========\n", "splitter = RecursiveCharacterTextSplitter(\n", " # 分隔符优先级：段落 → 换行 → 句号 → 空格 → 硬切\n", " separators=[\"\\n\\n\", \"\\n\", \"。\", \"！\", \"？\", \" \", \"\"],\n", " # 每个块最大 100 字符\n", " chunk_size=500,\n", " # 相邻块重叠 10 字符（chunk_size 的 20%）\n", " chunk_overlap=100,\n", " # 长度计算函数\n", " length_function=len\n", ")\n", "\n", "# (可优化点： 语义分块)\n", "\n", "\n", "valid_docs = splitter.split_documents(pages)" ] }, { "cell_type": "code", "execution_count": 7, "id": "3c6d3e2c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:213: PyMilvusDeprecationWarning: `utility.has_collection` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " if utility.has_collection(self.collection_name, using=self.alias):\n", "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:401: PyMilvusDeprecationWarning: `Collection` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " self.col = Collection(\n", "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:430: PyMilvusDeprecationWarning: `Collection.indexes` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " for x in self.col.indexes:\n", "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:450: PyMilvusDeprecationWarning: `Collection.create_index` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " self.col.create_index(\n", "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:432: PyMilvusDeprecationWarning: `Index.to_dict` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " return x.to_dict()\n", "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:506: PyMilvusDeprecationWarning: `utility.load_state` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " and utility.load_state(self.collection_name, using=self.alias)\n", "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:509: PyMilvusDeprecationWarning: `Collection.load` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " self.col.load(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Successfully stored 21 documents in Milvus collection 'car_info_collection'.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:629: PyMilvusDeprecationWarning: `Collection.insert` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " res = self.col.insert(insert_list, timeout=timeout, **kwargs)\n" ] } ], "source": [ "# 第四步 ：向量化（Milvus）\n", "from langchain_community.vectorstores import Milvus\n", "from langchain_community.embeddings import DashScopeEmbeddings\n", "\n", "embedding_model = DashScopeEmbeddings(\n", " model=\"text-embedding-v3\",\n", " dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\")\n", ")\n", "\n", "vectorstore = Milvus.from_documents(\n", " valid_docs,\n", " embedding_model,\n", " connection_args={\"uri\": \"http://127.0.0.1:19530\"},\n", " collection_name=\"hr_management_instructions_collection\",\n", ")\n", "\n", "print(f\"Successfully stored {len(valid_docs)} documents in Milvus collection 'car_info_collection'.\")" ] }, { "cell_type": "code", "execution_count": 11, "id": "46744f15", "metadata": {}, "outputs": [], "source": [ "from langchain_community.retrievers import BM25Retriever\n", "from langchain_classic.retrievers import EnsembleRetriever\n", "# ========== 第五步：创建检索器 ==========\n", "# retriever = vectorstore.as_retriever(search_kwargs={\"k\": 1})\n", "\n", "# 可优化： 混合检索\n", "bm25_retriever = BM25Retriever.from_documents(valid_docs)\n", "retriever = EnsembleRetriever(\n", " retrievers=[bm25_retriever, vectorstore.as_retriever(search_kwargs={\"k\": 1})],\n", " weights=[0.4, 0.6],\n", " normalize_scores=True\n", ")" ] }, { "cell_type": "code", "execution_count": 12, "id": "a053e4dc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Relevant docs: [Document(metadata={'producer': '', 'creator': '', 'creationdate': '', 'source': './../../resource/人事管理流程.docx', 'file_path': './../../resource/人事管理流程.docx', 'total_pages': 13, 'format': 'Office document', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'encryption': '', 'modDate': '', 'creationDate': '', 'page': 9, 'pk': 467492688208533558}, page_content='女）去世的员工请丧假，可享受2个工作日丧假；\\n3.10.3以上假期超过部分按事假处理；\\n销假管理\\n为方便行政人事部对员工假期的管理以及考勤统计，员工需于假期结\\n束后第一个工作日内，到行政人事部进行销假处理；需要续延假期的\\n应重新办理请假手续。\\n以下行为，公司有权予以辞退处理并不支付任何经济补偿金：\\n迟到/早退次数：月度合计达到5次者，或季度合计达到12次者，或半\\n年达到20次者，或年度达到36次及以上者，公司有权单方面予以辞退\\n处理。\\n员工（含试用期）月度事假累计5天，或季度累计10天，或半年累计'), Document(metadata={'producer': '', 'creator': '', 'creationdate': '', 'source': './../../resource/人事管理流程.docx', 'file_path': './../../resource/人事管理流程.docx', 'total_pages': 13, 'format': 'Office document', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'encryption': '', 'modDate': '', 'creationDate': '', 'page': 12}, page_content='2、员工填写固定资产领用清单，并仔细阅读本细则后签署，人事行\\n政部存档。'), Document(metadata={'producer': '', 'creator': '', 'creationdate': '', 'source': './../../resource/人事管理流程.docx', 'file_path': './../../resource/人事管理流程.docx', 'total_pages': 13, 'format': 'Office document', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'encryption': '', 'modDate': '', 'creationDate': '', 'page': 6}, page_content='3.2 迟到/早退/脱岗/未准假不在岗：指员工在上班期间未履行任何手\\n续不在工作岗位的，超过 1 小时（未请假）按旷工0.5天计算；超过4\\n小时以上（未请假）则按旷工 1 天计算。\\n3.3 旷工处罚：旷工当日工资不计薪另处以当日工资双倍的罚款。\\n3.4 自动离职：员工未按公司离职流程办理离职手续且未做工作交接\\n者，视为自动离职。自动离职员工，公司有权扣发一个月工资，并记\\n入员工个人档案。\\n第七条 加班\\n公司不支持加班，因特殊原因公司安排需要加班的，请在系统提交申\\n请审批。\\n请假规定与类别\\n1、请假审批流程：\\n1.1普通员工申请请假/加班/调休：由直属领导、事业部总监、人事\\n行政\\n主管审批。抄送考勤组、请假部门、人事行政经理；\\n1.2 主管及以上人员申请请假/调休/请假≥3天：由直属领导、事业部\\n总监、人事行政经理、总经理审批。抄送考勤组、请假部门、人事行\\n政主管、股东会成员；\\n1.3原则上所有请假/加班/调休必须在钉钉系统进行申请并批准后方\\n可执行；因紧急特殊原因请假/调休，可以电话获得部门经理同意后\\n24小时内申请，逾期未申请相关审批人员可不予准假，否则视为旷\\n工。'), Document(metadata={'producer': '', 'creator': '', 'creationdate': '', 'source': './../../resource/人事管理流程.docx', 'file_path': './../../resource/人事管理流程.docx', 'total_pages': 13, 'format': 'Office document', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'encryption': '', 'modDate': '', 'creationDate': '', 'page': 1}, page_content='1天内发送未录用通知短信\\n注：在正常编制范围内的人员，不需要提交《人事需求审批表》，可\\n直接招聘，招聘工作由用人部门与人事行政部共同完成。\\n二、入职\\n1、人员入职，人事行政部为新员工办理如下手续：\\n1.1 入职人员提交资料：身份证、银行卡、毕业证、职业资格证、与\\n原单位解除或终止劳动合同证明（此项没有填写无法提供离职证明承\\n诺书）原件审核、体检报告。\\n1.2 资料审核无误 入职面谈及安排工位 签订培训告知书\\n发放入职指引 办理意外保险 进入培训期(7天)\\n通过者\\n通过者\\n签订入职承诺书、劳动合同、保密协议与竞业协议；\\n签订入职承诺书、劳动合同、保密协议与竞业协议；\\n结束劳动关系\\n结束劳动关系\\n不通过\\n不通过\\n试用与转正\\n1、试用与转正流程'), Document(metadata={'producer': '', 'creator': '', 'creationdate': '', 'source': './../../resource/人事管理流程.docx', 'file_path': './../../resource/人事管理流程.docx', 'total_pages': 13, 'format': 'Office document', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'moddate': '', 'trapped': '', 'encryption': '', 'modDate': '', 'creationDate': '', 'page': 2}, page_content='不合格\\n不合格\\n合格者\\n合格者\\n试用期1-3个月 试用期合格与否用人部门提前7天告知人事行政部 员\\n工填写转正申请走审批流程 人事行政部备案\\n员工填写离职申请走审批流程 人事行政部办理离职\\n员工申请 主管审批 人事招聘主管审批 抄送人事经理/人事主管\\n员工申请（主管岗） 人事经理审批 总经理审批 抄送人事招聘/行政\\n主管\\n2、凡有以下情形者，均被视为不符合录用条件：\\n2.1不符合招录条件：提供的学历、个人简历、工作经历、技能证\\n明、体检证明等材料或者填写的《员工登记表》等内容与事实不符或\\n有虚假的或者不按入职要求提供资料的；\\n2.2 与原用人单位未依法解除、终止劳动合同或劳动关系的；或与原\\n用人单位存在竞业限制约定且在限制范围之内的；\\n2.3体检不合格者或患有职业病、传染病、精神疾病、其他身体健康\\n条件不符合工作岗位要求的；\\n2.4被发现在外兼职对本职工作造成影响，经书面通知后仍不纠正\\n的；\\n2.5未完成工作内容、工作指标或工作任务，或者未能通过相关的工\\n作考核的；\\n2.6严重失职、营私舞弊，给用人单位利益造损失达1000元以上的；')]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/Users/widya/Documents/IT/projects-python/.venv/lib/python3.11/site-packages/langchain_community/vectorstores/milvus.py:784: PyMilvusDeprecationWarning: `Collection.search` is an ORM-style PyMilvus API and will be removed in PyMilvus 3.1. Use `MilvusClient` instead.\n", " res = self.col.search(\n" ] } ], "source": [ "# ========== 第六步：提问 ==========\n", "query = \"我这个月请了5天假，会被辞退吗？\"\n", "\n", "# 查询侧优化ß\n", "\n", "relevant_docs = retriever.invoke(query)\n", "print(f\"Relevant docs: {relevant_docs}\")" ] }, { "cell_type": "code", "execution_count": 13, "id": "c73b682e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "是的，会被辞退。 \n", "原文规定：“员工（含试用期）月度事假累计5天……公司有权予以辞退处理并不支付任何经济补偿金”。\n" ] } ], "source": [ "from langchain_core.prompts import ChatPromptTemplate\n", "from langchain_community.chat_models import ChatTongyi\n", "from langchain_core.output_parsers import StrOutputParser\n", "\n", "\n", "# ========== 第七步：生成回答 ==========\n", "context = \"\\n\\n---\\n\\n\".join([d.page_content for d in relevant_docs])\n", "\n", "prompt = ChatPromptTemplate.from_template(\"\"\"\n", "你是一个专业的知识库助手。请根据以下上下文回答问题。\n", "\n", "**规则：**\n", "- 只基于提供的上下文回答，不要编造\n", "- 如果上下文中没有相关信息，直接说「根据现有资料，我找不到这个问题的答案」\n", "- 回答要简洁直接，引用原文时用引号\n", "\n", "**上下文：**\n", "{context}\n", "\n", "**问题：**\n", "{question}\n", "\"\"\")\n", "\n", "llm = ChatTongyi(model=\"qwen-plus\", dashscope_api_key=os.getenv(\"DASHSCOPE_API_KEY\"))\n", "chain = prompt | llm | StrOutputParser()\n", "\n", "answer = chain.invoke({\"context\": context, \"question\": query})\n", "print(answer)" ] } ], "metadata": { "kernelspec": { "display_name": "01_LANGCHAIN", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.0rc2" } }, "nbformat": 4, "nbformat_minor": 5 }