zengli
/
coding-agent


			
							1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996
							<!doctype html>
<html lang="zh" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
        <link rel="prev" href="4.10%20RAG%E4%B8%AD%E7%9A%84Query%E6%94%B9%E5%86%99%28%E6%89%A9%E5%B1%95%E8%B5%84%E6%96%99%29.html">
      
      
        <link rel="next" href="../%E2%91%A4%20RAG%E7%B3%BB%E7%BB%9F%E8%AF%84%E4%BC%B0/5.1%20RAG%E7%B3%BB%E7%BB%9F%E8%AF%84%E4%BC%B0%E5%B7%A5%E5%85%B7.html">
      
      
      <link rel="icon" href="../img/AI.jpg">
      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.21">
    
    
        <title>数据集生成与优化(扩展资料) - 企业知识库RAG智能问答项目V1.0</title>
      
    
      <link rel="stylesheet" href="../assets/stylesheets/main.2a3383ac.min.css">
      
      
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&amp;display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
      <link rel="stylesheet" href="../stylesheets/extra.css">
    
    <script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      
  </head>
  
  
    <body dir="ltr">
  
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#_1" class="md-skip">
          跳转至
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
    
<header class="md-header md-header--shadow" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="页眉">
    <a href="../index.html" title="企业知识库RAG智能问答项目V1.0" class="md-header__button md-logo" aria-label="企业知识库RAG智能问答项目V1.0" data-md-component="logo">
      
  <img src="../img/AI.jpg" alt="logo">

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      
      <svg xmlns="http://www.w3.org/2000/svg" viewbox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"></path></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            企业知识库RAG智能问答项目V1.0
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              数据集生成与优化(扩展资料)
            
          </span>
        </div>
      </div>
    </div>
    
    
      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
    
    
        <label class="md-header__button md-icon" for="__search">
          
          <svg xmlns="http://www.w3.org/2000/svg" viewbox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"></path></svg>
        </label>
        <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="搜索" placeholder="搜索" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewbox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"></path></svg>
        
        <svg xmlns="http://www.w3.org/2000/svg" viewbox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"></path></svg>
      </label>
      <nav class="md-search__options" aria-label="查找">
        
        <button type="reset" class="md-search__icon md-icon" title="清空当前内容" aria-label="清空当前内容" tabindex="-1">
          
          <svg xmlns="http://www.w3.org/2000/svg" viewbox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"></path></svg>
        </button>
      </nav>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            正在初始化搜索引擎
          </div>
          <ol class="md-search-result__list" role="presentation"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
      
    
  </nav>
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--primary" aria-label="导航栏" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../index.html" title="企业知识库RAG智能问答项目V1.0" class="md-nav__button md-logo" aria-label="企业知识库RAG智能问答项目V1.0" data-md-component="logo">
      
  <img src="../img/AI.jpg" alt="logo">

    </a>
    企业知识库RAG智能问答项目V1.0
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
    <li class="md-nav__item">
      <a href="../index.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    知识库内容
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2">
        
          
          <label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    ① 项目概述
    
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_2">
            <span class="md-nav__icon md-icon"></span>
            ① 项目概述
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../%E2%91%A0%20%E9%A1%B9%E7%9B%AE%E6%A6%82%E8%BF%B0/1.1%20%E9%A1%B9%E7%9B%AE%E6%9E%B6%E6%9E%84.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    1.1 项目架构
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A0%20%E9%A1%B9%E7%9B%AE%E6%A6%82%E8%BF%B0/1.2%20%E9%A1%B9%E7%9B%AE%E7%8E%AF%E5%A2%83%28window%29.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    1.2 项目环境(window)
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A0%20%E9%A1%B9%E7%9B%AE%E6%A6%82%E8%BF%B0/1.3%20%E9%A1%B9%E7%9B%AE%E7%8E%AF%E5%A2%83%28mac%29.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    1.3 项目环境(mac)
    
  </span>
  

      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3">
        
          
          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    ② 项目工具
    
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_3">
            <span class="md-nav__icon md-icon"></span>
            ② 项目工具
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
              
                
    <li class="md-nav__item">
      <a href="../%E2%91%A1%20%E9%A1%B9%E7%9B%AE%E5%B7%A5%E5%85%B7/2.1%20Ollama%E5%A4%A7%E6%A8%A1%E5%9E%8B%E9%AB%98%E6%95%88%E7%AE%A1%E7%90%86%E5%B7%A5%E5%85%B7.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    2.1 Ollama大模型高效管理工具
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A1%20%E9%A1%B9%E7%9B%AE%E5%B7%A5%E5%85%B7/2.2%20LangChain%E5%9F%BA%E7%A1%80%E7%9F%A5%E8%AF%86.md.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    2.2 LangChain基础知识.md
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A1%20%E9%A1%B9%E7%9B%AE%E5%B7%A5%E5%85%B7/2.3%20%E5%90%91%E9%87%8F%E7%B4%A2%E5%BC%95%EF%BC%9AHNSW%E3%80%81IVF%E4%B8%8EIVF_PQ.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    2.3 向量索引：HNSW、IVF与IVF_PQ
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A1%20%E9%A1%B9%E7%9B%AE%E5%B7%A5%E5%85%B7/2.4%20Milvus%E5%90%91%E9%87%8F%E6%95%B0%E6%8D%AE%E5%BA%93.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    2.4 Milvus向量数据库
    
  </span>
  

      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4">
        
          
          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    ③ 基于Mysql库的问答系统
    
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_4">
            <span class="md-nav__icon md-icon"></span>
            ③ 基于Mysql库的问答系统
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../%E2%91%A2%20%E5%9F%BA%E4%BA%8EMysql%E5%BA%93%E7%9A%84%E9%97%AE%E7%AD%94%E7%B3%BB%E7%BB%9F/3.1%20Python%E6%97%A5%E5%BF%97%E4%BB%8B%E7%BB%8D%E4%B8%8E%E5%BA%94%E7%94%A8.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    3.1 Python日志介绍与应用
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A2%20%E5%9F%BA%E4%BA%8EMysql%E5%BA%93%E7%9A%84%E9%97%AE%E7%AD%94%E7%B3%BB%E7%BB%9F/3.2%20Redis%E6%95%B0%E6%8D%AE%E5%BA%93%E7%AE%80%E4%BB%8B%E4%B8%8E%E4%BD%BF%E7%94%A8.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    3.2 Redis数据库简介与使用
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A2%20%E5%9F%BA%E4%BA%8EMysql%E5%BA%93%E7%9A%84%E9%97%AE%E7%AD%94%E7%B3%BB%E7%BB%9F/3.3%20%E5%9F%BA%E4%BA%8EMysql%E6%95%B0%E6%8D%AE%E5%BA%93%E5%AE%9E%E7%8E%B0%E9%97%AE%E7%AD%94%E7%B3%BB%E7%BB%9F.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    3.3 基于Mysql数据库实现问答系统
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A2%20%E5%9F%BA%E4%BA%8EMysql%E5%BA%93%E7%9A%84%E9%97%AE%E7%AD%94%E7%B3%BB%E7%BB%9F/3.4%20BM25%E7%AE%97%E6%B3%95%E7%AE%80%E4%BB%8B%E4%B8%8E%E5%BA%94%E7%94%A8.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    3.4 BM25算法简介与应用
    
  </span>
  

      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--active md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" checked>
        
          
          <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    ④ 基于Milvus库的问答系统
    
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="true">
          <label class="md-nav__title" for="__nav_5">
            <span class="md-nav__icon md-icon"></span>
            ④ 基于Milvus库的问答系统
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="4.1%20%E6%95%B4%E4%BD%93%E6%9E%B6%E6%9E%84%E4%B8%8E%E5%B7%A5%E7%A8%8B%E6%B5%81%E7%A8%8B.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.1 整体架构与工程流程
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="4.2%20%E5%9F%BA%E7%A1%80%E6%A8%A1%E5%9D%97.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.2 基础模块
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="4.3%20%E6%96%87%E6%A1%A3%E5%A4%84%E7%90%86%E6%A8%A1%E5%9D%97.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.3 文档处理模块
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="4.4%20%E5%90%91%E9%87%8F%E5%AD%98%E5%82%A8.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.4 向量存储
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item"><a href="4.5%20BERT%E5%9F%BA%E7%A1%80%E4%B8%8E%E4%BB%BB%E5%8A%A1%E5%BE%AE%E8%B0%83.html" class="md-nav__link"><span class="md-ellipsis">4.5 BERT基础与任务微调</span></a></li><li class="md-nav__item">
      <a href="4.6%20prompts%E8%AE%BE%E8%AE%A1%E4%B8%8Equery%E6%84%8F%E5%9B%BE%E8%AF%86%E5%88%AB.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.6 prompts设计与query意图识别
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="4.7%20%E6%A3%80%E7%B4%A2%E7%AD%96%E7%95%A5%E4%B8%8ERAG%E7%B3%BB%E7%BB%9F%E8%AE%BE%E8%AE%A1.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.7 检索策略与RAG系统设计
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="4.8%20RAG%E7%B3%BB%E7%BB%9F%E8%BF%90%E8%A1%8C.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.8 RAG系统运行
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="4.9%20%E6%96%87%E6%A1%A3%E8%A7%A3%E6%9E%90%E5%B7%A5%E5%85%B7%28%E6%89%A9%E5%B1%95%E8%B5%84%E6%96%99%29.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.9 文档解析工具(扩展资料)
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="4.10%20RAG%E4%B8%AD%E7%9A%84Query%E6%94%B9%E5%86%99%28%E6%89%A9%E5%B1%95%E8%B5%84%E6%96%99%29.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4.10 RAG中的Query改写(扩展资料)
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          
  
  <span class="md-ellipsis">
    数据集生成与优化(扩展资料)
    
  </span>
  

          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="%E6%95%B0%E6%8D%AE%E9%9B%86%E7%94%9F%E6%88%90%E4%B8%8E%E4%BC%98%E5%8C%96%28%E6%89%A9%E5%B1%95%E8%B5%84%E6%96%99%29.html" class="md-nav__link md-nav__link--active">
        
  
  <span class="md-ellipsis">
    数据集生成与优化(扩展资料)
    
  </span>
  

      </a>
      
        
<nav class="md-nav md-nav--secondary" aria-label="目录">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      目录
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#_1" class="md-nav__link">
    <span class="md-ellipsis">
      学习目标
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_2" class="md-nav__link">
    <span class="md-ellipsis">
      一、数据生成与优化
    </span>
  </a>
  
    <nav class="md-nav" aria-label="一、数据生成与优化">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#1" class="md-nav__link">
    <span class="md-ellipsis">
      1 功能概述
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#2" class="md-nav__link">
    <span class="md-ellipsis">
      2 完整代码
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#3" class="md-nav__link">
    <span class="md-ellipsis">
      3 实现细节
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#4" class="md-nav__link">
    <span class="md-ellipsis">
      4 说明
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#5" class="md-nav__link">
    <span class="md-ellipsis">
      5 执行示例
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_3" class="md-nav__link">
    <span class="md-ellipsis">
      二、数据集生成流程
    </span>
  </a>
  
    <nav class="md-nav" aria-label="二、数据集生成流程">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#1_1" class="md-nav__link">
    <span class="md-ellipsis">
      1 生成流程
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#2_1" class="md-nav__link">
    <span class="md-ellipsis">
      2 代码示例（使用数据集）
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_4" class="md-nav__link">
    <span class="md-ellipsis">
      三、数据集特点与优化
    </span>
  </a>
  
    <nav class="md-nav" aria-label="三、数据集特点与优化">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#1_2" class="md-nav__link">
    <span class="md-ellipsis">
      1 数据集特点
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#2_2" class="md-nav__link">
    <span class="md-ellipsis">
      2 优化效果
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_5" class="md-nav__link">
    <span class="md-ellipsis">
      章节小结
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6">
        
          
          <label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    ⑤ RAG系统评估
    
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_6">
            <span class="md-nav__icon md-icon"></span>
            ⑤ RAG系统评估
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../%E2%91%A4%20RAG%E7%B3%BB%E7%BB%9F%E8%AF%84%E4%BC%B0/5.1%20RAG%E7%B3%BB%E7%BB%9F%E8%AF%84%E4%BC%B0%E5%B7%A5%E5%85%B7.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    5.1 RAG系统评估工具
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A4%20RAG%E7%B3%BB%E7%BB%9F%E8%AF%84%E4%BC%B0/5.2%20%E5%AE%9E%E7%8E%B0EnterpriseKB-RAG%E7%B3%BB%E7%BB%9F%E7%9A%84%E8%AF%84%E4%BC%B0.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    5.2 实现企业知识库RAG系统的评估
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A4%20RAG%E7%B3%BB%E7%BB%9F%E8%AF%84%E4%BC%B0/5.3%20RAG%E8%AF%84%E4%BC%B0%E6%95%B0%E6%8D%AE%E9%9B%86%E7%9A%84%E6%9E%84%E9%80%A0%28%E6%89%A9%E5%B1%95%29.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    5.3 RAG评估数据集的构造(扩展)
    
  </span>
  

      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7">
        
          
          <label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    ⑥ 融合Mysql的RAG系统
    
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_7">
            <span class="md-nav__icon md-icon"></span>
            ⑥ 融合Mysql的RAG系统
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../%E2%91%A5%20%E8%9E%8D%E5%90%88Mysql%E7%9A%84RAG%E7%B3%BB%E7%BB%9F/6.1%20%E8%9E%8D%E5%90%88FQA%E5%92%8C%E7%9F%A5%E8%AF%86%E5%BA%93%E6%9F%A5%E8%AF%A2.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    6.1 融合FQA和知识库查询
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A5%20%E8%9E%8D%E5%90%88Mysql%E7%9A%84RAG%E7%B3%BB%E7%BB%9F/6.2%20%E8%9E%8D%E5%90%88FAQ%20%E5%92%8C%E7%9F%A5%E8%AF%86%E5%BA%93%E6%9F%A5%E8%AF%A2%EF%BC%88%E4%BC%98%E5%8C%96%E7%89%88%EF%BC%89.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    6.2 融合FAQ 和知识库查询（优化版）
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A5%20%E8%9E%8D%E5%90%88Mysql%E7%9A%84RAG%E7%B3%BB%E7%BB%9F/6.3%20%E5%9F%BA%E4%BA%8E%20FastAPI%20%E7%9A%84%20FAQ%20%E4%B8%8E%E7%9F%A5%E8%AF%86%E5%BA%93%E6%9F%A5%E8%AF%A2%E6%8E%A5%E5%8F%A3.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    6.3 基于 FastAPI 的 FAQ 与知识库查询接口
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A5%20%E8%9E%8D%E5%90%88Mysql%E7%9A%84RAG%E7%B3%BB%E7%BB%9F/6.4%20FastAPI%E5%92%8CFlask%E7%9A%84%E5%8C%BA%E5%88%AB%E5%92%8C%E8%81%94%E7%B3%BB%28%E6%89%A9%E5%B1%95%E8%B5%84%E6%96%99%29.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    6.4 FastAPI和Flask的区别和联系(扩展资料)
    
  </span>
  

      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

    <li class="md-nav__item md-nav__item--nested">
      
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8">
        
          
          <label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    ⑦ 企业级生产环境部署
    
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_8">
            <span class="md-nav__icon md-icon"></span>
            ⑦ 企业级生产环境部署
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
    <li class="md-nav__item">
      <a href="../%E2%91%A6%20%E4%BC%81%E4%B8%9A%E7%BA%A7%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E9%83%A8%E7%BD%B2/7.1%20Docker%E7%9A%84%E5%8E%9F%E7%90%86%E5%92%8C%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    7.1 Docker的原理和基本使用
    
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../%E2%91%A6%20%E4%BC%81%E4%B8%9A%E7%BA%A7%E7%94%9F%E4%BA%A7%E7%8E%AF%E5%A2%83%E9%83%A8%E7%BD%B2/7.2%20%E6%9C%8D%E5%8A%A1%E9%83%A8%E7%BD%B2.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    7.2 服务部署
    
  </span>
  

      </a>
    </li>
  

          </ul>
        </nav>
      
    </li>
  

  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--secondary" aria-label="目录">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      目录
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#_1" class="md-nav__link">
    <span class="md-ellipsis">
      学习目标
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_2" class="md-nav__link">
    <span class="md-ellipsis">
      一、数据生成与优化
    </span>
  </a>
  
    <nav class="md-nav" aria-label="一、数据生成与优化">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#1" class="md-nav__link">
    <span class="md-ellipsis">
      1 功能概述
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#2" class="md-nav__link">
    <span class="md-ellipsis">
      2 完整代码
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#3" class="md-nav__link">
    <span class="md-ellipsis">
      3 实现细节
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#4" class="md-nav__link">
    <span class="md-ellipsis">
      4 说明
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#5" class="md-nav__link">
    <span class="md-ellipsis">
      5 执行示例
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_3" class="md-nav__link">
    <span class="md-ellipsis">
      二、数据集生成流程
    </span>
  </a>
  
    <nav class="md-nav" aria-label="二、数据集生成流程">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#1_1" class="md-nav__link">
    <span class="md-ellipsis">
      1 生成流程
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#2_1" class="md-nav__link">
    <span class="md-ellipsis">
      2 代码示例（使用数据集）
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_4" class="md-nav__link">
    <span class="md-ellipsis">
      三、数据集特点与优化
    </span>
  </a>
  
    <nav class="md-nav" aria-label="三、数据集特点与优化">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#1_2" class="md-nav__link">
    <span class="md-ellipsis">
      1 数据集特点
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#2_2" class="md-nav__link">
    <span class="md-ellipsis">
      2 优化效果
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#_5" class="md-nav__link">
    <span class="md-ellipsis">
      章节小结
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
                  </div>
                </div>
              </div>
            
          
            <div class="md-content" data-md-component="content">
              <article class="md-content__inner md-typeset">
                
                  
  <h1>数据集生成与优化(扩展资料)</h1>

<h2 id="_1">学习目标<a class="headerlink" href="#_1" title="Permanent link">¶</a></h2>
<ul>
<li>1.掌握如何结合规则模板和 Qwen-Plus 模型生成高质量查询数据集。</li>
<li>2.通过 tqdm 进度条监控生成进度，并实现分阶段数据保存。</li>
</ul>
<hr>
<p><code>generate_query_dataset_hybrid.py</code> 是 EnterpriseKB-RAG 系统中用于生成训练数据集的核心脚本，旨在为 <code>QueryClassifier</code> 提供 6000 条高质量数据（“通用知识”和“企业知识”各 3000 条）。通过规则模板和 Qwen-Plus 模型的混合生成，结合进度条和分阶段保存，本脚本确保了生成效率和数据可靠性。本章节将详细讲解其功能、实现和应用。</p>
<h2 id="_2">一、数据生成与优化<a class="headerlink" href="#_2" title="Permanent link">¶</a></h2>
<h3 id="1">1 功能概述<a class="headerlink" href="#1" title="Permanent link">¶</a></h3>
<p><code>generate_query_dataset_hybrid.py</code> 提供以下功能：</p>
<ul>
<li><strong>规则生成</strong>：基于模板和同义词替换生成 3000 条数据（“通用知识”和“企业知识”各 1500 条）。</li>
<li><strong>大模型生成</strong>：利用 Qwen-Plus 生成 3000 条自然语言查询（各 1500 条）。</li>
<li><strong>进度监控</strong>：通过 <code>tqdm</code> 进度条可视化每个生成阶段。</li>
<li><strong>分阶段保存</strong>：每生成 1500 条数据保存一次，最终合并保存完整数据集。</li>
<li><strong>数据集输出</strong>：生成均衡的 6000 条数据，保存为 JSON 文件。</li>
</ul>
<h3 id="2">2 完整代码<a class="headerlink" href="#2" title="Permanent link">¶</a></h3>
<div class="highlight"><pre><span></span><code><span class="c1"># generate_query_dataset_hybrid.py</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">openai</span><span class="w"> </span><span class="kn">import</span> <span class="n">OpenAI</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dotenv</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_dotenv</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">tqdm</span><span class="w"> </span><span class="kn">import</span> <span class="n">tqdm</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>

<span class="c1"># 加载环境变量</span>
<span class="n">load_dotenv</span><span class="p">()</span>

<span class="c1"># 初始化Qwen-Plus客户端</span>
<span class="n">client</span> <span class="o">=</span> <span class="n">OpenAI</span><span class="p">(</span>
    <span class="n">api_key</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">getenv</span><span class="p">(</span><span class="s2">"DASHSCOPE_API_KEY"</span><span class="p">),</span>
    <span class="n">base_url</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">getenv</span><span class="p">(</span><span class="s2">"DASHSCOPE_BASE_URL"</span><span class="p">,</span> <span class="s2">"https://dashscope.aliyuncs.com/compatible-mode/v1"</span><span class="p">),</span>
<span class="p">)</span>

<span class="c1"># 同义词词典</span>
<span class="n">synonym_dict</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s2">"什么"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"什么"</span><span class="p">,</span> <span class="s2">"啥是"</span><span class="p">,</span> <span class="s2">"如何解释"</span><span class="p">,</span> <span class="s2">"具体是什么"</span><span class="p">,</span> <span class="s2">"请解释"</span><span class="p">],</span>
    <span class="s2">"知识文档"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"知识文档"</span><span class="p">,</span> <span class="s2">"内部赋能知识文档"</span><span class="p">,</span> <span class="s2">"制度安排"</span><span class="p">,</span> <span class="s2">"教学内容"</span><span class="p">,</span> <span class="s2">"课"</span><span class="p">],</span>
    <span class="s2">"报销标准"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"报销标准"</span><span class="p">,</span> <span class="s2">"费用"</span><span class="p">,</span> <span class="s2">"申请费"</span><span class="p">,</span> <span class="s2">"学习费用"</span><span class="p">,</span> <span class="s2">"价格"</span><span class="p">,</span> <span class="s2">"收费"</span><span class="p">],</span>
    <span class="s2">"大纲"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"大纲"</span><span class="p">,</span> <span class="s2">"知识库内容"</span><span class="p">,</span> <span class="s2">"教学计划"</span><span class="p">,</span> <span class="s2">"讲义"</span><span class="p">],</span>
    <span class="s2">"专家支持"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"专家支持"</span><span class="p">,</span> <span class="s2">"教师团队"</span><span class="p">,</span> <span class="s2">"讲师阵容"</span><span class="p">,</span> <span class="s2">"专家支持力量"</span><span class="p">,</span> <span class="s2">"支持人员"</span><span class="p">,</span> <span class="s2">"讲师"</span><span class="p">],</span>
    <span class="s2">"内部赋能"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"内部赋能"</span><span class="p">,</span> <span class="s2">"辅导"</span><span class="p">,</span> <span class="s2">"工作计划"</span><span class="p">,</span> <span class="s2">"知识库知识文档"</span><span class="p">],</span>
    <span class="s2">"在哪里"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"在哪里"</span><span class="p">,</span> <span class="s2">"位于何处"</span><span class="p">,</span> <span class="s2">"设在何地"</span><span class="p">,</span> <span class="s2">"在哪"</span><span class="p">],</span>
    <span class="s2">"介绍"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"介绍"</span><span class="p">,</span> <span class="s2">"说明"</span><span class="p">,</span> <span class="s2">"讲解"</span><span class="p">,</span> <span class="s2">"概述"</span><span class="p">,</span> <span class="s2">"讲讲"</span><span class="p">,</span> <span class="s2">"说说"</span><span class="p">],</span>
    <span class="s2">"请问"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"请问"</span><span class="p">,</span> <span class="s2">"能否告知"</span><span class="p">,</span> <span class="s2">"是否可以告诉我"</span><span class="p">,</span> <span class="s2">"麻烦说下"</span><span class="p">],</span>
    <span class="s2">"原理"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"原理"</span><span class="p">,</span> <span class="s2">"基本思想"</span><span class="p">,</span> <span class="s2">"工作机制"</span><span class="p">],</span>
    <span class="s2">"写一个"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"写一个"</span><span class="p">,</span> <span class="s2">"编写一个"</span><span class="p">,</span> <span class="s2">"生成一个"</span><span class="p">,</span> <span class="s2">"创建一个"</span><span class="p">],</span>
    <span class="s2">"等于多少"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"等于多少"</span><span class="p">,</span> <span class="s2">"是多少"</span><span class="p">,</span> <span class="s2">"结果是啥"</span><span class="p">,</span> <span class="s2">"得多少"</span><span class="p">],</span>
    <span class="s2">"如何"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"如何"</span><span class="p">,</span> <span class="s2">"怎样"</span><span class="p">,</span> <span class="s2">"咋样"</span><span class="p">],</span>
    <span class="s2">"需要"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"需要"</span><span class="p">,</span> <span class="s2">"要"</span><span class="p">,</span> <span class="s2">"得有"</span><span class="p">,</span> <span class="s2">"必须具备"</span><span class="p">],</span>
    <span class="s2">"基础"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"基础"</span><span class="p">,</span> <span class="s2">"前提"</span><span class="p">,</span> <span class="s2">"背景"</span><span class="p">,</span> <span class="s2">"基本知识"</span><span class="p">]</span>
<span class="p">}</span>

<span class="k">def</span><span class="w"> </span><span class="nf">apply_synonym_variation</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">replace_prob</span><span class="o">=</span><span class="mf">0.5</span><span class="p">):</span>
<span class="w">    </span><span class="sd">"""对文本进行同义词随机替换"""</span>
    <span class="k">for</span> <span class="n">word</span><span class="p">,</span> <span class="n">synonyms</span> <span class="ow">in</span> <span class="n">synonym_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
        <span class="n">pattern</span> <span class="o">=</span> <span class="sa">r</span><span class="s2">"\b"</span> <span class="o">+</span> <span class="n">re</span><span class="o">.</span><span class="n">escape</span><span class="p">(</span><span class="n">word</span><span class="p">)</span> <span class="o">+</span> <span class="sa">r</span><span class="s2">"\b"</span>
        <span class="k">def</span><span class="w"> </span><span class="nf">repl</span><span class="p">(</span><span class="n">match</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">()</span> <span class="o">&lt;</span> <span class="n">replace_prob</span><span class="p">:</span>
                <span class="k">return</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">synonyms</span><span class="p">)</span>
            <span class="k">return</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
        <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="n">pattern</span><span class="p">,</span> <span class="n">repl</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">text</span>

<span class="c1"># 规则生成部分</span>
<span class="k">def</span><span class="w"> </span><span class="nf">generate_generic_query_rule</span><span class="p">():</span>
<span class="w">    </span><span class="sd">"""规则生成通用知识查询"""</span>
    <span class="n">templates</span> <span class="o">=</span> <span class="p">[</span>
        <span class="s2">"什么</span><span class="si">{concept}</span><span class="s2">？"</span><span class="p">,</span>
        <span class="s2">"</span><span class="si">{concept}</span><span class="s2">的定义是什么？"</span><span class="p">,</span>
        <span class="s2">"请解释</span><span class="si">{concept}</span><span class="s2">的原理。"</span><span class="p">,</span>
        <span class="s2">"如何运用</span><span class="si">{concept}</span><span class="s2">？"</span><span class="p">,</span>
        <span class="s2">"计算</span><span class="si">{num1}</span><span class="s2">+</span><span class="si">{num2}</span><span class="s2">等于多少？"</span><span class="p">,</span>
        <span class="s2">"写一个</span><span class="si">{lang}</span><span class="s2">的</span><span class="si">{func}</span><span class="s2">函数"</span><span class="p">,</span>
        <span class="s2">"为什么</span><span class="si">{thing}</span><span class="s2">是</span><span class="si">{state}</span><span class="s2">？"</span>
    <span class="p">]</span>
    <span class="n">concepts</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"AI"</span><span class="p">,</span> <span class="s2">"Transformer模型"</span><span class="p">,</span> <span class="s2">"Python"</span><span class="p">,</span> <span class="s2">"递归"</span><span class="p">,</span> <span class="s2">"算法复杂度"</span><span class="p">,</span> <span class="s2">"数据结构"</span><span class="p">,</span> <span class="s2">"机器学习"</span><span class="p">]</span>
    <span class="n">langs</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"Python"</span><span class="p">,</span> <span class="s2">"Java"</span><span class="p">,</span> <span class="s2">"C++"</span><span class="p">]</span>
    <span class="n">funcs</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"排序"</span><span class="p">,</span> <span class="s2">"计算"</span><span class="p">,</span> <span class="s2">"打印"</span><span class="p">]</span>
    <span class="n">things</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"太阳"</span><span class="p">,</span> <span class="s2">"水"</span><span class="p">,</span> <span class="s2">"风"</span><span class="p">]</span>
    <span class="n">states</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"热的"</span><span class="p">,</span> <span class="s2">"流动的"</span><span class="p">,</span> <span class="s2">"无形的"</span><span class="p">]</span>
    <span class="n">nums</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">100</span><span class="p">))</span>

    <span class="n">t</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">templates</span><span class="p">)</span>
    <span class="k">if</span> <span class="s2">"</span><span class="si">{concept}</span><span class="s2">"</span> <span class="ow">in</span> <span class="n">t</span><span class="p">:</span>
        <span class="n">replacements</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"concept"</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">concepts</span><span class="p">)}</span>
    <span class="k">elif</span> <span class="s2">"</span><span class="si">{num1}</span><span class="s2">"</span> <span class="ow">in</span> <span class="n">t</span><span class="p">:</span>
        <span class="n">replacements</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"num1"</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">nums</span><span class="p">),</span> <span class="s2">"num2"</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">nums</span><span class="p">)}</span>
    <span class="k">elif</span> <span class="s2">"</span><span class="si">{lang}</span><span class="s2">"</span> <span class="ow">in</span> <span class="n">t</span><span class="p">:</span>
        <span class="n">replacements</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"lang"</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">langs</span><span class="p">),</span> <span class="s2">"func"</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">funcs</span><span class="p">)}</span>
    <span class="k">elif</span> <span class="s2">"</span><span class="si">{thing}</span><span class="s2">"</span> <span class="ow">in</span> <span class="n">t</span><span class="p">:</span>
        <span class="n">replacements</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"thing"</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">things</span><span class="p">),</span> <span class="s2">"state"</span><span class="p">:</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">states</span><span class="p">)}</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">replacements</span> <span class="o">=</span> <span class="p">{}</span>

    <span class="n">query</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">replacements</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">apply_synonym_variation</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>

<span class="k">def</span><span class="w"> </span><span class="nf">generate_professional_query_rule</span><span class="p">():</span>
<span class="w">    </span><span class="sd">"""规则生成企业知识查询"""</span>
    <span class="n">templates</span> <span class="o">=</span> <span class="p">[</span>
        <span class="s2">"请问</span><span class="si">{subject}</span><span class="s2">知识文档的报销标准是多少？"</span><span class="p">,</span>
        <span class="s2">"</span><span class="si">{subject}</span><span class="s2">的知识文档目录是什么？"</span><span class="p">,</span>
        <span class="s2">"</span><span class="si">{subject}</span><span class="s2">内部赋能的学习周期有多长？"</span><span class="p">,</span>
        <span class="s2">"请介绍一下</span><span class="si">{subject}</span><span class="s2">内部赋能的主要项目内容。"</span><span class="p">,</span>
        <span class="s2">"请问</span><span class="si">{subject}</span><span class="s2">内部赋能地点在哪里？"</span>
    <span class="p">]</span>
    <span class="n">subjects</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"JAVA"</span><span class="p">,</span> <span class="s2">"AI"</span><span class="p">,</span> <span class="s2">"测试"</span><span class="p">,</span> <span class="s2">"Web前端"</span><span class="p">,</span> <span class="s2">"Python"</span><span class="p">,</span> <span class="s2">"大数据"</span><span class="p">,</span> <span class="s2">"DevOps"</span><span class="p">]</span>
    <span class="n">t</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">templates</span><span class="p">)</span>
    <span class="n">query</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="o">=</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">subjects</span><span class="p">))</span>
    <span class="k">return</span> <span class="n">apply_synonym_variation</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>

<span class="c1"># 大模型生成部分</span>
<span class="k">def</span><span class="w"> </span><span class="nf">generate_with_qwen</span><span class="p">(</span><span class="n">prompt</span><span class="p">):</span>
<span class="w">    </span><span class="sd">"""调用Qwen-Plus生成查询，添加超时控制"""</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="n">completion</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">chat</span><span class="o">.</span><span class="n">completions</span><span class="o">.</span><span class="n">create</span><span class="p">(</span>
            <span class="n">model</span><span class="o">=</span><span class="s2">"qwen-plus"</span><span class="p">,</span>
            <span class="n">messages</span><span class="o">=</span><span class="p">[{</span><span class="s2">"role"</span><span class="p">:</span> <span class="s2">"user"</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">prompt</span><span class="p">}],</span>
            <span class="n">temperature</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
            <span class="n">timeout</span><span class="o">=</span><span class="mi">10</span>
        <span class="p">)</span>
        <span class="k">return</span> <span class="n">completion</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Qwen-Plus调用失败: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
        <span class="k">return</span> <span class="kc">None</span>

<span class="k">def</span><span class="w"> </span><span class="nf">generate_generic_query_qwen</span><span class="p">():</span>
<span class="w">    </span><span class="sd">"""使用Qwen-Plus生成通用知识查询"""</span>
    <span class="n">prompt</span> <span class="o">=</span> <span class="s2">"""</span>
<span class="s2">    你是一个用户，生成一个“通用知识”类的查询，涉及数学计算、代码生成/纠错、概念与原理或常识性问题。</span>
<span class="s2">    示例：</span>
<span class="s2">    - “3+5等于多少？”</span>
<span class="s2">    - “写一个Python排序函数”</span>
<span class="s2">    - “什么是神经网络？”</span>
<span class="s2">    - “太阳为什么是热的？”</span>
<span class="s2">    请生成一个类似的查询，直接返回查询文本，不要多余说明。</span>
<span class="s2">    """</span>
    <span class="k">return</span> <span class="n">generate_with_qwen</span><span class="p">(</span><span class="n">prompt</span><span class="p">)</span>

<span class="k">def</span><span class="w"> </span><span class="nf">generate_professional_query_qwen</span><span class="p">():</span>
<span class="w">    </span><span class="sd">"""使用Qwen-Plus生成企业知识查询"""</span>
    <span class="n">prompt</span> <span class="o">=</span> <span class="s2">"""</span>
<span class="s2">    你是一个用户，生成一个“企业知识”类的查询，涉及IT知识库内部赋能（如知识文档详情、专家支持、费用、周期、地点等）。</span>
<span class="s2">    示例：</span>
<span class="s2">    - “JAVA知识文档费用多少？”</span>
<span class="s2">    - “AI内部赋能有哪些支持人员？”</span>
<span class="s2">    - “测试知识文档什么时候开课？”</span>
<span class="s2">    请生成一个类似的查询，直接返回查询文本，不要多余说明。</span>
<span class="s2">    """</span>
    <span class="k">return</span> <span class="n">generate_with_qwen</span><span class="p">(</span><span class="n">prompt</span><span class="p">)</span>

<span class="c1"># 保存数据集</span>
<span class="k">def</span><span class="w"> </span><span class="nf">save_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">stage_name</span><span class="p">):</span>
<span class="w">    </span><span class="sd">"""保存数据集到指定文件"""</span>
    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
        <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">stage_name</span><span class="si">}</span><span class="s2">：已保存 </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span><span class="si">}</span><span class="s2"> 条数据到 </span><span class="si">{</span><span class="n">filename</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>

<span class="k">def</span><span class="w"> </span><span class="nf">generate_training_dataset</span><span class="p">(</span><span class="n">total_samples</span><span class="o">=</span><span class="mi">6000</span><span class="p">):</span>
<span class="w">    </span><span class="sd">"""生成6000条训练数据，规则和大模型各占一半，每1500条保存"""</span>
    <span class="n">num_per_category</span> <span class="o">=</span> <span class="n">total_samples</span> <span class="o">//</span> <span class="mi">2</span>  <span class="c1"># 3000</span>
    <span class="n">num_rule</span> <span class="o">=</span> <span class="n">num_per_category</span> <span class="o">//</span> <span class="mi">2</span>  <span class="c1"># 1500</span>
    <span class="n">num_qwen</span> <span class="o">=</span> <span class="n">num_per_category</span> <span class="o">-</span> <span class="n">num_rule</span>  <span class="c1"># 1500</span>

    <span class="n">generic_samples</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="n">professional_samples</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="n">generic_set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
    <span class="n">professional_set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>

    <span class="c1"># 规则生成 - 通用知识</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">"生成规则通用知识数据..."</span><span class="p">)</span>
    <span class="k">with</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">total</span><span class="o">=</span><span class="n">num_rule</span><span class="p">,</span> <span class="n">desc</span><span class="o">=</span><span class="s2">"Rule-based Generic"</span><span class="p">)</span> <span class="k">as</span> <span class="n">pbar</span><span class="p">:</span>
        <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">generic_samples</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">num_rule</span><span class="p">:</span>
            <span class="n">q</span> <span class="o">=</span> <span class="n">generate_generic_query_rule</span><span class="p">()</span>
            <span class="k">if</span> <span class="n">q</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">generic_set</span><span class="p">:</span>
                <span class="n">generic_set</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">q</span><span class="p">)</span>
                <span class="n">generic_samples</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">q</span><span class="p">,</span> <span class="s2">"label"</span><span class="p">:</span> <span class="s2">"通用知识"</span><span class="p">})</span>
                <span class="n">pbar</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
    <span class="n">save_dataset</span><span class="p">(</span><span class="n">generic_samples</span><span class="p">,</span> <span class="s2">"rule_generic_1500.json"</span><span class="p">,</span> <span class="s2">"规则通用知识"</span><span class="p">)</span>

    <span class="c1"># 规则生成 - 企业知识</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">"生成规则企业知识数据..."</span><span class="p">)</span>
    <span class="k">with</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">total</span><span class="o">=</span><span class="n">num_rule</span><span class="p">,</span> <span class="n">desc</span><span class="o">=</span><span class="s2">"Rule-based Professional"</span><span class="p">)</span> <span class="k">as</span> <span class="n">pbar</span><span class="p">:</span>
        <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">professional_samples</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">num_rule</span><span class="p">:</span>
            <span class="n">q</span> <span class="o">=</span> <span class="n">generate_professional_query_rule</span><span class="p">()</span>
            <span class="k">if</span> <span class="n">q</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">professional_set</span><span class="p">:</span>
                <span class="n">professional_set</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">q</span><span class="p">)</span>
                <span class="n">professional_samples</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">q</span><span class="p">,</span> <span class="s2">"label"</span><span class="p">:</span> <span class="s2">"企业知识"</span><span class="p">})</span>
                <span class="n">pbar</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
    <span class="n">save_dataset</span><span class="p">(</span><span class="n">professional_samples</span><span class="p">,</span> <span class="s2">"rule_professional_1500.json"</span><span class="p">,</span> <span class="s2">"规则企业知识"</span><span class="p">)</span>

    <span class="c1"># Qwen-Plus生成 - 通用知识</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">"生成Qwen-Plus通用知识数据..."</span><span class="p">)</span>
    <span class="k">with</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">total</span><span class="o">=</span><span class="n">num_qwen</span><span class="p">,</span> <span class="n">desc</span><span class="o">=</span><span class="s2">"Qwen-based Generic"</span><span class="p">)</span> <span class="k">as</span> <span class="n">pbar</span><span class="p">:</span>
        <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">generic_samples</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">num_per_category</span><span class="p">:</span>
            <span class="n">q</span> <span class="o">=</span> <span class="n">generate_generic_query_qwen</span><span class="p">()</span>
            <span class="k">if</span> <span class="n">q</span> <span class="ow">and</span> <span class="n">q</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">generic_set</span><span class="p">:</span>
                <span class="n">generic_set</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">q</span><span class="p">)</span>
                <span class="n">generic_samples</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">q</span><span class="p">,</span> <span class="s2">"label"</span><span class="p">:</span> <span class="s2">"通用知识"</span><span class="p">})</span>
                <span class="n">pbar</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mf">0.5</span><span class="p">)</span>  <span class="c1"># 避免 API 限流</span>
    <span class="n">save_dataset</span><span class="p">(</span><span class="n">generic_samples</span><span class="p">,</span> <span class="s2">"generic_3000.json"</span><span class="p">,</span> <span class="s2">"通用知识（规则+Qwen）"</span><span class="p">)</span>

    <span class="c1"># Qwen-Plus生成 - 企业知识</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">"生成Qwen-Plus企业知识数据..."</span><span class="p">)</span>
    <span class="k">with</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">total</span><span class="o">=</span><span class="n">num_qwen</span><span class="p">,</span> <span class="n">desc</span><span class="o">=</span><span class="s2">"Qwen-based Professional"</span><span class="p">)</span> <span class="k">as</span> <span class="n">pbar</span><span class="p">:</span>
        <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">professional_samples</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">num_per_category</span><span class="p">:</span>
            <span class="n">q</span> <span class="o">=</span> <span class="n">generate_professional_query_qwen</span><span class="p">()</span>
            <span class="k">if</span> <span class="n">q</span> <span class="ow">and</span> <span class="n">q</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">professional_set</span><span class="p">:</span>
                <span class="n">professional_set</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">q</span><span class="p">)</span>
                <span class="n">professional_samples</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s2">"query"</span><span class="p">:</span> <span class="n">q</span><span class="p">,</span> <span class="s2">"label"</span><span class="p">:</span> <span class="s2">"企业知识"</span><span class="p">})</span>
                <span class="n">pbar</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mf">0.5</span><span class="p">)</span>  <span class="c1"># 避免 API 限流</span>
    <span class="n">save_dataset</span><span class="p">(</span><span class="n">professional_samples</span><span class="p">,</span> <span class="s2">"professional_3000.json"</span><span class="p">,</span> <span class="s2">"企业知识（规则+Qwen）"</span><span class="p">)</span>

    <span class="c1"># 合并并混洗</span>
    <span class="n">dataset</span> <span class="o">=</span> <span class="n">generic_samples</span> <span class="o">+</span> <span class="n">professional_samples</span>
    <span class="n">random</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
    <span class="n">final_filename</span> <span class="o">=</span> <span class="s2">"training_dataset_hybrid_6000.json"</span>
    <span class="n">save_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">final_filename</span><span class="p">,</span> <span class="s2">"最终数据集"</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">dataset</span>

<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
    <span class="n">dataset</span> <span class="o">=</span> <span class="n">generate_training_dataset</span><span class="p">(</span><span class="n">total_samples</span><span class="o">=</span><span class="mi">6000</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"成功生成 </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span><span class="si">}</span><span class="s2"> 条训练数据，保存在 training_dataset_hybrid_6000.json 文件中。"</span><span class="p">)</span>
    <span class="c1"># 输出前10条作为示例</span>
    <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">dataset</span><span class="p">[:</span><span class="mi">10</span><span class="p">]:</span>
        <span class="nb">print</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
</code></pre></div>
<h3 id="3">3 实现细节<a class="headerlink" href="#3" title="Permanent link">¶</a></h3>
<ol>
<li><strong><code>apply_synonym_variation</code></strong>：</li>
<li><strong>作用</strong>：为规则生成查询增加多样性，通过同义词替换（如“什么” -&gt; “啥是”）。</li>
<li><strong>逻辑</strong>：正则表达式匹配词边界，50% 概率替换。</li>
<li><strong><code>generate_generic_query_rule</code> 和 <code>generate_professional_query_rule</code></strong>：</li>
<li><strong>作用</strong>：基于模板生成初始数据。</li>
<li><strong>设计</strong>：覆盖数学计算、代码生成、概念问题和 IT 内部赋能场景。</li>
<li><strong><code>generate_with_qwen</code></strong>：</li>
<li><strong>作用</strong>：封装 Qwen-Plus API 调用，生成自然语言查询。</li>
<li><strong>参数</strong>：<code>temperature=0.9</code> 确保多样性，<code>timeout=10</code> 防止卡顿。</li>
<li><strong><code>generate_generic_query_qwen</code> 和 <code>generate_professional_query_qwen</code></strong>：</li>
<li><strong>作用</strong>：通过精心设计的 Prompt 指导 Qwen-Plus 生成符合类别的查询。</li>
<li><strong>逻辑</strong>：提供示例，输出简洁的查询文本。</li>
<li><strong><code>save_dataset</code></strong>：</li>
<li><strong>作用</strong>：统一保存数据集，显示阶段名称和数据量。</li>
<li><strong>实现</strong>：支持 JSON 格式，保存中间和最终结果。</li>
<li><strong><code>generate_training_dataset</code></strong>：</li>
<li><strong>作用</strong>：整合规则和大模型生成流程。</li>
<li><strong>流程</strong>：<ul>
<li>规则生成 1500 条“通用知识”，保存。</li>
<li>规则生成 1500 条“企业知识”，保存。</li>
<li>Qwen-Plus 生成 1500 条“通用知识”，累计 3000 条保存。</li>
<li>Qwen-Plus 生成 1500 条“企业知识”，累计 3000 条保存。</li>
<li>合并混洗 6000 条，保存最终数据集。</li>
</ul>
</li>
<li><strong>进度条</strong>：<code>tqdm</code> 实时显示每个阶段的生成进度。</li>
</ol>
<h3 id="4">4 说明<a class="headerlink" href="#4" title="Permanent link">¶</a></h3>
<ul>
<li><strong>混合生成</strong>：规则生成高效可控，Qwen-Plus 生成自然真实。</li>
<li><strong>分阶段保存</strong>：每 1500 条保存一次，支持断点恢复。</li>
<li><strong>进度监控</strong>：<code>tqdm</code> 提供直观反馈，优化用户体验。</li>
</ul>
<h3 id="5">5 执行示例<a class="headerlink" href="#5" title="Permanent link">¶</a></h3>
<p>运行脚本时，输出类似：
</p><div class="highlight"><pre><span></span><code>生成规则通用知识数据...
Rule-based Generic: 100%|██████████| 1500/1500 [00:02&lt;00:00, 750it/s]
规则通用知识：已保存 1500 条数据到 rule_generic_1500.json
生成规则企业知识数据...
Rule-based Professional: 100%|██████████| 1500/1500 [00:02&lt;00:00, 700it/s]
规则企业知识：已保存 1500 条数据到 rule_professional_1500.json
生成Qwen-Plus通用知识数据...
Qwen-based Generic: 100%|██████████| 1500/1500 [05:00&lt;00:00, 5it/s]
通用知识（规则+Qwen）：已保存 3000 条数据到 generic_3000.json
生成Qwen-Plus企业知识数据...
Qwen-based Professional: 100%|██████████| 1500/1500 [05:05&lt;00:00, 4.9it/s]
企业知识（规则+Qwen）：已保存 3000 条数据到 professional_3000.json
最终数据集：已保存 6000 条数据到 training_dataset_hybrid_6000.json
成功生成 6000 条训练数据，保存在 training_dataset_hybrid_6000.json 文件中。
</code></pre></div>
<hr>
<h2 id="_3">二、数据集生成流程<a class="headerlink" href="#_3" title="Permanent link">¶</a></h2>
<h3 id="1_1">1 生成流程<a class="headerlink" href="#1_1" title="Permanent link">¶</a></h3>
<ol>
<li><strong>规则生成（3000 条）</strong>：</li>
<li>生成 1500 条“通用知识”数据，保存为 <code>rule_generic_1500.json</code>。</li>
<li>生成 1500 条“企业知识”数据，保存为 <code>rule_professional_1500.json</code>.</li>
<li>使用模板和同义词替换，确保多样性。</li>
<li><strong>Qwen-Plus 生成（3000 条）</strong>：</li>
<li>生成 1500 条“通用知识”数据，累计 3000 条保存为 <code>generic_3000.json</code>。</li>
<li>生成 1500 条“企业知识”数据，累计 3000 条保存为 <code>professional_3000.json</code>.</li>
<li>通过 Prompt 控制生成质量。</li>
<li><strong>数据整合</strong>：</li>
<li>合并 6000 条数据，随机混洗。</li>
<li>保存为 <code>training_dataset_hybrid_6000.json</code>.</li>
</ol>
<h3 id="2_1">2 代码示例（使用数据集）<a class="headerlink" href="#2_1" title="Permanent link">¶</a></h3>
<div class="highlight"><pre><span></span><code><span class="c1"># core/query_classifier.py</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">sklearn.feature_extraction.text</span><span class="w"> </span><span class="kn">import</span> <span class="n">TfidfVectorizer</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">sklearn.naive_bayes</span><span class="w"> </span><span class="kn">import</span> <span class="n">MultinomialNB</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">sklearn.pipeline</span><span class="w"> </span><span class="kn">import</span> <span class="n">Pipeline</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">joblib</span>

<span class="k">class</span><span class="w"> </span><span class="nc">QueryClassifier</span><span class="p">:</span>
    <span class="k">def</span><span class="w"> </span><span class="nf">train_model</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"training_dataset_hybrid_6000.json"</span><span class="p">,</span> <span class="s2">"r"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">data</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
        <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span><span class="p">[</span><span class="s2">"query"</span><span class="p">]</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">]</span>
        <span class="n">labels</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span><span class="p">[</span><span class="s2">"label"</span><span class="p">]</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="p">([</span>
            <span class="p">(</span><span class="s2">"tfidf"</span><span class="p">,</span> <span class="n">TfidfVectorizer</span><span class="p">()),</span>
            <span class="p">(</span><span class="s2">"classifier"</span><span class="p">,</span> <span class="n">MultinomialNB</span><span class="p">()),</span>
        <span class="p">])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">labels</span><span class="p">)</span>
        <span class="n">joblib</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">,</span> <span class="s2">"query_classifier_model.pkl"</span><span class="p">)</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">"模型训练完成并保存"</span><span class="p">)</span>
</code></pre></div>
<hr>
<h2 id="_4">三、数据集特点与优化<a class="headerlink" href="#_4" title="Permanent link">¶</a></h2>
<h3 id="1_2">1 数据集特点<a class="headerlink" href="#1_2" title="Permanent link">¶</a></h3>
<ul>
<li><strong>总数</strong>：6000 条（“通用知识” 3000 条，“企业知识” 3000 条）。</li>
<li><strong>来源</strong>：规则生成 50%（3000 条），Qwen-Plus 生成 50%（3000 条）。</li>
<li><strong>多样性</strong>：</li>
<li>规则生成：模板和同义词替换覆盖多种场景。</li>
<li>Qwen-Plus 生成：自然语言查询贴近真实用户输入。</li>
<li><strong>保存机制</strong>：每 1500 条保存，支持断点续传。</li>
<li><strong>可视化</strong>：<code>tqdm</code> 进度条提供实时反馈。</li>
</ul>
<h3 id="2_2">2 优化效果<a class="headerlink" href="#2_2" title="Permanent link">¶</a></h3>
<ul>
<li><strong>高效性</strong>：简化生成流程，规则生成瞬时完成。</li>
<li><strong>可靠性</strong>：分阶段保存确保数据安全。</li>
<li><strong>分类性能</strong>：均衡数据集提升 <code>QueryClassifier</code> 准确性。</li>
<li><strong>用户体验</strong>：进度条和保存日志增强交互性。</li>
</ul>
<hr>
<h2 id="_5">章节小结<a class="headerlink" href="#_5" title="Permanent link">¶</a></h2>
<p>本章节详细介绍了优化后的 <code>generate_query_dataset_hybrid.py</code>：
- <strong>功能</strong>：混合生成 6000 条数据，规则和 Qwen-Plus 各占一半。
- <strong>实现</strong>：通过模板生成 3000 条，Qwen-Plus 生成 3000 条，每 1500 条保存，进度条监控。
- <strong>作用</strong>：为 <code>QueryClassifier</code> 提供高质量数据，优化 EnterpriseKB-RAG 系统分类能力。</p>
<p>学习者掌握了高效的数据生成和保存方法，能够为 RAG 系统提供可靠支持。</p>


              </article>
            </div>
          
          
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
    <div class="md-copyright__highlight">
      © EnterpriseKB-RAG 2026. Internal knowledge base demo site.
    </div>
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
    
      <script id="__config" type="application/json">{"base": "..", "features": [], "search": "../assets/javascripts/workers/search.973d3a69.min.js", "tags": null, "translations": {"clipboard.copied": "\u5df2\u590d\u5236", "clipboard.copy": "\u590d\u5236", "search.result.more.one": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.more.other": "\u5728\u8be5\u9875\u4e0a\u8fd8\u6709 # \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.none": "\u6ca1\u6709\u627e\u5230\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.one": "\u627e\u5230 1 \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.other": "# \u4e2a\u7b26\u5408\u6761\u4ef6\u7684\u7ed3\u679c", "search.result.placeholder": "\u952e\u5165\u4ee5\u5f00\u59cb\u641c\u7d22", "search.result.term.missing": "\u7f3a\u5c11", "select.version": "\u9009\u62e9\u5f53\u524d\u7248\u672c"}, "version": null}</script>
    
    
      <script src="../assets/javascripts/bundle.f55a23d4.min.js"></script>
      
        <script src="../javascripts/extra.js"></script>
      
        <script src="../javascripts/mathjax.js"></script>
      
        <!--        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>-->
      
        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
      
    
  </body>
</html>