agent_backend/public/content/troubleshooting.json

[
  {
    "id": "01",
    "group": "影响、日志与观测",
    "title": "先确认影响范围，再选择止损动作",
    "summary": "建立本地事故演练记录，区分接口故障、导出积压和实例不可用。",
    "objectives": [
      "用用户可观察行为定义影响范围",
      "记录统一时间线和已验证事实",
      "为止损动作写出预期、回滚与恢复门槛"
    ],
    "concepts": [
      {
        "title": "错误数量不能单独决定影响",
        "body": "一个导出接口失败和全部笔记读取失败有不同影响。记录受影响操作、用户范围、开始时间、成功率和延迟，再确定处理优先级。"
      },
      {
        "title": "先区分事实与假设",
        "body": "“新增请求都超时”是观测，“数据库挂了”是待验证假设。每次只改变一个因素，避免多个操作互相覆盖证据。"
      },
      {
        "title": "恢复服务与定位根因分开记录",
        "body": "止损可以是暂停大导出受理、回退本次配置或缩小流量；它不自动证明根因。恢复后继续核对已受理任务和数据一致性。"
      }
    ],
    "steps": [
      {
        "title": "建立独立演练目录与时间基线",
        "body": "本模块在部署模块建立的 notes-service 隔离副本上演练；Linux 单元为 notes-api.service，基础端点为 /live（进程存活）与 /ready（依赖就绪）。先在副本新建 diagnostics_lab。Linux/WSL2 命令只针对隔离目标；容器部署从 Compose 目录使用 docker compose ps 定位 service，不猜容器名。第 11 课的 SSE/WS 是 API 模块扩展路由，须先确认已加入副本。",
        "code": "mkdir -p diagnostics_lab\ndate -u +%FT%TZ\nsystemctl show notes-api.service -p ActiveState -p SubState -p MainPID\ncurl -sS --connect-timeout 1 --max-time 3 -o /dev/null -w \"live status=%{http_code} total=%{time_total}\\n\" http://127.0.0.1:8000/live\ncurl -sS --connect-timeout 1 --max-time 3 -o /dev/null -w \"ready status=%{http_code} total=%{time_total}\\n\" http://127.0.0.1:8000/ready",
        "lang": "bash"
      },
      {
        "title": "建立影响矩阵",
        "body": "保存 incident.md，把每格填成实际观测。下表是待测模板，不是真实事故结论。持久导出沿异步任务模块的 tasks_lab 命令和 export_jobs 表核对；第 01 课内存 POST /exports 只是协议练习，并未自动加入部署 API。如果另行实现了持久 HTTP 适配层，再添加对应路由；202 只表示受理。流式扩展未加入本次部署时填不适用。",
        "code": "| 操作 | 当前观测 | 受影响范围 | 最后一次正常时间 |\n|---|---|---|---|\n| GET /live | 待测 | 待测 | 待测 |\n| GET /ready | 待测 | 待测 | 待测 |\n| GET /notes | 待测 | 待测 | 待测 |\n| submit_snapshot 提交持久导出 | 待测 | 待测 | 待测 |\n| export_jobs 状态与结果文件 | 待测 | 待测 | 待测 |\n| SSE /stream 与 WS /ws | 待测 | 待测 | 待测 |",
        "lang": "text",
        "filename": "incident.md"
      },
      {
        "title": "为每次动作写四个字段",
        "body": "先用只读查询验证假设，再决定是否在隔离副本中操作。比如只暂停本地导出受理，预期减小新增积压；回滚方式是恢复之前配置。当前代码若没有受理开关，先增加并测试该开关，不能把未实现的操作写成已执行。",
        "code": "{\n  \"timestamp_utc\": \"执行时填写\",\n  \"observation\": \"填写实际现象\",\n  \"hypothesis\": \"填写待验证原因\",\n  \"action\": \"planned\",\n  \"expected_change\": \"填写应变化的指标\",\n  \"rollback\": \"填写恢复原配置的方法\",\n  \"result\": \"not_run\"\n}",
        "lang": "json",
        "filename": "action-record.json"
      },
      {
        "title": "核对止损后仍有无遗漏任务",
        "body": "只读查询数据库任务状态与最老等待。恢复判定至少覆盖健康、笔记读取、一个新导出完成、旧任务去向；不能仅以进程重新启动作为结束标准。",
        "code": "SELECT status,count(*) AS jobs,\n       extract(epoch FROM now()-min(created_at)) AS oldest_age_seconds\nFROM export_jobs GROUP BY status;\n\nSELECT count(*) AS unpublished FROM export_outbox WHERE sent_at IS NULL;",
        "lang": "sql"
      },
      {
        "title": "维护带因果关系的时间线",
        "body": "每条记录包含时间、执行人、目标、动作和结果。日志或截图引用需先脱敏；记录中的时间使用UTC，操作系统和应用时钟偏差单独注明。最后保留尚未解释的事实。",
        "code": "# incident.md 时间线模板\n# UTC时间 | 目标实例/任务ID | 实际观测 | 动作 | 结果 | 下一验证\n# 未执行动作统一标planned，不用预计结果填入实际结果列。",
        "lang": "text",
        "filename": "incident.md"
      }
    ],
    "checks": [
      "影响范围按具体用户操作列出，而非只写服务异常。",
      "每次变更有目标、预期、回滚和实际结果。",
      "恢复门槛包含业务请求与既有任务去向，未执行项保持not_run。"
    ],
    "pitfalls": [
      {
        "symptom": "看到500立即重启所有组件",
        "cause": "未判断影响位置，重启可能扩大故障并丢掉现场。",
        "fix": "先保存进程、接口和依赖观测，仅对已验证目标采取有界动作。"
      },
      {
        "symptom": "健康检查成功就宣布恢复",
        "cause": "健康检查可能只确认进程存活，没有覆盖数据库和队列。",
        "fix": "增加真实读取与导出完成验收，区分liveness和业务可用性。"
      }
    ],
    "exercise": {
      "task": "“/live正常、/notes正常、导出202正常但10分钟未完成”应怎样描述影响？",
      "hint": "把用户完成时间与请求受理状态分开。",
      "answer": "描述为导出执行链路延迟或积压，读取功能仍可用；检查outbox、队列、worker、租约，不能仅凭202判定业务正常，也不能直接推断数据库整体不可用。"
    },
    "refs": [
      {
        "label": "Google SRE事故管理",
        "url": "https://sre.google/sre-book/managing-incidents/"
      },
      {
        "label": "Google SRE有效排障",
        "url": "https://sre.google/sre-book/effective-troubleshooting/"
      }
    ],
    "production": {
      "scenario": "用户反馈导出一直等待，但笔记列表仍正常。",
      "decisions": [
        "先降低用户影响，再定位根因，两者分别留证据。",
        "一次只调整一个因素，限定实例和任务ID。",
        "业务恢复要求验证旧任务和新请求，不能只看进程状态。"
      ],
      "failureDrill": "只暂停本地练习worker，提交一条导出，按模板判断影响，再恢复worker核对任务完成。",
      "acceptance": [
        "时间线区分事实、假设与操作。",
        "没有盲目批量重启或终止进程。",
        "恢复结论有业务验证支持。"
      ]
    }
  },
  {
    "id": "02",
    "group": "影响、日志与观测",
    "title": "用结构化日志与 request_id 关联一次请求",
    "summary": "建立低泄露风险的访问日志，追踪状态、耗时和异常类型。",
    "objectives": [
      "为每个HTTP请求提供可返回的request_id",
      "用JSON结构记录路由模板与状态",
      "避免日志输出凭据、正文和任意请求头"
    ],
    "concepts": [
      {
        "title": "请求ID用于关联，不代表身份",
        "body": "接收可信网关生成的ID或自行生成；客户端输入需要限制长度和字符，不能直接拼入日志行。request_id只帮助定位请求，不能用于权限判断。"
      },
      {
        "title": "记录路由模板和必要字段",
        "body": "优先记录GET、/notes/{note_id}、状态、耗时、实例与版本。完整URL可能包含查询凭据，笔记正文可能包含敏感内容，排障不要求默认记录全部数据。"
      },
      {
        "title": "HTTP完成与业务完成不同",
        "body": "HTTP响应完成可以记录访问耗时，导出任务另记job_id和业务阶段。SSE长连接需按首段和整体生命周期分别观察，不能把202访问日志当成导出完成日志。"
      }
    ],
    "steps": [
      {
        "title": "编写JSON日志中间件",
        "body": "在隔离副本建立request_logging.py。使用ASGI包装send观察响应完成，适用于普通响应与流。只接收有限字符的X-Request-ID，否则生成新值；响应头返回该ID。",
        "code": "import json\nimport logging\nimport re\nimport time\nfrom uuid import uuid4\n\nlogger = logging.getLogger(\"notes.requests\")\n\nclass RequestLogMiddleware:\n    def __init__(self, app):\n        self.app = app\n\n    async def __call__(self, scope, receive, send):\n        if scope[\"type\"] != \"http\":\n            return await self.app(scope, receive, send)\n        headers = dict(scope.get(\"headers\", []))\n        supplied = headers.get(b\"x-request-id\", b\"\").decode(\"ascii\", \"ignore\")\n        request_id = supplied if re.fullmatch(r\"[A-Za-z0-9_-]{1,64}\", supplied) else uuid4().hex\n        scope.setdefault(\"state\", {})[\"request_id\"] = request_id\n        started = time.perf_counter()\n        status = 500\n        error_type = None\n\n        async def observe_send(message):\n            nonlocal status\n            if message[\"type\"] == \"http.response.start\":\n                status = message[\"status\"]\n                clean = [(k, v) for k, v in message.get(\"headers\", []) if k.lower() != b\"x-request-id\"]\n                message = {**message, \"headers\": clean + [(b\"x-request-id\", request_id.encode())]}\n            await send(message)\n\n        try:\n            await self.app(scope, receive, observe_send)\n        except Exception as exc:\n            error_type = type(exc).__name__\n            raise\n        finally:\n            route = getattr(scope.get(\"route\"), \"path\", \"unmatched\")\n            logger.info(json.dumps({\n                \"event\": \"http_request\", \"request_id\": request_id,\n                \"method\": scope[\"method\"], \"route\": route, \"status\": status,\n                \"duration_ms\": round((time.perf_counter()-started)*1000, 2),\n                \"error_type\": error_type,\n            }, ensure_ascii=False))",
        "lang": "python",
        "filename": "request_logging.py"
      },
      {
        "title": "注册并验证响应头",
        "body": "在现有FastAPI app上注册一次，不创建第二个app。logging格式只输出message以保持单行JSON；如果部署已经统一日志配置，应合并配置而不是覆盖所有logger。",
        "code": "import logging\nfrom request_logging import RequestLogMiddleware\n\nlogging.basicConfig(level=logging.INFO, format=\"%(message)s\")\napp.add_middleware(RequestLogMiddleware)\n\n# 启动后：\n# curl -i -H \"X-Request-ID: drill-001\" http://127.0.0.1:8000/live",
        "lang": "python",
        "filename": "app/main.py"
      },
      {
        "title": "用合成记录练习检索",
        "body": "以下仅为示例日志。保存到sample.jsonl后用rg 'drill-001'查找。真实日志中应出现实际状态和耗时，不把下例数值作为当前服务性能结论。",
        "code": "{\"event\":\"http_request\",\"request_id\":\"drill-001\",\"method\":\"GET\",\"route\":\"/notes/{note_id}\",\"status\":404,\"duration_ms\":4.2,\"error_type\":null}",
        "lang": "json",
        "filename": "sample.jsonl"
      },
      {
        "title": "验证脱敏和异常路径",
        "body": "对本地请求添加测试用Authorization和带敏感占位词的查询字符串，确认自定义日志没有这些值。只使用fake-secret，不用真实凭据。再触发404与受控500，核对request_id和error_type；网关/Uvicorn默认访问日志需要另行检查查询字符串记录策略。",
        "code": "curl -sS -H 'Authorization: Bearer fake-secret' 'http://127.0.0.1:8000/live?token=fake-secret'\nrg -n 'fake-secret|Authorization|password' sample.jsonl\n# 对实际演练日志重复检查，预期自定义JSON访问日志不包含测试秘密。",
        "lang": "bash"
      },
      {
        "title": "把任务日志接到同一业务ID",
        "body": "提交导出时记录request_id和job_id的对应关系，worker记录job_id、message_id和lease_token。不要把整个request对象塞进Celery消息；跨进程关联下一课后续用trace上下文补充。",
        "code": "def log_export_accepted(request_id: str, job_id: str):\n    logger.info(json.dumps({\n        \"event\": \"export_accepted\",\n        \"request_id\": request_id,\n        \"job_id\": job_id,\n    }))",
        "lang": "python",
        "filename": "request_logging.py"
      }
    ],
    "checks": [
      "合法request_id能在响应头和JSON访问日志中对应。",
      "不存在路径使用unmatched，不记录完整查询串。",
      "fake-secret不进入自定义日志，任务受理与HTTP请求能够关联。"
    ],
    "pitfalls": [
      {
        "symptom": "日志不是一行一个JSON",
        "cause": "异常正文或字符串拼接引入未转义换行。",
        "fix": "用json.dumps构造完整记录，按日志系统约定处理堆栈字段。"
      },
      {
        "symptom": "自定义日志脱敏但代理日志仍泄露token",
        "cause": "只检查应用logger，没有检查访问日志链路。",
        "fix": "逐层审查代理、Uvicorn与应用日志，URL不承载长期凭据。"
      }
    ],
    "exercise": {
      "task": "传入超过64字符的X-Request-ID，确认服务器生成新ID而不是直接回显。",
      "hint": "测试输入只用重复字母，不需要任何真实秘密。",
      "answer": "返回ID长度应为生成的32位hex，并可在日志关联；非法输入不会进入日志字段。"
    },
    "refs": [
      {
        "label": "Python logging",
        "url": "https://docs.python.org/3.11/library/logging.html"
      },
      {
        "label": "ASGI HTTP事件",
        "url": "https://asgi.readthedocs.io/en/latest/specs/www.html"
      },
      {
        "label": "OWASP日志安全",
        "url": "https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html"
      }
    ],
    "production": {
      "scenario": "某个用户操作失败，需要从响应定位一次请求，又不能泄露笔记和密钥。",
      "decisions": [
        "关联ID、业务ID和错误类型足以支持多数定位。",
        "日志字段采用允许列表，敏感数据不靠事后人工删除保障。",
        "请求完成与异步任务完成使用不同事件名。"
      ],
      "failureDrill": "对本地接口发送fake-secret、非法ID及受控错误，检索全部日志层的实际输出。",
      "acceptance": [
        "请求能关联，凭据不会被自定义日志保存。",
        "日志是可解析的结构化记录。",
        "示例日志与真实观测明确区分。"
      ]
    }
  },
  {
    "id": "03",
    "group": "影响、日志与观测",
    "title": "定义 SLI、SLO 与低基数指标",
    "summary": "为请求成功率、耗时和导出等待建立指标，避免每个用户或任务产生新时间序列。",
    "objectives": [
      "用用户行为定义可测量SLI",
      "实现Prometheus计数与耗时分布",
      "用路由模板等有限标签控制基数"
    ],
    "concepts": [
      {
        "title": "SLI先定义分母与好事件",
        "body": "例如笔记读取的有效请求中成功完成的比例，必须说明是否排除主动取消和客户端无效请求。SLO是时间窗口内目标，不能把单次健康检查通过当作达标。"
      },
      {
        "title": "标签组合决定时间序列数量",
        "body": "request_id、job_id、用户ID和原始URL不适合作为指标标签。使用method、路由模板、状态类别；具体请求通过日志与trace定位。"
      },
      {
        "title": "分位数需要正确聚合",
        "body": "Histogram按桶计数，可在服务实例间聚合；平均耗时可能掩盖慢请求。低流量时分位数和错误率波动大，应同时看样本量，不能通过人为补成功请求掩盖空窗口。"
      }
    ],
    "steps": [
      {
        "title": "安装指标库并定义有限标签",
        "body": "在隔离虚拟环境安装prometheus-client。新建metrics_lab.py，单进程Uvicorn验证本课；多worker的指标汇总须另按Prometheus Python多进程说明配置，不能直接假定全局变量共享。",
        "code": "from prometheus_client import Counter, Histogram\n\nREQUESTS = Counter(\n    \"notes_http_requests_total\", \"HTTP requests\",\n    [\"method\", \"route\", \"status_class\"],\n)\nDURATION = Histogram(\n    \"notes_http_request_duration_seconds\", \"HTTP request lifecycle\",\n    [\"method\", \"route\"],\n    buckets=(0.01, 0.05, 0.1, 0.3, 1, 3, 10),\n)",
        "lang": "python",
        "filename": "metrics_lab.py"
      },
      {
        "title": "在实际请求完成时观测",
        "body": "将record_metrics加入第02课ASGI中间件finally，并传scope和状态；这里提供独立函数避免重复注册另一套访问中间件。SSE会记录整个连接时长，应另外统计首段到达时间。",
        "code": "from metrics_lab import REQUESTS, DURATION\n\ndef record_metrics(scope, status, elapsed):\n    if scope.get(\"path\") == \"/metrics\":\n        return\n    method = scope.get(\"method\", \"OTHER\")\n    if method not in {\"GET\",\"POST\",\"PUT\",\"PATCH\",\"DELETE\",\"OPTIONS\",\"HEAD\"}:\n        method = \"OTHER\"\n    route = getattr(scope.get(\"route\"), \"path\", \"unmatched\")\n    group = f\"{status // 100}xx\"\n    REQUESTS.labels(method, route, group).inc()\n    DURATION.labels(method, route).observe(elapsed)\n\n# RequestLogMiddleware的finally中：\n# record_metrics(scope, status, time.perf_counter()-started)",
        "lang": "python",
        "filename": "request_logging.py"
      },
      {
        "title": "暴露本地抓取入口",
        "body": "在现有app注册一次/metrics，使用Response直接返回指标文本。正式环境应限制观测端点访问范围。抓取后连续请求不同note_id，确认标签仍是/notes/{note_id}。",
        "code": "from fastapi import Response\nfrom prometheus_client import generate_latest, CONTENT_TYPE_LATEST\n\n@app.get(\"/metrics\", include_in_schema=False)\ndef metrics():\n    return Response(content=generate_latest(), headers={\"Content-Type\": CONTENT_TYPE_LATEST})",
        "lang": "python",
        "filename": "app/main.py"
      },
      {
        "title": "写出查询与业务目标",
        "body": "在本地Prometheus配置抓取目标后使用查询。下面示例看5xx比例和P95，均需先有流量。为GET /notes另写目标：例如最近7天有效读取99.5%在300ms内成功；数值是练习目标，是否适合业务需按实际基线确定。",
        "code": "sum(rate(notes_http_requests_total{status_class=\"5xx\"}[5m]))\n/\nsum(rate(notes_http_requests_total[5m]))\n\nhistogram_quantile(\n  0.95,\n  sum by (le, route) (rate(notes_http_request_duration_seconds_bucket[5m]))\n)",
        "lang": "promql",
        "filename": "metrics-queries.promql"
      },
      {
        "title": "验证基数与告警触发条件",
        "body": "用20个不同ID访问同一路由，检查/metrics没有20个原始路径标签。无请求的窗口标无数据；告警同时要求有效样本和持续时间。导出积压另由状态表采样为Gauge，不能把每个job作为标签。",
        "code": "curl -sS http://127.0.0.1:8000/metrics\n# 指标标签只允许有限方法、路由模板与状态类别。\n# 不应出现 request_id=、job_id=、user_id= 或带查询串的route。",
        "lang": "bash"
      }
    ],
    "checks": [
      "指标能被Prometheus解析，计数随真实请求增加。",
      "不同note_id共享同一路由标签，没有高基数ID标签。",
      "SLI写明分母与窗口，SLO数值被标为练习目标。"
    ],
    "pitfalls": [
      {
        "symptom": "指标内存持续增长",
        "cause": "把原始URL、用户或任务ID用作标签。",
        "fix": "换成有限路由模板，具体对象放日志和trace。"
      },
      {
        "symptom": "平均耗时正常却有用户一直等待",
        "cause": "平均值掩盖尾部延迟，导出202也没有覆盖完成时间。",
        "fix": "同时看P95/分桶和业务受理到完成耗时。"
      }
    ],
    "exercise": {
      "task": "写一个300ms内请求比例查询，并说明它还缺什么条件才能叫“快速且成功”。",
      "hint": "Histogram的le=0.3桶只看时长，本课时长指标未附带成功标签。",
      "answer": "sum(rate(..._bucket{le='0.3'}[5m]))/sum(rate(..._count[5m]))得到时长比例；若要快速且成功，应另记录成功请求的时长桶或专用好事件计数，不能直接把独立成功率与时长比例相乘。"
    },
    "refs": [
      {
        "label": "Prometheus标签命名",
        "url": "https://prometheus.io/docs/practices/naming/"
      },
      {
        "label": "Prometheus Histogram",
        "url": "https://prometheus.io/docs/practices/histograms/"
      },
      {
        "label": "Prometheus Python客户端",
        "url": "https://prometheus.github.io/client_python/"
      }
    ],
    "production": {
      "scenario": "接口平均延迟没有明显变化，但部分用户查询和导出等待越来越久。",
      "decisions": [
        "成功率、尾部耗时和异步完成时间分别建模。",
        "指标保持低基数，日志和trace承接单请求细节。",
        "SLO依据基线与用户需求制定，空窗口不伪造100%成功。"
      ],
      "failureDrill": "在本地只让少数请求延迟，再比较平均值与耗时桶；以不同ID请求验证时间序列数量稳定。",
      "acceptance": [
        "统计口径可复算。",
        "标签数量受控。",
        "告警有样本量与持续时间条件，不由一条慢请求直接推导事故。"
      ]
    }
  },
  {
    "id": "04",
    "group": "影响、日志与观测",
    "title": "把 trace 贯穿 HTTP、数据库与 Celery",
    "summary": "用 OpenTelemetry 关联API内部阶段和队列执行，保留outbox跨进程的上下文。",
    "objectives": [
      "配置本地trace导出并观察父子span",
      "为数据库操作增加不含正文的span",
      "跨outbox保存并恢复必要trace上下文"
    ],
    "concepts": [
      {
        "title": "trace显示阶段关系与耗时",
        "body": "一个请求可能包含数据库查询、HTTP调用和消息发布。span支持观察每段耗时，但采样trace不是全部请求计数，不能直接替代SLI指标。"
      },
      {
        "title": "跨进程需要传播上下文",
        "body": "HTTP请求头和Celery消息头可以携带trace上下文；outbox把发布延迟到另一个进程后，需要保存允许的追踪字段，再在publisher恢复。不能把Authorization或整个请求头带过去。"
      },
      {
        "title": "异步任务不一定延续请求寿命",
        "body": "长延迟任务可以用span link关联原请求。为本地演示先使用标准traceparent传播，生产需按采样与链路保留策略决定父子关系或链接。"
      }
    ],
    "steps": [
      {
        "title": "安装并配置本地trace输出",
        "body": "安装opentelemetry-api、opentelemetry-sdk、opentelemetry-instrumentation-fastapi、opentelemetry-instrumentation-httpx、opentelemetry-instrumentation-celery，并freeze记录兼容版本。新建trace_setup.py，Console仅供隔离调试；输出可能包含运行元信息，分享前仍需脱敏。",
        "code": "from opentelemetry import trace\nfrom opentelemetry.sdk.resources import Resource\nfrom opentelemetry.sdk.trace import TracerProvider\nfrom opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter\n\ndef setup_trace(service_name: str):\n    provider = TracerProvider(resource=Resource.create({\"service.name\": service_name}))\n    provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))\n    trace.set_tracer_provider(provider)\n    return provider",
        "lang": "python",
        "filename": "trace_setup.py"
      },
      {
        "title": "给API和数据库操作加span",
        "body": "在应用启动配置中调用setup_trace一次，instrument现有app。数据库span只记录有限操作名和行数，不把笔记正文或SQL参数设成属性；HTTPX自动插桩也需核对URL是否包含敏感查询参数。",
        "code": "from opentelemetry import trace\nfrom opentelemetry.instrumentation.fastapi import FastAPIInstrumentor\nfrom opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor\nfrom trace_setup import setup_trace\n\nsetup_trace(\"notes-service-api\")\nFastAPIInstrumentor.instrument_app(app, excluded_urls=\"live,ready,metrics\")\nHTTPXClientInstrumentor().instrument()\ntracer = trace.get_tracer(\"notes-service\")\n\ndef traced_read(conn, note_id):\n    with tracer.start_as_current_span(\"db.notes.read\") as span:\n        row = conn.execute(\"SELECT id,title FROM notes WHERE id=%s\", (note_id,)).fetchone()\n        span.set_attribute(\"app.rows_returned\", 1 if row else 0)\n        return row",
        "lang": "python",
        "filename": "app/main.py"
      },
      {
        "title": "把允许的追踪字段存入outbox",
        "body": "给export_outbox加trace_context jsonb列，默认{}。在提交事务中调用carrier_for_outbox并把返回值作为Jsonb写入同一事件。只保存标准追踪字段，不保存任意baggage或请求头。",
        "code": "from opentelemetry.propagate import inject\n\ndef carrier_for_outbox():\n    carrier = {}\n    inject(carrier)\n    return {key: value for key, value in carrier.items()\n            if key in {\"traceparent\", \"tracestate\"}}\n\n# SQL：\n# ALTER TABLE export_outbox ADD COLUMN IF NOT EXISTS\n# trace_context jsonb NOT NULL DEFAULT '{}';\n# INSERT outbox时将carrier_for_outbox()写入该列。",
        "lang": "python",
        "filename": "trace_carrier.py"
      },
      {
        "title": "在publisher与worker恢复上下文",
        "body": "publisher领取事件时返回trace_context，在send_task外围attach/extract，finally detach。Celery消费者使用官方插桩；prefork子进程在worker_process_init中初始化provider。producer也在其进程初始化CeleryInstrumentor，避免重复初始化。",
        "code": "from opentelemetry import context\nfrom opentelemetry.propagate import extract\n\ndef publish_with_context(event, celery_app):\n    token = context.attach(extract(event.get(\"trace_context\") or {}))\n    try:\n        return celery_app.send_task(\n            \"notes.export\", args=[str(event[\"job_id\"])],\n            task_id=str(event[\"id\"]), queue=event[\"queue_name\"], retry=False,\n        )\n    finally:\n        context.detach(token)\n\n# worker进程初始化模块，确保被Celery导入：\nfrom celery.signals import worker_process_init\nfrom opentelemetry.instrumentation.celery import CeleryInstrumentor\nfrom trace_setup import setup_trace\n\n@worker_process_init.connect(weak=False)\ndef init_worker_trace(**kwargs):\n    setup_trace(\"notes-service-worker\")\n    CeleryInstrumentor().instrument()",
        "lang": "python",
        "filename": "trace_worker.py"
      },
      {
        "title": "比较同步与延迟发布链路",
        "body": "发送一条本地导出请求，记录request_id→job_id映射和trace_id。先正常发布，再暂停publisher后恢复，观察上下文仍可关联。日志保留job_id，不能在trace采样丢弃时失去业务定位能力。",
        "code": "# 待填写的演练字段\n# request_id:\n# job_id:\n# outbox_id:\n# api_trace_id:\n# publish_trace_id:\n# worker_trace_id:\n# relation_observed:\n# result: not_run",
        "lang": "text",
        "filename": "trace-verification.md"
      }
    ],
    "checks": [
      "API内部数据库span可定位并未记录笔记正文。",
      "outbox只保存允许的追踪字段。",
      "publisher恢复上下文，worker链路可以关联，日志仍保留业务ID。"
    ],
    "pitfalls": [
      {
        "symptom": "HTTP有trace，worker却变成另一条无关联链路",
        "cause": "outbox异步发布丢了当前请求上下文，或producer未插桩。",
        "fix": "保存并恢复标准trace字段，初始化publisher和worker插桩。"
      },
      {
        "symptom": "每次任务执行都初始化TracerProvider",
        "cause": "重复初始化造成警告或多个导出器。",
        "fix": "每个进程只初始化一次，prefork在子进程启动时设置。"
      }
    ],
    "exercise": {
      "task": "删除一条实验outbox事件的trace_context后发布，说明业务任务是否应该失败。",
      "hint": "可观测性关联与业务输入是不同职责。",
      "answer": "任务仍按job_id正常执行，只失去原trace关联；publisher可开始新trace并通过job_id日志定位，不能因缺少trace上下文拒绝已受理业务。"
    },
    "refs": [
      {
        "label": "OpenTelemetry Python",
        "url": "https://opentelemetry.io/docs/languages/python/instrumentation/"
      },
      {
        "label": "FastAPI插桩",
        "url": "https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/fastapi/fastapi.html"
      },
      {
        "label": "Celery插桩",
        "url": "https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/celery/celery.html"
      }
    ],
    "production": {
      "scenario": "一次请求很快返回202，但导出很久才完成，需要定位等待发生在哪里。",
      "decisions": [
        "同时关联request_id、job_id、消息ID与trace。",
        "outbox只持久必要追踪字段，外部输入按信任边界处理。",
        "采样策略不能代替任务状态与指标，缺trace不阻断业务。"
      ],
      "failureDrill": "暂停本地publisher后恢复，比较保留和缺少trace_context两种事件的链路关联。",
      "acceptance": [
        "可以区分API执行、待发布、排队和worker执行阶段。",
        "凭据与正文没有进入trace属性。",
        "缺少追踪上下文仍能根据业务ID恢复任务。"
      ]
    }
  },
  {
    "id": "05",
    "group": "进程、内存与依赖",
    "title": "区分 CPU 忙、事件循环阻塞与线程池等待",
    "summary": "用进程指标、事件循环延迟和有界故障对照，定位Python服务变慢的位置。",
    "objectives": [
      "记录进程CPU与线程数量",
      "观察事件循环调度延迟",
      "区分阻塞I/O、纯Python计算与执行器排队"
    ],
    "concepts": [
      {
        "title": "CPU低不代表应用没有阻塞",
        "body": "async路由中time.sleep或同步网络等待会阻塞事件循环，但CPU可能很低。CPU持续高则需进一步区分Python计算、序列化与底层库工作。"
      },
      {
        "title": "事件循环延迟显示调度是否及时",
        "body": "周期协程实际恢复时间晚于预期，说明事件循环无法及时调度。它是现象指标，仍要结合堆栈、CPU与依赖耗时定位原因。"
      },
      {
        "title": "线程池排队有独立等待成本",
        "body": "阻塞函数移到线程后，线程数不足仍可能排队；无限增加线程会扩大连接和内存消耗。任务数量、工作时间与下游额度共同决定容量。"
      }
    ],
    "steps": [
      {
        "title": "采集一个明确进程的基线",
        "body": "先按部署名称读取MainPID，再确认PID非0、对应本次notes-service。pidstat来自sysstat，若未安装可先用ps。命令只读，不输出环境变量或完整命令行。",
        "code": "API_PID=$(systemctl show notes-api.service -p MainPID --value)\nps -p \"$API_PID\" -o pid,ppid,comm,etime,%cpu,%mem,nlwp\npidstat -p \"$API_PID\" 1 5\nps -L -p \"$API_PID\" -o pid,tid,comm,pcpu",
        "lang": "bash"
      },
      {
        "title": "在隔离应用记录事件循环延迟",
        "body": "把watch_loop作为现有lifespan管理的任务，启动时create_task保存引用，退出时cancel并await。不要另建第二个FastAPI实例覆盖已有路由。下面只定义监测函数，日志数值来自实际运行。",
        "code": "import asyncio\nimport json\n\nasync def watch_loop():\n    loop = asyncio.get_running_loop()\n    while True:\n        expected = loop.time() + 0.1\n        await asyncio.sleep(0.1)\n        lag = max(0, loop.time() - expected)\n        print(json.dumps({\"event\": \"event_loop_lag\", \"seconds\": lag}))\n\n# 现有lifespan中：\n# monitor = asyncio.create_task(watch_loop())\n# try:\n#     yield\n# finally:\n#     monitor.cancel()\n#     with contextlib.suppress(asyncio.CancelledError):\n#         await monitor",
        "lang": "python",
        "filename": "loop_watch.py"
      },
      {
        "title": "只在隔离副本复现两种阻塞",
        "body": "添加受开关控制的实验路由，默认404。开启LAB_ENABLE_FAULTS=1只用于本地副本。分别请求mode=sleep与mode=cpu，同时观察/live、CPU和loop lag；每次工作上限0.5秒，不启动无限循环。",
        "code": "import os\nimport time\nfrom typing import Literal\nfrom fastapi import HTTPException\n\n@app.get(\"/lab/block\")\nasync def block_demo(mode: Literal[\"sleep\", \"cpu\"] = \"sleep\"):\n    if os.getenv(\"LAB_ENABLE_FAULTS\") != \"1\":\n        raise HTTPException(404, \"not enabled\")\n    if mode == \"sleep\":\n        time.sleep(0.5)\n    else:\n        until = time.perf_counter() + 0.5\n        while time.perf_counter() < until:\n            pass\n    return {\"mode\": mode}",
        "lang": "python",
        "filename": "app/main.py"
      },
      {
        "title": "记录修复前后同样负载",
        "body": "sleep改为await asyncio.sleep；同步SDK按异步任务课使用受限线程，CPU工作分块或移到受限进程。保持同样请求数与参数比较，不同时调整worker数量和代码。若需要堆栈，先在隔离环境预先启用faulthandler，不能向未注册信号处理的进程随意发送信号。",
        "code": "# 在同样的低并发负载下各记录：\n# CPU%、loop lag、HTTP耗时、线程数量、工作队列等待。\n# 恢复后移除/lab/block，关闭LAB_ENABLE_FAULTS。\n# 不把“线程数增加”本身当作修复证据。",
        "lang": "text",
        "filename": "cpu-verification.md"
      }
    ],
    "checks": [
      "sleep阻塞与CPU循环都可增加loop lag，但CPU表现不同。",
      "修复使用可等待I/O后，同样负载下/live不再被同步sleep拖住。",
      "线程池容量与排队情况有记录，未用无限并发掩盖问题。"
    ],
    "pitfalls": [
      {
        "symptom": "看到CPU高就直接增加Uvicorn worker",
        "cause": "可能超过CPU配额或放大数据库连接。",
        "fix": "先判断可并行部分与资源预算，再做单变量容量实验。"
      },
      {
        "symptom": "对生产进程直接发送SIGUSR1期待堆栈",
        "cause": "目标可能未注册处理器，信号行为未确认。",
        "fix": "仅在已配置并验证的隔离进程使用诊断信号，优先读取现有指标与日志。"
      }
    ],
    "exercise": {
      "task": "描述“CPU低、loop lag高、请求都一起卡住”的首要检查方向。",
      "hint": "寻找async调用路径中的同步等待，而非先假定算力不足。",
      "answer": "检查time.sleep、同步HTTP/数据库调用、同步文件操作；结合当前堆栈和依赖耗时验证，再改为异步调用或受限线程。"
    },
    "refs": [
      {
        "label": "Python asyncio调试",
        "url": "https://docs.python.org/3.11/library/asyncio-dev.html"
      },
      {
        "label": "Python faulthandler",
        "url": "https://docs.python.org/3.11/library/faulthandler.html"
      },
      {
        "label": "Python执行器",
        "url": "https://docs.python.org/3.11/library/concurrent.futures.html"
      }
    ],
    "production": {
      "scenario": "高峰期所有API偶尔停顿，主机CPU却没有满。",
      "decisions": [
        "事件循环延迟与CPU一起观察。",
        "诊断信号和分析工具先在隔离环境验证权限与开销。",
        "修复时限制线程、进程和下游连接总量。"
      ],
      "failureDrill": "本地分别注入0.5秒sleep和CPU循环，比较指标后恢复可等待实现。",
      "acceptance": [
        "能够用证据区分计算和等待。",
        "相同负载下修复前后可比较。",
        "故障路由默认关闭且最终移除。"
      ]
    }
  },
  {
    "id": "06",
    "group": "进程、内存与依赖",
    "title": "区分 Python 分配、RSS 与容器 OOM",
    "summary": "用小规模内存实验理解不同观测口径，避免把缓存保留或原生内存误判为Python泄漏。",
    "objectives": [
      "读取进程RSS与容器内存限制",
      "用tracemalloc比较Python分配来源",
      "通过OOM证据区分内存终止与其他退出"
    ],
    "concepts": [
      {
        "title": "RSS不等于Python活对象大小",
        "body": "RSS包含解释器、原生库、内存分配器和映射等；对象释放后RSS不一定立即下降。tracemalloc跟踪其启用后可追踪的Python内存分配，不能解释所有原生内存。"
      },
      {
        "title": "容器限制与主机余量不同",
        "body": "容器可能因自身memory.max触发OOM，即使主机仍有可用内存。查看cgroup事件和容器退出状态，不能只看free输出。"
      },
      {
        "title": "诊断也需要容量预算",
        "body": "堆快照、频繁采样和大量日志会增加开销。本课只分配约10MiB演示，不故意耗尽宿主机内存，也不通过drop_caches改变现场。"
      }
    ],
    "steps": [
      {
        "title": "读取进程与容器口径",
        "body": "Linux systemd 环境使用第 05 课确认的 API_PID。若使用 Docker，在部署模块的 Compose 目录运行容器命令，先用 api service 获取唯一容器 ID；多副本时逐个选择。docker inspect 仅取所需字段，不导出含环境变量的完整配置。容器 PID 与宿主 PID 不可直接互换。Docker stats 与 RSS 口径不同，记录时保留来源。",
        "code": "# Linux systemd 环境：\nps -p \"$API_PID\" -o pid,comm,rss,vsz,%mem\ncat /proc/\"$API_PID\"/cgroup\n\n# Docker Compose 环境：在当前练习 Compose 目录执行。\nAPI_CONTAINER=$(docker compose ps -q api)\nif [ -z \"$API_CONTAINER\" ]; then\n  printf \"api service has no running container\\n\" >&2\nelse\n  docker stats --no-stream \"$API_CONTAINER\"\n  docker inspect --format '{{.State.Status}} OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}}' \"$API_CONTAINER\"\nfi",
        "lang": "bash"
      },
      {
        "title": "运行有限Python分配实验",
        "body": "新建memory_probe.py，只在隔离环境执行。tracemalloc在分配前启动；先比before和during，再删除对象观察after。打印分配位置和大小，不打印对象内容。",
        "code": "import gc\nimport tracemalloc\n\ntracemalloc.start(10)\nbefore = tracemalloc.take_snapshot()\nbuffers = [bytearray(512 * 1024) for _ in range(20)]\nduring = tracemalloc.take_snapshot()\nfor stat in during.compare_to(before, \"lineno\")[:5]:\n    print(stat)\ndel buffers\ngc.collect()\nafter = tracemalloc.take_snapshot()\ncurrent, peak = tracemalloc.get_traced_memory()\nprint({\"current_bytes\": current, \"peak_bytes\": peak})\nfor stat in after.compare_to(before, \"lineno\")[:5]:\n    print(stat)",
        "lang": "python",
        "filename": "memory_probe.py"
      },
      {
        "title": "关联OOM事件与进程退出",
        "body": "在cgroup v2容器内部检查memory.current、memory.max和memory.events；不同部署挂载方式可能不同，先确认cgroup路径。ExitCode=137也可能是人为SIGKILL，需结合OOMKilled和oom_kill计数，不单凭137下结论。",
        "code": "# 在已确认使用cgroup v2的隔离容器内：\ncat /sys/fs/cgroup/memory.current\ncat /sys/fs/cgroup/memory.max\ncat /sys/fs/cgroup/memory.events\n# 只读取；不写drop_caches，不修改宿主机内存限制。",
        "lang": "bash"
      },
      {
        "title": "设计增长复现与修复对照",
        "body": "对同样的有限请求批次做多轮快照，观察相同分配位置是否持续增加。若tracemalloc稳定但RSS增长，进一步查原生库、连接缓冲和子进程；不要直接判为Python对象泄漏。修复后比较稳态与峰值。",
        "code": "{\n  \"sample_type\": \"planned_memory_observation\",\n  \"round\": null,\n  \"request_count\": null,\n  \"rss_bytes\": null,\n  \"tracemalloc_current_bytes\": null,\n  \"tracemalloc_peak_bytes\": null,\n  \"cgroup_oom_kill_delta\": null,\n  \"result\": \"not_run\"\n}",
        "lang": "json",
        "filename": "memory-record.json"
      }
    ],
    "checks": [
      "实验分配约10MiB，释放后可追踪活跃分配明显下降。",
      "RSS、tracemalloc与容器限制分别记录，未混用单位。",
      "OOM结论有事件或状态证据，非仅依据退出码。"
    ],
    "pitfalls": [
      {
        "symptom": "删除对象后RSS没降就认定泄漏",
        "cause": "分配器可能保留内存供复用，RSS不等于活对象。",
        "fix": "比较多轮稳态和tracemalloc增长来源，再判断是否持续无界增长。"
      },
      {
        "symptom": "容器反复退出但主机free正常",
        "cause": "容器自身内存上限被触发。",
        "fix": "检查memory.max和memory.events，结合OOMKilled与具体负载调整。"
      }
    ],
    "exercise": {
      "task": "如果tracemalloc当前值稳定、RSS持续增长，应先收集哪两类信息？",
      "hint": "考虑不被tracemalloc完整覆盖的分配与进程范围。",
      "answer": "检查原生库/网络缓冲及映射，核对是否包含子进程或容器总内存；同时记录同一负载下的增长曲线，不能直接用gc.collect当修复。"
    },
    "refs": [
      {
        "label": "Python tracemalloc",
        "url": "https://docs.python.org/3.11/library/tracemalloc.html"
      },
      {
        "label": "Linux cgroup v2",
        "url": "https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html"
      },
      {
        "label": "Docker内存限制",
        "url": "https://docs.docker.com/engine/containers/resource_constraints/"
      }
    ],
    "production": {
      "scenario": "导出运行一段时间后容器重启，需要确认是内存泄漏、峰值超限还是其他终止。",
      "decisions": [
        "对齐进程、容器和主机三种口径。",
        "诊断实验有严格内存上限，禁止耗尽宿主资源。",
        "修复依据增长来源，内存上限调整不能替代无界队列修复。"
      ],
      "failureDrill": "仅运行本课10MiB有限分配脚本，对比快照与RSS；OOM部分使用已有隔离演练证据或标not_run。",
      "acceptance": [
        "没有人为触发宿主OOM或清理缓存干扰现场。",
        "每个内存指标注明单位和范围。",
        "结论区分可追踪Python分配与原生/容器内存。"
      ]
    }
  },
  {
    "id": "07",
    "group": "进程、内存与依赖",
    "title": "分开排查 DNS、TCP、TLS 与连接池等待",
    "summary": "把一次外部调用拆成阶段，核对客户端连接预算和部署网络命名空间。",
    "objectives": [
      "区分名称解析、连接建立与HTTP响应失败",
      "配置HTTPX连接池和阶段超时",
      "避免凭据进入抓包或命令输出"
    ],
    "concepts": [
      {
        "title": "不同阶段失败需要不同证据",
        "body": "DNS失败还没有获得目标地址，TCP连接拒绝通常没有进入HTTP，TLS错误发生在加密连接建立阶段，HTTP504可能来自网关或应用。不能把所有错误都叫网络慢。"
      },
      {
        "title": "localhost属于当前网络命名空间",
        "body": "宿主、容器和WSL中的127.0.0.1可能指向不同位置。任务broker宿主映射6384，容器内通常通过broker:6379；缓存宿主6380，不能混用端口。"
      },
      {
        "title": "连接池耗尽也表现为等待",
        "body": "HTTPX PoolTimeout是等待池连接超时，不表示目标服务器连接失败。限制max_connections，并复用客户端，避免每请求建立独立连接池。"
      }
    ],
    "steps": [
      {
        "title": "核对解析、监听与目标端口",
        "body": "命令在发生问题的同一容器或WSL环境执行。getent使用实际依赖主机名；ss只看必要监听与汇总信息。不要打印包含密码的完整DATABASE_URL或Celery URL。",
        "code": "getent hosts broker\nss -lnt\nss -s\n# 在宿主检查本地映射；不带任何凭据：\ncurl -sS --connect-timeout 1 --max-time 3 -o /dev/null -w 'code=%{http_code} connect=%{time_connect} total=%{time_total}\\n' http://127.0.0.1:8000/live",
        "lang": "bash"
      },
      {
        "title": "限制连接数并分开阶段超时",
        "body": "新建http_budget.py。HTTPX异步客户端在应用lifespan创建并关闭；以下脚本独立运行以观察异常类型。只请求本地/slow，不传真实凭据。",
        "code": "import asyncio\nimport httpx\n\nasync def main():\n    limits = httpx.Limits(max_connections=2, max_keepalive_connections=2)\n    timeout = httpx.Timeout(connect=1, read=3, write=3, pool=0.1)\n    async with httpx.AsyncClient(\n        base_url=\"http://127.0.0.1:8000\", limits=limits, timeout=timeout\n    ) as client:\n        async def one():\n            try:\n                response = await client.get(\"/slow\", params={\"delay\": 1})\n                return response.status_code\n            except httpx.HTTPError as exc:\n                return type(exc).__name__\n        print(await asyncio.gather(*(one() for _ in range(5))))\n\nif __name__ == \"__main__\":\n    asyncio.run(main())",
        "lang": "python",
        "filename": "http_budget.py"
      },
      {
        "title": "区分池等待与连接失败",
        "body": "先运行上一步，超过两个同时请求时可能出现PoolTimeout。再把独立脚本base_url改成本地确认未监听的端口，观察ConnectError；恢复后把read改0.1秒观察ReadTimeout。一次只改一项，记录实际异常类型。",
        "code": "# 演练记录字段：\n# phase: pool / connect / read\n# configuration_changed:\n# exception_type_observed:\n# elapsed_seconds:\n# restored_to_baseline: false\n# 上例是记录模板，不是已观察到的结果。",
        "lang": "text",
        "filename": "network-record.md"
      },
      {
        "title": "只读取必要网络证据",
        "body": "若必须定位TLS，用无凭据的测试请求核对证书主机名与链路；不要以verify=False作为修复。抓包只在隔离且无敏感流量的环境按最小范围使用，本模块默认以阶段耗时和连接统计定位，不导出完整HTTP载荷。",
        "code": "# curl阶段耗时，可用于已确认的无凭据测试URL：\n# -w 'dns=%{time_namelookup} connect=%{time_connect} tls=%{time_appconnect} first_byte=%{time_starttransfer} total=%{time_total}'\n# 这些是从开始累计的时间，不是可以直接相加的独立阶段耗时。",
        "lang": "text",
        "filename": "network-notes.md"
      }
    ],
    "checks": [
      "错误能定位到解析、连接、读响应或池等待阶段。",
      "HTTPX客户端有连接上限且在生命周期结束时关闭。",
      "宿主与容器端口映射清楚，日志没有输出带密码的URL。"
    ],
    "pitfalls": [
      {
        "symptom": "容器用127.0.0.1:6384连接broker失败",
        "cause": "6384是宿主映射，容器localhost不是broker服务。",
        "fix": "在同网络容器使用broker:6379，在宿主使用127.0.0.1:6384。"
      },
      {
        "symptom": "PoolTimeout后不断扩容下游",
        "cause": "真正等待的是本进程连接池，未确认下游是否瓶颈。",
        "fix": "检查连接泄漏、流响应是否关闭、并发需求与池预算，再逐项调整。"
      }
    ],
    "exercise": {
      "task": "在两个连接额度下发五个耗时1秒请求，解释为什么pool=0.1可能先失败。",
      "hint": "池等待发生在获得连接前，与read=3不是同一个计时阶段。",
      "answer": "前两个请求占用连接，后续请求等待超过0.1秒可触发PoolTimeout，不能因此认定下游超过3秒未响应。"
    },
    "refs": [
      {
        "label": "HTTPX阶段超时",
        "url": "https://www.python-httpx.org/advanced/timeouts/"
      },
      {
        "label": "HTTPX连接限制",
        "url": "https://www.python-httpx.org/advanced/resource-limits/"
      },
      {
        "label": "curl时间变量",
        "url": "https://curl.se/docs/manpage.html"
      }
    ],
    "production": {
      "scenario": "导出调用存储接口时偶发超时，扩容后错误反而增加。",
      "decisions": [
        "先区分命名空间、网络阶段和连接池等待。",
        "连接预算与worker并发一起规划。",
        "TLS校验和凭据保密不是排障时可以随意关闭的步骤。"
      ],
      "failureDrill": "仅对本地/slow控制池上限与read超时，分别观察PoolTimeout与ReadTimeout并恢复配置。",
      "acceptance": [
        "每种异常有对应证据。",
        "客户端连接和流资源会关闭。",
        "未通过泄露凭据抓包或禁用证书校验得出结论。"
      ]
    }
  },
  {
    "id": "08",
    "group": "进程、内存与依赖",
    "title": "定位慢 SQL、锁等待与连接池耗尽",
    "summary": "从 pg_stat_activity 进入具体等待关系，使用短事务和隔离锁实验验证原因。",
    "objectives": [
      "读取活动会话与阻塞PID",
      "区分SQL执行慢和等待连接/锁",
      "用参数化EXPLAIN与事务级超时限制诊断"
    ],
    "concepts": [
      {
        "title": "数据库耗时包含多个阶段",
        "body": "应用先等连接池，再发送SQL，数据库还可能等锁或I/O。只看到接口慢，不足以判断需要加索引。"
      },
      {
        "title": "等待者与阻塞者要关联",
        "body": "pg_stat_activity显示状态、事务年龄和wait_event，pg_blocking_pids定位阻塞来源。先看进程与关系，再决定是否需要查看脱敏SQL。"
      },
      {
        "title": "诊断SQL也可能执行实际工作",
        "body": "EXPLAIN ANALYZE会真的执行查询，写操作还会产生副作用。先用EXPLAIN；隔离环境才能用已限制范围的ANALYZE，并设置合理statement_timeout和lock_timeout。"
      }
    ],
    "steps": [
      {
        "title": "读取活动会话和等待信息",
        "body": "连接PostgreSQL16 notes_lab，app_user，宿主5433。通过安全配置的DATABASE_URL或pgpass连接，避免把密码贴入命令行。普通账户可见信息受权限限制，不把缺失字段当作没有活动。",
        "code": "SELECT pid,usename,application_name,state,\n       now()-xact_start AS transaction_age,\n       now()-query_start AS query_age,\n       wait_event_type,wait_event,\n       pg_blocking_pids(pid) AS blocked_by\nFROM pg_stat_activity\nWHERE datname='notes_lab' AND pid<>pg_backend_pid()\nORDER BY xact_start NULLS LAST;",
        "lang": "sql"
      },
      {
        "title": "在两会话复现有界锁等待",
        "body": "只选本地演练笔记ID。会话A开启事务并锁定该行；会话B设置1秒lock_timeout后尝试锁同一行。观察锁等待错误，随后两个会话都ROLLBACK，释放故障条件。",
        "code": "-- 会话A：将1替换为专门用于本次演练的笔记ID\nBEGIN;\nSELECT id FROM notes WHERE id=1 FOR UPDATE;\n-- 暂不提交，在会话B执行下面语句后回到A：ROLLBACK;\n\n-- 会话B\nBEGIN;\nSET LOCAL lock_timeout='1s';\nSELECT id FROM notes WHERE id=1 FOR UPDATE;\n-- 超时后：ROLLBACK;",
        "lang": "sql"
      },
      {
        "title": "先看计划，再在隔离数据上测执行",
        "body": "下面EXPLAIN不实际执行SELECT。若需要ANALYZE，先确认只读、数据规模有限且本地可承受，再以事务级statement_timeout执行。不要在事故中直接对未知大表运行昂贵诊断。",
        "code": "EXPLAIN (COSTS, VERBOSE)\nSELECT id,title FROM notes WHERE id=1;\n\n-- 仅隔离练习：\nBEGIN;\nSET LOCAL statement_timeout='2s';\nEXPLAIN (ANALYZE, BUFFERS)\nSELECT id,title FROM notes WHERE id=1;\nROLLBACK;",
        "lang": "sql"
      },
      {
        "title": "观察应用连接池预算",
        "body": "若使用psycopg_pool，记录get_stats中的等待、请求和连接数，字段按安装版本核对。连接预算需乘以API worker和Celery进程数；不要只把每个池的max_size调大。等待连接时数据库可能没有相应新查询。",
        "code": "# 使用现有psycopg_pool对象，只记录统计数字：\n# stats = pool.get_stats()\n# logger.info({\"event\": \"db_pool_stats\", \"stats\": stats})\n\n# 检查应用事务边界：\n# 获取连接 → 短SQL/事务 → 提交或回滚 → 归还连接\n# 不把网络上传、长await或CPU导出放在持有事务的区间。",
        "lang": "python",
        "filename": "pool_observation.py"
      },
      {
        "title": "恢复后核对锁和业务请求",
        "body": "回滚会话A后重试读取，应恢复正常；再次查询活动会话确认没有本次idle in transaction。记录谁持锁、谁等待、锁何时释放。未经确认不要终止任意数据库backend。",
        "code": "SELECT pid,state,now()-xact_start AS transaction_age\nFROM pg_stat_activity\nWHERE datname='notes_lab' AND state='idle in transaction';",
        "lang": "sql"
      }
    ],
    "checks": [
      "会话B在受控时间内超时，能关联会话A为阻塞者。",
      "回滚后锁释放，读取恢复。",
      "应用池等待与数据库锁等待分别记录，没有盲目增大连接数。"
    ],
    "pitfalls": [
      {
        "symptom": "接口慢但pg_stat_activity没有对应查询",
        "cause": "请求可能还在应用连接池排队。",
        "fix": "检查连接归还、池等待统计和进程总连接预算。"
      },
      {
        "symptom": "锁超时后继续在同一事务执行又报错",
        "cause": "PostgreSQL事务已进入失败状态。",
        "fix": "先ROLLBACK，再重新开始需要的事务；不要忽略异常后继续使用。"
      }
    ],
    "exercise": {
      "task": "说明为什么一个API进程池max_size=10，扩成4个进程后数据库连接可能接近40。",
      "hint": "进程之间通常不共享同一个连接池对象。",
      "answer": "每个进程拥有自己的上限，总预算需加上Celery、publisher、recovery和管理连接；扩worker前先核对数据库容量。"
    },
    "refs": [
      {
        "label": "PostgreSQL活动统计",
        "url": "https://www.postgresql.org/docs/16/monitoring-stats.html"
      },
      {
        "label": "PostgreSQL锁",
        "url": "https://www.postgresql.org/docs/16/explicit-locking.html"
      },
      {
        "label": "PostgreSQL EXPLAIN",
        "url": "https://www.postgresql.org/docs/16/using-explain.html"
      },
      {
        "label": "Psycopg连接池",
        "url": "https://www.psycopg.org/psycopg3/docs/advanced/pool.html"
      }
    ],
    "production": {
      "scenario": "发布后笔记读取突然变慢，数据库CPU却不高。",
      "decisions": [
        "先查池等待和锁关系，再评估SQL计划。",
        "诊断操作设超时，锁实验限定专门笔记和可回滚事务。",
        "连接上限按全进程总量规划。"
      ],
      "failureDrill": "使用两条本地数据库连接持锁与超时，再回滚并确认恢复，不终止其他会话。",
      "acceptance": [
        "等待关系可解释。",
        "所有演练事务最终回滚。",
        "没有将EXPLAIN ANALYZE描述为纯静态分析。"
      ]
    }
  },
  {
    "id": "09",
    "group": "队列、实时通信与恢复",
    "title": "排查 Redis 热点、淘汰与缓存降级",
    "summary": "先确认实例用途，再观察命中率、内存和延迟，保护数据库避免缓存故障扩散。",
    "objectives": [
      "区分缓存实例与任务broker",
      "读取命中、淘汰、连接与内存指标",
      "给缓存失效设计受限回源路径"
    ],
    "concepts": [
      {
        "title": "缓存与broker不能用同一恢复策略",
        "body": "缓存可按业务降级回源，broker承载待交付消息，不能通过清空键空间恢复。模块统一缓存宿主6380、独立任务broker6384；先核对用途再操作。"
      },
      {
        "title": "累计指标需要按窗口取增量",
        "body": "keyspace_hits和keyspace_misses是累计计数，计算窗口命中率要使用增量。evicted_keys增加表示内存策略淘汰，expired_keys增加表示TTL到期，两者原因不同。"
      },
      {
        "title": "回源必须有容量上限",
        "body": "缓存不可用后所有请求直接打数据库可能造成级联故障。设置缓存短超时、回源并发限制和必要的过载拒绝；多进程总量不能只按单进程Semaphore理解。"
      }
    ],
    "steps": [
      {
        "title": "安全连接指定缓存实例",
        "body": "在Linux/WSL2终端通过read隐藏输入本地缓存密码，不把密码写进-a参数。以下只读命令指向6380；broker6384需要其独立密码，不能复用缓存故障动作。",
        "code": "read -rsp 'Local cache password: ' REDISCLI_AUTH\nexport REDISCLI_AUTH\nredis-cli -h 127.0.0.1 -p 6380 PING\nredis-cli -h 127.0.0.1 -p 6380 INFO stats\nredis-cli -h 127.0.0.1 -p 6380 INFO memory\nredis-cli -h 127.0.0.1 -p 6380 INFO clients",
        "lang": "bash"
      },
      {
        "title": "检查策略与延迟来源",
        "body": "CONFIG GET是只读但可能被ACL禁止；权限不足需要按观测权限配置处理，不绕过认证。SLOWLOG可能包含命令参数，先查长度；只有无敏感数据的隔离实例才查看具体条目。不要运行KEYS *或MONITOR扫描实际业务内容。",
        "code": "redis-cli -h 127.0.0.1 -p 6380 CONFIG GET maxmemory maxmemory-policy\nredis-cli -h 127.0.0.1 -p 6380 LATENCY LATEST\nredis-cli -h 127.0.0.1 -p 6380 SLOWLOG LEN\n# 在无敏感数据的隔离实例才进一步检查SLOWLOG内容。\nunset REDISCLI_AUTH",
        "lang": "bash"
      },
      {
        "title": "给缓存访问配置受限回源",
        "body": "下面函数使用已有redis异步客户端和read_db异步函数，客户端需设置socket_connect_timeout/socket_timeout，例如各0.2秒。缺失或连接失败时回源，缓存写失败不改变已成功读取的数据。此示例只用于单进程容量实验；真实权限必须进入缓存key或查询范围。",
        "code": "import asyncio\nimport json\nfrom redis.exceptions import ConnectionError as RedisConnectionError\nfrom redis.exceptions import TimeoutError as RedisTimeoutError\n\ndb_slots = asyncio.Semaphore(4)\n\nasync def cached_note(note_id, cache, read_db):\n    key = f\"lab:notes:{note_id}\"\n    try:\n        raw = await cache.get(key)\n        if raw is not None:\n            return json.loads(raw)\n    except (RedisConnectionError, RedisTimeoutError, json.JSONDecodeError):\n        pass\n    async with asyncio.timeout(1):\n        async with db_slots:\n            row = await read_db(note_id)\n    if row is not None:\n        try:\n            await cache.set(key, json.dumps(row), ex=30)\n        except (RedisConnectionError, RedisTimeoutError):\n            pass\n    return row",
        "lang": "python",
        "filename": "cache_fallback.py"
      },
      {
        "title": "以增量比较缓存故障前后",
        "body": "仅停止本地缓存实例，保持broker和数据库运行，低并发读取相同笔记。记录缓存异常、数据库请求数、池等待和HTTP状态；恢复缓存后观察命中回升。回源超时应由API转换为明确503，不返回虚构笔记或无限等待。",
        "code": "{\n  \"sample_type\": \"cache_drill_template\",\n  \"window_seconds\": null,\n  \"hits_delta\": null,\n  \"misses_delta\": null,\n  \"evicted_keys_delta\": null,\n  \"db_requests_delta\": null,\n  \"http_503_count\": null,\n  \"result\": \"not_run\"\n}",
        "lang": "json",
        "filename": "cache-drill.json"
      }
    ],
    "checks": [
      "实例地址与用途明确，6384broker未被缓存故障动作影响。",
      "窗口命中率使用计数增量，能区分TTL到期与内存淘汰。",
      "缓存故障时回源有时间与并发预算，过载明确失败。"
    ],
    "pitfalls": [
      {
        "symptom": "缓存故障后数据库也被打满",
        "cause": "无上限回源造成级联放大。",
        "fix": "先限制回源并发和请求速率，必要时拒绝非关键读取。"
      },
      {
        "symptom": "对broker执行缓存清理",
        "cause": "混淆实例用途，可能删除未处理消息。",
        "fix": "独立实例与凭据，先核对地址和角色，不以flushdb/purge作为排障默认动作。"
      }
    ],
    "exercise": {
      "task": "命中100、未命中20增至命中140、未命中30，计算该窗口命中率。",
      "hint": "使用变化量40和10，不使用启动以来累计值。",
      "answer": "窗口命中率=40/(40+10)=80%；同时记录窗口请求量50，不能把低样本波动误判为长期趋势。"
    },
    "refs": [
      {
        "label": "Redis INFO",
        "url": "https://redis.io/docs/latest/commands/info/"
      },
      {
        "label": "Redis淘汰策略",
        "url": "https://redis.io/docs/latest/develop/reference/eviction/"
      },
      {
        "label": "Redis延迟排查",
        "url": "https://redis.io/docs/latest/operate/oss_and_stack/management/optimization/latency/"
      }
    ],
    "production": {
      "scenario": "热门笔记读取突然变慢，缓存命中下降且数据库连接开始排队。",
      "decisions": [
        "先确认是过期、淘汰、热点还是Redis不可用。",
        "缓存降级需要保护数据库容量。",
        "任务broker与缓存使用独立实例和恢复手册。"
      ],
      "failureDrill": "只暂停本地缓存6380，以有限并发请求观察回源，再恢复并核对命中和数据库负载。",
      "acceptance": [
        "没有清空broker或扫描敏感键值。",
        "降级不突破数据库预算。",
        "缓存恢复后业务结果仍与数据库一致。"
      ]
    }
  },
  {
    "id": "10",
    "group": "队列、实时通信与恢复",
    "title": "定位 Celery 积压、重试风暴与租约恢复",
    "summary": "把受理、outbox发布、broker排队和worker执行分开检查，再选择恢复动作。",
    "objectives": [
      "按任务阶段定位积压位置",
      "区分业务任务重试与重复交付",
      "基于持久状态与租约进行受控恢复"
    ],
    "concepts": [
      {
        "title": "积压可能发生在broker之前",
        "body": "export_outbox未发布也属于用户等待；只看Celery active会漏掉publisher故障。queued任务即使暂时没有pending outbox，也可能已在broker或存在消息丢失窗口，需要派发记录和恢复规则判断。"
      },
      {
        "title": "重试与重投有不同来源",
        "body": "任务主动retry可能带退避；broker重复交付可能携带旧消息。持久attempts、next_attempt_at、deadline和领取CAS必须一起限制，不能只看Celery request.retries。"
      },
      {
        "title": "恢复先处理原因，再处理队列",
        "body": "下游仍不可用时增加worker或批量重放会放大负载。先停止错误输入或恢复依赖，再按有限批次处理到期任务；不要默认purge清空消息或强杀所有worker。"
      }
    ],
    "steps": [
      {
        "title": "读取worker与路由状态",
        "body": "在tasks_lab工作目录，读取正确Celery环境：独立broker宿主6384，容器内broker:6379，入口celery_app:app。inspect可能包含任务参数，本模块后期只传job_id；分享输出前核对。",
        "code": "python -m celery -A celery_app:app status\npython -m celery -A celery_app:app inspect active\npython -m celery -A celery_app:app inspect reserved\npython -m celery -A celery_app:app inspect scheduled\npython -m celery -A celery_app:app inspect active_queues",
        "lang": "bash"
      },
      {
        "title": "把数据库阶段并排查询",
        "body": "使用SQL同时看业务状态、待发布事件和过期租约。queued很老只是排查线索，不足以直接断言消息已经丢失；按异步任务模块的派发记录与恢复门槛处理。",
        "code": "SELECT status,count(*) AS total,\n       extract(epoch FROM now()-min(created_at)) AS oldest_seconds\nFROM export_jobs GROUP BY status;\n\nSELECT queue_name,count(*) AS pending,\n       extract(epoch FROM now()-min(created_at)) AS oldest_seconds\nFROM export_outbox WHERE sent_at IS NULL GROUP BY queue_name;\n\nSELECT id,attempts,lease_until,error_code\nFROM export_jobs WHERE status='running' AND lease_until<now()\nORDER BY lease_until LIMIT 20;",
        "lang": "sql"
      },
      {
        "title": "按一条任务核对重试链",
        "body": "从实际失败日志选定一个本地job_id，查询其事件ID、attempts和状态转换。区分多次消息到达与多次成功领取；重复消息到达不应绕过next_attempt_at或为活跃租约增加执行副作用。不要只看同一task_id的最终缓存值。",
        "code": "-- 将参数绑定为本次演练job_id，避免拼接外部输入。\n-- SELECT id,status,attempts,deadline_at,lease_token,lease_until,error_code\n-- FROM export_jobs WHERE id=%s;\n-- SELECT id,job_id,attempts,sent_at,available_at\n-- FROM export_outbox WHERE job_id=%s ORDER BY created_at;\n-- 如果已应用任务模块派发与退避字段，同时核对相应时间条件。",
        "lang": "sql"
      },
      {
        "title": "恢复一个有证据的本地故障",
        "body": "先确认下游恢复、容量可用和代码修复，再运行任务模块有限批次recovery与publisher。单步脚本不常驻；部署中的循环wrapper有独立节奏与停机管理。失败耗尽任务仅在确认修复后用replay_job逐条生成新job，保留旧失败证据。",
        "code": "python recovery.py\npython publisher.py\n# 对已修复且明确的failed任务：\n# python replay_job.py <本次已核对的失败job_id>\n# 不运行celery purge，不通过重置attempts解除失败限制。",
        "lang": "bash"
      },
      {
        "title": "验证恢复不是短暂下降",
        "body": "恢复后持续观察最老等待、完成速率、重试次数和下游错误，覆盖至少一个任务截止时间与恢复扫描周期。增加worker后若数据库池等待上升，应停止扩大并发并重新核算预算。",
        "code": "# 恢复验收记录：\n# 影响队列：\n# 下游修复证据：\n# 受控恢复任务ID：\n# 最老等待变化：\n# 重试/重投变化：\n# 新导出完成结果：\n# 未恢复任务及原因：\n# result: not_run",
        "lang": "text",
        "filename": "queue-recovery.md"
      }
    ],
    "checks": [
      "能判断积压发生在outbox、broker、worker还是业务失败隔离。",
      "重投和重试没有绕过持久尝试预算与退避时间。",
      "恢复有下游正常和最终结果证据，旧失败记录被保留。"
    ],
    "pitfalls": [
      {
        "symptom": "worker在线但没有任务执行",
        "cause": "路由队列不匹配、publisher未发布或任务未注册。",
        "fix": "核对active_queues、registered和outbox，再查执行容量。"
      },
      {
        "symptom": "重试风暴时继续增加worker",
        "cause": "下游不可用，更多worker只产生更多失败。",
        "fix": "先止住重复输入或恢复依赖，保留退避、总预算和入口容量。"
      }
    ],
    "exercise": {
      "task": "outbox数量上升、worker空闲、broker可连接时，优先查哪两个位置？",
      "hint": "消息可能没有被publisher成功发送，或发到了无人消费的队列。",
      "answer": "检查publisher错误/领取租约/标记过程，以及outbox queue_name与worker active_queues是否一致；不能直接判断worker算力不足。"
    },
    "refs": [
      {
        "label": "Celery监控",
        "url": "https://docs.celeryq.dev/en/stable/userguide/monitoring.html"
      },
      {
        "label": "Celery优化",
        "url": "https://docs.celeryq.dev/en/stable/userguide/optimizing.html"
      },
      {
        "label": "Celery Redis交付",
        "url": "https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html"
      }
    ],
    "production": {
      "scenario": "任务队列积压，团队准备增加worker，但导出持续重试。",
      "decisions": [
        "按阶段定位，数据库状态为业务真源。",
        "下游恢复前不通过并发扩容放大失败。",
        "恢复按持久租约、退避和次数预算执行，重放生成新业务ID。"
      ],
      "failureDrill": "仅暂停本地publisher后提交，再恢复；另让隔离worker一次失败，对照两种积压阶段和恢复路径。",
      "acceptance": [
        "未清空消息掩盖故障。",
        "重复交付不能越过状态约束。",
        "恢复后新旧导出均有明确结果或失败去向。"
      ]
    }
  },
  {
    "id": "11",
    "group": "队列、实时通信与恢复",
    "title": "端到端排查 SSE 缓冲与 WebSocket 断开",
    "summary": "对比直连、代理和浏览器，分别验证首段到达、消息边界和连接生命周期。",
    "objectives": [
      "区分HTTP响应头到达与首个业务片段到达",
      "定位代理缓冲与客户端整包读取",
      "核对WebSocket升级、空闲超时和重连行为"
    ],
    "concepts": [
      {
        "title": "流式发送不保证每段立刻被用户看到",
        "body": "服务端生成、ASGI发送、代理转发、客户端读取和页面渲染都可能缓冲。先比较各段链路，再修改对应配置。"
      },
      {
        "title": "SSE与WebSocket的代理要求不同",
        "body": "SSE是持续HTTP响应，关注缓冲、压缩和读超时；WebSocket在HTTP/1.1握手时需要正确升级头，之后关注空闲超时与心跳。不能用普通GET成功证明WebSocket链路正常。"
      },
      {
        "title": "断开与重连属于业务行为",
        "body": "用户停止后应关闭连接和定时器，不自动重新生成同一内容。网络断开可有限退避重连，但不自动重发未经确认的业务消息。"
      }
    ],
    "steps": [
      {
        "title": "直连API测首段到达",
        "body": "新建stream_probe.py，只发送hello测试文本。按实际到达data行记录时间，不能用curl time_starttransfer代替首个业务片段，因为响应头可能很早已发送。",
        "code": "from time import perf_counter\nimport httpx\n\ndef probe(base_url):\n    started = perf_counter()\n    with httpx.Client(base_url=base_url, timeout=5) as client:\n        with client.stream(\"GET\", \"/stream\", params={\"prompt\": \"hello\"}) as response:\n            print(\"status\", response.status_code, response.headers.get(\"content-type\"))\n            response.raise_for_status()\n            for line in response.iter_lines():\n                if line.startswith(\"data:\"):\n                    print(round(perf_counter()-started, 3), line, flush=True)\n\nif __name__ == \"__main__\":\n    probe(\"http://127.0.0.1:8000\")",
        "lang": "python",
        "filename": "stream_probe.py"
      },
      {
        "title": "保持请求不变对比代理",
        "body": "按Docker/Linux模块的实际代理端口修改base_url；下面8080仅表示本地演练映射。若直连逐段、代理一次到达，检查代理缓冲；若两者正常而浏览器延后，查response.text/json或渲染缓存。",
        "code": "curl -N --max-time 5 'http://127.0.0.1:8000/stream?prompt=hello'\ncurl -N --max-time 5 'http://127.0.0.1:8080/stream?prompt=hello'\n# Windows终端用curl.exe；代理端口以本地部署配置为准。",
        "lang": "bash"
      },
      {
        "title": "只修改隔离代理中的对应location",
        "body": "以下片段放入已有nginx实验配置。示例代理与API同宿主；容器部署用实际API服务名替换127.0.0.1。先nginx -t通过，再按部署模块重载隔离代理。proxy_read_timeout是两次读取间隔限制，不是总请求时长。",
        "code": "# server内部的SSE location\nlocation /stream {\n    proxy_pass http://127.0.0.1:8000;\n    proxy_http_version 1.1;\n    proxy_buffering off;\n    proxy_read_timeout 65s;\n}\n\n# http级别：\n# map $http_upgrade $connection_upgrade {\n#     default upgrade;\n#     '' close;\n# }\n# server内部的WebSocket location：\n# location /ws {\n#     proxy_pass http://127.0.0.1:8000;\n#     proxy_http_version 1.1;\n#     proxy_set_header Upgrade $http_upgrade;\n#     proxy_set_header Connection $connection_upgrade;\n#     proxy_read_timeout 65s;\n# }",
        "lang": "nginx",
        "filename": "nginx-lab.conf"
      },
      {
        "title": "在浏览器核对升级和关闭",
        "body": "在隔离页面Console连接已确认的代理WS地址，观察Network中的握手与Frames/Messages。HTTPS页面使用wss。收到open后发送hello和ping，再主动关闭；按API课程有限退避逻辑验证重连。",
        "code": "const socket = new WebSocket(\"ws://127.0.0.1:8080/ws\");\nsocket.onopen = () => {\n  socket.send(\"hello\");\n  socket.send(\"ping\");\n};\nsocket.onmessage = event => console.log(JSON.parse(event.data));\nsocket.onerror = () => console.log(\"socket error\");\nsocket.onclose = event => console.log(\"closed\", event.code, event.reason);\n// 收到消息后主动停止：socket.close(1000, \"drill complete\");",
        "lang": "javascript",
        "filename": "浏览器Console"
      },
      {
        "title": "核对主动停止与真实清理",
        "body": "SSE收到done后关闭EventSource；fetch用AbortController停止读取，服务端取消后释放资源。WebSocket主动停止后等待超过最大重连间隔，确认无新连接。恢复所有临时代理配置，再跑一次完整SSE和echo。",
        "code": "# 记录实际观测，勿填写预计时间：\n# direct_first_data_seconds:\n# proxy_first_data_seconds:\n# browser_first_render_seconds:\n# websocket_handshake_result:\n# stop_followed_by_reconnect: 未测\n# server_cleanup_observed: 未测\n# result: not_run",
        "lang": "text",
        "filename": "stream-verification.md"
      }
    ],
    "checks": [
      "直连、代理、浏览器三段各有独立到达记录。",
      "WebSocket能完成握手与echo/pong，不仅是普通HTTP返回200。",
      "主动停止后没有意外重连或重复生成，服务端清理可观察。"
    ],
    "pitfalls": [
      {
        "symptom": "终端逐段显示，网页一次显示全部",
        "cause": "前端调用了response.text/json或等待完整文本再渲染。",
        "fix": "使用流式读取和按协议边界解析，分段更新UI。"
      },
      {
        "symptom": "WebSocket连接约固定时间断开",
        "cause": "代理空闲超时或心跳缺失可能触发，但需结合日志验证。",
        "fix": "核对代理读超时与实际消息间隔，采用有界应用心跳和停止规则。"
      }
    ],
    "exercise": {
      "task": "如果response headers立刻到达、三条data都在1.2秒同时出现，能断言服务器没有yield吗？",
      "hint": "中间层或客户端可能缓冲，先做直连对照。",
      "answer": "不能。比较直连data到达时间与代理结果，再检查生成器、代理缓冲和客户端读取；响应头时间不证明每段业务数据的传输情况。"
    },
    "refs": [
      {
        "label": "Nginx代理缓冲",
        "url": "https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering"
      },
      {
        "label": "Nginx WebSocket",
        "url": "https://nginx.org/en/docs/http/websocket.html"
      },
      {
        "label": "HTTPX流读取",
        "url": "https://www.python-httpx.org/quickstart/"
      },
      {
        "label": "MDN EventSource",
        "url": "https://developer.mozilla.org/en-US/docs/Web/API/EventSource"
      }
    ],
    "production": {
      "scenario": "本地API流式正常，部署后文本一次出现或WebSocket频繁断开。",
      "decisions": [
        "按直连、代理、客户端逐段定位。",
        "配置变更限定相关location，测试语法后再重载隔离实例。",
        "重连不会自动重复不可逆业务消息。"
      ],
      "failureDrill": "只在隔离代理开启/关闭响应缓冲做对照，随后测试空闲与主动停止两类断开。",
      "acceptance": [
        "到达时间来自真实业务片段。",
        "代理与浏览器行为分别验证。",
        "故障配置恢复，停止逻辑无残留连接。"
      ]
    }
  },
  {
    "id": "12",
    "group": "队列、实时通信与恢复",
    "title": "完成故障演练、恢复门槛与 runbook",
    "summary": "把排障步骤整理为可交接流程，保存证据、恢复条件和后续修复项。",
    "objectives": [
      "执行单变量故障矩阵并保留实际观测",
      "用明确业务门槛判断恢复",
      "编写可执行的runbook与复盘行动项"
    ],
    "concepts": [
      {
        "title": "演练要有开始与结束条件",
        "body": "每个故障限定实例、任务ID、持续时间和恢复方法。达到预定风险边界就停止，不能为了复现而无限增加负载。"
      },
      {
        "title": "恢复门槛应覆盖业务与数据",
        "body": "服务恢复包括新请求正常、积压受控、已受理任务去向清楚、无错误发布结果。重启成功或监控颜色变绿只是部分证据。"
      },
      {
        "title": "复盘区分触发因素与系统缺口",
        "body": "记录什么触发、为什么扩大、哪些控制有效、哪里缺少观测或恢复能力。行动项要有负责人、验收条件和优先级，不只写“加强监控”。"
      }
    ],
    "steps": [
      {
        "title": "准备隔离演练清单",
        "body": "每个案例用全新测试数据，先保存无故障基线。下面矩阵是待执行计划，不代表已经发生事故或完成测试。禁止把生产凭据、真实笔记正文放入公开材料。",
        "code": "| 案例 | 限定故障 | 主要证据 | 恢复动作 |\n|---|---|---|---|\n| 请求阻塞 | 本地0.5秒同步sleep | loop lag、CPU、HTTP耗时 | 恢复await实现 |\n| 数据库锁 | 两会话同一演练笔记 | blocked_by、lock_timeout | 两会话ROLLBACK |\n| 缓存不可用 | 只停缓存6380 | 回源量、503、池等待 | 恢复缓存 |\n| 发布积压 | 只停publisher | outbox最老等待 | 恢复有界publisher |\n| worker租约到期 | 一个记录过的job | token、attempts、状态 | 有界恢复扫描 |\n| SSE缓冲 | 隔离代理location | 首data时间对照 | 恢复缓冲配置 |",
        "lang": "text",
        "filename": "drill-plan.md"
      },
      {
        "title": "定义停止与恢复门槛",
        "body": "将以下阈值改为本地机器可承受值，再执行。示例阈值只用于练习，不作为线上SLO建议。超出范围停止故障注入并回到基线，不继续加压寻找更大异常。",
        "code": "{\n  \"scope\": \"isolated notes-service only\",\n  \"max_test_requests\": 20,\n  \"max_fault_duration_seconds\": 30,\n  \"max_controlled_python_allocation_mib\": 10,\n  \"stop_if\": [\"出现非演练数据写入风险\", \"无法确定目标实例\", \"恢复命令失败\"],\n  \"recovery_gate\": [\n    \"live、ready与笔记读取按基线返回\",\n    \"一个新导出完成且文件符合快照\",\n    \"旧演练任务均有结果或明确失败原因\",\n    \"outbox和租约等待回到可解释范围\"\n  ],\n  \"result\": \"not_run\"\n}",
        "lang": "json",
        "filename": "drill-boundary.json"
      },
      {
        "title": "按时间线只执行一个故障",
        "body": "每次记录UTC时间、目标、request_id/job_id、配置前值、操作、指标变化和恢复前值。日志示例标synthetic，实际输出标observed。任何未执行项保留not_run，不能根据理论预计填写成功。",
        "code": "{\n  \"record_type\": \"template\",\n  \"case\": \"db_lock_wait\",\n  \"started_at_utc\": null,\n  \"target\": null,\n  \"request_id\": null,\n  \"job_id\": null,\n  \"observations\": [],\n  \"restored_at_utc\": null,\n  \"recovery_checks\": [],\n  \"result\": \"not_run\"\n}",
        "lang": "json",
        "filename": "drill-result.json"
      },
      {
        "title": "用固定结构编写runbook",
        "body": "每条runbook提供症状、只读检查、分支判断、限定动作、回滚和验收。命令中的PID、容器名、数据库和端口必须先核对；不要放“重启全部”“清空队列”之类无范围操作。",
        "code": "# Runbook：导出已受理但长期未完成\n## 识别\n记录job_id、受理时间、当前数据库状态。\n## 只读检查\noutbox未发布？队列有人消费？租约有效？attempts/deadline是否耗尽？\n## 分支\n未发布→查publisher；无人消费→查路由；租约过期→按恢复规则处理。\n## 动作\n只处理已确认的本地实例/任务，保持次数、退避与容量约束。\n## 回滚\n恢复本次修改前配置，停止新故障输入。\n## 验收\n核对新任务、旧任务与结果文件；未恢复项明确列出。",
        "lang": "text",
        "filename": "runbook-export.md"
      },
      {
        "title": "输出复盘行动项与证据索引",
        "body": "复盘不只记录最后一个报错。为触发原因、扩大因素、恢复机制和观测缺口分别填写证据，并把修复项写成可验收工作。示例中的行动项需要实际演练支持后才采用。",
        "code": "| 类型 | 发现 | 证据 | 行动项 | 验收 |\n|---|---|---|---|---|\n| 触发 | 待实际观测 | 时间线链接 | 待定 | 可重复复现 |\n| 扩大因素 | 待实际观测 | 指标/trace | 待定 | 相同故障影响受限 |\n| 恢复缺口 | 待实际观测 | 任务状态 | 待定 | 恢复时间可测 |\n| 观测缺口 | 待实际观测 | 缺失字段 | 待定 | 可定位具体阶段 |",
        "lang": "text",
        "filename": "postmortem-actions.md"
      }
    ],
    "checks": [
      "每次故障均有范围、停止条件和实际恢复记录。",
      "恢复验证覆盖新请求、旧任务与输出一致性。",
      "runbook有分支与验收，复盘行动项能被具体检查。"
    ],
    "pitfalls": [
      {
        "symptom": "演练结束但临时故障开关还开着",
        "cause": "没有记录配置前值和恢复清单。",
        "fix": "每次演练最后核对开关、代理配置、进程和事务，跑无故障回归。"
      },
      {
        "symptom": "复盘只写某人误操作",
        "cause": "忽略缺少权限边界、幂等、容量或观测导致的扩大。",
        "fix": "把触发因素和系统控制缺口分别写清，并给可验收修复项。"
      }
    ],
    "exercise": {
      "task": "把“加强监控”改成一个可验收行动项。",
      "hint": "指定指标、范围、触发条件、关联信息和验证方式。",
      "answer": "例如：为export_outbox最老未发布年龄增加告警，限定服务标签，持续超过本地设定阈值时提供publisher日志入口；用暂停本地publisher的演练验证触发与恢复，数值按实际业务基线设置。"
    },
    "refs": [
      {
        "label": "Google SRE复盘",
        "url": "https://sre.google/sre-book/postmortem-culture/"
      },
      {
        "label": "Google SRE事故管理",
        "url": "https://sre.google/sre-book/managing-incidents/"
      },
      {
        "label": "Google SRE有效排障",
        "url": "https://sre.google/sre-book/effective-troubleshooting/"
      }
    ],
    "production": {
      "scenario": "将notes-service交付维护，希望另一位开发者能按文档定位并恢复常见故障。",
      "decisions": [
        "演练在独立环境，目标和恢复动作先明确。",
        "证据与预计行为分开，未运行不标通过。",
        "复盘行动项落实到观测、容量、一致性和恢复能力。"
      ],
      "failureDrill": "逐项执行本课矩阵，每次恢复基线后才进入下一项，最终检查所有临时开关与事务已关闭。",
      "acceptance": [
        "交接者可按runbook走到具体分支。",
        "已受理任务没有不明去向。",
        "所有完成声明有实际记录，未执行部分明确列出。"
      ]
    }
  }
]