| 123456789101112131415161718192021222324252627282930313233343536373839404142434445 |
- groups:
- - name: zhifu
- rules:
- - alert: ServiceUnavailable
- expr: up{job=~"platform|inference|operations|replica-gateway"} == 0
- for: 30s
- labels: {severity: critical}
- annotations:
- summary: "服务监控采集失败"
- description: "检查目标进程、网络及依赖服务。"
- - alert: WorkerUnavailable
- expr: zhifu_workers_alive == 0
- for: 15s
- labels: {severity: critical}
- annotations:
- summary: "工作进程心跳中断"
- description: "检查对应 role 的进程日志并恢复服务。"
- - alert: QueueWaiting
- expr: zhifu_oldest_pending_seconds > 120
- for: 1m
- labels: {severity: warning}
- annotations:
- summary: "任务等待超过两分钟"
- description: "检查模型额度、工作进程及外部业务响应。"
- - alert: HttpServerErrors
- expr: sum by(job) (rate(http_requests_total{status=~"5.."}[5m])) / clamp_min(sum by(job) (rate(http_requests_total[5m])), 0.01) > 0.05
- for: 1m
- labels: {severity: critical}
- annotations:
- summary: "HTTP 服务错误比例超过百分之五"
- description: "依据请求编号检查应用日志及依赖状态。"
- - alert: InferenceSaturation
- expr: sum(rate(http_requests_total{job="inference",status="429"}[5m])) / clamp_min(sum(rate(http_requests_total{job="inference"}[5m])), 0.01) > 0.1
- for: 1m
- labels: {severity: warning}
- annotations:
- summary: "向量服务繁忙请求超过百分之十"
- description: "检查计算资源、请求批量和服务副本数量。"
- - alert: AgentFailures
- expr: zhifu_jobs_15m{kind="agent",status="FAILED"} >= 5
- for: 1m
- labels: {severity: warning}
- annotations:
- summary: "最近十五分钟 Agent 失败任务达到五项"
- description: "查询任务 error_code 和模型服务状态。"
|
