zhifu-platform/scripts/prepare_agent_evaluation.py

import hashlib
import json
from pathlib import Path

from zhifu.evaluation import AgentManifest, Dataset, Judgment, RetrievalManifest, source_digest


def main():
    path = Path("evaluations/agent-scenarios.json")
    content = path.read_bytes()
    dataset = Dataset.model_validate_json(content)
    output = Path(".work/agent-evaluation")
    output.mkdir(parents=True, exist_ok=True)
    # 同一情境的不同表述属于同一集合，避免重复表述跨集合影响评测结果。
    cases = [
        {
            **case,
            "status": "NOT_RUN",
            "label_review": "REQUIRED",
        }
        for case in dataset.cases()
    ]
    report = {
        "version": dataset.version,
        "origin": dataset.origin,
        "sha256": hashlib.sha256(content).hexdigest(),
        "source_sha256": source_digest(),
        "count": len(cases),
        "generation_attempts": 0,
        "cases": cases,
    }
    (output / "cases.json").write_text(
        json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
    )
    for name, model in (("agent-manifest", AgentManifest), ("judgment", Judgment),
                        ("retrieval-manifest", RetrievalManifest)):
        (output / f"{name}.schema.json").write_text(
            json.dumps(model.model_json_schema(), ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
    print(json.dumps({"count": len(cases), "generation_attempts": 0, "output": str(output)}))


if __name__ == "__main__":
    main()