From 69754d89e689ceb2e3f00dae6d57779dacb2d1b7 Mon Sep 17 00:00:00 2001 From: AsyncKurisu <1750981157@qq.com> Date: Wed, 29 Jul 2026 15:54:40 +0800 Subject: [PATCH 1/2] feat: add eval-optimize-loop example with report and gate --- .../optimization/eval_optimize_loop/DESIGN.md | 25 + .../optimization/eval_optimize_loop/README.md | 160 +++ .../optimization/eval_optimize_loop/gate.json | 15 + .../eval_optimize_loop/optimizer.json | 39 + .../output/optimization_report.json | 1160 +++++++++++++++++ .../output/optimization_report.md | 133 ++ .../eval_optimize_loop/pipeline/__init__.py | 22 + .../pipeline/attribution.py | 220 ++++ .../eval_optimize_loop/pipeline/delta.py | 289 ++++ .../eval_optimize_loop/pipeline/gate.py | 259 ++++ .../pipeline/optimization.py | 320 +++++ .../eval_optimize_loop/pipeline/pipeline.py | 729 +++++++++++ .../eval_optimize_loop/pipeline/report.py | 166 +++ .../eval_optimize_loop/pipeline/types.py | 464 +++++++ .../eval_optimize_loop/prompts/skill.md | 1 + .../eval_optimize_loop/prompts/system.md | 1 + .../eval_optimize_loop/run_pipeline.py | 63 + .../eval_optimize_loop/tests/__init__.py | 5 + .../eval_optimize_loop/tests/conftest.py | 306 +++++ .../tests/test_attribution.py | 121 ++ .../eval_optimize_loop/tests/test_delta.py | 54 + .../eval_optimize_loop/tests/test_gate.py | 122 ++ .../tests/test_pipeline_reports.py | 93 ++ .../tests/test_real_mode.py | 49 + .../eval_optimize_loop/train.evalset.json | 97 ++ .../eval_optimize_loop/val.evalset.json | 97 ++ 26 files changed, 5010 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/gate.json create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/output/optimization_report.json create mode 100644 examples/optimization/eval_optimize_loop/output/optimization_report.md create mode 100644 examples/optimization/eval_optimize_loop/pipeline/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/attribution.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/delta.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/gate.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/optimization.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/report.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/types.py create mode 100644 examples/optimization/eval_optimize_loop/prompts/skill.md create mode 100644 examples/optimization/eval_optimize_loop/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/tests/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/tests/conftest.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_attribution.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_delta.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_gate.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py create mode 100644 examples/optimization/eval_optimize_loop/tests/test_real_mode.py create mode 100644 examples/optimization/eval_optimize_loop/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/val.evalset.json diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 000000000..4022a49d1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,25 @@ +# 方案设计说明 + +## 整体流程 + +Pipeline 读取训练集、验证集、Prompt 源文件、优化配置和 Gate 配置，先分别运行 baseline 评测，再对失败 case 做错误归因。随后优化器基于 baseline 结果生成候选 Prompt，Pipeline 使用候选 Prompt 重新评测训练集和验证集，并生成逐 case delta。最终 Gate 只根据候选复评结果和 delta 判断是否接受候选，报告同时输出 JSON 与 Markdown。 + +## 失败归因方法 + +错误归因由 `FailureAttributor` 负责，输入是标准化后的 case 结果、trace、metric details 和 evaluator metadata。规则优先识别工具调用名称或顺序错误、工具参数不匹配、最终回复不一致、LLM rubric 未达标、召回类指标失败和格式约束失败。每个失败 case 都会写入 `failure_analysis`，其中包含分类、置信度、解释和 evidence。无法稳定判断时归为 `unknown`，但仍保留原始失败线索，方便人工复核。 + +## 优化方法 + +优化目标是 `prompts/system.md` 与 `prompts/skill.md`。real 模式复用 `AgentOptimizer` 和 `TargetPrompt`，并固定 `update_source=False`，只生成候选 Prompt，不覆盖源文件。fake 模式使用确定性优化器，根据错误归因摘要追加修复指令，用于无 API Key 的可复现演示。候选 Prompt 必须重新跑评测，不能只依赖优化器内部的 aggregate 分数。 + +## 接受策略 + +Gate 以验证集为核心依据。默认策略要求验证集达到最小分数提升、不能新增失败、不能出现回归、关键 case 不允许退化、成本不能超过预算。每条规则都会生成 `GateRuleResult`，报告中保留规则名、是否通过、严重程度、说明和 evidence。最终决策为 `accept` 或 `reject`，并给出推荐动作。 + +## 防止过拟合策略 + +Pipeline 同时评测训练集和验证集，但 Gate 不因训练集提升而直接接受候选。当训练集提升达到阈值而验证集下降时，会标记为过拟合并拒绝。默认 fake 样例包含“训练集收益 + 验证集关键 case 退化”的情况，用于展示闭环如何避免把只优化训练样本的 Prompt 推向生产。 + +## 复现信息 + +`metadata` 负责保留最小复现信息：示例根目录、复现命令、输出路径和运行模式。这样既能定位这次运行的产物，又不会把输入文件 hash、配置脱敏快照和 Prompt diff 维护成单独一层。JSON 用于自动化消费，Markdown 用于人工复盘，两者都写入固定 `output/` 目录，便于比较和归档。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 000000000..4d3eb3b90 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,160 @@ +# Evaluation + Optimization 自动闭环示例 + +## 示例目标 + +本示例演示如何在 `examples/optimization/eval_optimize_loop/` 中组合 `AgentEvaluator` 与 `AgentOptimizer`，完成 baseline 评测、错误归因、Prompt 优化、候选验证和 Gate 决策的完整闭环。实现全部保留在 example 内，不修改 `trpc_agent_sdk` 公共 API。 + +## 目录结构 + +```text +examples/optimization/eval_optimize_loop/ + README.md + DESIGN.md + gate.json + optimizer.json + output/ + prompts/ + system.md + skill.md + run_pipeline.py + train.evalset.json + val.evalset.json + pipeline/ + __init__.py + attribution.py + delta.py + gate.py + optimization.py + pipeline.py + report.py + types.py + tests/ + conftest.py + test_*.py +``` + +## 运行方式 + +先进入示例目录： + +```bash +cd examples/optimization/eval_optimize_loop +``` + +fake 模式可无 API Key 运行完整流程： + +```bash +python run_pipeline.py --mode fake +``` + +real 模式接入 `AgentEvaluator` 与 `AgentOptimizer`，需要可用模型凭证： + +```bash +python run_pipeline.py --mode real +``` + +运行后固定生成： + +- `output/optimization_report.json` +- `output/optimization_report.md` + +## 输入与输出 + +输入包括： + +- `train.evalset.json` +- `val.evalset.json` +- `optimizer.json` +- `gate.json` +- `prompts/system.md` +- `prompts/skill.md` + +输出包括： + +- `optimization_report.json`：给机器读取的结构化报告 +- `optimization_report.md`：给开发者阅读的审计报告 + +## 报告内容 + +`optimization_report.json` 包含以下部分： + +- `run`：运行时间、模式、随机种子 +- `inputs`：输入文件相对路径 +- `config`：gate 与 optimizer 的有效配置快照 +- `baseline`：训练集与验证集 baseline 结果 +- `failure_attribution`：baseline 与 candidate 的错误归因汇总 +- `candidate`：候选 Prompt 的重新评测结果 +- `delta`：baseline 与 candidate 的逐 case 差异 +- `gate_decision`：接受或拒绝候选的规则结果 +- `optimization`：优化轮次、原因、成本、耗时、Prompt 版本 +- `metadata`：示例根目录、复现命令、输出路径 + +`optimization_report.md` 则按人类阅读顺序展示： + +- 运行摘要 +- Baseline 表现 +- 错误归因汇总 +- 优化过程 +- 候选验证 +- 逐 case delta +- Gate 决策 +- 审计与复现 + +## fake 模式语义与目的 + +fake 模式用于在没有真实 API Key 时跑通完整闭环，目的不是模拟生产答案，而是稳定演示“评测 - 归因 - 优化 - 复评 - Gate”的业务链路。当前示例故意覆盖三类场景： + +- 可优化成功：部分失败 case 被候选 Prompt 修复 +- 优化无效：部分 case 仍保持失败 +- 优化后退化：验证集关键 case 被候选 Prompt 破坏 + +因此 fake 报告默认会展示候选局部有收益，但最终仍因验证集退化或关键 case 退化而被拒绝。 + +## 优化目标 + +当前示例优化两个目标 Prompt： + +- `system_prompt` +- `skill` + +优化过程只生成候选版本，不覆盖源文件。Gate 只根据候选 Prompt 的独立复评结果做判断，不直接使用优化器自身的 aggregate 分数。 + +## 样例和场景 + +公开样例包含 6 条 case： + +- 3 条训练 case +- 3 条验证 case + +它们共同覆盖以下行为： + +- baseline 通过 +- baseline 失败 +- 候选修复成功 +- 候选保持失败 +- 候选引入退化 + +## Gate 策略 + +默认 Gate 要求： + +- 验证集必须有足够分数提升 +- 不能新增验证集失败 +- 不能出现验证集回归 +- 关键 case 不允许退化 +- 成本不能超预算 +- 训练集提升但验证集下降时判定为过拟合 + +Gate 仅基于候选复评和 delta 做决策，不以优化器内部输出替代验证结果。 + +## real 模式说明 + +real 模式会调用现有 `AgentEvaluator` 与 `AgentOptimizer`。为了保证示例不回写源 Prompt，优化始终使用 `update_source=False`。如果缺少真实凭证，推荐使用 fake 模式。 + +## 测试与格式检查 + +```bash +pytest tests -q +yapf --diff pipeline/*.py run_pipeline.py tests/*.py +flake8 pipeline/*.py run_pipeline.py tests/*.py +``` diff --git a/examples/optimization/eval_optimize_loop/gate.json b/examples/optimization/eval_optimize_loop/gate.json new file mode 100644 index 000000000..ec4a63eaf --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gate.json @@ -0,0 +1,15 @@ +{ + "min_val_score_gain": 0.05, + "min_val_pass_rate_gain": 0.0, + "allow_new_failures": false, + "allow_regressions": false, + "critical_case_ids": ["val_multiply_pass"], + "max_cost_delta": 0.0, + "max_cost_ratio": 1.2, + "require_validation_improvement": true, + "overfit_policy": { + "enabled": true, + "train_gain_min": 0.05, + "val_drop_tolerance": 0.0 + } +} diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 000000000..a3a33ee81 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,39 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 1024, + "temperature": 0.2 + } + }, + "max_metric_calls": 12 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/output/optimization_report.json b/examples/optimization/eval_optimize_loop/output/optimization_report.json new file mode 100644 index 000000000..18300bbc3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/output/optimization_report.json @@ -0,0 +1,1160 @@ +{ + "schema_version": "eval-optimize-loop-v1", + "run": { + "mode": "real", + "started_at": "2026-07-29T07:46:53.491932+00:00", + "finished_at": "2026-07-29T07:47:25.010475+00:00", + "duration_seconds": 31.518541, + "seed": 42 + }, + "inputs": { + "train_evalset_path": "train.evalset.json", + "val_evalset_path": "val.evalset.json", + "optimizer_config_path": "optimizer.json", + "gate_config_path": "gate.json", + "prompt_paths": { + "system_prompt": "prompts/system.md", + "skill": "prompts/skill.md" + } + }, + "config": { + "gate": { + "min_val_score_gain": 0.05, + "min_val_pass_rate_gain": 0.0, + "allow_new_failures": false, + "allow_regressions": false, + "critical_case_ids": [ + "val_multiply_pass" + ], + "max_cost_delta": 0.0, + "max_cost_ratio": 1.2, + "require_validation_improvement": true, + "overfit_policy": { + "enabled": true, + "train_gain_min": 0.05, + "val_drop_tolerance": 0.0 + } + }, + "optimizer": { + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 1024, + "temperature": 0.2 + } + }, + "max_metric_calls": 12 + } + } + } + }, + "baseline": { + "train": { + "eval_set_id": "eval_optimize_loop_train", + "case_count": 3, + "passed_count": 2, + "failed_count": 1, + "failure_attribution_summary": { + "final_answer_mismatch": 1, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + }, + "cases": [ + { + "id": "train_addition_pass", + "metric_score": 1.0, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "failure_reason": "", + "trace": { + "user": "小明有 4 个苹果，又买了 7 个，现在一共有多少个苹果？", + "expected": "答案：11 个", + "actual": "答案：11 个", + "tool_calls": [] + }, + "latency": 0.0003851220244541764, + "cost": 0.0, + "failure_analysis": null + }, + { + "id": "train_discount_fail", + "metric_score": 0.0, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "passed": false, + "failure_reason": "final_response_avg_score failed.", + "trace": { + "user": "一件衣服原价 200 元，打 8 折后多少钱？", + "expected": "答案：160 元", + "actual": "答案：180 元", + "tool_calls": [] + }, + "latency": 0.0002015280188061297, + "cost": 0.0, + "failure_analysis": { + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "train_discount_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：160 元", + "actual": "答案：180 元", + "failure_reason": "final_response_avg_score failed." + } + } + }, + { + "id": "train_percent_pass", + "metric_score": 1.0, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "failure_reason": "", + "trace": { + "user": "40 名学生中 25% 戴眼镜，戴眼镜的有多少人？", + "expected": "答案：10 人", + "actual": "计算 40 的 25%，答案：10 人", + "tool_calls": [] + }, + "latency": 0.00017735798610374331, + "cost": 0.0, + "failure_analysis": null + } + ] + }, + "val": { + "eval_set_id": "eval_optimize_loop_val", + "case_count": 3, + "passed_count": 1, + "failed_count": 2, + "failure_attribution_summary": { + "final_answer_mismatch": 2, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + }, + "cases": [ + { + "id": "val_multiply_pass", + "metric_score": 1.0, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "failure_reason": "", + "trace": { + "user": "教室里有 5 排座位，每排 8 个，一共有多少个座位？", + "expected": "答案：40 个", + "actual": "答案：40 个", + "tool_calls": [] + }, + "latency": 0.00013485999079421163, + "cost": 0.0, + "failure_analysis": null + }, + { + "id": "val_water_fail", + "metric_score": 0.0, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "passed": false, + "failure_reason": "final_response_avg_score failed.", + "trace": { + "user": "1 升水重 1 千克，3.5 升水重多少千克？", + "expected": "答案：3.5 千克", + "actual": "答案：4 千克", + "tool_calls": [] + }, + "latency": 0.00013199500972405076, + "cost": 0.0, + "failure_analysis": { + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_water_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：3.5 千克", + "actual": "答案：4 千克", + "failure_reason": "final_response_avg_score failed." + } + } + }, + { + "id": "val_percent_fail", + "metric_score": 0.0, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "passed": false, + "failure_reason": "final_response_avg_score failed.", + "trace": { + "user": "班里 30 人，其中 60% 是女生，有多少名女生？", + "expected": "答案：18 人", + "actual": "答案：20 人", + "tool_calls": [] + }, + "latency": 0.0001270869979634881, + "cost": 0.0, + "failure_analysis": { + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_percent_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：18 人", + "actual": "答案：20 人", + "failure_reason": "final_response_avg_score failed." + } + } + } + ] + } + }, + "failure_attribution": { + "train_summary": { + "baseline": { + "final_answer_mismatch": 1, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + }, + "candidate": { + "final_answer_mismatch": 1, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + } + }, + "val_summary": { + "baseline": { + "final_answer_mismatch": 2, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + }, + "candidate": { + "final_answer_mismatch": 2, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + } + }, + "overall_summary": { + "final_answer_mismatch": 6, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + }, + "failed_cases": [ + { + "variant": "baseline", + "split": "train", + "case_id": "train_discount_fail", + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "train_discount_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：160 元", + "actual": "答案：180 元", + "failure_reason": "final_response_avg_score failed." + } + }, + { + "variant": "baseline", + "split": "val", + "case_id": "val_water_fail", + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_water_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：3.5 千克", + "actual": "答案：4 千克", + "failure_reason": "final_response_avg_score failed." + } + }, + { + "variant": "baseline", + "split": "val", + "case_id": "val_percent_fail", + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_percent_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：18 人", + "actual": "答案：20 人", + "failure_reason": "final_response_avg_score failed." + } + }, + { + "variant": "candidate", + "split": "train", + "case_id": "train_discount_fail", + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "train_discount_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：160 元", + "actual": "答案：180 元", + "failure_reason": "final_response_avg_score failed." + } + }, + { + "variant": "candidate", + "split": "val", + "case_id": "val_water_fail", + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_water_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：3.5 千克", + "actual": "答案：4 千克", + "failure_reason": "final_response_avg_score failed." + } + }, + { + "variant": "candidate", + "split": "val", + "case_id": "val_percent_fail", + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_percent_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：18 人", + "actual": "答案：20 人", + "failure_reason": "final_response_avg_score failed." + } + } + ] + }, + "candidate": { + "train": { + "eval_set_id": "eval_optimize_loop_train", + "case_count": 3, + "passed_count": 2, + "failed_count": 1, + "failure_attribution_summary": { + "final_answer_mismatch": 1, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + }, + "cases": [ + { + "id": "train_addition_pass", + "metric_score": 1.0, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "failure_reason": "", + "trace": { + "user": "小明有 4 个苹果，又买了 7 个，现在一共有多少个苹果？", + "expected": "答案：11 个", + "actual": "答案：11 个", + "tool_calls": [] + }, + "latency": 0.0006882919697090983, + "cost": 0.0, + "failure_analysis": null + }, + { + "id": "train_discount_fail", + "metric_score": 0.0, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "passed": false, + "failure_reason": "final_response_avg_score failed.", + "trace": { + "user": "一件衣服原价 200 元，打 8 折后多少钱？", + "expected": "答案：160 元", + "actual": "答案：180 元", + "tool_calls": [] + }, + "latency": 0.0005271550035104156, + "cost": 0.0, + "failure_analysis": { + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "train_discount_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：160 元", + "actual": "答案：180 元", + "failure_reason": "final_response_avg_score failed." + } + } + }, + { + "id": "train_percent_pass", + "metric_score": 1.0, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "failure_reason": "", + "trace": { + "user": "40 名学生中 25% 戴眼镜，戴眼镜的有多少人？", + "expected": "答案：10 人", + "actual": "计算 40 的 25%，答案：10 人", + "tool_calls": [] + }, + "latency": 0.0004119730438105762, + "cost": 0.0, + "failure_analysis": null + } + ] + }, + "val": { + "eval_set_id": "eval_optimize_loop_val", + "case_count": 3, + "passed_count": 1, + "failed_count": 2, + "failure_attribution_summary": { + "final_answer_mismatch": 2, + "tool_call_error": 0, + "parameter_error": 0, + "llm_rubric_failed": 0, + "retrieval_failure": 0, + "format_error": 0, + "unknown": 0 + }, + "cases": [ + { + "id": "val_multiply_pass", + "metric_score": 1.0, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "failure_reason": "", + "trace": { + "user": "教室里有 5 排座位，每排 8 个，一共有多少个座位？", + "expected": "答案：40 个", + "actual": "答案：40 个", + "tool_calls": [] + }, + "latency": 0.0001963640097528696, + "cost": 0.0, + "failure_analysis": null + }, + { + "id": "val_water_fail", + "metric_score": 0.0, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "passed": false, + "failure_reason": "final_response_avg_score failed.", + "trace": { + "user": "1 升水重 1 千克，3.5 升水重多少千克？", + "expected": "答案：3.5 千克", + "actual": "答案：4 千克", + "tool_calls": [] + }, + "latency": 0.0001838289899751544, + "cost": 0.0, + "failure_analysis": { + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_water_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：3.5 千克", + "actual": "答案：4 千克", + "failure_reason": "final_response_avg_score failed." + } + } + }, + { + "id": "val_percent_fail", + "metric_score": 0.0, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "passed": false, + "failure_reason": "final_response_avg_score failed.", + "trace": { + "user": "班里 30 人，其中 60% 是女生，有多少名女生？", + "expected": "答案：18 人", + "actual": "答案：20 人", + "tool_calls": [] + }, + "latency": 0.00017802900401875377, + "cost": 0.0, + "failure_analysis": { + "category": "final_answer_mismatch", + "confidence": 0.9, + "explanation": "Final response did not match the reference answer.", + "evidence": { + "case_id": "val_percent_fail", + "failed_metrics": [ + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "eval_status": "FAILED", + "reason": null + } + ], + "expected": "答案：18 人", + "actual": "答案：20 人", + "failure_reason": "final_response_avg_score failed." + } + } + } + ] + }, + "prompts": { + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + } + }, + "delta": { + "summary": { + "overall_change_type": "unchanged", + "train": { + "split": "train", + "baseline_score": 0.666667, + "candidate_score": 0.666667, + "score_delta": 0.0, + "baseline_pass_rate": 0.666667, + "candidate_pass_rate": 0.666667, + "pass_rate_delta": 0.0, + "case_count": 3, + "new_pass_count": 0, + "new_fail_count": 0, + "regression_count": 0, + "improvement_count": 0, + "missing_candidate_case_ids": [], + "extra_candidate_case_ids": [] + }, + "val": { + "split": "val", + "baseline_score": 0.333333, + "candidate_score": 0.333333, + "score_delta": 0.0, + "baseline_pass_rate": 0.333333, + "candidate_pass_rate": 0.333333, + "pass_rate_delta": 0.0, + "case_count": 3, + "new_pass_count": 0, + "new_fail_count": 0, + "regression_count": 0, + "improvement_count": 0, + "missing_candidate_case_ids": [], + "extra_candidate_case_ids": [] + }, + "regression_count": 0, + "improvement_count": 0, + "new_pass_count": 0, + "new_fail_count": 0, + "missing_candidate_case_ids": [], + "extra_candidate_case_ids": [] + }, + "train": { + "split": "train", + "baseline_score": 0.666667, + "candidate_score": 0.666667, + "score_delta": 0.0, + "baseline_pass_rate": 0.666667, + "candidate_pass_rate": 0.666667, + "pass_rate_delta": 0.0, + "case_deltas": [ + { + "id": "train_addition_pass", + "split": "train", + "change_type": "unchanged", + "baseline": { + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 0.0003851220244541764, + "cost": 0.0 + }, + "candidate": { + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 0.0006882919697090983, + "cost": 0.0 + }, + "score_delta": 0.0, + "metric_deltas": [ + { + "metric_name": "final_response_avg_score", + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0, + "baseline_passed": true, + "candidate_passed": true, + "status_transition": "passed_to_passed" + } + ], + "latency_delta": 0.000303, + "cost_delta": 0.0, + "regression": false, + "improvement": false, + "notes": [] + }, + { + "id": "train_discount_fail", + "split": "train", + "change_type": "unchanged", + "baseline": { + "passed": false, + "metric_score": 0.0, + "failure_category": "final_answer_mismatch", + "latency": 0.0002015280188061297, + "cost": 0.0 + }, + "candidate": { + "passed": false, + "metric_score": 0.0, + "failure_category": "final_answer_mismatch", + "latency": 0.0005271550035104156, + "cost": 0.0 + }, + "score_delta": 0.0, + "metric_deltas": [ + { + "metric_name": "final_response_avg_score", + "baseline_score": 0.0, + "candidate_score": 0.0, + "score_delta": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "status_transition": "failed_to_failed" + } + ], + "latency_delta": 0.000326, + "cost_delta": 0.0, + "regression": false, + "improvement": false, + "notes": [] + }, + { + "id": "train_percent_pass", + "split": "train", + "change_type": "unchanged", + "baseline": { + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 0.00017735798610374331, + "cost": 0.0 + }, + "candidate": { + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 0.0004119730438105762, + "cost": 0.0 + }, + "score_delta": 0.0, + "metric_deltas": [ + { + "metric_name": "final_response_avg_score", + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0, + "baseline_passed": true, + "candidate_passed": true, + "status_transition": "passed_to_passed" + } + ], + "latency_delta": 0.000235, + "cost_delta": 0.0, + "regression": false, + "improvement": false, + "notes": [] + } + ], + "missing_candidate_case_ids": [], + "extra_candidate_case_ids": [] + }, + "val": { + "split": "val", + "baseline_score": 0.333333, + "candidate_score": 0.333333, + "score_delta": 0.0, + "baseline_pass_rate": 0.333333, + "candidate_pass_rate": 0.333333, + "pass_rate_delta": 0.0, + "case_deltas": [ + { + "id": "val_multiply_pass", + "split": "val", + "change_type": "unchanged", + "baseline": { + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 0.00013485999079421163, + "cost": 0.0 + }, + "candidate": { + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 0.0001963640097528696, + "cost": 0.0 + }, + "score_delta": 0.0, + "metric_deltas": [ + { + "metric_name": "final_response_avg_score", + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0, + "baseline_passed": true, + "candidate_passed": true, + "status_transition": "passed_to_passed" + } + ], + "latency_delta": 6.2e-05, + "cost_delta": 0.0, + "regression": false, + "improvement": false, + "notes": [] + }, + { + "id": "val_percent_fail", + "split": "val", + "change_type": "unchanged", + "baseline": { + "passed": false, + "metric_score": 0.0, + "failure_category": "final_answer_mismatch", + "latency": 0.0001270869979634881, + "cost": 0.0 + }, + "candidate": { + "passed": false, + "metric_score": 0.0, + "failure_category": "final_answer_mismatch", + "latency": 0.00017802900401875377, + "cost": 0.0 + }, + "score_delta": 0.0, + "metric_deltas": [ + { + "metric_name": "final_response_avg_score", + "baseline_score": 0.0, + "candidate_score": 0.0, + "score_delta": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "status_transition": "failed_to_failed" + } + ], + "latency_delta": 5.1e-05, + "cost_delta": 0.0, + "regression": false, + "improvement": false, + "notes": [] + }, + { + "id": "val_water_fail", + "split": "val", + "change_type": "unchanged", + "baseline": { + "passed": false, + "metric_score": 0.0, + "failure_category": "final_answer_mismatch", + "latency": 0.00013199500972405076, + "cost": 0.0 + }, + "candidate": { + "passed": false, + "metric_score": 0.0, + "failure_category": "final_answer_mismatch", + "latency": 0.0001838289899751544, + "cost": 0.0 + }, + "score_delta": 0.0, + "metric_deltas": [ + { + "metric_name": "final_response_avg_score", + "baseline_score": 0.0, + "candidate_score": 0.0, + "score_delta": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "status_transition": "failed_to_failed" + } + ], + "latency_delta": 5.2e-05, + "cost_delta": 0.0, + "regression": false, + "improvement": false, + "notes": [] + } + ], + "missing_candidate_case_ids": [], + "extra_candidate_case_ids": [] + } + }, + "gate_decision": { + "decision": "reject", + "accepted": false, + "reason": "Validation score gain is below the minimum threshold.", + "rule_results": [ + { + "rule_name": "validation_score_gain", + "passed": false, + "severity": "required", + "message": "Validation score gain is below the minimum threshold.", + "evidence": { + "score_delta": 0.0, + "min_val_score_gain": 0.05, + "baseline_val_score": 0.333333, + "candidate_val_score": 0.333333 + } + }, + { + "rule_name": "validation_pass_rate_gain", + "passed": true, + "severity": "required", + "message": "Validation pass-rate gain satisfies the minimum threshold.", + "evidence": { + "pass_rate_delta": 0.0, + "min_val_pass_rate_gain": 0.0, + "baseline_val_pass_rate": 0.333333, + "candidate_val_pass_rate": 0.333333 + } + }, + { + "rule_name": "new_failures", + "passed": true, + "severity": "required", + "message": "No new validation failures were introduced.", + "evidence": { + "allow_new_failures": false, + "new_fail_case_ids": [] + } + }, + { + "rule_name": "validation_regressions", + "passed": true, + "severity": "required", + "message": "No validation regressions were detected.", + "evidence": { + "allow_regressions": false, + "regression_case_ids": [] + } + }, + { + "rule_name": "critical_case_regressions", + "passed": true, + "severity": "required", + "message": "No critical validation cases regressed.", + "evidence": { + "critical_case_ids": [ + "val_multiply_pass" + ], + "critical_regression_case_ids": [] + } + }, + { + "rule_name": "cost_budget", + "passed": true, + "severity": "required", + "message": "Candidate cost is within the configured budget.", + "evidence": { + "baseline_cost": 0.0, + "candidate_cost": 0.0, + "cost_delta": 0.0, + "cost_ratio": null, + "max_cost_delta": 0.0, + "max_cost_ratio": 1.2 + } + }, + { + "rule_name": "overfit_detection", + "passed": true, + "severity": "required", + "message": "No train-only overfit pattern was detected.", + "evidence": { + "enabled": true, + "train_score_delta": 0.0, + "val_score_delta": 0.0, + "train_gain_min": 0.05, + "val_drop_tolerance": 0.0 + } + } + ], + "summary": { + "baseline_val_score": 0.333333, + "candidate_val_score": 0.333333, + "val_score_delta": 0.0, + "val_pass_rate_delta": 0.0, + "train_score_delta": 0.0, + "new_fail_count": 0, + "regression_count": 0, + "critical_regression_count": 0, + "overfit_detected": false, + "baseline_cost": 0.0, + "candidate_cost": 0.0, + "cost_delta": 0.0, + "cost_ratio": null, + "total_optimization_cost": 0.0 + }, + "config": { + "min_val_score_gain": 0.05, + "min_val_pass_rate_gain": 0.0, + "allow_new_failures": false, + "allow_regressions": false, + "critical_case_ids": [ + "val_multiply_pass" + ], + "max_cost_delta": 0.0, + "max_cost_ratio": 1.2, + "require_validation_improvement": true, + "overfit_policy": { + "enabled": true, + "train_gain_min": 0.05, + "val_drop_tolerance": 0.0 + } + }, + "recommended_action": "keep_baseline_prompts", + "generated_at": "2026-07-29T07:47:25.010467+00:00" + }, + "optimization": { + "target_prompt_names": [ + "system_prompt", + "skill" + ], + "baseline_prompts": { + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + }, + "best_prompts": { + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + }, + "rounds": [ + { + "round": 1, + "optimized_field_names": [ + "system_prompt" + ], + "before": { + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + }, + "after": { + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + }, + "reason": "no candidate produced this round", + "accepted": false, + "validation_pass_rate": 0.0, + "duration_seconds": 19.865978, + "cost": 0.0 + }, + { + "round": 2, + "optimized_field_names": [ + "skill" + ], + "before": { + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + }, + "after": { + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + }, + "reason": "no candidate produced this round", + "accepted": false, + "validation_pass_rate": 0.0, + "duration_seconds": 10.44344, + "cost": 0.0 + } + ], + "total_rounds": 2, + "total_cost": 0.0, + "duration_seconds": 30.389949, + "seed": 42, + "reason": "Optimizer finished with status=SUCCEEDED, finish_reason=no_improvement, improvement=0.0.", + "artifacts": { + "mode": "real", + "source_prompts_updated": false, + "optimizer_artifact_dir": "output/optimizer_artifacts", + "train_call_agent_evalset_path": "output/optimizer_inputs/train.call_agent.evalset.json", + "val_call_agent_evalset_path": "output/optimizer_inputs/val.call_agent.evalset.json" + } + }, + "metadata": { + "example_root": ".", + "reproduction_command": "python run_pipeline.py --mode real", + "output_dir": "output", + "output_paths": { + "json": "output/optimization_report.json", + "markdown": "output/optimization_report.md" + } + } +} diff --git a/examples/optimization/eval_optimize_loop/output/optimization_report.md b/examples/optimization/eval_optimize_loop/output/optimization_report.md new file mode 100644 index 000000000..e7f89d6c5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/output/optimization_report.md @@ -0,0 +1,133 @@ +# Evaluation Optimization Pipeline 报告 + +## 运行摘要 + +- 决策：**REJECT** +- 推荐动作：`keep_baseline_prompts` +- 模式：`real` +- Schema：`eval-optimize-loop-v1` +- 验证集分数变化：0.0000 +- 验证集通过率变化：0.0000 +- 主要原因：Validation score gain is below the minimum threshold. + +## Baseline 表现 + +- 训练集：2/3 通过，通过率 0.6667 +- 验证集：1/3 通过，通过率 0.3333 + +## 错误归因汇总 + +### 训练集 baseline + +| category | count | +| --- | ---: | +| final_answer_mismatch | 1 | +| tool_call_error | 0 | +| parameter_error | 0 | +| llm_rubric_failed | 0 | +| retrieval_failure | 0 | +| format_error | 0 | +| unknown | 0 | + +### 训练集 candidate + +| category | count | +| --- | ---: | +| final_answer_mismatch | 1 | +| tool_call_error | 0 | +| parameter_error | 0 | +| llm_rubric_failed | 0 | +| retrieval_failure | 0 | +| format_error | 0 | +| unknown | 0 | + +### 验证集 baseline + +| category | count | +| --- | ---: | +| final_answer_mismatch | 2 | +| tool_call_error | 0 | +| parameter_error | 0 | +| llm_rubric_failed | 0 | +| retrieval_failure | 0 | +| format_error | 0 | +| unknown | 0 | + +### 验证集 candidate + +| category | count | +| --- | ---: | +| final_answer_mismatch | 2 | +| tool_call_error | 0 | +| parameter_error | 0 | +| llm_rubric_failed | 0 | +| retrieval_failure | 0 | +| format_error | 0 | +| unknown | 0 | + +## 优化过程 + +- 目标 Prompt：system_prompt, skill +- 优化轮数：2 +- 优化成本：0.0 +- 随机种子：42 + +### Round 1 + +- 修改字段：system_prompt +- 是否接受为候选：False +- 原因：no candidate produced this round + +### Round 2 + +- 修改字段：skill +- 是否接受为候选：False +- 原因：no candidate produced this round + +## 候选验证 + +- 训练集候选：2/3 通过，通过率 0.6667 +- 验证集候选：1/3 通过，通过率 0.3333 + +## 逐 Case Delta + +| split | case id | baseline | candidate | change | failure transition | regression | improvement | +| --- | --- | ---: | ---: | --- | --- | --- | --- | +| train | train_addition_pass | 1.0000 | 1.0000 | unchanged | None -> None | False | False | +| train | train_discount_fail | 0.0000 | 0.0000 | unchanged | final_answer_mismatch -> final_answer_mismatch | False | False | +| train | train_percent_pass | 1.0000 | 1.0000 | unchanged | None -> None | False | False | +| val | val_multiply_pass | 1.0000 | 1.0000 | unchanged | None -> None | False | False | +| val | val_percent_fail | 0.0000 | 0.0000 | unchanged | final_answer_mismatch -> final_answer_mismatch | False | False | +| val | val_water_fail | 0.0000 | 0.0000 | unchanged | final_answer_mismatch -> final_answer_mismatch | False | False | + +## Gate 决策 + +- 最终决策：**REJECT** +- 推荐动作：`keep_baseline_prompts` +- 拒绝/接受理由：Validation score gain is below the minimum threshold. +- 最小验证集分数提升：0.05 +- 允许新增失败：False +- 允许验证集回归：False +- 关键 Case：val_multiply_pass + +| rule | passed | severity | message | +| --- | --- | --- | --- | +| validation_score_gain | False | required | Validation score gain is below the minimum threshold. | +| validation_pass_rate_gain | True | required | Validation pass-rate gain satisfies the minimum threshold. | +| new_failures | True | required | No new validation failures were introduced. | +| validation_regressions | True | required | No validation regressions were detected. | +| critical_case_regressions | True | required | No critical validation cases regressed. | +| cost_budget | True | required | Candidate cost is within the configured budget. | +| overfit_detection | True | required | No train-only overfit pattern was detected. | + +## 元数据与复现 + +- 示例根目录：`.` +- 复现命令：`python run_pipeline.py --mode real` +- 输出 JSON：`output/optimization_report.json` +- 输出 Markdown：`output/optimization_report.md` + +| key | value | +| --- | --- | +| example_root | . | +| output_dir | output | diff --git a/examples/optimization/eval_optimize_loop/pipeline/__init__.py b/examples/optimization/eval_optimize_loop/pipeline/__init__.py new file mode 100644 index 000000000..a23606dea --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/__init__.py @@ -0,0 +1,22 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Evaluation/optimization loop helpers for the eval-optimize-loop example.""" + +from .attribution import FailureAttributor +from .delta import DeltaAnalyzer +from .gate import GateEvaluator +from .optimization import PromptOptimizer +from .pipeline import BaselinePipeline +from .pipeline import EvalOptimizePipeline + +__all__ = [ + "BaselinePipeline", + "DeltaAnalyzer", + "EvalOptimizePipeline", + "FailureAttributor", + "GateEvaluator", + "PromptOptimizer", +] diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py new file mode 100644 index 000000000..a8286775e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -0,0 +1,220 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Rule-based failure attribution for the eval-optimize-loop example.""" + +from __future__ import annotations + +from typing import Any + +from .types import BaselineCaseRecord +from .types import BaselineSplitResult +from .types import FailureAnalysis + + +class FailureAttributor: + """Assign a single explainable failure category to each failed case.""" + + def annotate_split(self, split: BaselineSplitResult) -> BaselineSplitResult: + """Attach failure_analysis to failed cases in a split.""" + for case in split.cases: + case.failure_analysis = self.analyze_case(case) + return split + + def analyze_case(self, case: BaselineCaseRecord) -> FailureAnalysis | None: + """Return the best-effort attribution for one failed case.""" + if case.passed: + return None + + metadata = case.evaluator_metadata or {} + failed_metrics = _failed_metric_results(metadata) + expected_tool_calls = _tool_calls(metadata.get("expected_tool_calls")) + actual_tool_calls = _tool_calls(metadata.get("actual_tool_calls")) + + if _has_failed_metric(failed_metrics, "tool_trajectory"): + return self._tool_analysis(case, expected_tool_calls, actual_tool_calls, failed_metrics) + if _has_failed_metric(failed_metrics, "knowledge_recall", "retrieval", "recall"): + return _analysis( + "retrieval_failure", + 0.85, + "Knowledge recall or retrieval quality metric was below threshold.", + case, + failed_metrics, + ) + if _has_failed_metric(failed_metrics, "llm_rubric"): + return _analysis( + "llm_rubric_failed", + 0.85, + "LLM rubric metric was below threshold.", + case, + failed_metrics, + ) + if _has_format_signal(failed_metrics): + return _analysis( + "format_error", + 0.8, + "Output format did not satisfy the configured constraint.", + case, + failed_metrics, + ) + if _has_failed_metric(failed_metrics, "final_response", "response_match"): + return _analysis( + "final_answer_mismatch", + 0.9, + "Final response did not match the reference answer.", + case, + failed_metrics, + ) + return _analysis( + "unknown", + 0.2, + _unknown_explanation(metadata), + case, + failed_metrics, + ) + + def _tool_analysis( + self, + case: BaselineCaseRecord, + expected_tool_calls: list[dict[str, Any]], + actual_tool_calls: list[dict[str, Any]], + failed_metrics: list[dict[str, Any]], + ) -> FailureAnalysis: + if _same_tool_names(expected_tool_calls, actual_tool_calls) and expected_tool_calls != actual_tool_calls: + return _analysis( + "parameter_error", + 0.9, + "Tool calls used the expected tools, but arguments differed from the reference trace.", + case, + failed_metrics, + extra_evidence={ + "expected_tool_calls": expected_tool_calls, + "actual_tool_calls": actual_tool_calls, + }, + ) + return _analysis( + "tool_call_error", + 0.9, + "Tool call names, count, or order differed from the reference trace.", + case, + failed_metrics, + extra_evidence={ + "expected_tool_calls": expected_tool_calls, + "actual_tool_calls": actual_tool_calls, + }, + ) + + +def _analysis( + category: str, + confidence: float, + explanation: str, + case: BaselineCaseRecord, + failed_metrics: list[dict[str, Any]], + *, + extra_evidence: dict[str, Any] | None = None, +) -> FailureAnalysis: + evidence = { + "case_id": case.id, + "failed_metrics": [_metric_evidence(metric) for metric in failed_metrics], + "expected": case.trace.get("expected", ""), + "actual": case.trace.get("actual", ""), + "failure_reason": case.failure_reason, + } + if extra_evidence: + evidence.update(extra_evidence) + return FailureAnalysis( + category=category, + confidence=confidence, + explanation=explanation, + evidence=evidence, + ) + + +def _failed_metric_results(metadata: dict[str, Any]) -> list[dict[str, Any]]: + failed = metadata.get("failed_metric_results") + if isinstance(failed, list): + return [metric for metric in failed if isinstance(metric, dict)] + + metrics = metadata.get("overall_metric_results") + if not isinstance(metrics, list): + return [] + + failed_metrics = [] + for metric in metrics: + if not isinstance(metric, dict): + continue + status = str(metric.get("eval_status") or "").upper() + score = metric.get("score") + threshold = metric.get("threshold") + if status and status != "PASSED": + failed_metrics.append(metric) + elif isinstance(score, (int, float)) and isinstance(threshold, (int, float)) and score < threshold: + failed_metrics.append(metric) + return failed_metrics + + +def _has_failed_metric(metrics: list[dict[str, Any]], *needles: str) -> bool: + for metric in metrics: + haystack = _metric_text(metric) + if any(needle in haystack for needle in needles): + return True + return False + + +def _has_format_signal(metrics: list[dict[str, Any]]) -> bool: + format_needles = ("format", "json", "schema", "regex", "pattern") + for metric in metrics: + metric_name = str(metric.get("metric_name") or "").lower() + if "final_response" in metric_name or "response_match" in metric_name: + continue + if any(needle in _metric_text(metric) for needle in format_needles): + return True + return False + + +def _metric_text(metric: dict[str, Any]) -> str: + parts = [ + metric.get("metric_name"), + metric.get("reason"), + metric.get("criterion"), + ] + return " ".join(str(part).lower() for part in parts if part) + + +def _metric_evidence(metric: dict[str, Any]) -> dict[str, Any]: + return { + "metric_name": metric.get("metric_name"), + "score": metric.get("score"), + "threshold": metric.get("threshold"), + "eval_status": metric.get("eval_status"), + "reason": metric.get("reason"), + } + + +def _tool_calls(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + calls = [] + for item in value: + if isinstance(item, dict): + calls.append({ + "name": item.get("name"), + "args": item.get("args") or {}, + }) + return calls + + +def _same_tool_names(expected: list[dict[str, Any]], actual: list[dict[str, Any]]) -> bool: + if len(expected) != len(actual): + return False + return [call.get("name") for call in expected] == [call.get("name") for call in actual] + + +def _unknown_explanation(metadata: dict[str, Any]) -> str: + error_message = metadata.get("error_message") + if error_message: + return f"Evaluation did not expose enough structured signals: {error_message}" + return "Evaluation did not expose enough structured signals for a specific attribution." diff --git a/examples/optimization/eval_optimize_loop/pipeline/delta.py b/examples/optimization/eval_optimize_loop/pipeline/delta.py new file mode 100644 index 000000000..d5e7fa242 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/delta.py @@ -0,0 +1,289 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Candidate regression delta analysis for the eval-optimize-loop example.""" + +from __future__ import annotations + +from typing import Any + +from .types import BaselineCaseRecord +from .types import BaselineSplitResult +from .types import CaseDelta +from .types import MetricDelta +from .types import OptimizationDelta +from .types import SplitDeltaSummary + + +class DeltaAnalyzer: + """Compare baseline and candidate evaluation snapshots without re-running evaluators.""" + + def analyze( + self, + *, + train_baseline: BaselineSplitResult, + train_candidate: BaselineSplitResult, + val_baseline: BaselineSplitResult, + val_candidate: BaselineSplitResult, + ) -> OptimizationDelta: + return OptimizationDelta( + train=self.analyze_split( + split="train", + baseline=train_baseline, + candidate=train_candidate, + ), + val=self.analyze_split( + split="val", + baseline=val_baseline, + candidate=val_candidate, + ), + ) + + def analyze_split( + self, + *, + split: str, + baseline: BaselineSplitResult, + candidate: BaselineSplitResult, + ) -> SplitDeltaSummary: + baseline_by_id = {case.id: case for case in baseline.cases} + candidate_by_id = {case.id: case for case in candidate.cases} + baseline_ids = set(baseline_by_id) + candidate_ids = set(candidate_by_id) + missing_ids = sorted(baseline_ids - candidate_ids) + extra_ids = sorted(candidate_ids - baseline_ids) + + case_deltas = [ + _case_delta( + split=split, + baseline=baseline_by_id[case_id], + candidate=candidate_by_id.get(case_id), + ) for case_id in sorted(baseline_ids) + ] + case_deltas.extend( + _extra_candidate_delta( + split=split, + candidate=candidate_by_id[case_id], + ) for case_id in extra_ids) + + return SplitDeltaSummary( + split=split, + baseline_score=_average_score(baseline.cases), + candidate_score=_average_score(candidate.cases), + score_delta=_round(_average_score(candidate.cases) - _average_score(baseline.cases)), + baseline_pass_rate=_pass_rate(baseline.cases), + candidate_pass_rate=_pass_rate(candidate.cases), + pass_rate_delta=_round(_pass_rate(candidate.cases) - _pass_rate(baseline.cases)), + case_deltas=case_deltas, + missing_candidate_case_ids=missing_ids, + extra_candidate_case_ids=extra_ids, + ) + + +def _case_delta( + *, + split: str, + baseline: BaselineCaseRecord, + candidate: BaselineCaseRecord | None, +) -> CaseDelta: + if candidate is None: + return CaseDelta( + id=baseline.id, + split=split, + change_type="missing_candidate", + baseline=_case_ref(baseline), + candidate=_missing_ref(), + score_delta=None, + metric_deltas=_metric_deltas(baseline, None), + latency_delta=None, + cost_delta=None, + regression=True, + improvement=False, + notes=["Candidate evaluation is missing this baseline case."], + ) + + metric_deltas = _metric_deltas(baseline, candidate) + change_type = _change_type(baseline, candidate) + metric_regression = any(metric.status_transition == "passed_to_failed" for metric in metric_deltas) + metric_improvement = any(metric.status_transition == "failed_to_passed" for metric in metric_deltas) + regression = change_type in {"new_fail", "score_down"} or metric_regression + improvement = change_type in {"new_pass", "score_up"} or metric_improvement + return CaseDelta( + id=baseline.id, + split=split, + change_type=change_type, + baseline=_case_ref(baseline), + candidate=_case_ref(candidate), + score_delta=_round(candidate.metric_score - baseline.metric_score), + metric_deltas=metric_deltas, + latency_delta=_round(candidate.latency - baseline.latency), + cost_delta=_round(candidate.cost - baseline.cost), + regression=regression, + improvement=improvement, + ) + + +def _extra_candidate_delta(*, split: str, candidate: BaselineCaseRecord) -> CaseDelta: + return CaseDelta( + id=candidate.id, + split=split, + change_type="extra_candidate", + baseline=_missing_ref(), + candidate=_case_ref(candidate), + score_delta=None, + metric_deltas=_metric_deltas(None, candidate), + latency_delta=None, + cost_delta=None, + regression=False, + improvement=True, + notes=["Candidate evaluation produced a case that is not present in baseline."], + ) + + +def _metric_deltas( + baseline: BaselineCaseRecord | None, + candidate: BaselineCaseRecord | None, +) -> list[MetricDelta]: + baseline_metrics = _metric_statuses(baseline) + candidate_metrics = _metric_statuses(candidate) + metric_names = sorted(set(baseline_metrics) | set(candidate_metrics)) + return [ + MetricDelta( + metric_name=name, + baseline_score=baseline_metrics.get(name, {}).get("score"), + candidate_score=candidate_metrics.get(name, {}).get("score"), + score_delta=_metric_score_delta(baseline_metrics.get(name), candidate_metrics.get(name)), + baseline_passed=baseline_metrics.get(name, {}).get("passed"), + candidate_passed=candidate_metrics.get(name, {}).get("passed"), + status_transition=_status_transition( + baseline_metrics.get(name, {}).get("passed"), + candidate_metrics.get(name, {}).get("passed"), + ), + ) for name in metric_names + ] + + +def _metric_statuses(case: BaselineCaseRecord | None) -> dict[str, dict[str, Any]]: + if case is None: + return {} + metadata = case.evaluator_metadata or {} + metadata_metrics = metadata.get("overall_metric_results") or [] + statuses: dict[str, dict[str, Any]] = {} + if isinstance(metadata_metrics, list): + for metric in metadata_metrics: + if not isinstance(metric, dict): + continue + name = str(metric.get("metric_name") or "") + if not name: + continue + statuses[name] = { + "score": metric.get("score"), + "passed": _metric_passed(metric), + } + for name, score in case.metric_scores.items(): + statuses.setdefault(name, { + "score": score, + "passed": score >= 1.0, + }) + return statuses + + +def _metric_passed(metric: dict[str, Any]) -> bool | None: + status = str(metric.get("eval_status") or "").upper() + if status == "PASSED": + return True + if status in {"FAILED", "NOT_EVALUATED"}: + return False + score = metric.get("score") + threshold = metric.get("threshold") + if isinstance(score, (int, float)) and isinstance(threshold, (int, float)): + return score >= threshold + return None + + +def _metric_score_delta( + baseline: dict[str, Any] | None, + candidate: dict[str, Any] | None, +) -> float | None: + if baseline is None or candidate is None: + return None + baseline_score = baseline.get("score") + candidate_score = candidate.get("score") + if not isinstance(baseline_score, (int, float)) or not isinstance(candidate_score, (int, float)): + return None + return _round(candidate_score - baseline_score) + + +def _status_transition( + baseline_passed: bool | None, + candidate_passed: bool | None, +) -> str: + if baseline_passed is None and candidate_passed is None: + return "missing_both" + if baseline_passed is None: + return "missing_baseline" + if candidate_passed is None: + return "missing_candidate" + if baseline_passed and candidate_passed: + return "passed_to_passed" + if baseline_passed and not candidate_passed: + return "passed_to_failed" + if not baseline_passed and candidate_passed: + return "failed_to_passed" + return "failed_to_failed" + + +def _change_type(baseline: BaselineCaseRecord, candidate: BaselineCaseRecord) -> str: + if not baseline.passed and candidate.passed: + return "new_pass" + if baseline.passed and not candidate.passed: + return "new_fail" + if candidate.metric_score > baseline.metric_score: + return "score_up" + if candidate.metric_score < baseline.metric_score: + return "score_down" + return "unchanged" + + +def _case_ref(case: BaselineCaseRecord) -> dict[str, Any]: + return { + "passed": case.passed, + "metric_score": case.metric_score, + "failure_category": _failure_category(case), + "latency": case.latency, + "cost": case.cost, + } + + +def _missing_ref() -> dict[str, Any]: + return { + "passed": None, + "metric_score": None, + "failure_category": None, + "latency": None, + "cost": None, + } + + +def _failure_category(case: BaselineCaseRecord) -> str | None: + if case.failure_analysis is None: + return None + return case.failure_analysis.category + + +def _average_score(cases: list[BaselineCaseRecord]) -> float: + if not cases: + return 0.0 + return _round(sum(case.metric_score for case in cases) / len(cases)) + + +def _pass_rate(cases: list[BaselineCaseRecord]) -> float: + if not cases: + return 0.0 + return _round(sum(1 for case in cases if case.passed) / len(cases)) + + +def _round(value: float) -> float: + return round(value, 6) diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py new file mode 100644 index 000000000..6c6ec0757 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -0,0 +1,259 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Acceptance gate for candidate prompt deltas.""" + +from __future__ import annotations + +from datetime import datetime +from datetime import timezone +from typing import Any + +from .types import CaseDelta +from .types import GateConfig +from .types import GateDecision +from .types import GateRuleResult +from .types import OptimizationDelta +from .types import OptimizationRunRecord + + +class GateEvaluator: + """Apply example-local acceptance rules to an already computed delta.""" + + def __init__(self, config: GateConfig | None = None) -> None: + self.config = config or GateConfig() + + def evaluate( + self, + *, + delta: OptimizationDelta, + optimization: OptimizationRunRecord | None = None, + ) -> GateDecision: + summary = self._summary(delta, optimization) + rule_results = [ + self._validation_score_rule(delta), + self._validation_pass_rate_rule(delta), + self._new_failure_rule(delta), + self._regression_rule(delta), + self._critical_case_rule(delta), + self._cost_rule(summary), + self._overfit_rule(delta), + ] + accepted = all(result.passed for result in rule_results) + return GateDecision( + decision="accept" if accepted else "reject", + accepted=accepted, + reason=_decision_reason(accepted, rule_results), + rule_results=rule_results, + summary=summary, + config=self.config, + recommended_action="use_candidate_prompts" if accepted else "keep_baseline_prompts", + generated_at=datetime.now(timezone.utc).isoformat(), + ) + + def _summary( + self, + delta: OptimizationDelta, + optimization: OptimizationRunRecord | None, + ) -> dict[str, Any]: + critical_regressions = _critical_regressions(delta.val.case_deltas, self.config.critical_case_ids) + baseline_cost, candidate_cost = _costs(delta) + cost_delta = _round(candidate_cost - baseline_cost) + return { + "baseline_val_score": delta.val.baseline_score, + "candidate_val_score": delta.val.candidate_score, + "val_score_delta": delta.val.score_delta, + "val_pass_rate_delta": delta.val.pass_rate_delta, + "train_score_delta": delta.train.score_delta, + "new_fail_count": _count_change(delta.val.case_deltas, "new_fail"), + "regression_count": sum(1 for case in delta.val.case_deltas if case.regression), + "critical_regression_count": len(critical_regressions), + "overfit_detected": _overfit_detected(delta, self.config), + "baseline_cost": baseline_cost, + "candidate_cost": candidate_cost, + "cost_delta": cost_delta, + "cost_ratio": _cost_ratio(baseline_cost, candidate_cost), + "total_optimization_cost": optimization.total_cost if optimization else 0.0, + } + + def _validation_score_rule(self, delta: OptimizationDelta) -> GateRuleResult: + if not self.config.require_validation_improvement: + return GateRuleResult( + rule_name="validation_score_gain", + passed=True, + severity="required", + message="Validation score improvement is not required by gate config.", + evidence={"required": False}, + ) + passed = delta.val.score_delta >= self.config.min_val_score_gain + return GateRuleResult( + rule_name="validation_score_gain", + passed=passed, + severity="required", + message=("Validation score gain satisfies the minimum threshold." + if passed else "Validation score gain is below the minimum threshold."), + evidence={ + "score_delta": delta.val.score_delta, + "min_val_score_gain": self.config.min_val_score_gain, + "baseline_val_score": delta.val.baseline_score, + "candidate_val_score": delta.val.candidate_score, + }, + ) + + def _validation_pass_rate_rule(self, delta: OptimizationDelta) -> GateRuleResult: + passed = delta.val.pass_rate_delta >= self.config.min_val_pass_rate_gain + return GateRuleResult( + rule_name="validation_pass_rate_gain", + passed=passed, + severity="required", + message=("Validation pass-rate gain satisfies the minimum threshold." + if passed else "Validation pass-rate gain is below the minimum threshold."), + evidence={ + "pass_rate_delta": delta.val.pass_rate_delta, + "min_val_pass_rate_gain": self.config.min_val_pass_rate_gain, + "baseline_val_pass_rate": delta.val.baseline_pass_rate, + "candidate_val_pass_rate": delta.val.candidate_pass_rate, + }, + ) + + def _new_failure_rule(self, delta: OptimizationDelta) -> GateRuleResult: + new_fail_ids = [case.id for case in delta.val.case_deltas if case.change_type == "new_fail"] + passed = self.config.allow_new_failures or not new_fail_ids + return GateRuleResult( + rule_name="new_failures", + passed=passed, + severity="required", + message=("No new validation failures were introduced." + if passed else "Candidate introduced new validation failures."), + evidence={ + "allow_new_failures": self.config.allow_new_failures, + "new_fail_case_ids": new_fail_ids, + }, + ) + + def _regression_rule(self, delta: OptimizationDelta) -> GateRuleResult: + regression_ids = [case.id for case in delta.val.case_deltas if case.regression] + passed = self.config.allow_regressions or not regression_ids + return GateRuleResult( + rule_name="validation_regressions", + passed=passed, + severity="required", + message=("No validation regressions were detected." + if passed else "Candidate introduced validation regressions."), + evidence={ + "allow_regressions": self.config.allow_regressions, + "regression_case_ids": regression_ids, + }, + ) + + def _critical_case_rule(self, delta: OptimizationDelta) -> GateRuleResult: + critical_regressions = _critical_regressions(delta.val.case_deltas, self.config.critical_case_ids) + passed = not critical_regressions + return GateRuleResult( + rule_name="critical_case_regressions", + passed=passed, + severity="required", + message=("No critical validation cases regressed." + if passed else "One or more critical validation cases regressed."), + evidence={ + "critical_case_ids": self.config.critical_case_ids, + "critical_regression_case_ids": critical_regressions, + }, + ) + + def _cost_rule(self, summary: dict[str, Any]) -> GateRuleResult: + cost_delta = float(summary["cost_delta"]) + cost_ratio = summary["cost_ratio"] + delta_ok = cost_delta <= self.config.max_cost_delta + ratio_ok = cost_ratio is None or cost_ratio <= self.config.max_cost_ratio + passed = delta_ok and ratio_ok + return GateRuleResult( + rule_name="cost_budget", + passed=passed, + severity="required", + message=("Candidate cost is within the configured budget." + if passed else "Candidate cost exceeds the configured budget."), + evidence={ + "baseline_cost": summary["baseline_cost"], + "candidate_cost": summary["candidate_cost"], + "cost_delta": cost_delta, + "cost_ratio": cost_ratio, + "max_cost_delta": self.config.max_cost_delta, + "max_cost_ratio": self.config.max_cost_ratio, + }, + ) + + def _overfit_rule(self, delta: OptimizationDelta) -> GateRuleResult: + overfit = _overfit_detected(delta, self.config) + return GateRuleResult( + rule_name="overfit_detection", + passed=not overfit, + severity="required", + message=("No train-only overfit pattern was detected." + if not overfit else "Candidate appears to improve train while degrading validation."), + evidence={ + "enabled": bool(self.config.overfit_policy.get("enabled", True)), + "train_score_delta": delta.train.score_delta, + "val_score_delta": delta.val.score_delta, + "train_gain_min": float(self.config.overfit_policy.get("train_gain_min", 0.05)), + "val_drop_tolerance": float(self.config.overfit_policy.get("val_drop_tolerance", 0.0)), + }, + ) + + +def _decision_reason(accepted: bool, rule_results: list[GateRuleResult]) -> str: + if accepted: + return "Validation improved without gate-blocking regressions." + failed = [result.message for result in rule_results if not result.passed] + return " ".join(failed) + + +def _critical_regressions(case_deltas: list[CaseDelta], critical_case_ids: list[str]) -> list[str]: + critical = set(critical_case_ids) + if not critical: + return [] + return [ + case.id for case in case_deltas if case.id in critical and ( + case.change_type in {"new_fail", "score_down", "missing_candidate"} or _has_metric_regression(case)) + ] + + +def _has_metric_regression(case: CaseDelta) -> bool: + return any(metric.status_transition == "passed_to_failed" for metric in case.metric_deltas) + + +def _overfit_detected(delta: OptimizationDelta, config: GateConfig) -> bool: + policy = config.overfit_policy + if not bool(policy.get("enabled", True)): + return False + train_gain_min = float(policy.get("train_gain_min", 0.05)) + val_drop_tolerance = float(policy.get("val_drop_tolerance", 0.0)) + return delta.train.score_delta >= train_gain_min and delta.val.score_delta < -val_drop_tolerance + + +def _count_change(case_deltas: list[CaseDelta], change_type: str) -> int: + return sum(1 for case in case_deltas if case.change_type == change_type) + + +def _costs(delta: OptimizationDelta) -> tuple[float, float]: + cases = delta.train.case_deltas + delta.val.case_deltas + baseline_cost = sum(_cost(case.baseline) for case in cases) + candidate_cost = sum(_cost(case.candidate) for case in cases) + return _round(baseline_cost), _round(candidate_cost) + + +def _cost(reference: dict[str, Any]) -> float: + value = reference.get("cost") + return float(value) if isinstance(value, (int, float)) else 0.0 + + +def _cost_ratio(baseline_cost: float, candidate_cost: float) -> float | None: + if baseline_cost == 0: + return None + return _round(candidate_cost / baseline_cost) + + +def _round(value: float) -> float: + return round(value, 6) diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimization.py b/examples/optimization/eval_optimize_loop/pipeline/optimization.py new file mode 100644 index 000000000..06a9dc556 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/optimization.py @@ -0,0 +1,320 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Prompt optimization helpers for the eval-optimize-loop example.""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any +from typing import Awaitable +from typing import Callable + +from .types import BaselineSplitResult +from .types import OptimizationRoundRecord +from .types import OptimizationRunRecord + +CallAgent = Callable[[str], Awaitable[str]] +TARGET_PROMPT_PATHS = { + "system_prompt": "prompts/system.md", + "skill": "prompts/skill.md", +} +FINAL_ANSWER_FIX_MARKER = "[fake_optimizer:final_answer_mismatch]" + +_KNOWN_ANSWERS = { + "小明有 4 个苹果，又买了 7 个，现在一共有多少个苹果？": "答案：11 个", + "一件衣服原价 200 元，打 8 折后多少钱？": "答案：160 元", + "40 名学生中 25% 戴眼镜，戴眼镜的有多少人？": "答案：10 人", + "教室里有 5 排座位，每排 8 个，一共有多少个座位？": "答案：42 个", + "1 升水重 1 千克，3.5 升水重多少千克？": "答案：3.5 千克", + "班里 30 人，其中 60% 是女生，有多少名女生？": "答案：20 人", +} + +_BASELINE_ANSWERS = { + "小明有 4 个苹果，又买了 7 个，现在一共有多少个苹果？": "答案：11 个", + "一件衣服原价 200 元，打 8 折后多少钱？": "答案：180 元", + "40 名学生中 25% 戴眼镜，戴眼镜的有多少人？": "计算 40 的 25%，答案：10 人", + "教室里有 5 排座位，每排 8 个，一共有多少个座位？": "答案：40 个", + "1 升水重 1 千克，3.5 升水重多少千克？": "答案：4 千克", + "班里 30 人，其中 60% 是女生，有多少名女生？": "答案：20 人", +} + + +class PromptOptimizer: + """Run fake or real prompt optimization for this example.""" + + def __init__( + self, + *, + example_root: Path, + output_dir: Path, + optimizer_config_path: Path, + train_evalset_path: Path, + val_evalset_path: Path, + ) -> None: + self.example_root = example_root + self.output_dir = output_dir + self.optimizer_config_path = optimizer_config_path + self.train_evalset_path = train_evalset_path + self.val_evalset_path = val_evalset_path + + def prompt_paths(self) -> dict[str, Path]: + return {name: self.example_root / rel for name, rel in TARGET_PROMPT_PATHS.items()} + + def read_baseline_prompts(self) -> dict[str, str]: + return {name: path.read_text(encoding="utf-8") for name, path in self.prompt_paths().items()} + + async def optimize( + self, + *, + mode: str, + optimizer_payload: dict[str, Any], + train_baseline: BaselineSplitResult, + val_baseline: BaselineSplitResult, + ) -> OptimizationRunRecord: + if mode == "real": + return await self._run_real_optimizer(optimizer_payload) + return self._run_fake_optimizer( + optimizer_payload=optimizer_payload, + train_baseline=train_baseline, + val_baseline=val_baseline, + ) + + def write_call_agent_evalsets(self) -> tuple[Path, Path]: + """Persist temporary non-trace evalsets for optimizer call_agent mode.""" + input_dir = self.output_dir / "optimizer_inputs" + input_dir.mkdir(parents=True, exist_ok=True) + train_path = input_dir / "train.call_agent.evalset.json" + val_path = input_dir / "val.call_agent.evalset.json" + _write_json(train_path, _call_agent_evalset_payload(_load_json(self.train_evalset_path))) + _write_json(val_path, _call_agent_evalset_payload(_load_json(self.val_evalset_path))) + return train_path, val_path + + def call_agent_from_prompts(self, prompts: dict[str, str]) -> CallAgent: + + async def call_agent(query: str) -> str: + return deterministic_answer(query, prompts) + + return call_agent + + def _run_fake_optimizer( + self, + *, + optimizer_payload: dict[str, Any], + train_baseline: BaselineSplitResult, + val_baseline: BaselineSplitResult, + ) -> OptimizationRunRecord: + started = time.perf_counter() + baseline_prompts = self.read_baseline_prompts() + seed = _seed_from_optimizer(optimizer_payload) + reason = _reason_from_failures(train_baseline, val_baseline) + best_prompts = _fake_best_prompts(baseline_prompts, reason) + round_record = OptimizationRoundRecord( + round=1, + optimized_field_names=["system_prompt", "skill"], + before=baseline_prompts, + after=best_prompts, + reason=reason, + accepted=True, + validation_pass_rate=0.0, + duration_seconds=0.0, + cost=0.0, + ) + return OptimizationRunRecord( + target_prompt_names=list(baseline_prompts), + baseline_prompts=baseline_prompts, + best_prompts=best_prompts, + rounds=[round_record], + total_rounds=1, + total_cost=0.0, + duration_seconds=time.perf_counter() - started, + seed=seed, + reason=reason, + artifacts={ + "mode": "fake", + "source_prompts_updated": False, + }, + ) + + async def _run_real_optimizer(self, optimizer_payload: dict[str, Any]) -> OptimizationRunRecord: + from trpc_agent_sdk.evaluation import AgentOptimizer + from trpc_agent_sdk.evaluation import TargetPrompt + + started = time.perf_counter() + baseline_prompts = self.read_baseline_prompts() + train_path, val_path = self.write_call_agent_evalsets() + artifact_dir = self.output_dir / "optimizer_artifacts" + target_prompt = TargetPrompt() + for name, path in self.prompt_paths().items(): + target_prompt.add_path(name, str(path)) + + result = await AgentOptimizer.optimize( + config_path=str(self.optimizer_config_path), + call_agent=self.call_agent_from_prompts_file(), + target_prompt=target_prompt, + train_dataset_path=str(train_path), + validation_dataset_path=str(val_path), + output_dir=str(artifact_dir), + update_source=False, + verbose=0, + ) + return _optimization_record_from_sdk_result( + result, + baseline_prompts=baseline_prompts, + seed=_seed_from_optimizer(optimizer_payload), + duration_seconds=time.perf_counter() - started, + artifacts={ + "mode": "real", + "source_prompts_updated": False, + "optimizer_artifact_dir": _relative_to_example(artifact_dir, self.example_root), + "train_call_agent_evalset_path": _relative_to_example(train_path, self.example_root), + "val_call_agent_evalset_path": _relative_to_example(val_path, self.example_root), + }, + ) + + def call_agent_from_prompts_file(self) -> CallAgent: + + async def call_agent(query: str) -> str: + return deterministic_answer(query, self.read_baseline_prompts()) + + return call_agent + + +def deterministic_answer(query: str, prompts: dict[str, str]) -> str: + prompt_text = "\n".join(prompts.values()).lower() + if FINAL_ANSWER_FIX_MARKER.lower() in prompt_text or "verify arithmetic" in prompt_text: + return _KNOWN_ANSWERS.get(query, "") + return _BASELINE_ANSWERS.get(query, "") + + +def _fake_best_prompts(baseline_prompts: dict[str, str], reason: str) -> dict[str, str]: + best = dict(baseline_prompts) + best["system_prompt"] = "".join([ + best["system_prompt"].rstrip(), + "\n\n", + "Fake optimizer guidance: verify arithmetic before answering, ", + f"preserve the requested answer format, and address {reason}. ", + FINAL_ANSWER_FIX_MARKER, + "\n", + ]) + best["skill"] = "".join([ + best["skill"].rstrip(), + "\n\n", + "When a baseline failure is attributed to final_answer_mismatch, recompute the numeric result ", + "and keep the expected unit in the final answer.\n", + ]) + return best + + +def _reason_from_failures(train: BaselineSplitResult, val: BaselineSplitResult) -> str: + counts: dict[str, int] = {} + for summary in (train.failure_attribution_summary(), val.failure_attribution_summary()): + for category, count in summary.items(): + if count: + counts[category] = counts.get(category, 0) + count + if not counts: + return "No baseline failure attribution categories were found; keep prompts stable." + parts = [f"{category}={count}" for category, count in sorted(counts.items())] + return "Address failure attribution categories: " + ", ".join(parts) + + +def _optimization_record_from_sdk_result( + result: Any, + *, + baseline_prompts: dict[str, str], + seed: int | None, + duration_seconds: float, + artifacts: dict[str, Any], +) -> OptimizationRunRecord: + best_prompts = dict(getattr(result, "best_prompts", None) or baseline_prompts) + rounds = [] + before = dict(getattr(result, "baseline_prompts", None) or baseline_prompts) + for raw_round in getattr(result, "rounds", []) or []: + after = dict(getattr(raw_round, "candidate_prompts", None) or before) + rounds.append( + OptimizationRoundRecord( + round=int(getattr(raw_round, "round", + len(rounds) + 1)), + optimized_field_names=list(getattr(raw_round, "optimized_field_names", []) or []), + before=before, + after=after, + reason=_round_reason(raw_round), + accepted=bool(getattr(raw_round, "accepted", False)), + validation_pass_rate=float(getattr(raw_round, "validation_pass_rate", 0.0) or 0.0), + duration_seconds=float(getattr(raw_round, "duration_seconds", 0.0) or 0.0), + cost=float(getattr(raw_round, "round_llm_cost", 0.0) or 0.0), + )) + before = after + + return OptimizationRunRecord( + target_prompt_names=list(best_prompts), + baseline_prompts=dict(getattr(result, "baseline_prompts", None) or baseline_prompts), + best_prompts=best_prompts, + rounds=rounds, + total_rounds=int(getattr(result, "total_rounds", len(rounds)) or len(rounds)), + total_cost=float(getattr(result, "total_llm_cost", 0.0) or 0.0), + duration_seconds=float(getattr(result, "duration_seconds", duration_seconds) or duration_seconds), + seed=seed, + reason=_result_reason(result), + artifacts=artifacts, + ) + + +def _round_reason(raw_round: Any) -> str: + reason = getattr(raw_round, "acceptance_reason", None) or getattr(raw_round, "skip_reason", None) + if reason: + return str(reason) + diagnosis = getattr(raw_round, "per_field_diagnosis", None) + if isinstance(diagnosis, dict) and diagnosis: + return "; ".join(f"{name}: {text}" for name, text in diagnosis.items()) + error = getattr(raw_round, "error_message", None) + if error: + return str(error) + return "Optimizer produced a candidate prompt." + + +def _result_reason(result: Any) -> str: + status = getattr(result, "status", "") + finish_reason = getattr(result, "finish_reason", "") + improvement = getattr(result, "pass_rate_improvement", None) + return f"Optimizer finished with status={status}, finish_reason={finish_reason}, improvement={improvement}." + + +def _call_agent_evalset_payload(payload: dict[str, Any]) -> dict[str, Any]: + converted = dict(payload) + cases = [] + for case in payload.get("eval_cases", []): + if not isinstance(case, dict): + continue + converted_case = dict(case) + converted_case.pop("eval_mode", None) + converted_case.pop("actual_conversation", None) + converted_case.pop("actualConversation", None) + cases.append(converted_case) + converted["eval_cases"] = cases + return converted + + +def _seed_from_optimizer(payload: dict[str, Any]) -> int | None: + algorithm = ((payload.get("optimize") or {}).get("algorithm") or {}) + seed = algorithm.get("seed") if isinstance(algorithm, dict) else None + return int(seed) if isinstance(seed, int) else None + + +def _load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + +def _relative_to_example(path: Path, example_root: Path) -> str: + try: + return str(path.resolve().relative_to(example_root.resolve())) + except ValueError: + return str(path) diff --git a/examples/optimization/eval_optimize_loop/pipeline/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline/pipeline.py new file mode 100644 index 000000000..07b7412bd --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/pipeline.py @@ -0,0 +1,729 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Evaluation, optimization, validation, and gate pipeline.""" + +from __future__ import annotations + +import json +import re +import time +from datetime import datetime +from datetime import timezone +from pathlib import Path +from typing import Any + +from .attribution import FailureAttributor +from .delta import DeltaAnalyzer +from .gate import GateEvaluator +from .optimization import PromptOptimizer +from .optimization import deterministic_answer +from .report import MARKDOWN_REPORT_FILENAME +from .report import REPORT_FILENAME +from .report import write_optimization_report +from .types import BaselineCaseRecord +from .types import BaselineOptimizationReport +from .types import BaselineSplitResult +from .types import CandidateEvaluation +from .types import ReportPaths + + +class EvalOptimizePipeline: + """Run the full evaluation and optimization loop.""" + + def __init__( + self, + *, + train_evalset_path: Path, + val_evalset_path: Path, + optimizer_config_path: Path, + gate_config_path: Path | None = None, + output_dir: Path | None = None, + mode: str = "fake", + ) -> None: + if mode not in {"real", "fake"}: + raise ValueError(f"mode must be 'real' or 'fake', got {mode!r}") + self.train_evalset_path = train_evalset_path + self.val_evalset_path = val_evalset_path + self.optimizer_config_path = optimizer_config_path + self.gate_config_path = gate_config_path or optimizer_config_path.resolve().parent / "gate.json" + self.mode = mode + self.example_root = optimizer_config_path.resolve().parent + self.output_dir = output_dir or self.example_root / "output" + + async def run(self) -> ReportPaths: + """Run the loop and persist both report formats.""" + started_at = datetime.now(timezone.utc) + started = time.perf_counter() + optimizer_payload = _load_optimizer_payload(self.optimizer_config_path) + gate_config = _load_gate_config(self.gate_config_path) + prompt_optimizer = PromptOptimizer( + example_root=self.example_root, + output_dir=self.output_dir, + optimizer_config_path=self.optimizer_config_path, + train_evalset_path=self.train_evalset_path, + val_evalset_path=self.val_evalset_path, + ) + + if self.mode == "real": + eval_config = _load_real_eval_config(optimizer_payload) + train_evalset = _load_real_evalset(self.train_evalset_path) + val_evalset = _load_real_evalset(self.val_evalset_path) + train = await self._run_real_split(train_evalset, eval_config) + val = await self._run_real_split(val_evalset, eval_config) + else: + metrics = _fake_metrics_from_optimizer(optimizer_payload) + train_evalset = _load_json(self.train_evalset_path) + val_evalset = _load_json(self.val_evalset_path) + train = self._run_fake_split(train_evalset, metrics) + val = self._run_fake_split(val_evalset, metrics) + + attributor = FailureAttributor() + attributor.annotate_split(train) + attributor.annotate_split(val) + + optimization = await prompt_optimizer.optimize( + mode=self.mode, + optimizer_payload=optimizer_payload, + train_baseline=train, + val_baseline=val, + ) + + if self.mode == "real": + train_path, val_path = prompt_optimizer.write_call_agent_evalsets() + candidate_train = await self._run_call_agent_split( + _load_real_evalset(train_path), + eval_config, + prompt_optimizer.call_agent_from_prompts(optimization.best_prompts), + ) + candidate_val = await self._run_call_agent_split( + _load_real_evalset(val_path), + eval_config, + prompt_optimizer.call_agent_from_prompts(optimization.best_prompts), + ) + else: + candidate_train = self._run_fake_split(train_evalset, metrics, candidate_prompts=optimization.best_prompts) + candidate_val = self._run_fake_split(val_evalset, metrics, candidate_prompts=optimization.best_prompts) + + attributor.annotate_split(candidate_train) + attributor.annotate_split(candidate_val) + delta = DeltaAnalyzer().analyze( + train_baseline=train, + train_candidate=candidate_train, + val_baseline=val, + val_candidate=candidate_val, + ) + gate_decision = GateEvaluator(gate_config).evaluate(delta=delta, optimization=optimization) + finished_at = datetime.now(timezone.utc) + duration_seconds = time.perf_counter() - started + output_paths = { + "json": _relative_to_example(self.output_dir / REPORT_FILENAME, self.example_root), + "markdown": _relative_to_example(self.output_dir / MARKDOWN_REPORT_FILENAME, self.example_root), + } + + report = BaselineOptimizationReport( + mode=self.mode, + train=train, + val=val, + run={ + "mode": self.mode, + "started_at": started_at.isoformat(), + "finished_at": finished_at.isoformat(), + "duration_seconds": round(duration_seconds, 6), + "seed": optimization.seed, + }, + inputs={ + "train_evalset_path": _relative_to_example(self.train_evalset_path, self.example_root), + "val_evalset_path": _relative_to_example(self.val_evalset_path, self.example_root), + "optimizer_config_path": _relative_to_example(self.optimizer_config_path, self.example_root), + "gate_config_path": _relative_to_example(self.gate_config_path, self.example_root), + "prompt_paths": { + name: _relative_to_example(path, self.example_root) + for name, path in prompt_optimizer.prompt_paths().items() + }, + }, + config={ + "gate": gate_config.to_dict(), + "optimizer": optimizer_payload, + }, + failure_attribution=_failure_attribution_payload( + baseline_train=train, + baseline_val=val, + candidate_train=candidate_train, + candidate_val=candidate_val, + ), + candidate=CandidateEvaluation( + train=candidate_train, + val=candidate_val, + prompts=optimization.best_prompts, + ), + optimization=optimization, + delta=delta, + gate_decision=gate_decision, + metadata={ + "example_root": ".", + "reproduction_command": f"python run_pipeline.py --mode {self.mode}", + "output_dir": _relative_to_example(self.output_dir, self.example_root), + "output_paths": output_paths, + }, + ) + return write_optimization_report(report, self.output_dir) + + async def _run_real_split( + self, + eval_set: Any, + eval_config: Any, + ) -> BaselineSplitResult: + from trpc_agent_sdk.evaluation import AgentEvaluator + from trpc_agent_sdk.evaluation import EvalSet + + records: list[BaselineCaseRecord] = [] + for case in eval_set.eval_cases: + one_case_set = EvalSet( + eval_set_id=eval_set.eval_set_id, + app_name=eval_set.app_name, + name=eval_set.name, + description=eval_set.description, + eval_cases=[case], + ) + started = time.perf_counter() + _, _, _, eval_results = await AgentEvaluator.evaluate_eval_set( + one_case_set, + eval_config=eval_config, + print_detailed_results=False, + ) + latency = time.perf_counter() - started + runs = eval_results.get(case.eval_id, []) + if not runs: + records.append(_missing_result_record(case, latency)) + continue + records.append(_record_from_eval_case_result(runs[0], latency)) + return BaselineSplitResult(eval_set_id=eval_set.eval_set_id, cases=records) + + def _run_fake_split( + self, + eval_set: dict[str, Any], + metrics: list[dict[str, Any]], + candidate_prompts: dict[str, str] | None = None, + ) -> BaselineSplitResult: + records = [] + for case in eval_set.get("eval_cases", []): + started = time.perf_counter() + records.append( + _fake_record( + case, + metrics, + time.perf_counter() - started, + candidate_prompts=candidate_prompts, + )) + return BaselineSplitResult(eval_set_id=str(eval_set.get("eval_set_id", "")), cases=records) + + async def _run_call_agent_split( + self, + eval_set: Any, + eval_config: Any, + call_agent: Any, + ) -> BaselineSplitResult: + from trpc_agent_sdk.evaluation import AgentEvaluator + from trpc_agent_sdk.evaluation import EvalSet + + records: list[BaselineCaseRecord] = [] + for case in eval_set.eval_cases: + one_case_set = EvalSet( + eval_set_id=eval_set.eval_set_id, + app_name=eval_set.app_name, + name=eval_set.name, + description=eval_set.description, + eval_cases=[case], + ) + started = time.perf_counter() + _, _, _, eval_results = await AgentEvaluator.evaluate_eval_set( + one_case_set, + call_agent=call_agent, + eval_config=eval_config, + print_detailed_results=False, + ) + latency = time.perf_counter() - started + runs = eval_results.get(case.eval_id, []) + if not runs: + records.append(_missing_result_record(case, latency)) + continue + records.append(_record_from_eval_case_result(runs[0], latency)) + return BaselineSplitResult(eval_set_id=eval_set.eval_set_id, cases=records) + + +def _load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _relative_to_example(path: Path, example_root: Path) -> str: + try: + return str(path.resolve().relative_to(example_root.resolve())) + except ValueError: + return str(path) + + +def _failure_attribution_payload( + *, + baseline_train: BaselineSplitResult, + baseline_val: BaselineSplitResult, + candidate_train: BaselineSplitResult, + candidate_val: BaselineSplitResult, +) -> dict[str, Any]: + splits = { + ("baseline", "train"): baseline_train, + ("baseline", "val"): baseline_val, + ("candidate", "train"): candidate_train, + ("candidate", "val"): candidate_val, + } + failed_cases = [] + overall_summary: dict[str, int] = {} + for (variant, split), result in splits.items(): + for category, count in result.failure_attribution_summary().items(): + overall_summary[category] = overall_summary.get(category, 0) + count + for case in result.cases: + if not case.failure_analysis: + continue + failed_cases.append({ + "variant": variant, + "split": split, + "case_id": case.id, + "category": case.failure_analysis.category, + "confidence": case.failure_analysis.confidence, + "explanation": case.failure_analysis.explanation, + "evidence": case.failure_analysis.evidence, + }) + return { + "train_summary": { + "baseline": baseline_train.failure_attribution_summary(), + "candidate": candidate_train.failure_attribution_summary(), + }, + "val_summary": { + "baseline": baseline_val.failure_attribution_summary(), + "candidate": candidate_val.failure_attribution_summary(), + }, + "overall_summary": overall_summary, + "failed_cases": failed_cases, + } + + +def _load_optimizer_payload(path: Path) -> dict[str, Any]: + payload = _load_json(path) + evaluate = payload.get("evaluate") + if not isinstance(evaluate, dict): + raise ValueError(f"{path} must contain an object field named 'evaluate'") + return payload + + +def _load_gate_config(path: Path) -> Any: + from .types import GateConfig + + if not path.exists(): + return GateConfig() + payload = _load_json(path) + if not isinstance(payload, dict): + raise ValueError(f"{path} must contain a JSON object") + return GateConfig.from_dict(payload) + + +def _load_real_evalset(path: Path) -> Any: + from trpc_agent_sdk.evaluation import EvalSet + + return EvalSet.model_validate_json(path.read_text(encoding="utf-8")) + + +def _load_real_eval_config(payload: dict[str, Any]) -> Any: + from trpc_agent_sdk.evaluation import EvalConfig + + return EvalConfig.model_validate(payload["evaluate"]) + + +def _fake_metrics_from_optimizer(payload: dict[str, Any]) -> list[dict[str, Any]]: + evaluate = payload["evaluate"] + metrics = evaluate.get("metrics") + if isinstance(metrics, list): + return [metric for metric in metrics if isinstance(metric, dict)] + criteria = evaluate.get("criteria") or {} + if not isinstance(criteria, dict): + return [] + return [{ + "metric_name": name, + "threshold": value if isinstance(value, (int, float)) else value.get("threshold", 1.0), + "criterion": value.get("criterion") if isinstance(value, dict) else None, + } for name, value in criteria.items()] + + +def _record_from_eval_case_result(result: Any, latency: float) -> BaselineCaseRecord: + metric_scores = { + metric.metric_name: float(metric.score) + for metric in result.overall_eval_metric_results if metric.score is not None + } + metric_score = _mean(metric_scores.values()) + passed = _status_name(result.final_eval_status) == "PASSED" + return BaselineCaseRecord( + id=result.eval_id, + metric_score=metric_score, + metric_scores=metric_scores, + passed=passed, + failure_reason="" if passed else _failure_reason_from_result(result), + trace=_trace_from_result(result), + latency=latency, + cost=0.0, + evaluator_metadata=_evaluator_metadata_from_result(result), + ) + + +def _missing_result_record(case: Any, latency: float) -> BaselineCaseRecord: + return BaselineCaseRecord( + id=str(getattr(case, "eval_id", "")), + metric_score=0.0, + metric_scores={}, + passed=False, + failure_reason="No EvalCaseResult was produced.", + trace=_trace_from_case(case), + latency=latency, + cost=0.0, + evaluator_metadata=_missing_result_metadata(case), + ) + + +def _failure_reason_from_result(result: Any) -> str: + if result.error_message: + return result.error_message + for metric in result.overall_eval_metric_results: + if _status_name(metric.eval_status) == "PASSED": + continue + reason = metric.details.reason if metric.details and metric.details.reason else None + if reason: + return reason + return f"{metric.metric_name} failed." + return "Case did not pass." + + +def _trace_from_result(result: Any) -> dict[str, Any]: + per_invocations = result.eval_metric_result_per_invocation or [] + if not per_invocations: + return {} + first = per_invocations[0] + return _trace_from_invocations( + expected=first.expected_invocation, + actual=first.actual_invocation, + ) + + +def _trace_from_case(case: Any) -> dict[str, Any]: + expected = case.conversation[0] if case.conversation else None + actual = case.actual_conversation[0] if case.actual_conversation else expected + return _trace_from_invocations(expected=expected, actual=actual) + + +def _trace_from_invocations( + *, + expected: Any | None, + actual: Any | None, +) -> dict[str, Any]: + from trpc_agent_sdk.evaluation import get_all_tool_calls + + return { + "user": + _content_text(actual.user_content if actual else expected.user_content if expected else None), + "expected": + _content_text(expected.final_response if expected else None), + "actual": + _content_text(actual.final_response if actual else None), + "tool_calls": [{ + "name": call.name, + "args": call.args or {}, + } for call in get_all_tool_calls(actual.intermediate_data if actual else None)], + } + + +def _evaluator_metadata_from_result(result: Any) -> dict[str, Any]: + failed_metrics = [] + overall_metrics = [] + for metric in result.overall_eval_metric_results: + metric_entry = _metric_result_entry(metric) + overall_metrics.append(metric_entry) + if metric_entry["eval_status"] != "PASSED": + failed_metrics.append(metric_entry) + + expected_invocation, actual_invocation = _first_invocation_pair(result) + return { + "error_message": + result.error_message, + "final_eval_status": + _status_name(result.final_eval_status), + "overall_metric_results": + overall_metrics, + "failed_metric_results": + failed_metrics, + "expected_tool_calls": + _tool_calls_from_invocation(expected_invocation.intermediate_data if expected_invocation else None), + "actual_tool_calls": + _tool_calls_from_invocation(actual_invocation.intermediate_data if actual_invocation else None), + "expected_final_response": + _content_text(expected_invocation.final_response if expected_invocation else None), + "actual_final_response": + _content_text(actual_invocation.final_response if actual_invocation else None), + } + + +def _first_invocation_pair(result: Any) -> tuple[Any | None, Any | None]: + per_invocations = result.eval_metric_result_per_invocation or [] + if not per_invocations: + return None, None + first = per_invocations[0] + return first.expected_invocation, first.actual_invocation + + +def _metric_result_entry(metric: Any) -> dict[str, Any]: + details = metric.details + return { + "metric_name": metric.metric_name, + "score": float(metric.score) if metric.score is not None else None, + "threshold": float(metric.threshold) if metric.threshold is not None else None, + "eval_status": _status_name(metric.eval_status), + "reason": details.reason if details else None, + "rubric_scores": getattr(details, "rubric_scores", None) if details else None, + } + + +def _tool_calls_from_invocation(intermediate_data: Any | None) -> list[dict[str, Any]]: + from trpc_agent_sdk.evaluation import get_all_tool_calls + + return [{ + "name": call.name, + "args": call.args or {}, + } for call in get_all_tool_calls(intermediate_data)] + + +def _missing_result_metadata(case: Any) -> dict[str, Any]: + expected_invocation = _first_case_invocation(case, "conversation") + actual_invocation = _first_case_invocation(case, "actual_conversation") + return { + "error_message": + "No EvalCaseResult was produced.", + "failed_metric_results": [], + "overall_metric_results": [], + "expected_tool_calls": + _tool_calls_from_invocation(expected_invocation.intermediate_data if expected_invocation else None), + "actual_tool_calls": + _tool_calls_from_invocation(actual_invocation.intermediate_data if actual_invocation else None), + "expected_final_response": + _content_text(expected_invocation.final_response if expected_invocation else None), + "actual_final_response": + _content_text(actual_invocation.final_response if actual_invocation else None), + } + + +def _first_case_invocation(case: Any, field_name: str) -> Any | None: + invocations = getattr(case, field_name, None) + if not invocations: + return None + return invocations[0] + + +def _fake_record( + case: dict[str, Any], + metrics: list[dict[str, Any]], + latency: float, + *, + candidate_prompts: dict[str, str] | None = None, +) -> BaselineCaseRecord: + conversation = case.get("conversation") or [] + actual_conversation = case.get("actual_conversation") or case.get("actualConversation") or [] + expected = conversation[0] if conversation else None + if candidate_prompts: + actual = _candidate_invocation(case, expected, candidate_prompts) + else: + actual = actual_conversation[0] if actual_conversation else expected + metric_scores: dict[str, float] = {} + failed_metrics: list[str] = [] + metric_results: list[dict[str, Any]] = [] + + for metric in metrics: + score = _fake_metric_score(metric, expected=expected, actual=actual) + metric_name = str(metric.get("metric_name") or metric.get("metricName") or "") + metric_scores[metric_name] = score + threshold = float(metric.get("threshold", 1.0)) + metric_result = { + "metric_name": metric_name, + "score": score, + "threshold": threshold, + "eval_status": "PASSED" if score >= threshold else "FAILED", + "reason": _fake_metric_reason(metric_name, score, threshold, expected=expected, actual=actual), + "rubric_scores": None, + } + metric_results.append(metric_result) + if score < threshold: + failed_metrics.append(metric_name) + + passed = not failed_metrics + return BaselineCaseRecord( + id=str(case.get("eval_id", "")), + metric_score=_mean(metric_scores.values()), + metric_scores=metric_scores, + passed=passed, + failure_reason="" if passed else f"Failed metrics: {', '.join(failed_metrics)}.", + trace=_fake_trace(expected=expected, actual=actual), + latency=latency, + cost=0.0, + evaluator_metadata={ + "error_message": None, + "final_eval_status": "PASSED" if passed else "FAILED", + "overall_metric_results": metric_results, + "failed_metric_results": [entry for entry in metric_results if entry["eval_status"] != "PASSED"], + "expected_tool_calls": _raw_tool_calls(expected), + "actual_tool_calls": _raw_tool_calls(actual), + "expected_final_response": _raw_content_text(expected.get("final_response") if expected else None), + "actual_final_response": _raw_content_text(actual.get("final_response") if actual else None), + }, + ) + + +def _candidate_invocation( + case: dict[str, Any], + expected: dict[str, Any] | None, + candidate_prompts: dict[str, str], +) -> dict[str, Any]: + source = expected or {} + user_content = source.get("user_content") or {} + query = _raw_content_text(user_content) + return { + "invocation_id": f"{case.get('eval_id', 'candidate')}-candidate", + "user_content": user_content, + "final_response": { + "parts": [{ + "text": deterministic_answer(query, candidate_prompts), + }], + "role": "model", + }, + } + + +def _fake_metric_score( + metric: dict[str, Any], + *, + expected: dict[str, Any] | None, + actual: dict[str, Any] | None, +) -> float: + metric_name = str(metric.get("metric_name") or metric.get("metricName") or "") + if metric_name == "final_response_avg_score": + expected_text = _raw_content_text(expected.get("final_response") if expected else None) + actual_text = _raw_content_text(actual.get("final_response") if actual else None) + return 1.0 if _text_matches(metric, actual_text, expected_text) else 0.0 + if metric_name == "tool_trajectory_avg_score": + expected_calls = _raw_tool_calls(expected) + actual_calls = _raw_tool_calls(actual) + return 1.0 if actual_calls == expected_calls else 0.0 + return 1.0 + + +def _fake_metric_reason( + metric_name: str, + score: float, + threshold: float, + *, + expected: dict[str, Any] | None, + actual: dict[str, Any] | None, +) -> str | None: + if score >= threshold: + return None + if metric_name == "final_response_avg_score": + return "Final response does not match the reference answer." + if "tool_trajectory" in metric_name: + return "Tool trajectory does not match the reference trace." + if "rubric" in metric_name: + return "LLM rubric score is below threshold." + if "recall" in metric_name: + return "Knowledge recall is below threshold." + if "format" in metric_name or "json" in metric_name: + return "Output format does not satisfy the requested constraint." + actual_text = _raw_content_text(actual.get("final_response") if actual else None) + expected_text = _raw_content_text(expected.get("final_response") if expected else None) + return f"Metric {metric_name} failed for actual={actual_text!r} expected={expected_text!r}." + + +def _text_matches(metric: dict[str, Any], actual: str, expected: str) -> bool: + strategy = _final_response_text_strategy(metric) + match = str(strategy.get("match", "exact")).lower() + case_insensitive = bool(strategy.get("case_insensitive", False)) + if case_insensitive: + actual = actual.lower() + expected = expected.lower() + if match == "contains": + return expected in actual + if match == "regex": + flags = re.IGNORECASE if case_insensitive else 0 + return re.search(expected, actual, flags=flags) is not None + return actual == expected + + +def _final_response_text_strategy(metric: dict[str, Any]) -> dict[str, Any]: + criterion = metric.get("criterion") or {} + final_response = criterion.get("final_response") or criterion.get("finalResponse") or {} + text = final_response.get("text") or {} + return text if isinstance(text, dict) else {} + + +def _content_text(content: Any | None) -> str: + if not content or not content.parts: + return "" + return "\n".join(part.text or "" for part in content.parts if part.text).strip() + + +def _fake_trace( + *, + expected: dict[str, Any] | None, + actual: dict[str, Any] | None, +) -> dict[str, Any]: + user_content = (actual or expected or {}).get("user_content") + return { + "user": _raw_content_text(user_content), + "expected": _raw_content_text(expected.get("final_response") if expected else None), + "actual": _raw_content_text(actual.get("final_response") if actual else None), + "tool_calls": _raw_tool_calls(actual), + } + + +def _raw_content_text(content: dict[str, Any] | None) -> str: + if not content: + return "" + parts = content.get("parts") or [] + return "\n".join(str(part.get("text") or "") for part in parts + if isinstance(part, dict) and part.get("text")).strip() + + +def _raw_tool_calls(invocation: dict[str, Any] | None) -> list[dict[str, Any]]: + if not invocation: + return [] + intermediate = invocation.get("intermediate_data") or invocation.get("intermediateData") or {} + tool_uses = intermediate.get("tool_uses") or intermediate.get("toolUses") or [] + calls = [] + for call in tool_uses: + if not isinstance(call, dict): + continue + calls.append({ + "name": call.get("name"), + "args": call.get("args") or {}, + }) + return calls + + +def _status_name(status: Any) -> str: + name = getattr(status, "name", None) + if isinstance(name, str): + return name + return str(status) + + +def _mean(values: Any) -> float: + numbers = list(values) + if not numbers: + return 0.0 + return sum(numbers) / len(numbers) + + +BaselinePipeline = EvalOptimizePipeline diff --git a/examples/optimization/eval_optimize_loop/pipeline/report.py b/examples/optimization/eval_optimize_loop/pipeline/report.py new file mode 100644 index 000000000..73ad8f241 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/report.py @@ -0,0 +1,166 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Report writing for the eval-optimize-loop example.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .types import BaselineOptimizationReport +from .types import ReportPaths + +REPORT_FILENAME = "optimization_report.json" +MARKDOWN_REPORT_FILENAME = "optimization_report.md" + + +def write_optimization_report(report: BaselineOptimizationReport, output_dir: Path) -> ReportPaths: + """Persist machine-readable and human-readable optimization reports.""" + output_dir.mkdir(parents=True, exist_ok=True) + report_dict = report.to_dict() + report_path = output_dir / REPORT_FILENAME + markdown_path = output_dir / MARKDOWN_REPORT_FILENAME + _atomic_write_text(report_path, json.dumps(report_dict, ensure_ascii=False, indent=2) + "\n") + _atomic_write_text(markdown_path, render_markdown_report(report_dict)) + return ReportPaths(json_path=report_path, markdown_path=markdown_path) + + +def render_markdown_report(report: dict[str, Any]) -> str: + gate = report["gate_decision"] + delta = report["delta"] + optimization = report["optimization"] + baseline = report["baseline"] + candidate = report["candidate"] + failure_attribution = report["failure_attribution"] + gate_config = report["config"]["gate"] + metadata = report["metadata"] + lines = [ + "# Evaluation Optimization Pipeline 报告", + "", + "## 运行摘要", + "", + f"- 决策：**{gate['decision'].upper()}**", + f"- 推荐动作：`{gate['recommended_action']}`", + f"- 模式：`{report['run']['mode']}`", + f"- Schema：`{report['schema_version']}`", + f"- 验证集分数变化：{_fmt(delta['val']['score_delta'])}", + f"- 验证集通过率变化：{_fmt(delta['val']['pass_rate_delta'])}", + f"- 主要原因：{gate['reason']}", + "", + "## Baseline 表现", + "", + _split_summary("训练集", baseline["train"]), + _split_summary("验证集", baseline["val"]), + "", + "## 错误归因汇总", + "", + _failure_table("训练集 baseline", failure_attribution["train_summary"]["baseline"]), + "", + _failure_table("训练集 candidate", failure_attribution["train_summary"]["candidate"]), + "", + _failure_table("验证集 baseline", failure_attribution["val_summary"]["baseline"]), + "", + _failure_table("验证集 candidate", failure_attribution["val_summary"]["candidate"]), + "", + "## 优化过程", + "", + f"- 目标 Prompt：{', '.join(optimization['target_prompt_names'])}", + f"- 优化轮数：{optimization['total_rounds']}", + f"- 优化成本：{optimization['total_cost']}", + f"- 随机种子：{optimization['seed']}", + ] + for round_record in optimization["rounds"]: + lines.extend([ + "", + f"### Round {round_record['round']}", + "", + f"- 修改字段：{', '.join(round_record['optimized_field_names'])}", + f"- 是否接受为候选：{round_record['accepted']}", + f"- 原因：{round_record['reason']}", + ]) + lines.extend([ + "", + "## 候选验证", + "", + _split_summary("训练集候选", candidate["train"]), + _split_summary("验证集候选", candidate["val"]), + "", + "## 逐 Case Delta", + "", + "| split | case id | baseline | candidate | change | failure transition | regression | improvement |", + "| --- | --- | ---: | ---: | --- | --- | --- | --- |", + ]) + for split_name in ("train", "val"): + for case_delta in delta[split_name]["case_deltas"]: + lines.append(_case_delta_row(case_delta)) + lines.extend([ + "", + "## Gate 决策", + "", + f"- 最终决策：**{gate['decision'].upper()}**", + f"- 推荐动作：`{gate['recommended_action']}`", + f"- 拒绝/接受理由：{gate['reason']}", + f"- 最小验证集分数提升：{gate_config['min_val_score_gain']}", + f"- 允许新增失败：{gate_config['allow_new_failures']}", + f"- 允许验证集回归：{gate_config['allow_regressions']}", + f"- 关键 Case：{', '.join(gate_config['critical_case_ids']) or '无'}", + "", + "| rule | passed | severity | message |", + "| --- | --- | --- | --- |", + ]) + for rule in gate["rule_results"]: + lines.append(f"| {rule['rule_name']} | {rule['passed']} | {rule['severity']} | {rule['message']} |") + lines.extend([ + "", + "## 元数据与复现", + "", + f"- 示例根目录：`{metadata['example_root']}`", + f"- 复现命令：`{metadata['reproduction_command']}`", + f"- 输出 JSON：`{metadata['output_paths']['json']}`", + f"- 输出 Markdown：`{metadata['output_paths']['markdown']}`", + "", + "| key | value |", + "| --- | --- |", + f"| example_root | {metadata['example_root']} |", + f"| output_dir | {metadata['output_dir']} |", + ]) + lines.append("") + return "\n".join(lines) + + +def _atomic_write_text(path: Path, content: str) -> None: + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_text(content, encoding="utf-8") + tmp_path.replace(path) + + +def _split_summary(label: str, split: dict[str, Any]) -> str: + pass_rate = split["passed_count"] / split["case_count"] if split["case_count"] else 0.0 + return f"- {label}：{split['passed_count']}/{split['case_count']} 通过，通过率 {_fmt(pass_rate)}" + + +def _failure_table(label: str, summary: dict[str, int]) -> str: + lines = [f"### {label}", "", "| category | count |", "| --- | ---: |"] + for category, count in summary.items(): + lines.append(f"| {category} | {count} |") + return "\n".join(lines) + + +def _case_delta_row(case_delta: dict[str, Any]) -> str: + failure_transition = "{} -> {}".format( + case_delta["baseline"].get("failure_category"), + case_delta["candidate"].get("failure_category"), + ) + return (f"| {case_delta['split']} | {case_delta['id']} | {_fmt(case_delta['baseline'].get('metric_score'))} | " + f"{_fmt(case_delta['candidate'].get('metric_score'))} | {case_delta['change_type']} | " + f"{failure_transition} | {case_delta['regression']} | {case_delta['improvement']} |") + + +def _fmt(value: Any) -> str: + if isinstance(value, float): + return f"{value:.4f}" + return str(value) diff --git a/examples/optimization/eval_optimize_loop/pipeline/types.py b/examples/optimization/eval_optimize_loop/pipeline/types.py new file mode 100644 index 000000000..5642d32f6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/types.py @@ -0,0 +1,464 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Plain Python report types for the eval-optimize-loop example.""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +from typing import Any + +FAILURE_CATEGORIES = [ + "final_answer_mismatch", + "tool_call_error", + "parameter_error", + "llm_rubric_failed", + "retrieval_failure", + "format_error", + "unknown", +] + + +@dataclass +class FailureAnalysis: + """Rule-based attribution for one failed case.""" + + category: str + confidence: float + explanation: str + evidence: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "category": self.category, + "confidence": self.confidence, + "explanation": self.explanation, + "evidence": self.evidence, + } + + +@dataclass +class BaselineCaseRecord: + """One case-level baseline result serialized into optimization_report.json.""" + + id: str + metric_score: float + metric_scores: dict[str, float] + passed: bool + failure_reason: str + trace: dict[str, Any] + latency: float + cost: float = 0.0 + failure_analysis: FailureAnalysis | None = None + evaluator_metadata: dict[str, Any] = field(default_factory=dict, repr=False) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "metric_score": self.metric_score, + "metric_scores": self.metric_scores, + "passed": self.passed, + "failure_reason": self.failure_reason, + "trace": self.trace, + "latency": self.latency, + "cost": self.cost, + "failure_analysis": self.failure_analysis.to_dict() if self.failure_analysis else None, + } + + +@dataclass +class BaselineSplitResult: + """Baseline results for one evalset split.""" + + eval_set_id: str + cases: list[BaselineCaseRecord] = field(default_factory=list) + + def failure_attribution_summary(self) -> dict[str, int]: + summary = {category: 0 for category in FAILURE_CATEGORIES} + for case in self.cases: + if case.failure_analysis: + summary[case.failure_analysis.category] = summary.get(case.failure_analysis.category, 0) + 1 + return summary + + def to_dict(self) -> dict[str, Any]: + passed_count = sum(1 for case in self.cases if case.passed) + failed_count = len(self.cases) - passed_count + return { + "eval_set_id": self.eval_set_id, + "case_count": len(self.cases), + "passed_count": passed_count, + "failed_count": failed_count, + "failure_attribution_summary": self.failure_attribution_summary(), + "cases": [case.to_dict() for case in self.cases], + } + + +@dataclass +class CandidateEvaluation: + """Candidate prompt evaluation snapshot serialized into optimization_report.json.""" + + train: BaselineSplitResult + val: BaselineSplitResult + prompts: dict[str, str] + + def to_dict(self) -> dict[str, Any]: + return { + "train": self.train.to_dict(), + "val": self.val.to_dict(), + "prompts": self.prompts, + } + + +@dataclass +class OptimizationRoundRecord: + """One prompt optimization round for the report.""" + + round: int + optimized_field_names: list[str] + before: dict[str, str] + after: dict[str, str] + reason: str + accepted: bool + validation_pass_rate: float = 0.0 + duration_seconds: float = 0.0 + cost: float = 0.0 + + def to_dict(self) -> dict[str, Any]: + return { + "round": self.round, + "optimized_field_names": self.optimized_field_names, + "before": self.before, + "after": self.after, + "reason": self.reason, + "accepted": self.accepted, + "validation_pass_rate": self.validation_pass_rate, + "duration_seconds": self.duration_seconds, + "cost": self.cost, + } + + +@dataclass +class OptimizationRunRecord: + """Prompt optimization loop summary.""" + + target_prompt_names: list[str] + baseline_prompts: dict[str, str] + best_prompts: dict[str, str] + rounds: list[OptimizationRoundRecord] + total_rounds: int + total_cost: float + duration_seconds: float + seed: int | None + reason: str + artifacts: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "target_prompt_names": self.target_prompt_names, + "baseline_prompts": self.baseline_prompts, + "best_prompts": self.best_prompts, + "rounds": [record.to_dict() for record in self.rounds], + "total_rounds": self.total_rounds, + "total_cost": self.total_cost, + "duration_seconds": self.duration_seconds, + "seed": self.seed, + "reason": self.reason, + "artifacts": self.artifacts, + } + + +@dataclass +class MetricDelta: + """Metric-level comparison for one case.""" + + metric_name: str + baseline_score: float | None + candidate_score: float | None + score_delta: float | None + baseline_passed: bool | None + candidate_passed: bool | None + status_transition: str + + def to_dict(self) -> dict[str, Any]: + return { + "metric_name": self.metric_name, + "baseline_score": self.baseline_score, + "candidate_score": self.candidate_score, + "score_delta": self.score_delta, + "baseline_passed": self.baseline_passed, + "candidate_passed": self.candidate_passed, + "status_transition": self.status_transition, + } + + +@dataclass +class CaseDelta: + """Case-level baseline vs candidate comparison.""" + + id: str + split: str + change_type: str + baseline: dict[str, Any] + candidate: dict[str, Any] + score_delta: float | None + metric_deltas: list[MetricDelta] + latency_delta: float | None + cost_delta: float | None + regression: bool + improvement: bool + notes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "split": self.split, + "change_type": self.change_type, + "baseline": self.baseline, + "candidate": self.candidate, + "score_delta": self.score_delta, + "metric_deltas": [metric.to_dict() for metric in self.metric_deltas], + "latency_delta": self.latency_delta, + "cost_delta": self.cost_delta, + "regression": self.regression, + "improvement": self.improvement, + "notes": self.notes, + } + + +@dataclass +class SplitDeltaSummary: + """Aggregate and case-level delta for one eval split.""" + + split: str + baseline_score: float + candidate_score: float + score_delta: float + baseline_pass_rate: float + candidate_pass_rate: float + pass_rate_delta: float + case_deltas: list[CaseDelta] + missing_candidate_case_ids: list[str] = field(default_factory=list) + extra_candidate_case_ids: list[str] = field(default_factory=list) + + def to_summary_dict(self) -> dict[str, Any]: + return { + "split": self.split, + "baseline_score": self.baseline_score, + "candidate_score": self.candidate_score, + "score_delta": self.score_delta, + "baseline_pass_rate": self.baseline_pass_rate, + "candidate_pass_rate": self.candidate_pass_rate, + "pass_rate_delta": self.pass_rate_delta, + "case_count": len(self.case_deltas), + "new_pass_count": sum(1 for case in self.case_deltas if case.change_type == "new_pass"), + "new_fail_count": sum(1 for case in self.case_deltas if case.change_type == "new_fail"), + "regression_count": sum(1 for case in self.case_deltas if case.regression), + "improvement_count": sum(1 for case in self.case_deltas if case.improvement), + "missing_candidate_case_ids": self.missing_candidate_case_ids, + "extra_candidate_case_ids": self.extra_candidate_case_ids, + } + + def to_dict(self) -> dict[str, Any]: + return { + "split": self.split, + "baseline_score": self.baseline_score, + "candidate_score": self.candidate_score, + "score_delta": self.score_delta, + "baseline_pass_rate": self.baseline_pass_rate, + "candidate_pass_rate": self.candidate_pass_rate, + "pass_rate_delta": self.pass_rate_delta, + "case_deltas": [case.to_dict() for case in self.case_deltas], + "missing_candidate_case_ids": self.missing_candidate_case_ids, + "extra_candidate_case_ids": self.extra_candidate_case_ids, + } + + +@dataclass +class OptimizationDelta: + """Top-level train/validation delta section.""" + + train: SplitDeltaSummary + val: SplitDeltaSummary + + def to_dict(self) -> dict[str, Any]: + all_cases = self.train.case_deltas + self.val.case_deltas + missing = self.train.missing_candidate_case_ids + self.val.missing_candidate_case_ids + extra = self.train.extra_candidate_case_ids + self.val.extra_candidate_case_ids + regression_count = sum(1 for case in all_cases if case.regression) + improvement_count = sum(1 for case in all_cases if case.improvement) + return { + "summary": { + "overall_change_type": _overall_change_type(regression_count, improvement_count), + "train": self.train.to_summary_dict(), + "val": self.val.to_summary_dict(), + "regression_count": regression_count, + "improvement_count": improvement_count, + "new_pass_count": sum(1 for case in all_cases if case.change_type == "new_pass"), + "new_fail_count": sum(1 for case in all_cases if case.change_type == "new_fail"), + "missing_candidate_case_ids": missing, + "extra_candidate_case_ids": extra, + }, + "train": self.train.to_dict(), + "val": self.val.to_dict(), + } + + +@dataclass +class GateConfig: + """Example-private gate policy loaded from gate.json.""" + + min_val_score_gain: float = 0.05 + min_val_pass_rate_gain: float = 0.0 + allow_new_failures: bool = False + allow_regressions: bool = False + critical_case_ids: list[str] = field(default_factory=list) + max_cost_delta: float = 0.0 + max_cost_ratio: float = 1.2 + require_validation_improvement: bool = True + overfit_policy: dict[str, Any] = field(default_factory=lambda: { + "enabled": True, + "train_gain_min": 0.05, + "val_drop_tolerance": 0.0, + }) + + @classmethod + def from_dict(cls, payload: dict[str, Any] | None) -> "GateConfig": + payload = payload or {} + defaults = cls() + overfit_policy = dict(defaults.overfit_policy) + if isinstance(payload.get("overfit_policy"), dict): + overfit_policy.update(payload["overfit_policy"]) + critical_case_ids = payload.get("critical_case_ids", []) + if not isinstance(critical_case_ids, list): + critical_case_ids = [] + return cls( + min_val_score_gain=float(payload.get("min_val_score_gain", defaults.min_val_score_gain)), + min_val_pass_rate_gain=float(payload.get("min_val_pass_rate_gain", defaults.min_val_pass_rate_gain)), + allow_new_failures=bool(payload.get("allow_new_failures", defaults.allow_new_failures)), + allow_regressions=bool(payload.get("allow_regressions", defaults.allow_regressions)), + critical_case_ids=[str(case_id) for case_id in critical_case_ids], + max_cost_delta=float(payload.get("max_cost_delta", defaults.max_cost_delta)), + max_cost_ratio=float(payload.get("max_cost_ratio", defaults.max_cost_ratio)), + require_validation_improvement=bool( + payload.get("require_validation_improvement", defaults.require_validation_improvement)), + overfit_policy=overfit_policy, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "min_val_score_gain": self.min_val_score_gain, + "min_val_pass_rate_gain": self.min_val_pass_rate_gain, + "allow_new_failures": self.allow_new_failures, + "allow_regressions": self.allow_regressions, + "critical_case_ids": self.critical_case_ids, + "max_cost_delta": self.max_cost_delta, + "max_cost_ratio": self.max_cost_ratio, + "require_validation_improvement": self.require_validation_improvement, + "overfit_policy": self.overfit_policy, + } + + +@dataclass +class GateRuleResult: + """One auditable gate rule outcome.""" + + rule_name: str + passed: bool + severity: str + message: str + evidence: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "rule_name": self.rule_name, + "passed": self.passed, + "severity": self.severity, + "message": self.message, + "evidence": self.evidence, + } + + +@dataclass +class GateDecision: + """Final candidate accept/reject decision.""" + + decision: str + accepted: bool + reason: str + rule_results: list[GateRuleResult] + summary: dict[str, Any] + config: GateConfig + recommended_action: str + generated_at: str + + def to_dict(self) -> dict[str, Any]: + return { + "decision": self.decision, + "accepted": self.accepted, + "reason": self.reason, + "rule_results": [result.to_dict() for result in self.rule_results], + "summary": self.summary, + "config": self.config.to_dict(), + "recommended_action": self.recommended_action, + "generated_at": self.generated_at, + } + + +def _overall_change_type(regression_count: int, improvement_count: int) -> str: + if regression_count and improvement_count: + return "mixed" + if regression_count: + return "regressed" + if improvement_count: + return "improved" + return "unchanged" + + +@dataclass +class BaselineOptimizationReport: + """Full evaluation and optimization report.""" + + mode: str + train: BaselineSplitResult + val: BaselineSplitResult + metadata: dict[str, Any] + run: dict[str, Any] = field(default_factory=dict) + inputs: dict[str, Any] = field(default_factory=dict) + config: dict[str, Any] = field(default_factory=dict) + failure_attribution: dict[str, Any] = field(default_factory=dict) + candidate: CandidateEvaluation | None = None + optimization: OptimizationRunRecord | None = None + delta: OptimizationDelta | None = None + gate_decision: GateDecision | None = None + schema_version: str = "eval-optimize-loop-v1" + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "run": self.run or { + "mode": self.mode + }, + "inputs": self.inputs, + "config": self.config, + "baseline": { + "train": self.train.to_dict(), + "val": self.val.to_dict(), + }, + "failure_attribution": self.failure_attribution, + "candidate": self.candidate.to_dict() if self.candidate else None, + "delta": self.delta.to_dict() if self.delta else None, + "gate_decision": self.gate_decision.to_dict() if self.gate_decision else None, + "optimization": self.optimization.to_dict() if self.optimization else None, + "metadata": self.metadata, + } + + +@dataclass +class ReportPaths: + """Locations of the generated report artifacts.""" + + json_path: Any + markdown_path: Any diff --git a/examples/optimization/eval_optimize_loop/prompts/skill.md b/examples/optimization/eval_optimize_loop/prompts/skill.md new file mode 100644 index 000000000..7a0911360 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/skill.md @@ -0,0 +1 @@ +For word problems, compute the numeric result first, then preserve the expected unit. diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 000000000..7c4440a75 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1 @@ +You are a concise math assistant. Return the final answer in the form "答案： ". diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 000000000..a0aa011be --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,63 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Run the evaluation and optimization loop example.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +from pipeline import EvalOptimizePipeline # noqa: E402 + +TRAIN_PATH = _HERE / "train.evalset.json" +VAL_PATH = _HERE / "val.evalset.json" +OPTIMIZER_PATH = _HERE / "optimizer.json" +GATE_PATH = _HERE / "gate.json" +OUTPUT_DIR = _HERE / "output" + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run evaluation, optimization, candidate validation, and gate.") + parser.add_argument( + "--mode", + choices=("real", "fake"), + default="fake", + help="real uses AgentOptimizer; fake uses deterministic local optimization.", + ) + parser.add_argument( + "--gate-config", + default=str(GATE_PATH), + help="Path to the example-private gate.json config.", + ) + return parser.parse_args() + + +async def main() -> None: + args = _parse_args() + pipeline = EvalOptimizePipeline( + train_evalset_path=TRAIN_PATH, + val_evalset_path=VAL_PATH, + optimizer_config_path=OPTIMIZER_PATH, + gate_config_path=Path(args.gate_config), + output_dir=OUTPUT_DIR, + mode=args.mode, + ) + report_paths = await pipeline.run() + print(f"optimization_report.json written to: {report_paths.json_path}") + print(f"optimization_report.md written to: {report_paths.markdown_path}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/optimization/eval_optimize_loop/tests/__init__.py b/examples/optimization/eval_optimize_loop/tests/__init__.py new file mode 100644 index 000000000..bc6e483f9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/__init__.py @@ -0,0 +1,5 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. diff --git a/examples/optimization/eval_optimize_loop/tests/conftest.py b/examples/optimization/eval_optimize_loop/tests/conftest.py new file mode 100644 index 000000000..726b5b6ab --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/conftest.py @@ -0,0 +1,306 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Shared fixtures for the eval-optimize-loop tests.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +_EXAMPLE_ROOT = Path(__file__).resolve().parents[1] +if str(_EXAMPLE_ROOT) not in sys.path: + sys.path.insert(0, str(_EXAMPLE_ROOT)) + +from pipeline import DeltaAnalyzer # type: ignore # noqa: E402 +from pipeline.types import BaselineCaseRecord # type: ignore # noqa: E402 +from pipeline.types import BaselineSplitResult # type: ignore # noqa: E402 + + +@pytest.fixture +def example_root() -> Path: + return _EXAMPLE_ROOT + + +def metric_result( + metric_name: str, + *, + score: float = 0.0, + threshold: float = 1.0, + eval_status: str = "FAILED", + reason: str | None = None, +) -> dict[str, Any]: + return { + "metric_name": metric_name, + "score": score, + "threshold": threshold, + "eval_status": eval_status, + "reason": reason, + } + + +def case_record( + case_id: str, + *, + passed: bool, + metric_results: list[dict[str, Any]], + expected_tool_calls: list[dict[str, Any]] | None = None, + actual_tool_calls: list[dict[str, Any]] | None = None, + expected_final: str = "expected", + actual_final: str = "actual", + failure_reason: str = "failed", +) -> BaselineCaseRecord: + if metric_results: + metric_score = sum(float(entry["score"]) for entry in metric_results) / len(metric_results) + else: + metric_score = 1.0 if passed else 0.0 + return BaselineCaseRecord( + id=case_id, + metric_score=metric_score, + metric_scores={entry["metric_name"]: float(entry["score"]) + for entry in metric_results}, + passed=passed, + failure_reason="" if passed else failure_reason, + trace={ + "user": "question", + "expected": expected_final, + "actual": actual_final, + "tool_calls": actual_tool_calls or [], + }, + latency=0.01, + cost=0.0, + evaluator_metadata={ + "error_message": None, + "final_eval_status": "PASSED" if passed else "FAILED", + "overall_metric_results": metric_results, + "failed_metric_results": [entry for entry in metric_results if entry["eval_status"] != "PASSED"], + "expected_tool_calls": expected_tool_calls or [], + "actual_tool_calls": actual_tool_calls or [], + "expected_final_response": expected_final, + "actual_final_response": actual_final, + }, + ) + + +def delta_from_scores( + *, + train_pairs: list[tuple[str, bool, float, bool, float]], + val_pairs: list[tuple[str, bool, float, bool, float]], + baseline_cost: float = 0.0, + candidate_cost: float = 0.0, +) -> Any: + return DeltaAnalyzer().analyze( + train_baseline=split_from_pairs("train_base", train_pairs, baseline=True, cost=baseline_cost), + train_candidate=split_from_pairs("train_candidate", train_pairs, baseline=False, cost=candidate_cost), + val_baseline=split_from_pairs("val_base", val_pairs, baseline=True, cost=baseline_cost), + val_candidate=split_from_pairs("val_candidate", val_pairs, baseline=False, cost=candidate_cost), + ) + + +def split_from_pairs( + eval_set_id: str, + pairs: list[tuple[str, bool, float, bool, float]], + *, + baseline: bool, + cost: float, +) -> BaselineSplitResult: + records = [] + for case_id, baseline_passed, baseline_score, candidate_passed, candidate_score in pairs: + passed = baseline_passed if baseline else candidate_passed + score = baseline_score if baseline else candidate_score + case = case_record( + case_id, + passed=passed, + metric_results=[metric_result( + "m", + score=score, + eval_status="PASSED" if passed else "FAILED", + )], + ) + case.cost = cost + records.append(case) + return BaselineSplitResult(eval_set_id=eval_set_id, cases=records) + + +def rule(decision: Any, rule_name: str) -> Any: + return next(result for result in decision.rule_results if result.rule_name == rule_name) + + +def install_fake_evaluation_sdk(monkeypatch: Any, example_root: Path) -> dict[str, Any]: + from pipeline.optimization import FINAL_ANSWER_FIX_MARKER # type: ignore # noqa: E402 + + captured: dict[str, Any] = {} + baseline_prompts = { + "system_prompt": (example_root / "prompts" / "system.md").read_text(encoding="utf-8"), + "skill": (example_root / "prompts" / "skill.md").read_text(encoding="utf-8"), + } + best_prompts = { + "system_prompt": baseline_prompts["system_prompt"] + f"\n{FINAL_ANSWER_FIX_MARKER}\n", + "skill": baseline_prompts["skill"], + } + + async def fake_optimize(**kwargs: Any) -> Any: + captured.update(kwargs) + return SimpleNamespace( + best_prompts=best_prompts, + baseline_prompts=baseline_prompts, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=["system_prompt"], + candidate_prompts=best_prompts, + acceptance_reason="stub accepted final_answer_mismatch fix", + accepted=True, + validation_pass_rate=1.0, + duration_seconds=0.01, + round_llm_cost=0.0, + ) + ], + total_rounds=1, + total_llm_cost=0.0, + duration_seconds=0.01, + status="SUCCEEDED", + finish_reason="stubbed", + pass_rate_improvement=1.0, + ) + + class FakeAgentOptimizer: + + optimize = staticmethod(fake_optimize) + + class FakeTargetPrompt: + + def __init__(self) -> None: + self._names: list[str] = [] + + def add_path(self, name: str, path: str) -> "FakeTargetPrompt": + self._names.append(name) + return self + + def names(self) -> list[str]: + return self._names + + class FakeEvalConfig: + + @classmethod + def model_validate(cls, payload: dict[str, Any]) -> dict[str, Any]: + return payload + + class FakeEvalSet: + + def __init__( + self, + *, + eval_set_id: str, + app_name: str = "", + name: str = "", + description: str = "", + eval_cases: list[Any] | None = None, + ) -> None: + self.eval_set_id = eval_set_id + self.app_name = app_name + self.name = name + self.description = description + self.eval_cases = eval_cases or [] + + @classmethod + def model_validate_json(cls, content: str) -> "FakeEvalSet": + payload = json.loads(content) + cases = [_fake_eval_case(case) for case in payload["eval_cases"]] + return cls( + eval_set_id=payload["eval_set_id"], + app_name=payload.get("app_name", ""), + name=payload.get("name", ""), + description=payload.get("description", ""), + eval_cases=cases, + ) + + class FakeAgentEvaluator: + + @staticmethod + async def evaluate_eval_set(eval_set: FakeEvalSet, + **kwargs: Any) -> tuple[None, list[str], list[str], dict[str, list[Any]]]: + call_agent = kwargs.get("call_agent") + results = {} + for case in eval_set.eval_cases: + user_text = content_text(case.conversation[0].user_content) + expected_text = content_text(case.conversation[0].final_response) + actual_text = await call_agent(user_text) if call_agent else content_text( + case.actual_conversation[0].final_response) + passed = expected_text in actual_text + metric = SimpleNamespace( + metric_name="final_response_avg_score", + score=1.0 if passed else 0.0, + threshold=1.0, + eval_status=SimpleNamespace(name="PASSED" if passed else "FAILED"), + details=SimpleNamespace(reason=None if passed else "Final response does not match."), + ) + result = SimpleNamespace( + eval_id=case.eval_id, + final_eval_status=SimpleNamespace(name="PASSED" if passed else "FAILED"), + error_message=None, + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[ + SimpleNamespace( + expected_invocation=case.conversation[0], + actual_invocation=SimpleNamespace( + user_content=case.conversation[0].user_content, + final_response=fake_content(actual_text), + intermediate_data=None, + ), + ) + ], + ) + results[case.eval_id] = [result] + return None, [], [], results + + monkeypatch.setitem( + sys.modules, + "trpc_agent_sdk.evaluation", + SimpleNamespace( + AgentEvaluator=FakeAgentEvaluator, + AgentOptimizer=FakeAgentOptimizer, + EvalConfig=FakeEvalConfig, + EvalSet=FakeEvalSet, + TargetPrompt=FakeTargetPrompt, + get_all_tool_calls=lambda intermediate_data: [], + ), + ) + return captured + + +def _fake_eval_case(case: dict[str, Any]) -> Any: + return SimpleNamespace( + eval_id=case["eval_id"], + conversation=[_fake_invocation(invocation) for invocation in case.get("conversation", [])], + actual_conversation=[_fake_invocation(invocation) for invocation in case.get("actual_conversation", [])], + ) + + +def _fake_invocation(invocation: dict[str, Any]) -> Any: + return SimpleNamespace( + user_content=fake_content(_raw_json_content_text(invocation.get("user_content"))), + final_response=fake_content(_raw_json_content_text(invocation.get("final_response"))), + intermediate_data=None, + ) + + +def fake_content(text: str) -> Any: + return SimpleNamespace(parts=[SimpleNamespace(text=text)]) + + +def content_text(content: Any) -> str: + return "\n".join(part.text or "" for part in content.parts if part.text).strip() + + +def _raw_json_content_text(content: dict[str, Any] | None) -> str: + if not content: + return "" + return "\n".join(str(part.get("text") or "") for part in content.get("parts", []) if part.get("text")).strip() diff --git a/examples/optimization/eval_optimize_loop/tests/test_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_attribution.py new file mode 100644 index 000000000..3fe91d795 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_attribution.py @@ -0,0 +1,121 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Failure attribution tests.""" + +from __future__ import annotations + +from pipeline import FailureAttributor + +from .conftest import case_record +from .conftest import metric_result + + +def test_attributor_classifies_all_supported_failure_categories() -> None: + attributor = FailureAttributor() + cases = [ + ( + case_record( + "final_answer", + passed=False, + metric_results=[ + metric_result( + "final_response_avg_score", + reason="Final response does not match the reference answer.", + ) + ], + ), + "final_answer_mismatch", + ), + ( + case_record( + "tool_call", + passed=False, + metric_results=[metric_result("tool_trajectory_avg_score")], + expected_tool_calls=[{ + "name": "search", + "args": { + "query": "apple" + } + }], + actual_tool_calls=[{ + "name": "lookup", + "args": { + "query": "apple" + } + }], + ), + "tool_call_error", + ), + ( + case_record( + "parameter", + passed=False, + metric_results=[metric_result("tool_trajectory_avg_score")], + expected_tool_calls=[{ + "name": "search", + "args": { + "query": "apple" + } + }], + actual_tool_calls=[{ + "name": "search", + "args": { + "query": "banana" + } + }], + ), + "parameter_error", + ), + ( + case_record( + "rubric", + passed=False, + metric_results=[metric_result("llm_rubric_response", reason="LLM rubric score is below threshold.")], + ), + "llm_rubric_failed", + ), + ( + case_record( + "retrieval", + passed=False, + metric_results=[metric_result("llm_rubric_knowledge_recall", reason="Knowledge recall failed.")], + ), + "retrieval_failure", + ), + ( + case_record( + "format", + passed=False, + metric_results=[metric_result("response_format_score", reason="Output format is invalid.")], + ), + "format_error", + ), + ( + case_record( + "unknown", + passed=False, + metric_results=[metric_result("custom_metric", reason=None)], + ), + "unknown", + ), + ] + + for case, expected_category in cases: + analysis = attributor.analyze_case(case) + assert analysis is not None + assert analysis.category == expected_category + assert analysis.explanation + assert analysis.evidence["case_id"] == case.id + + +def test_attributor_returns_none_for_passed_case() -> None: + case = case_record( + "passed", + passed=True, + metric_results=[metric_result("final_response_avg_score", score=1.0, eval_status="PASSED")], + ) + + assert FailureAttributor().analyze_case(case) is None diff --git a/examples/optimization/eval_optimize_loop/tests/test_delta.py b/examples/optimization/eval_optimize_loop/tests/test_delta.py new file mode 100644 index 000000000..51703f0c7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_delta.py @@ -0,0 +1,54 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Delta analysis tests.""" + +from __future__ import annotations + +from pipeline import DeltaAnalyzer +from pipeline.types import BaselineSplitResult + +from .conftest import case_record +from .conftest import metric_result + + +def test_delta_analyzer_covers_regression_improvement_and_case_set_mismatch() -> None: + baseline = BaselineSplitResult( + eval_set_id="synthetic_base", + cases=[ + case_record("new_fail", passed=True, metric_results=[metric_result("m", score=1.0, eval_status="PASSED")]), + case_record("score_up", passed=True, metric_results=[metric_result("m", score=0.5, eval_status="PASSED")]), + case_record("score_down", passed=True, metric_results=[metric_result("m", score=0.9, + eval_status="PASSED")]), + case_record("unchanged", passed=True, metric_results=[metric_result("m", score=1.0, eval_status="PASSED")]), + case_record("missing", passed=True, metric_results=[metric_result("m", score=1.0, eval_status="PASSED")]), + ], + ) + candidate = BaselineSplitResult( + eval_set_id="synthetic_candidate", + cases=[ + case_record("new_fail", passed=False, metric_results=[metric_result("m", score=0.0, eval_status="FAILED")]), + case_record("score_up", passed=True, metric_results=[metric_result("m", score=0.8, eval_status="PASSED")]), + case_record("score_down", passed=True, metric_results=[metric_result("m", score=0.4, + eval_status="PASSED")]), + case_record("unchanged", passed=True, metric_results=[metric_result("m", score=1.0, eval_status="PASSED")]), + case_record("extra", passed=True, metric_results=[metric_result("m", score=1.0, eval_status="PASSED")]), + ], + ) + + split_delta = DeltaAnalyzer().analyze_split(split="train", baseline=baseline, candidate=candidate) + by_id = {case.id: case for case in split_delta.case_deltas} + + assert by_id["new_fail"].change_type == "new_fail" + assert by_id["new_fail"].regression is True + assert by_id["score_up"].change_type == "score_up" + assert by_id["score_up"].improvement is True + assert by_id["score_down"].change_type == "score_down" + assert by_id["score_down"].regression is True + assert by_id["unchanged"].change_type == "unchanged" + assert by_id["missing"].change_type == "missing_candidate" + assert by_id["extra"].change_type == "extra_candidate" + assert split_delta.missing_candidate_case_ids == ["missing"] + assert split_delta.extra_candidate_case_ids == ["extra"] diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py new file mode 100644 index 000000000..20d56dbac --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -0,0 +1,122 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Gate decision tests.""" + +from __future__ import annotations + +from pipeline import GateEvaluator +from pipeline.types import GateConfig + +from .conftest import delta_from_scores +from .conftest import rule + + +def test_gate_accepts_validation_improvement_without_regressions() -> None: + delta = delta_from_scores( + train_pairs=[("train_new_pass", False, 0.0, True, 1.0)], + val_pairs=[("val_new_pass", False, 0.0, True, 1.0)], + ) + + decision = GateEvaluator(GateConfig(min_val_score_gain=0.05)).evaluate(delta=delta) + + assert decision.accepted is True + assert decision.decision == "accept" + assert decision.summary["val_score_delta"] == 1.0 + assert decision.summary["overfit_detected"] is False + + +def test_gate_rejects_when_validation_gain_is_below_threshold() -> None: + delta = delta_from_scores( + train_pairs=[("train_score_up", True, 0.8, True, 1.0)], + val_pairs=[("val_score_up", True, 0.95, True, 1.0)], + ) + + decision = GateEvaluator(GateConfig(min_val_score_gain=0.1)).evaluate(delta=delta) + + assert decision.accepted is False + assert decision.decision == "reject" + assert rule(decision, "validation_score_gain").passed is False + + +def test_gate_rejects_new_validation_failure() -> None: + delta = delta_from_scores( + train_pairs=[("train_ok", True, 1.0, True, 1.0)], + val_pairs=[("val_new_fail", True, 1.0, False, 0.0)], + ) + + decision = GateEvaluator(GateConfig(min_val_score_gain=-1.0, allow_new_failures=False)).evaluate(delta=delta) + + assert decision.accepted is False + assert rule(decision, "new_failures").passed is False + assert decision.summary["new_fail_count"] == 1 + + +def test_gate_rejects_validation_regression_and_metric_pass_to_fail() -> None: + delta = delta_from_scores( + train_pairs=[("train_ok", True, 1.0, True, 1.0)], + val_pairs=[("val_score_down", True, 1.0, True, 0.5)], + ) + + decision = GateEvaluator(GateConfig(min_val_score_gain=-1.0, allow_regressions=False)).evaluate(delta=delta) + + assert decision.accepted is False + assert rule(decision, "validation_regressions").passed is False + assert decision.summary["regression_count"] == 1 + + +def test_gate_rejects_critical_case_regression() -> None: + delta = delta_from_scores( + train_pairs=[("train_ok", True, 1.0, True, 1.0)], + val_pairs=[("critical_case", True, 1.0, True, 0.5)], + ) + + decision = GateEvaluator( + GateConfig( + min_val_score_gain=-1.0, + allow_regressions=True, + critical_case_ids=["critical_case"], + )).evaluate(delta=delta) + + assert decision.accepted is False + assert rule(decision, "critical_case_regressions").passed is False + assert decision.summary["critical_regression_count"] == 1 + + +def test_gate_rejects_train_gain_with_validation_drop_as_overfit() -> None: + delta = delta_from_scores( + train_pairs=[("train_new_pass", False, 0.0, True, 1.0)], + val_pairs=[("val_score_down", True, 1.0, True, 0.5)], + ) + + decision = GateEvaluator( + GateConfig( + min_val_score_gain=-1.0, + allow_regressions=True, + overfit_policy={ + "enabled": True, + "train_gain_min": 0.05, + "val_drop_tolerance": 0.0, + }, + )).evaluate(delta=delta) + + assert decision.accepted is False + assert decision.summary["overfit_detected"] is True + assert rule(decision, "overfit_detection").passed is False + + +def test_gate_rejects_cost_budget_overrun() -> None: + delta = delta_from_scores( + train_pairs=[("train_ok", True, 1.0, True, 1.0)], + val_pairs=[("val_ok", True, 0.8, True, 1.0)], + baseline_cost=1.0, + candidate_cost=2.0, + ) + + decision = GateEvaluator(GateConfig(max_cost_delta=0.1, max_cost_ratio=1.2)).evaluate(delta=delta) + + assert decision.accepted is False + assert rule(decision, "cost_budget").passed is False + assert decision.summary["cost_delta"] > 0.1 diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py new file mode 100644 index 000000000..32b5a886d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py @@ -0,0 +1,93 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""End-to-end report tests.""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from pipeline import EvalOptimizePipeline + + +@pytest.mark.asyncio +async def test_fake_pipeline_writes_business_reports_with_gate_delta_and_audit(example_root: Any) -> None: + output_dir = example_root / "output" + system_before = (example_root / "prompts" / "system.md").read_text(encoding="utf-8") + skill_before = (example_root / "prompts" / "skill.md").read_text(encoding="utf-8") + + report_paths = await EvalOptimizePipeline( + train_evalset_path=example_root / "train.evalset.json", + val_evalset_path=example_root / "val.evalset.json", + optimizer_config_path=example_root / "optimizer.json", + gate_config_path=example_root / "gate.json", + mode="fake", + ).run() + + assert report_paths.json_path == output_dir / "optimization_report.json" + assert report_paths.markdown_path == output_dir / "optimization_report.md" + assert not (output_dir / "baseline_evaluation.json").exists() + assert not (output_dir / "candidate_evaluation.json").exists() + + report = json.loads(report_paths.json_path.read_text(encoding="utf-8")) + markdown = report_paths.markdown_path.read_text(encoding="utf-8") + assert report["schema_version"] == "eval-optimize-loop-v1" + assert list(report)[:6] == ["schema_version", "run", "inputs", "config", "baseline", "failure_attribution"] + assert report["config"]["gate"] == report["gate_decision"]["config"] + assert "failure_attribution" in report + assert _contains_no_absolute_workspace_path(report) + assert report["metadata"]["example_root"] == "." + assert report["metadata"]["reproduction_command"] == "python run_pipeline.py --mode fake" + assert report["metadata"]["output_paths"] == { + "json": "output/optimization_report.json", + "markdown": "output/optimization_report.md", + } + + train = report["baseline"]["train"] + val = report["baseline"]["val"] + assert train["failure_attribution_summary"]["final_answer_mismatch"] == 1 + assert val["failure_attribution_summary"]["final_answer_mismatch"] == 2 + assert report["failure_attribution"]["overall_summary"]["final_answer_mismatch"] == 5 + assert {case["variant"] for case in report["failure_attribution"]["failed_cases"]} == {"baseline", "candidate"} + + candidate = report["candidate"] + assert candidate["train"]["case_count"] == 3 + assert candidate["val"]["case_count"] == 3 + assert candidate["train"]["failed_count"] == 0 + assert candidate["val"]["failed_count"] == 2 + assert set(candidate["prompts"]) == {"system_prompt", "skill"} + + delta = report["delta"] + assert delta["summary"]["overall_change_type"] == "mixed" + assert delta["summary"]["new_pass_count"] == 2 + assert delta["summary"]["new_fail_count"] == 1 + assert delta["summary"]["regression_count"] == 1 + assert delta["train"]["score_delta"] > 0 + assert delta["val"]["score_delta"] == 0.0 + val_changes = [case["change_type"] for case in delta["val"]["case_deltas"]] + assert val_changes.count("new_pass") == 1 + assert val_changes.count("new_fail") == 1 + assert val_changes.count("unchanged") == 1 + + gate = report["gate_decision"] + assert gate["decision"] == "reject" + assert gate["accepted"] is False + assert gate["recommended_action"] == "keep_baseline_prompts" + assert gate["summary"]["critical_regression_count"] == 1 + + assert "# Evaluation Optimization Pipeline 报告" in markdown + assert "## 错误归因汇总" in markdown + assert "## Gate 决策" in markdown + assert "REJECT" in markdown + assert "python run_pipeline.py --mode fake" in markdown + assert (example_root / "prompts" / "system.md").read_text(encoding="utf-8") == system_before + assert (example_root / "prompts" / "skill.md").read_text(encoding="utf-8") == skill_before + + +def _contains_no_absolute_workspace_path(value: Any) -> bool: + return "/home/kazenke/" not in json.dumps(value, ensure_ascii=False) diff --git a/examples/optimization/eval_optimize_loop/tests/test_real_mode.py b/examples/optimization/eval_optimize_loop/tests/test_real_mode.py new file mode 100644 index 000000000..9871418d5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_real_mode.py @@ -0,0 +1,49 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Real-mode integration seam tests.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from pipeline import EvalOptimizePipeline + +from .conftest import install_fake_evaluation_sdk + + +@pytest.mark.asyncio +async def test_real_mode_uses_agent_optimizer_without_updating_source( + example_root: Path, + tmp_path: Path, + monkeypatch: Any, +) -> None: + captured = install_fake_evaluation_sdk(monkeypatch, example_root) + output_dir = tmp_path / "real" + + report_paths = await EvalOptimizePipeline( + train_evalset_path=example_root / "train.evalset.json", + val_evalset_path=example_root / "val.evalset.json", + optimizer_config_path=example_root / "optimizer.json", + gate_config_path=example_root / "gate.json", + output_dir=output_dir, + mode="real", + ).run() + + report = json.loads(report_paths.json_path.read_text(encoding="utf-8")) + assert report["schema_version"] == "eval-optimize-loop-v1" + assert report["candidate"]["val"]["case_count"] == 3 + assert report["delta"]["summary"]["new_pass_count"] == 2 + assert report["gate_decision"]["decision"] == "reject" + assert report["optimization"]["rounds"][0]["reason"] == "stub accepted final_answer_mismatch fix" + assert captured["update_source"] is False + assert sorted(captured["target_prompt"].names()) == ["skill", "system_prompt"] + assert Path(captured["train_dataset_path"]).parent == output_dir / "optimizer_inputs" + assert Path(captured["validation_dataset_path"]).parent == output_dir / "optimizer_inputs" + assert report["metadata"]["reproduction_command"] == "python run_pipeline.py --mode real" diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 000000000..fae50ede6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,97 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "Eval optimize loop train set", + "description": "Trace-mode training cases for evaluation and prompt optimization.", + "eval_cases": [ + { + "eval_id": "train_addition_pass", + "eval_mode": "trace", + "actual_conversation": [ + { + "invocation_id": "train-actual-1", + "user_content": { + "parts": [{"text": "小明有 4 个苹果，又买了 7 个，现在一共有多少个苹果？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：11 个"}], + "role": "model" + } + } + ], + "conversation": [ + { + "invocation_id": "train-expected-1", + "user_content": { + "parts": [{"text": "小明有 4 个苹果，又买了 7 个，现在一共有多少个苹果？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：11 个"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "train_discount_fail", + "eval_mode": "trace", + "actual_conversation": [ + { + "invocation_id": "train-actual-2", + "user_content": { + "parts": [{"text": "一件衣服原价 200 元，打 8 折后多少钱？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：180 元"}], + "role": "model" + } + } + ], + "conversation": [ + { + "invocation_id": "train-expected-2", + "user_content": { + "parts": [{"text": "一件衣服原价 200 元，打 8 折后多少钱？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：160 元"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "train_percent_pass", + "eval_mode": "trace", + "actual_conversation": [ + { + "invocation_id": "train-actual-3", + "user_content": { + "parts": [{"text": "40 名学生中 25% 戴眼镜，戴眼镜的有多少人？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "计算 40 的 25%，答案：10 人"}], + "role": "model" + } + } + ], + "conversation": [ + { + "invocation_id": "train-expected-3", + "user_content": { + "parts": [{"text": "40 名学生中 25% 戴眼镜，戴眼镜的有多少人？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：10 人"}], + "role": "model" + } + } + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 000000000..2f30ebe97 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,97 @@ +{ + "eval_set_id": "eval_optimize_loop_val", + "name": "Eval optimize loop validation set", + "description": "Trace-mode validation cases for candidate regression checks.", + "eval_cases": [ + { + "eval_id": "val_multiply_pass", + "eval_mode": "trace", + "actual_conversation": [ + { + "invocation_id": "val-actual-1", + "user_content": { + "parts": [{"text": "教室里有 5 排座位，每排 8 个，一共有多少个座位？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：40 个"}], + "role": "model" + } + } + ], + "conversation": [ + { + "invocation_id": "val-expected-1", + "user_content": { + "parts": [{"text": "教室里有 5 排座位，每排 8 个，一共有多少个座位？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：40 个"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "val_water_fail", + "eval_mode": "trace", + "actual_conversation": [ + { + "invocation_id": "val-actual-2", + "user_content": { + "parts": [{"text": "1 升水重 1 千克，3.5 升水重多少千克？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：4 千克"}], + "role": "model" + } + } + ], + "conversation": [ + { + "invocation_id": "val-expected-2", + "user_content": { + "parts": [{"text": "1 升水重 1 千克，3.5 升水重多少千克？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：3.5 千克"}], + "role": "model" + } + } + ] + }, + { + "eval_id": "val_percent_fail", + "eval_mode": "trace", + "actual_conversation": [ + { + "invocation_id": "val-actual-3", + "user_content": { + "parts": [{"text": "班里 30 人，其中 60% 是女生，有多少名女生？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：20 人"}], + "role": "model" + } + } + ], + "conversation": [ + { + "invocation_id": "val-expected-3", + "user_content": { + "parts": [{"text": "班里 30 人，其中 60% 是女生，有多少名女生？"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案：18 人"}], + "role": "model" + } + } + ] + } + ] +} From 12dc6ec5b9897f3365b9f8eaa1d7a75b341e71f1 Mon Sep 17 00:00:00 2001 From: AsyncKurisu <1750981157@qq.com> Date: Wed, 29 Jul 2026 17:12:28 +0800 Subject: [PATCH 2/2] Switch the JSON to fake mode --- .../optimization/eval_optimize_loop/README.md | 2 + .../output/optimization_report.json | 427 ++++++++---------- .../output/optimization_report.md | 38 +- .../pipeline/optimization.py | 22 +- .../eval_optimize_loop/pipeline/pipeline.py | 17 +- .../tests/test_pipeline_reports.py | 116 ++++- .../tests/test_real_mode.py | 6 + 7 files changed, 347 insertions(+), 281 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 4d3eb3b90..c363ff258 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -58,6 +58,8 @@ python run_pipeline.py --mode real - `output/optimization_report.json` - `output/optimization_report.md` +仓库中提交的 `output/` 产物固定作为 fake 模式参考报告。如果运行 `--mode real` 覆盖了这两个文件，提交前请重新执行 `python run_pipeline.py --mode fake`。 + ## 输入与输出 输入包括： diff --git a/examples/optimization/eval_optimize_loop/output/optimization_report.json b/examples/optimization/eval_optimize_loop/output/optimization_report.json index 18300bbc3..40be1f6de 100644 --- a/examples/optimization/eval_optimize_loop/output/optimization_report.json +++ b/examples/optimization/eval_optimize_loop/output/optimization_report.json @@ -1,10 +1,10 @@ { "schema_version": "eval-optimize-loop-v1", "run": { - "mode": "real", - "started_at": "2026-07-29T07:46:53.491932+00:00", - "finished_at": "2026-07-29T07:47:25.010475+00:00", - "duration_seconds": 31.518541, + "mode": "fake", + "started_at": "2026-07-29T09:10:49.915072+00:00", + "finished_at": "2026-07-29T09:10:49.915505+00:00", + "duration_seconds": 0.00043, "seed": 42 }, "inputs": { @@ -105,7 +105,7 @@ "actual": "答案：11 个", "tool_calls": [] }, - "latency": 0.0003851220244541764, + "latency": 1.5296973288059235e-07, "cost": 0.0, "failure_analysis": null }, @@ -116,14 +116,14 @@ "final_response_avg_score": 0.0 }, "passed": false, - "failure_reason": "final_response_avg_score failed.", + "failure_reason": "Failed metrics: final_response_avg_score.", "trace": { "user": "一件衣服原价 200 元，打 8 折后多少钱？", "expected": "答案：160 元", "actual": "答案：180 元", "tool_calls": [] }, - "latency": 0.0002015280188061297, + "latency": 1.5401747077703476e-07, "cost": 0.0, "failure_analysis": { "category": "final_answer_mismatch", @@ -137,12 +137,12 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：160 元", "actual": "答案：180 元", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } } }, @@ -160,7 +160,7 @@ "actual": "计算 40 的 25%，答案：10 人", "tool_calls": [] }, - "latency": 0.00017735798610374331, + "latency": 7.497146725654602e-08, "cost": 0.0, "failure_analysis": null } @@ -195,7 +195,7 @@ "actual": "答案：40 个", "tool_calls": [] }, - "latency": 0.00013485999079421163, + "latency": 6.798654794692993e-08, "cost": 0.0, "failure_analysis": null }, @@ -206,14 +206,14 @@ "final_response_avg_score": 0.0 }, "passed": false, - "failure_reason": "final_response_avg_score failed.", + "failure_reason": "Failed metrics: final_response_avg_score.", "trace": { "user": "1 升水重 1 千克，3.5 升水重多少千克？", "expected": "答案：3.5 千克", "actual": "答案：4 千克", "tool_calls": [] }, - "latency": 0.00013199500972405076, + "latency": 6.600748747587204e-08, "cost": 0.0, "failure_analysis": { "category": "final_answer_mismatch", @@ -227,12 +227,12 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：3.5 千克", "actual": "答案：4 千克", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } } }, @@ -243,14 +243,14 @@ "final_response_avg_score": 0.0 }, "passed": false, - "failure_reason": "final_response_avg_score failed.", + "failure_reason": "Failed metrics: final_response_avg_score.", "trace": { "user": "班里 30 人，其中 60% 是女生，有多少名女生？", "expected": "答案：18 人", "actual": "答案：20 人", "tool_calls": [] }, - "latency": 0.0001270869979634881, + "latency": 4.901085048913956e-08, "cost": 0.0, "failure_analysis": { "category": "final_answer_mismatch", @@ -264,12 +264,12 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：18 人", "actual": "答案：20 人", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } } } @@ -288,7 +288,7 @@ "unknown": 0 }, "candidate": { - "final_answer_mismatch": 1, + "final_answer_mismatch": 0, "tool_call_error": 0, "parameter_error": 0, "llm_rubric_failed": 0, @@ -318,7 +318,7 @@ } }, "overall_summary": { - "final_answer_mismatch": 6, + "final_answer_mismatch": 5, "tool_call_error": 0, "parameter_error": 0, "llm_rubric_failed": 0, @@ -342,12 +342,12 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：160 元", "actual": "答案：180 元", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } }, { @@ -365,12 +365,12 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：3.5 千克", "actual": "答案：4 千克", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } }, { @@ -388,58 +388,35 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：18 人", "actual": "答案：20 人", - "failure_reason": "final_response_avg_score failed." - } - }, - { - "variant": "candidate", - "split": "train", - "case_id": "train_discount_fail", - "category": "final_answer_mismatch", - "confidence": 0.9, - "explanation": "Final response did not match the reference answer.", - "evidence": { - "case_id": "train_discount_fail", - "failed_metrics": [ - { - "metric_name": "final_response_avg_score", - "score": 0.0, - "threshold": 1.0, - "eval_status": "FAILED", - "reason": null - } - ], - "expected": "答案：160 元", - "actual": "答案：180 元", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } }, { "variant": "candidate", "split": "val", - "case_id": "val_water_fail", + "case_id": "val_multiply_pass", "category": "final_answer_mismatch", "confidence": 0.9, "explanation": "Final response did not match the reference answer.", "evidence": { - "case_id": "val_water_fail", + "case_id": "val_multiply_pass", "failed_metrics": [ { "metric_name": "final_response_avg_score", "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], - "expected": "答案：3.5 千克", - "actual": "答案：4 千克", - "failure_reason": "final_response_avg_score failed." + "expected": "答案：40 个", + "actual": "答案：42 个", + "failure_reason": "Failed metrics: final_response_avg_score." } }, { @@ -457,12 +434,12 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：18 人", "actual": "答案：20 人", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } } ] @@ -471,10 +448,10 @@ "train": { "eval_set_id": "eval_optimize_loop_train", "case_count": 3, - "passed_count": 2, - "failed_count": 1, + "passed_count": 3, + "failed_count": 0, "failure_attribution_summary": { - "final_answer_mismatch": 1, + "final_answer_mismatch": 0, "tool_call_error": 0, "parameter_error": 0, "llm_rubric_failed": 0, @@ -497,46 +474,27 @@ "actual": "答案：11 个", "tool_calls": [] }, - "latency": 0.0006882919697090983, + "latency": 8.69622454047203e-08, "cost": 0.0, "failure_analysis": null }, { "id": "train_discount_fail", - "metric_score": 0.0, + "metric_score": 1.0, "metric_scores": { - "final_response_avg_score": 0.0 + "final_response_avg_score": 1.0 }, - "passed": false, - "failure_reason": "final_response_avg_score failed.", + "passed": true, + "failure_reason": "", "trace": { "user": "一件衣服原价 200 元，打 8 折后多少钱？", "expected": "答案：160 元", - "actual": "答案：180 元", + "actual": "答案：160 元", "tool_calls": [] }, - "latency": 0.0005271550035104156, + "latency": 6.30388967692852e-08, "cost": 0.0, - "failure_analysis": { - "category": "final_answer_mismatch", - "confidence": 0.9, - "explanation": "Final response did not match the reference answer.", - "evidence": { - "case_id": "train_discount_fail", - "failed_metrics": [ - { - "metric_name": "final_response_avg_score", - "score": 0.0, - "threshold": 1.0, - "eval_status": "FAILED", - "reason": null - } - ], - "expected": "答案：160 元", - "actual": "答案：180 元", - "failure_reason": "final_response_avg_score failed." - } - } + "failure_analysis": null }, { "id": "train_percent_pass", @@ -549,10 +507,10 @@ "trace": { "user": "40 名学生中 25% 戴眼镜，戴眼镜的有多少人？", "expected": "答案：10 人", - "actual": "计算 40 的 25%，答案：10 人", + "actual": "答案：10 人", "tool_calls": [] }, - "latency": 0.0004119730438105762, + "latency": 6.199115887284279e-08, "cost": 0.0, "failure_analysis": null } @@ -575,59 +533,59 @@ "cases": [ { "id": "val_multiply_pass", - "metric_score": 1.0, - "metric_scores": { - "final_response_avg_score": 1.0 - }, - "passed": true, - "failure_reason": "", - "trace": { - "user": "教室里有 5 排座位，每排 8 个，一共有多少个座位？", - "expected": "答案：40 个", - "actual": "答案：40 个", - "tool_calls": [] - }, - "latency": 0.0001963640097528696, - "cost": 0.0, - "failure_analysis": null - }, - { - "id": "val_water_fail", "metric_score": 0.0, "metric_scores": { "final_response_avg_score": 0.0 }, "passed": false, - "failure_reason": "final_response_avg_score failed.", + "failure_reason": "Failed metrics: final_response_avg_score.", "trace": { - "user": "1 升水重 1 千克，3.5 升水重多少千克？", - "expected": "答案：3.5 千克", - "actual": "答案：4 千克", + "user": "教室里有 5 排座位，每排 8 个，一共有多少个座位？", + "expected": "答案：40 个", + "actual": "答案：42 个", "tool_calls": [] }, - "latency": 0.0001838289899751544, + "latency": 5.00003807246685e-08, "cost": 0.0, "failure_analysis": { "category": "final_answer_mismatch", "confidence": 0.9, "explanation": "Final response did not match the reference answer.", "evidence": { - "case_id": "val_water_fail", + "case_id": "val_multiply_pass", "failed_metrics": [ { "metric_name": "final_response_avg_score", "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], - "expected": "答案：3.5 千克", - "actual": "答案：4 千克", - "failure_reason": "final_response_avg_score failed." + "expected": "答案：40 个", + "actual": "答案：42 个", + "failure_reason": "Failed metrics: final_response_avg_score." } } }, + { + "id": "val_water_fail", + "metric_score": 1.0, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "failure_reason": "", + "trace": { + "user": "1 升水重 1 千克，3.5 升水重多少千克？", + "expected": "答案：3.5 千克", + "actual": "答案：3.5 千克", + "tool_calls": [] + }, + "latency": 4.0046870708465576e-08, + "cost": 0.0, + "failure_analysis": null + }, { "id": "val_percent_fail", "metric_score": 0.0, @@ -635,14 +593,14 @@ "final_response_avg_score": 0.0 }, "passed": false, - "failure_reason": "final_response_avg_score failed.", + "failure_reason": "Failed metrics: final_response_avg_score.", "trace": { "user": "班里 30 人，其中 60% 是女生，有多少名女生？", "expected": "答案：18 人", "actual": "答案：20 人", "tool_calls": [] }, - "latency": 0.00017802900401875377, + "latency": 3.597233444452286e-08, "cost": 0.0, "failure_analysis": { "category": "final_answer_mismatch", @@ -656,38 +614,38 @@ "score": 0.0, "threshold": 1.0, "eval_status": "FAILED", - "reason": null + "reason": "Final response does not match the reference answer." } ], "expected": "答案：18 人", "actual": "答案：20 人", - "failure_reason": "final_response_avg_score failed." + "failure_reason": "Failed metrics: final_response_avg_score." } } } ] }, "prompts": { - "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", - "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n\nFake optimizer guidance: verify arithmetic before answering, preserve the requested answer format, and address Address failure attribution categories: final_answer_mismatch=3. [fake_optimizer:final_answer_mismatch]\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n\nWhen a baseline failure is attributed to final_answer_mismatch, recompute the numeric result and keep the expected unit in the final answer.\n" } }, "delta": { "summary": { - "overall_change_type": "unchanged", + "overall_change_type": "mixed", "train": { "split": "train", "baseline_score": 0.666667, - "candidate_score": 0.666667, - "score_delta": 0.0, + "candidate_score": 1.0, + "score_delta": 0.333333, "baseline_pass_rate": 0.666667, - "candidate_pass_rate": 0.666667, - "pass_rate_delta": 0.0, + "candidate_pass_rate": 1.0, + "pass_rate_delta": 0.333333, "case_count": 3, - "new_pass_count": 0, + "new_pass_count": 1, "new_fail_count": 0, "regression_count": 0, - "improvement_count": 0, + "improvement_count": 1, "missing_candidate_case_ids": [], "extra_candidate_case_ids": [] }, @@ -700,28 +658,28 @@ "candidate_pass_rate": 0.333333, "pass_rate_delta": 0.0, "case_count": 3, - "new_pass_count": 0, - "new_fail_count": 0, - "regression_count": 0, - "improvement_count": 0, + "new_pass_count": 1, + "new_fail_count": 1, + "regression_count": 1, + "improvement_count": 1, "missing_candidate_case_ids": [], "extra_candidate_case_ids": [] }, - "regression_count": 0, - "improvement_count": 0, - "new_pass_count": 0, - "new_fail_count": 0, + "regression_count": 1, + "improvement_count": 2, + "new_pass_count": 2, + "new_fail_count": 1, "missing_candidate_case_ids": [], "extra_candidate_case_ids": [] }, "train": { "split": "train", "baseline_score": 0.666667, - "candidate_score": 0.666667, - "score_delta": 0.0, + "candidate_score": 1.0, + "score_delta": 0.333333, "baseline_pass_rate": 0.666667, - "candidate_pass_rate": 0.666667, - "pass_rate_delta": 0.0, + "candidate_pass_rate": 1.0, + "pass_rate_delta": 0.333333, "case_deltas": [ { "id": "train_addition_pass", @@ -731,14 +689,14 @@ "passed": true, "metric_score": 1.0, "failure_category": null, - "latency": 0.0003851220244541764, + "latency": 1.5296973288059235e-07, "cost": 0.0 }, "candidate": { "passed": true, "metric_score": 1.0, "failure_category": null, - "latency": 0.0006882919697090983, + "latency": 8.69622454047203e-08, "cost": 0.0 }, "score_delta": 0.0, @@ -753,7 +711,7 @@ "status_transition": "passed_to_passed" } ], - "latency_delta": 0.000303, + "latency_delta": -0.0, "cost_delta": 0.0, "regression": false, "improvement": false, @@ -762,37 +720,37 @@ { "id": "train_discount_fail", "split": "train", - "change_type": "unchanged", + "change_type": "new_pass", "baseline": { "passed": false, "metric_score": 0.0, "failure_category": "final_answer_mismatch", - "latency": 0.0002015280188061297, + "latency": 1.5401747077703476e-07, "cost": 0.0 }, "candidate": { - "passed": false, - "metric_score": 0.0, - "failure_category": "final_answer_mismatch", - "latency": 0.0005271550035104156, + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 6.30388967692852e-08, "cost": 0.0 }, - "score_delta": 0.0, + "score_delta": 1.0, "metric_deltas": [ { "metric_name": "final_response_avg_score", "baseline_score": 0.0, - "candidate_score": 0.0, - "score_delta": 0.0, + "candidate_score": 1.0, + "score_delta": 1.0, "baseline_passed": false, - "candidate_passed": false, - "status_transition": "failed_to_failed" + "candidate_passed": true, + "status_transition": "failed_to_passed" } ], - "latency_delta": 0.000326, + "latency_delta": -0.0, "cost_delta": 0.0, "regression": false, - "improvement": false, + "improvement": true, "notes": [] }, { @@ -803,14 +761,14 @@ "passed": true, "metric_score": 1.0, "failure_category": null, - "latency": 0.00017735798610374331, + "latency": 7.497146725654602e-08, "cost": 0.0 }, "candidate": { "passed": true, "metric_score": 1.0, "failure_category": null, - "latency": 0.0004119730438105762, + "latency": 6.199115887284279e-08, "cost": 0.0 }, "score_delta": 0.0, @@ -825,7 +783,7 @@ "status_transition": "passed_to_passed" } ], - "latency_delta": 0.000235, + "latency_delta": -0.0, "cost_delta": 0.0, "regression": false, "improvement": false, @@ -847,36 +805,36 @@ { "id": "val_multiply_pass", "split": "val", - "change_type": "unchanged", + "change_type": "new_fail", "baseline": { "passed": true, "metric_score": 1.0, "failure_category": null, - "latency": 0.00013485999079421163, + "latency": 6.798654794692993e-08, "cost": 0.0 }, "candidate": { - "passed": true, - "metric_score": 1.0, - "failure_category": null, - "latency": 0.0001963640097528696, + "passed": false, + "metric_score": 0.0, + "failure_category": "final_answer_mismatch", + "latency": 5.00003807246685e-08, "cost": 0.0 }, - "score_delta": 0.0, + "score_delta": -1.0, "metric_deltas": [ { "metric_name": "final_response_avg_score", "baseline_score": 1.0, - "candidate_score": 1.0, - "score_delta": 0.0, + "candidate_score": 0.0, + "score_delta": -1.0, "baseline_passed": true, - "candidate_passed": true, - "status_transition": "passed_to_passed" + "candidate_passed": false, + "status_transition": "passed_to_failed" } ], - "latency_delta": 6.2e-05, + "latency_delta": -0.0, "cost_delta": 0.0, - "regression": false, + "regression": true, "improvement": false, "notes": [] }, @@ -888,14 +846,14 @@ "passed": false, "metric_score": 0.0, "failure_category": "final_answer_mismatch", - "latency": 0.0001270869979634881, + "latency": 4.901085048913956e-08, "cost": 0.0 }, "candidate": { "passed": false, "metric_score": 0.0, "failure_category": "final_answer_mismatch", - "latency": 0.00017802900401875377, + "latency": 3.597233444452286e-08, "cost": 0.0 }, "score_delta": 0.0, @@ -910,7 +868,7 @@ "status_transition": "failed_to_failed" } ], - "latency_delta": 5.1e-05, + "latency_delta": -0.0, "cost_delta": 0.0, "regression": false, "improvement": false, @@ -919,37 +877,37 @@ { "id": "val_water_fail", "split": "val", - "change_type": "unchanged", + "change_type": "new_pass", "baseline": { "passed": false, "metric_score": 0.0, "failure_category": "final_answer_mismatch", - "latency": 0.00013199500972405076, + "latency": 6.600748747587204e-08, "cost": 0.0 }, "candidate": { - "passed": false, - "metric_score": 0.0, - "failure_category": "final_answer_mismatch", - "latency": 0.0001838289899751544, + "passed": true, + "metric_score": 1.0, + "failure_category": null, + "latency": 4.0046870708465576e-08, "cost": 0.0 }, - "score_delta": 0.0, + "score_delta": 1.0, "metric_deltas": [ { "metric_name": "final_response_avg_score", "baseline_score": 0.0, - "candidate_score": 0.0, - "score_delta": 0.0, + "candidate_score": 1.0, + "score_delta": 1.0, "baseline_passed": false, - "candidate_passed": false, - "status_transition": "failed_to_failed" + "candidate_passed": true, + "status_transition": "failed_to_passed" } ], - "latency_delta": 5.2e-05, + "latency_delta": -0.0, "cost_delta": 0.0, "regression": false, - "improvement": false, + "improvement": true, "notes": [] } ], @@ -960,7 +918,7 @@ "gate_decision": { "decision": "reject", "accepted": false, - "reason": "Validation score gain is below the minimum threshold.", + "reason": "Validation score gain is below the minimum threshold. Candidate introduced new validation failures. Candidate introduced validation regressions. One or more critical validation cases regressed.", "rule_results": [ { "rule_name": "validation_score_gain", @@ -988,34 +946,40 @@ }, { "rule_name": "new_failures", - "passed": true, + "passed": false, "severity": "required", - "message": "No new validation failures were introduced.", + "message": "Candidate introduced new validation failures.", "evidence": { "allow_new_failures": false, - "new_fail_case_ids": [] + "new_fail_case_ids": [ + "val_multiply_pass" + ] } }, { "rule_name": "validation_regressions", - "passed": true, + "passed": false, "severity": "required", - "message": "No validation regressions were detected.", + "message": "Candidate introduced validation regressions.", "evidence": { "allow_regressions": false, - "regression_case_ids": [] + "regression_case_ids": [ + "val_multiply_pass" + ] } }, { "rule_name": "critical_case_regressions", - "passed": true, + "passed": false, "severity": "required", - "message": "No critical validation cases regressed.", + "message": "One or more critical validation cases regressed.", "evidence": { "critical_case_ids": [ "val_multiply_pass" ], - "critical_regression_case_ids": [] + "critical_regression_case_ids": [ + "val_multiply_pass" + ] } }, { @@ -1039,7 +1003,7 @@ "message": "No train-only overfit pattern was detected.", "evidence": { "enabled": true, - "train_score_delta": 0.0, + "train_score_delta": 0.333333, "val_score_delta": 0.0, "train_gain_min": 0.05, "val_drop_tolerance": 0.0 @@ -1051,10 +1015,10 @@ "candidate_val_score": 0.333333, "val_score_delta": 0.0, "val_pass_rate_delta": 0.0, - "train_score_delta": 0.0, - "new_fail_count": 0, - "regression_count": 0, - "critical_regression_count": 0, + "train_score_delta": 0.333333, + "new_fail_count": 1, + "regression_count": 1, + "critical_regression_count": 1, "overfit_detected": false, "baseline_cost": 0.0, "candidate_cost": 0.0, @@ -1080,7 +1044,7 @@ } }, "recommended_action": "keep_baseline_prompts", - "generated_at": "2026-07-29T07:47:25.010467+00:00" + "generated_at": "2026-07-29T09:10:49.915499+00:00" }, "optimization": { "target_prompt_names": [ @@ -1092,32 +1056,14 @@ "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" }, "best_prompts": { - "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", - "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n\nFake optimizer guidance: verify arithmetic before answering, preserve the requested answer format, and address Address failure attribution categories: final_answer_mismatch=3. [fake_optimizer:final_answer_mismatch]\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n\nWhen a baseline failure is attributed to final_answer_mismatch, recompute the numeric result and keep the expected unit in the final answer.\n" }, "rounds": [ { "round": 1, "optimized_field_names": [ - "system_prompt" - ], - "before": { - "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", - "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" - }, - "after": { - "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", - "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" - }, - "reason": "no candidate produced this round", - "accepted": false, - "validation_pass_rate": 0.0, - "duration_seconds": 19.865978, - "cost": 0.0 - }, - { - "round": 2, - "optimized_field_names": [ + "system_prompt", "skill" ], "before": { @@ -1125,32 +1071,29 @@ "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" }, "after": { - "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n", - "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n" + "system_prompt": "You are a concise math assistant. Return the final answer in the form \"答案： \".\n\nFake optimizer guidance: verify arithmetic before answering, preserve the requested answer format, and address Address failure attribution categories: final_answer_mismatch=3. [fake_optimizer:final_answer_mismatch]\n", + "skill": "For word problems, compute the numeric result first, then preserve the expected unit.\n\nWhen a baseline failure is attributed to final_answer_mismatch, recompute the numeric result and keep the expected unit in the final answer.\n" }, - "reason": "no candidate produced this round", - "accepted": false, + "reason": "Address failure attribution categories: final_answer_mismatch=3", + "accepted": true, "validation_pass_rate": 0.0, - "duration_seconds": 10.44344, + "duration_seconds": 0.0, "cost": 0.0 } ], - "total_rounds": 2, + "total_rounds": 1, "total_cost": 0.0, - "duration_seconds": 30.389949, + "duration_seconds": 4.211196210235357e-05, "seed": 42, - "reason": "Optimizer finished with status=SUCCEEDED, finish_reason=no_improvement, improvement=0.0.", + "reason": "Address failure attribution categories: final_answer_mismatch=3", "artifacts": { - "mode": "real", - "source_prompts_updated": false, - "optimizer_artifact_dir": "output/optimizer_artifacts", - "train_call_agent_evalset_path": "output/optimizer_inputs/train.call_agent.evalset.json", - "val_call_agent_evalset_path": "output/optimizer_inputs/val.call_agent.evalset.json" + "mode": "fake", + "source_prompts_updated": false } }, "metadata": { "example_root": ".", - "reproduction_command": "python run_pipeline.py --mode real", + "reproduction_command": "python run_pipeline.py --mode fake", "output_dir": "output", "output_paths": { "json": "output/optimization_report.json", diff --git a/examples/optimization/eval_optimize_loop/output/optimization_report.md b/examples/optimization/eval_optimize_loop/output/optimization_report.md index e7f89d6c5..4dd30099f 100644 --- a/examples/optimization/eval_optimize_loop/output/optimization_report.md +++ b/examples/optimization/eval_optimize_loop/output/optimization_report.md @@ -4,11 +4,11 @@ - 决策：**REJECT** - 推荐动作：`keep_baseline_prompts` -- 模式：`real` +- 模式：`fake` - Schema：`eval-optimize-loop-v1` - 验证集分数变化：0.0000 - 验证集通过率变化：0.0000 -- 主要原因：Validation score gain is below the minimum threshold. +- 主要原因：Validation score gain is below the minimum threshold. Candidate introduced new validation failures. Candidate introduced validation regressions. One or more critical validation cases regressed. ## Baseline 表现 @@ -33,7 +33,7 @@ | category | count | | --- | ---: | -| final_answer_mismatch | 1 | +| final_answer_mismatch | 0 | | tool_call_error | 0 | | parameter_error | 0 | | llm_rubric_failed | 0 | @@ -68,25 +68,19 @@ ## 优化过程 - 目标 Prompt：system_prompt, skill -- 优化轮数：2 +- 优化轮数：1 - 优化成本：0.0 - 随机种子：42 ### Round 1 -- 修改字段：system_prompt -- 是否接受为候选：False -- 原因：no candidate produced this round - -### Round 2 - -- 修改字段：skill -- 是否接受为候选：False -- 原因：no candidate produced this round +- 修改字段：system_prompt, skill +- 是否接受为候选：True +- 原因：Address failure attribution categories: final_answer_mismatch=3 ## 候选验证 -- 训练集候选：2/3 通过，通过率 0.6667 +- 训练集候选：3/3 通过，通过率 1.0000 - 验证集候选：1/3 通过，通过率 0.3333 ## 逐 Case Delta @@ -94,17 +88,17 @@ | split | case id | baseline | candidate | change | failure transition | regression | improvement | | --- | --- | ---: | ---: | --- | --- | --- | --- | | train | train_addition_pass | 1.0000 | 1.0000 | unchanged | None -> None | False | False | -| train | train_discount_fail | 0.0000 | 0.0000 | unchanged | final_answer_mismatch -> final_answer_mismatch | False | False | +| train | train_discount_fail | 0.0000 | 1.0000 | new_pass | final_answer_mismatch -> None | False | True | | train | train_percent_pass | 1.0000 | 1.0000 | unchanged | None -> None | False | False | -| val | val_multiply_pass | 1.0000 | 1.0000 | unchanged | None -> None | False | False | +| val | val_multiply_pass | 1.0000 | 0.0000 | new_fail | None -> final_answer_mismatch | True | False | | val | val_percent_fail | 0.0000 | 0.0000 | unchanged | final_answer_mismatch -> final_answer_mismatch | False | False | -| val | val_water_fail | 0.0000 | 0.0000 | unchanged | final_answer_mismatch -> final_answer_mismatch | False | False | +| val | val_water_fail | 0.0000 | 1.0000 | new_pass | final_answer_mismatch -> None | False | True | ## Gate 决策 - 最终决策：**REJECT** - 推荐动作：`keep_baseline_prompts` -- 拒绝/接受理由：Validation score gain is below the minimum threshold. +- 拒绝/接受理由：Validation score gain is below the minimum threshold. Candidate introduced new validation failures. Candidate introduced validation regressions. One or more critical validation cases regressed. - 最小验证集分数提升：0.05 - 允许新增失败：False - 允许验证集回归：False @@ -114,16 +108,16 @@ | --- | --- | --- | --- | | validation_score_gain | False | required | Validation score gain is below the minimum threshold. | | validation_pass_rate_gain | True | required | Validation pass-rate gain satisfies the minimum threshold. | -| new_failures | True | required | No new validation failures were introduced. | -| validation_regressions | True | required | No validation regressions were detected. | -| critical_case_regressions | True | required | No critical validation cases regressed. | +| new_failures | False | required | Candidate introduced new validation failures. | +| validation_regressions | False | required | Candidate introduced validation regressions. | +| critical_case_regressions | False | required | One or more critical validation cases regressed. | | cost_budget | True | required | Candidate cost is within the configured budget. | | overfit_detection | True | required | No train-only overfit pattern was detected. | ## 元数据与复现 - 示例根目录：`.` -- 复现命令：`python run_pipeline.py --mode real` +- 复现命令：`python run_pipeline.py --mode fake` - 输出 JSON：`output/optimization_report.json` - 输出 Markdown：`output/optimization_report.md` diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimization.py b/examples/optimization/eval_optimize_loop/pipeline/optimization.py index 06a9dc556..e9baf5f43 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/optimization.py +++ b/examples/optimization/eval_optimize_loop/pipeline/optimization.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import os import time from pathlib import Path from typing import Any @@ -75,9 +76,15 @@ async def optimize( optimizer_payload: dict[str, Any], train_baseline: BaselineSplitResult, val_baseline: BaselineSplitResult, + call_agent_train_path: Path | None = None, + call_agent_val_path: Path | None = None, ) -> OptimizationRunRecord: if mode == "real": - return await self._run_real_optimizer(optimizer_payload) + return await self._run_real_optimizer( + optimizer_payload, + train_path=call_agent_train_path, + val_path=call_agent_val_path, + ) return self._run_fake_optimizer( optimizer_payload=optimizer_payload, train_baseline=train_baseline, @@ -140,13 +147,20 @@ def _run_fake_optimizer( }, ) - async def _run_real_optimizer(self, optimizer_payload: dict[str, Any]) -> OptimizationRunRecord: + async def _run_real_optimizer( + self, + optimizer_payload: dict[str, Any], + *, + train_path: Path | None, + val_path: Path | None, + ) -> OptimizationRunRecord: from trpc_agent_sdk.evaluation import AgentOptimizer from trpc_agent_sdk.evaluation import TargetPrompt started = time.perf_counter() baseline_prompts = self.read_baseline_prompts() - train_path, val_path = self.write_call_agent_evalsets() + if train_path is None or val_path is None: + raise ValueError("real mode requires call_agent evalset paths") artifact_dir = self.output_dir / "optimizer_artifacts" target_prompt = TargetPrompt() for name, path in self.prompt_paths().items(): @@ -317,4 +331,4 @@ def _relative_to_example(path: Path, example_root: Path) -> str: try: return str(path.resolve().relative_to(example_root.resolve())) except ValueError: - return str(path) + return os.path.relpath(path.resolve(), example_root.resolve()) diff --git a/examples/optimization/eval_optimize_loop/pipeline/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline/pipeline.py index 07b7412bd..34e6a8b4f 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/pipeline.py +++ b/examples/optimization/eval_optimize_loop/pipeline/pipeline.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import os import re import time from datetime import datetime @@ -73,12 +74,15 @@ async def run(self) -> ReportPaths: val_evalset = _load_real_evalset(self.val_evalset_path) train = await self._run_real_split(train_evalset, eval_config) val = await self._run_real_split(val_evalset, eval_config) + call_agent_train_path, call_agent_val_path = prompt_optimizer.write_call_agent_evalsets() else: metrics = _fake_metrics_from_optimizer(optimizer_payload) train_evalset = _load_json(self.train_evalset_path) val_evalset = _load_json(self.val_evalset_path) train = self._run_fake_split(train_evalset, metrics) val = self._run_fake_split(val_evalset, metrics) + call_agent_train_path = None + call_agent_val_path = None attributor = FailureAttributor() attributor.annotate_split(train) @@ -89,17 +93,18 @@ async def run(self) -> ReportPaths: optimizer_payload=optimizer_payload, train_baseline=train, val_baseline=val, + call_agent_train_path=call_agent_train_path, + call_agent_val_path=call_agent_val_path, ) if self.mode == "real": - train_path, val_path = prompt_optimizer.write_call_agent_evalsets() candidate_train = await self._run_call_agent_split( - _load_real_evalset(train_path), + _load_real_evalset(call_agent_train_path), eval_config, prompt_optimizer.call_agent_from_prompts(optimization.best_prompts), ) candidate_val = await self._run_call_agent_split( - _load_real_evalset(val_path), + _load_real_evalset(call_agent_val_path), eval_config, prompt_optimizer.call_agent_from_prompts(optimization.best_prompts), ) @@ -262,7 +267,7 @@ def _relative_to_example(path: Path, example_root: Path) -> str: try: return str(path.resolve().relative_to(example_root.resolve())) except ValueError: - return str(path) + return os.path.relpath(path.resolve(), example_root.resolve()) def _failure_attribution_payload( @@ -415,7 +420,7 @@ def _trace_from_result(result: Any) -> dict[str, Any]: def _trace_from_case(case: Any) -> dict[str, Any]: expected = case.conversation[0] if case.conversation else None - actual = case.actual_conversation[0] if case.actual_conversation else expected + actual = case.actual_conversation[0] if case.actual_conversation else None return _trace_from_invocations(expected=expected, actual=actual) @@ -538,7 +543,7 @@ def _fake_record( if candidate_prompts: actual = _candidate_invocation(case, expected, candidate_prompts) else: - actual = actual_conversation[0] if actual_conversation else expected + actual = actual_conversation[0] if actual_conversation else None metric_scores: dict[str, float] = {} failed_metrics: list[str] = [] metric_results: list[dict[str, Any]] = [] diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py index 32b5a886d..194f0d446 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline_reports.py @@ -8,6 +8,8 @@ from __future__ import annotations import json +import os +from pathlib import Path from typing import Any import pytest @@ -16,8 +18,11 @@ @pytest.mark.asyncio -async def test_fake_pipeline_writes_business_reports_with_gate_delta_and_audit(example_root: Any) -> None: - output_dir = example_root / "output" +async def test_fake_pipeline_writes_business_reports_with_gate_delta_and_metadata( + example_root: Path, + tmp_path: Path, +) -> None: + output_dir = tmp_path / "output" system_before = (example_root / "prompts" / "system.md").read_text(encoding="utf-8") skill_before = (example_root / "prompts" / "skill.md").read_text(encoding="utf-8") @@ -26,6 +31,7 @@ async def test_fake_pipeline_writes_business_reports_with_gate_delta_and_audit(e val_evalset_path=example_root / "val.evalset.json", optimizer_config_path=example_root / "optimizer.json", gate_config_path=example_root / "gate.json", + output_dir=output_dir, mode="fake", ).run() @@ -40,12 +46,12 @@ async def test_fake_pipeline_writes_business_reports_with_gate_delta_and_audit(e assert list(report)[:6] == ["schema_version", "run", "inputs", "config", "baseline", "failure_attribution"] assert report["config"]["gate"] == report["gate_decision"]["config"] assert "failure_attribution" in report - assert _contains_no_absolute_workspace_path(report) + assert _path_like_values_are_relative(report) assert report["metadata"]["example_root"] == "." assert report["metadata"]["reproduction_command"] == "python run_pipeline.py --mode fake" assert report["metadata"]["output_paths"] == { - "json": "output/optimization_report.json", - "markdown": "output/optimization_report.md", + "json": _relative_to_example(output_dir / "optimization_report.json", example_root), + "markdown": _relative_to_example(output_dir / "optimization_report.md", example_root), } train = report["baseline"]["train"] @@ -89,5 +95,101 @@ async def test_fake_pipeline_writes_business_reports_with_gate_delta_and_audit(e assert (example_root / "prompts" / "skill.md").read_text(encoding="utf-8") == skill_before -def _contains_no_absolute_workspace_path(value: Any) -> bool: - return "/home/kazenke/" not in json.dumps(value, ensure_ascii=False) +def test_committed_output_report_is_fake_canonical(example_root: Path) -> None: + report_path = example_root / "output" / "optimization_report.json" + markdown_path = example_root / "output" / "optimization_report.md" + + report = json.loads(report_path.read_text(encoding="utf-8")) + markdown = markdown_path.read_text(encoding="utf-8") + + assert report["run"]["mode"] == "fake" + assert "audit" not in report + assert report["failure_attribution"]["overall_summary"]["final_answer_mismatch"] == 5 + assert report["candidate"]["train"]["failed_count"] == 0 + assert report["delta"]["summary"]["overall_change_type"] == "mixed" + assert report["delta"]["summary"]["new_pass_count"] == 2 + assert report["delta"]["summary"]["new_fail_count"] == 1 + assert report["gate_decision"]["summary"]["critical_regression_count"] == 1 + assert _path_like_values_are_relative(report) + assert "python run_pipeline.py --mode fake" in markdown + + +def test_fake_split_records_empty_actual_when_trace_actual_is_missing(example_root: Path, tmp_path: Path) -> None: + pipeline = EvalOptimizePipeline( + train_evalset_path=example_root / "train.evalset.json", + val_evalset_path=example_root / "val.evalset.json", + optimizer_config_path=example_root / "optimizer.json", + gate_config_path=example_root / "gate.json", + output_dir=tmp_path / "output", + mode="fake", + ) + eval_set = { + "eval_set_id": + "missing_actual", + "eval_cases": [{ + "eval_id": + "case_without_actual", + "conversation": [{ + "user_content": { + "parts": [{ + "text": "question" + }] + }, + "final_response": { + "parts": [{ + "text": "expected answer" + }] + }, + }], + }], + } + metrics = [{ + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains" + } + } + }, + }] + + result = pipeline._run_fake_split(eval_set, metrics) + case = result.cases[0] + + assert case.passed is False + assert case.trace["expected"] == "expected answer" + assert case.trace["actual"] == "" + assert case.evaluator_metadata["actual_final_response"] == "" + + +def _path_like_values_are_relative(value: Any) -> bool: + return not list(_absolute_path_like_values(value)) + + +def _absolute_path_like_values(value: Any, key: str = "") -> list[str]: + if isinstance(value, dict): + absolute_paths = [] + for child_key, child_value in value.items(): + absolute_paths.extend(_absolute_path_like_values(child_value, str(child_key))) + return absolute_paths + if isinstance(value, list): + absolute_paths = [] + for item in value: + absolute_paths.extend(_absolute_path_like_values(item, key)) + return absolute_paths + if isinstance(value, str) and _is_path_like_key(key) and Path(value).is_absolute(): + return [value] + return [] + + +def _is_path_like_key(key: str) -> bool: + return key in {"example_root", "output_dir", "optimizer_artifact_dir"} or key.endswith(("_path", "_paths")) + + +def _relative_to_example(path: Path, example_root: Path) -> str: + try: + return str(path.resolve().relative_to(example_root.resolve())) + except ValueError: + return os.path.relpath(path.resolve(), example_root.resolve()) diff --git a/examples/optimization/eval_optimize_loop/tests/test_real_mode.py b/examples/optimization/eval_optimize_loop/tests/test_real_mode.py index 9871418d5..f47c03099 100644 --- a/examples/optimization/eval_optimize_loop/tests/test_real_mode.py +++ b/examples/optimization/eval_optimize_loop/tests/test_real_mode.py @@ -46,4 +46,10 @@ async def test_real_mode_uses_agent_optimizer_without_updating_source( assert sorted(captured["target_prompt"].names()) == ["skill", "system_prompt"] assert Path(captured["train_dataset_path"]).parent == output_dir / "optimizer_inputs" assert Path(captured["validation_dataset_path"]).parent == output_dir / "optimizer_inputs" + train_artifact = report["optimization"]["artifacts"]["train_call_agent_evalset_path"] + val_artifact = report["optimization"]["artifacts"]["val_call_agent_evalset_path"] + assert not Path(train_artifact).is_absolute() + assert not Path(val_artifact).is_absolute() + assert train_artifact.endswith("optimizer_inputs/train.call_agent.evalset.json") + assert val_artifact.endswith("optimizer_inputs/val.call_agent.evalset.json") assert report["metadata"]["reproduction_command"] == "python run_pipeline.py --mode real"