Files
AgentLab/backend/tests/test_eval_stats.py
T
ThearchyHelios e29adf80d0 评测执行的测试:执行、评分、统计与对比、清理、定时与通知
为什么:照细案 01 的测试清单把这一波的行为钉住,顺带修了跑测试时撞出来的两处。

- test_eval_runner:后台排队、并发不超过 max_parallel 也不超过全局并发的四分之一、同一题集同时只能一次、停止后只重做
  没做完的(评完的不动)、重启标中断不续跑、单次金额上限触顶不再开新题、评测每日账单独记(不占裁判、核查的额度)、
  连续 3 题助手调用失败止损、每题超时取消正在跑的运行、停在审批上的作答取消掉(评测不代批)、接口的估算与进度、设置里
  eval 一组和 /settings/eval/spend
- test_eval_score:五种结论、写出精度和百分数、维度配对与弱匹配、多行 / 名单 / 没有数据 / 反问、路径和期望先于数字、
  dual 两版、允许的相对误差和双盲复算是同一个函数;读封存的答案、逐题 E1 用账本的比对函数;重评只读封存、不调模型
- test_eval_stats:Wilson、精确 McNemar、P50/P90 和手算对照;总表的各项;转移与四种归因;指纹差异;配对表;选基准
- test_eval_cleanup:保留规则;forget 只删一条引用都不剩的文件;清理后总表还在、结果和底下的运行删掉、共用的工件留着
- test_eval_schedule:定时接口收题集(拒工作流、输入模板、Webhook)、到点建一次评测并写 eval_run_id、重叠跳过、错过的按
  宽限补;有退步推「评测有退步」、没评完推「评测失败」,消息里没有期望值和答案里的数
- test_eval_e2e 补:开评拍目录快照、记表结构前后;打开裁判时结论裁判走按需裁判那条路、花费记在评测账上,文字要点单列
- 修:期望反问的题在建图、运行失败(含超时)时把原因写全;改非工作流目标的定时任务时提交后不再读过期对象
2026-10-16 16:49:47 +08:00

163 lines
9.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""评测的统计与对比(app/eval/stats.py、app/eval/compare.py;细案 01 做法 11、12,评测-D10)。纯计算,和手算对照。
- Wilson 95% 区间、精确 McNemar 双侧 p 值、P50 / P90;
- 总表:准确率和可追溯准确率分开报(分母只算能判的)、作答路径、出具档位、复核、look、按角色的用量、引用率、逐题 E1、
重复 k 次的题给通过率;
- 和上一次比:逐题转移(退步 / 进步 / 不变 / 不可比)和归因(数据变、做法变、只参数变、路径变),列出指纹里哪几项变了;
- A/B 配对表:都对 / 都错 / 只 A 对 / 只 B 对,不能判的不进表;
- 选基准:点名的 > 同一变体里钉住的 > 同一变体、比这次先发起、已经结束的最近一次。
"""
from __future__ import annotations
from datetime import timedelta
import pytest
import app.db.models # noqa: F401 - conftest 按已导入的模型建表
from app.eval import compare, stats
def test_wilson_and_mcnemar_match_hand_computation():
assert stats.wilson(8, 10) == (0.4902, 0.9433)
assert stats.wilson(0, 0) is None
low, high = stats.wilson(0, 5)
assert low == 0.0 and 0.43 < high < 0.44
assert stats.mcnemar_exact(1, 0) == 1.0
assert stats.mcnemar_exact(6, 0) == 0.03125
assert stats.mcnemar_exact(7, 2) == 0.179688
assert stats.mcnemar_exact(0, 0) == 1.0
assert stats.percentile([5, 1, 3, 2, 4], 50) == 3 and stats.percentile([5, 1, 3, 2, 4], 90) == 5
assert stats.percentile([], 50) is None
def _row(case: str, verdict: str | None, *, path: str = "ran", status: str = "ok", repeat: int = 0, **extra) -> dict:
scores = [{"id": "e1", "kind": "scalar", "required": True,
"outcome": {"pass": "traced", "fail": "wrong"}.get(verdict or "", "ungradable")}]
return {"case_id": case, "case_seq": int(case[-1]) if case[-1].isdigit() else 1, "case_hash": f"h-{case}",
"verdict": verdict, "path": path, "status": status, "repeat": repeat, "scores": scores,
"traced": verdict == "pass", **extra}
def test_summary_counts_only_gradable_items_and_splits_roles():
usage = {"assistant": {"by_model": {"m": {"calls": 3, "cost_usd": 0.03}}, "calls": 3, "cost_usd": 0.03,
"input_tokens": 10, "output_tokens": 5, "unpriced_calls": 0},
"writer": {"by_model": {}, "calls": 1, "cost_usd": 0.01, "input_tokens": 1, "output_tokens": 1,
"unpriced_calls": 1},
"judge": {"cost_usd": 0.0, "calls": 0}}
rows = [
_row("c1", "pass", tier="formal", usage=usage, looks={"model": 3, "system": 0, "limit_hit": True},
timings={"build_ms": 1000, "total_ms": 4000, "review_ms": 30_000}, review={"verdict": "annotated",
"severity": "degraded"},
answer_stats={"numbers": 2, "numbers_cited": 1, "uncited_numbers": 1}, attempts=1,
sql_match={"status": "proven", "aspects": []}),
_row("c2", "fail", usage=usage, timings={"build_ms": 3000, "total_ms": 6000},
sql_match={"status": "related", "aspects": ["time"]}),
_row("c3", "ungradable"),
_row("c4", None, path=None, status="skipped"),
]
rows[0]["scores"][0]["outcome"] = "traced"
summary = stats.summarize(rows)
assert summary["accuracy"] == {"passed": 1, "n": 2, "rate": 0.5, "ci": stats.wilson(1, 2)}
assert summary["traced_accuracy"]["passed"] == 1
assert summary["verdicts"] == {"pass": 1, "fail": 1, "ungradable": 1, "skipped": 0}
assert summary["paths"] == {"ran": 3} and summary["tiers"]["formal"] == 1 and summary["tiers"]["explore"] == 2
assert summary["review"]["retries"] == 1 and summary["review"]["slow"] == 1
assert summary["review"]["severity"] == {"degraded": 1}
assert summary["looks"]["model"] == 3 and summary["looks"]["limit_hit"] == round(1 / 3, 4)
assert summary["timings"]["build_ms"] == {"p50": 1000, "p90": 3000, "n": 2}
assert summary["usage"]["assistant"]["calls"] == 6 and summary["usage"]["writer"]["unpriced_calls"] == 2
assert summary["cost_usd"] == 0.08 and summary["unpriced_calls"] == 2
assert summary["usage"]["gold"] == {"cost_usd": 0.0, "calls": 0}, "期望探针不调模型"
assert summary["citations"]["rate"] == 0.5
assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1}
def test_repeats_give_a_pass_rate_per_case():
rows = [_row("c1", "pass", repeat=0), _row("c1", "fail", repeat=1), _row("c1", "pass", repeat=2)]
summary = stats.summarize(rows)
assert summary["pass_rates"]["c1"]["passed"] == 2 and summary["pass_rates"]["c1"]["n"] == 3
assert compare.case_outcome(rows)["verdict"] == "pass", "多数决"
def test_transitions_and_their_causes():
sql_a = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-09-01'"]}]
sql_b = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-08-01'"]}]
sql_c = [{"text": "SELECT SUM(x) FROM t WHERE e = ?", "literals": ["'2026-09-01'"]}]
base = [_row("c1", "pass", gold_key="g1", sql_shapes=sql_a), _row("c2", "pass", gold_key="g2", sql_shapes=sql_a),
_row("c3", "fail", gold_key="g3", sql_shapes=sql_a), _row("c4", "pass", gold_key="g4", path="ran"),
_row("c5", "pass", gold_key="g5"), _row("c6", "fail", gold_key="g6")]
target = [_row("c1", "fail", gold_key="g1-new", sql_shapes=sql_a), _row("c2", "fail", gold_key="g2", sql_shapes=sql_c),
_row("c3", "pass", gold_key="g3", sql_shapes=sql_b), _row("c4", "fail", gold_key="g4", path="reply"),
{**_row("c5", "fail", gold_key="g5"), "case_hash": "changed"}, _row("c6", "ungradable", gold_key="g6"),
_row("c7", "pass")]
moved = compare.transitions(base, target)
by = {i["case_id"]: i for i in moved["items"]}
assert (by["c1"]["transition"], by["c1"]["cause"]) == ("regressed", "data")
assert (by["c2"]["transition"], by["c2"]["cause"]) == ("regressed", "method")
assert (by["c3"]["transition"], by["c3"]["cause"]) == ("improved", "params")
assert (by["c4"]["transition"], by["c4"]["cause"]) == ("regressed", "path")
assert by["c5"]["transition"] == "incomparable" and "题目改过" in by["c5"]["reason"]
assert by["c6"]["transition"] == "incomparable" and by["c7"]["transition"] == "incomparable"
assert moved["counts"] == {"regressed": 3, "improved": 1, "same": 0, "incomparable": 3}
assert compare.sql_change(sql_a, sql_a) == "same"
def test_fingerprint_diff_lists_what_changed():
a = {"assistant_model": "m1", "writer_model": None, "catalog": "live:aaa", "prompt": "p1", "schemas": {"s": "x"},
"cases": "c", "scorer_version": 1}
b = {**a, "assistant_model": "m2", "schemas": {"s": "y"}}
diff = compare.fingerprint_diff(a, b)
assert [d["key"] for d in diff] == ["assistant_model", "schemas"]
assert diff[0] == {"key": "assistant_model", "label": "助手模型", "before": "m1", "after": "m2"}
assert "before" not in diff[1], "表结构只说变没变"
def test_the_pair_table():
a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")]
b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")]
table = compare.pair_table([("当前目录", a), ("关目录", b)])
[pair] = table["pairs"]
assert (pair["both"], pair["neither"], pair["only_a"], pair["only_b"], pair["excluded"]) == (1, 1, 1, 0, 1)
assert pair["p_value"] == 1.0 and pair["note"] == compare.NO_DIFFERENCE
assert table["arms"][0]["accuracy"]["rate"] == round(2 / 3, 4)
async def test_picking_the_baseline():
from app.db.base import SessionLocal, utcnow
from app.db.models_eval import EvalRun, EvalSuite
now = utcnow()
async with SessionLocal() as session:
suite = EvalSuite(name="虚构题集-基准")
session.add(suite)
await session.flush()
def run(minutes: int, key: str, status: str = "succeeded", **extra) -> EvalRun:
row = EvalRun(suite_id=suite.id, status=status, fingerprint={"variant_key": key},
created_at=now - timedelta(minutes=minutes), **extra)
session.add(row)
return row
old = run(50, "k1")
pinned = run(40, "k1", pinned=True)
latest = run(30, "k1")
run(20, "k2")
run(15, "k1", status="failed")
target = run(10, "k1")
later = run(5, "k1")
await session.flush()
assert (await compare.pick_baseline(session, target)).id == pinned.id, "同一变体里钉住的优先"
pinned.pinned = False
await session.flush()
assert (await compare.pick_baseline(session, target)).id == latest.id, "比这次先发起、已结束的最近一次"
assert (await compare.pick_baseline(session, target, explicit=old.id)).id == old.id
assert (await compare.pick_baseline(session, target, explicit=later.id)).id == later.id
await session.delete(suite)
await session.commit()
@pytest.mark.parametrize("n, k", [(1, 0), (3, 3)])
def test_rates_carry_their_sample_size(n, k):
r = stats.rate(k, n)
assert r["n"] == n and r["passed"] == k and r["ci"] == stats.wilson(k, n)