mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 02:52:00 +00:00
评测总表照实列出没开评的题、按评完的题估每题花费;对比测试单独成文件,补超时止损的测试
为什么:总表里「没开评、停下、出错」的题以前只体现在计数里,准确率旁边看不出来;下一次开跑前的估算按每题平均花费算, 把没开评的题也摊进去会估低。 - stats.summarize 加 not_run;per_case_cost_usd 只按评完了的题平均 - test_eval_compare.py:逐题转移与归因、指纹差异、A/B 配对表、选基准(从 test_eval_stats 拆出来,对上细案的测试清单) - test_eval_runner:最近 10 题里 7 题超时就止损
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
"""评测的汇总与统计(细案 01 做法 11、12;评测-D10)。所有汇总都在后端算好,界面直接显示(界面层不放逻辑:同一个数不能
|
||||
有两套算法)。纯计算,输入是一次评测的逐题结果(EvalResult 的列,result_row 的形状)。
|
||||
|
||||
- 准确率和可追溯准确率:分母是能判的题(通过 + 不通过);不能评分、跳过的单列,既不算对也不算错。都带 Wilson 95% 区间和
|
||||
样本数——题集小、模型有随机性,差两三题就可能被读成「变好了」「变差了」(评测-D10:只展示、不卡门)。
|
||||
- 准确率和可追溯准确率:分母是能判的题(通过 + 不通过);不能评分、跳过的单列,既不算对也不算错,没开评、停下、出错的
|
||||
另列 not_run。都带 Wilson 95% 区间和样本数——题集小、模型有随机性,差两三题就可能被读成「变好了」「变差了」(评测-D10:只展示、不卡门)。
|
||||
- 作答路径、出具档位(声明了出具的按 formal / degraded / withheld,没声明的单列「探索问答」)、复核结论(verdict × severity)
|
||||
和自动重来的次数、复核用时超过问数据页等待(25 秒)的题数。
|
||||
- 结论裁判(打开了才有):supported / partial / contradicted / insufficient / 未判的句数;文字要点的判断单列。
|
||||
@@ -79,6 +79,8 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) -
|
||||
from app.eval.score import SCORER_VERSION
|
||||
|
||||
done = [r for r in results if r.get("status") in ("ok", "skipped", "failed", "cancelled", "partial")]
|
||||
#: 评完了的(status=ok);没开评、停下、出错的单列成 not_run
|
||||
evaluated = [r for r in done if r.get("status") == "ok"]
|
||||
verdicts = Counter(r.get("verdict") or "none" for r in done)
|
||||
graded = [r for r in done if r.get("verdict") in ("pass", "fail")]
|
||||
passed = [r for r in graded if r.get("verdict") == "pass"]
|
||||
@@ -116,6 +118,7 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) -
|
||||
"items": len(results), "done": len(done),
|
||||
"cases": len({r.get("case_id") for r in results}),
|
||||
"verdicts": {k: verdicts.get(k, 0) for k in ("pass", "fail", "ungradable", "skipped")},
|
||||
"not_run": len(done) - len(evaluated),
|
||||
"accuracy": rate(len(passed), len(graded)),
|
||||
"traced_accuracy": rate(len(traced), len(graded)),
|
||||
"outcomes": dict(outcomes),
|
||||
@@ -141,7 +144,8 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) -
|
||||
"timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()},
|
||||
"usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}},
|
||||
"cost_usd": cost, "unpriced_calls": unpriced,
|
||||
"per_case_cost_usd": round(cost / len(done), 6) if done else None,
|
||||
# 每题平均花费(下一次开跑前的估算按它):只按评完了的题平均,没开评的不摊
|
||||
"per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated else None,
|
||||
"citations": {"numbers": numbers, "numbers_cited": cited,
|
||||
"uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats),
|
||||
"unresolved": sum(int(s.get("unresolved") or 0) for s in stats),
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""和上一次比、A/B 对照(app/eval/compare.py;细案 01 做法 12,评测-D10)。纯计算,选基准读库。
|
||||
|
||||
- 逐题转移(退步 / 进步 / 不变 / 不可比)和归因(数据变、做法变、只参数变、路径变),列出指纹里哪几项变了;
|
||||
- A/B 配对表:都对 / 都错 / 只 A 对 / 只 B 对,不能判的不进表,精确 McNemar;
|
||||
- 选基准:点名的 > 同一变体里钉住的 > 同一变体、比这次先发起、已经结束的最近一次。
|
||||
题目、模型名都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import app.db.models # noqa: F401 - conftest 按已导入的模型建表
|
||||
from app.eval import compare
|
||||
|
||||
|
||||
def _row(case: str, verdict: str | None, *, path: str = "ran", status: str = "ok", repeat: int = 0, **extra) -> dict:
|
||||
return {"case_id": case, "case_seq": int(case[-1]) if case[-1].isdigit() else 1, "case_hash": f"h-{case}",
|
||||
"verdict": verdict, "path": path, "status": status, "repeat": repeat, **extra}
|
||||
|
||||
|
||||
def test_transitions_and_their_causes():
|
||||
sql_a = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-09-01'"]}]
|
||||
sql_b = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-08-01'"]}]
|
||||
sql_c = [{"text": "SELECT SUM(x) FROM t WHERE e = ?", "literals": ["'2026-09-01'"]}]
|
||||
base = [_row("c1", "pass", gold_key="g1", sql_shapes=sql_a), _row("c2", "pass", gold_key="g2", sql_shapes=sql_a),
|
||||
_row("c3", "fail", gold_key="g3", sql_shapes=sql_a), _row("c4", "pass", gold_key="g4", path="ran"),
|
||||
_row("c5", "pass", gold_key="g5"), _row("c6", "fail", gold_key="g6")]
|
||||
target = [_row("c1", "fail", gold_key="g1-new", sql_shapes=sql_a), _row("c2", "fail", gold_key="g2", sql_shapes=sql_c),
|
||||
_row("c3", "pass", gold_key="g3", sql_shapes=sql_b), _row("c4", "fail", gold_key="g4", path="reply"),
|
||||
{**_row("c5", "fail", gold_key="g5"), "case_hash": "changed"}, _row("c6", "ungradable", gold_key="g6"),
|
||||
_row("c7", "pass")]
|
||||
moved = compare.transitions(base, target)
|
||||
by = {i["case_id"]: i for i in moved["items"]}
|
||||
assert (by["c1"]["transition"], by["c1"]["cause"]) == ("regressed", "data")
|
||||
assert (by["c2"]["transition"], by["c2"]["cause"]) == ("regressed", "method")
|
||||
assert (by["c3"]["transition"], by["c3"]["cause"]) == ("improved", "params")
|
||||
assert (by["c4"]["transition"], by["c4"]["cause"]) == ("regressed", "path")
|
||||
assert by["c5"]["transition"] == "incomparable" and "题目改过" in by["c5"]["reason"]
|
||||
assert by["c6"]["transition"] == "incomparable" and by["c7"]["transition"] == "incomparable"
|
||||
assert moved["counts"] == {"regressed": 3, "improved": 1, "same": 0, "incomparable": 3}
|
||||
assert compare.sql_change(sql_a, sql_a) == "same"
|
||||
|
||||
|
||||
def test_fingerprint_diff_lists_what_changed():
|
||||
a = {"assistant_model": "m1", "writer_model": None, "catalog": "live:aaa", "prompt": "p1", "schemas": {"s": "x"},
|
||||
"cases": "c", "scorer_version": 1}
|
||||
b = {**a, "assistant_model": "m2", "schemas": {"s": "y"}}
|
||||
diff = compare.fingerprint_diff(a, b)
|
||||
assert [d["key"] for d in diff] == ["assistant_model", "schemas"]
|
||||
assert diff[0] == {"key": "assistant_model", "label": "助手模型", "before": "m1", "after": "m2"}
|
||||
assert "before" not in diff[1], "表结构只说变没变"
|
||||
|
||||
|
||||
def test_the_pair_table():
|
||||
a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")]
|
||||
b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")]
|
||||
table = compare.pair_table([("当前目录", a), ("关目录", b)])
|
||||
[pair] = table["pairs"]
|
||||
assert (pair["both"], pair["neither"], pair["only_a"], pair["only_b"], pair["excluded"]) == (1, 1, 1, 0, 1)
|
||||
assert pair["p_value"] == 1.0 and pair["note"] == compare.NO_DIFFERENCE
|
||||
assert table["arms"][0]["accuracy"]["rate"] == round(2 / 3, 4)
|
||||
|
||||
|
||||
async def test_picking_the_baseline():
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models_eval import EvalRun, EvalSuite
|
||||
|
||||
now = utcnow()
|
||||
async with SessionLocal() as session:
|
||||
suite = EvalSuite(name="虚构题集-基准")
|
||||
session.add(suite)
|
||||
await session.flush()
|
||||
|
||||
def run(minutes: int, key: str, status: str = "succeeded", **extra) -> EvalRun:
|
||||
row = EvalRun(suite_id=suite.id, status=status, fingerprint={"variant_key": key},
|
||||
created_at=now - timedelta(minutes=minutes), **extra)
|
||||
session.add(row)
|
||||
return row
|
||||
|
||||
old = run(50, "k1")
|
||||
pinned = run(40, "k1", pinned=True)
|
||||
latest = run(30, "k1")
|
||||
run(20, "k2")
|
||||
run(15, "k1", status="failed")
|
||||
target = run(10, "k1")
|
||||
later = run(5, "k1")
|
||||
await session.flush()
|
||||
assert (await compare.pick_baseline(session, target)).id == pinned.id, "同一变体里钉住的优先"
|
||||
pinned.pinned = False
|
||||
await session.flush()
|
||||
assert (await compare.pick_baseline(session, target)).id == latest.id, "比这次先发起、已结束的最近一次"
|
||||
assert (await compare.pick_baseline(session, target, explicit=old.id)).id == old.id
|
||||
assert (await compare.pick_baseline(session, target, explicit=later.id)).id == later.id
|
||||
await session.delete(suite)
|
||||
await session.commit()
|
||||
@@ -230,6 +230,22 @@ async def test_consecutive_assistant_failures_stop_the_evaluation(monkeypatch):
|
||||
assert row.stop_reason.startswith("连续 3 题助手调用失败") and "502" in row.stop_reason
|
||||
|
||||
|
||||
async def test_too_many_timeouts_stop_the_evaluation(monkeypatch):
|
||||
async def drive(self):
|
||||
self.turn.path, self.turn.timed_out, self.turn.reason = "build_failed", True, "超过每题时限"
|
||||
return self.turn
|
||||
|
||||
monkeypatch.setattr(answer_mod.Driver, "drive", drive)
|
||||
suite_id = await es.make_suite("none", cases=ask_cases(9), budget={"max_parallel": 1})
|
||||
[run] = await start(suite_id)
|
||||
await es.wait_eval(run.id)
|
||||
row = await get(run.id)
|
||||
statuses = [r.status for r in await results(run.id)]
|
||||
assert statuses == ["ok"] * runner.STOP_TIMEOUTS + ["skipped"] * (9 - runner.STOP_TIMEOUTS)
|
||||
assert row.stop_reason == runner.STOP_TIMEOUT_TEXT.format(window=runner.STOP_WINDOW, n=runner.STOP_TIMEOUTS)
|
||||
assert row.summary["not_run"] == 9 - runner.STOP_TIMEOUTS
|
||||
|
||||
|
||||
async def test_a_case_over_its_time_limit_stops_its_run(monkeypatch):
|
||||
stopped: list[str | None] = []
|
||||
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
"""评测的统计与对比(app/eval/stats.py、app/eval/compare.py;细案 01 做法 11、12,评测-D10)。纯计算,和手算对照。
|
||||
"""评测的统计(app/eval/stats.py;细案 01 做法 11,评测-D10)。纯计算,和手算对照。
|
||||
|
||||
- Wilson 95% 区间、精确 McNemar 双侧 p 值、P50 / P90;
|
||||
- 总表:准确率和可追溯准确率分开报(分母只算能判的)、作答路径、出具档位、复核、look、按角色的用量、引用率、逐题 E1、
|
||||
重复 k 次的题给通过率;
|
||||
- 和上一次比:逐题转移(退步 / 进步 / 不变 / 不可比)和归因(数据变、做法变、只参数变、路径变),列出指纹里哪几项变了;
|
||||
- A/B 配对表:都对 / 都错 / 只 A 对 / 只 B 对,不能判的不进表;
|
||||
- 选基准:点名的 > 同一变体里钉住的 > 同一变体、比这次先发起、已经结束的最近一次。
|
||||
重复 k 次的题给通过率。
|
||||
和上一次比、A/B 配对表、选基准在 test_eval_compare.py。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -59,7 +57,7 @@ def test_summary_counts_only_gradable_items_and_splits_roles():
|
||||
summary = stats.summarize(rows)
|
||||
assert summary["accuracy"] == {"passed": 1, "n": 2, "rate": 0.5, "ci": stats.wilson(1, 2)}
|
||||
assert summary["traced_accuracy"]["passed"] == 1
|
||||
assert summary["verdicts"] == {"pass": 1, "fail": 1, "ungradable": 1, "skipped": 0}
|
||||
assert summary["verdicts"] == {"pass": 1, "fail": 1, "ungradable": 1, "skipped": 0} and summary["not_run"] == 1
|
||||
assert summary["paths"] == {"ran": 3} and summary["tiers"]["formal"] == 1 and summary["tiers"]["explore"] == 2
|
||||
assert summary["review"]["retries"] == 1 and summary["review"]["slow"] == 1
|
||||
assert summary["review"]["severity"] == {"degraded": 1}
|
||||
@@ -67,6 +65,7 @@ def test_summary_counts_only_gradable_items_and_splits_roles():
|
||||
assert summary["timings"]["build_ms"] == {"p50": 1000, "p90": 3000, "n": 2}
|
||||
assert summary["usage"]["assistant"]["calls"] == 6 and summary["usage"]["writer"]["unpriced_calls"] == 2
|
||||
assert summary["cost_usd"] == 0.08 and summary["unpriced_calls"] == 2
|
||||
assert summary["per_case_cost_usd"] == round(0.08 / 3, 6), "按评完了的题平均,没开评的不摊"
|
||||
assert summary["usage"]["gold"] == {"cost_usd": 0.0, "calls": 0}, "期望探针不调模型"
|
||||
assert summary["citations"]["rate"] == 0.5
|
||||
assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1}
|
||||
@@ -79,83 +78,6 @@ def test_repeats_give_a_pass_rate_per_case():
|
||||
assert compare.case_outcome(rows)["verdict"] == "pass", "多数决"
|
||||
|
||||
|
||||
def test_transitions_and_their_causes():
|
||||
sql_a = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-09-01'"]}]
|
||||
sql_b = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-08-01'"]}]
|
||||
sql_c = [{"text": "SELECT SUM(x) FROM t WHERE e = ?", "literals": ["'2026-09-01'"]}]
|
||||
base = [_row("c1", "pass", gold_key="g1", sql_shapes=sql_a), _row("c2", "pass", gold_key="g2", sql_shapes=sql_a),
|
||||
_row("c3", "fail", gold_key="g3", sql_shapes=sql_a), _row("c4", "pass", gold_key="g4", path="ran"),
|
||||
_row("c5", "pass", gold_key="g5"), _row("c6", "fail", gold_key="g6")]
|
||||
target = [_row("c1", "fail", gold_key="g1-new", sql_shapes=sql_a), _row("c2", "fail", gold_key="g2", sql_shapes=sql_c),
|
||||
_row("c3", "pass", gold_key="g3", sql_shapes=sql_b), _row("c4", "fail", gold_key="g4", path="reply"),
|
||||
{**_row("c5", "fail", gold_key="g5"), "case_hash": "changed"}, _row("c6", "ungradable", gold_key="g6"),
|
||||
_row("c7", "pass")]
|
||||
moved = compare.transitions(base, target)
|
||||
by = {i["case_id"]: i for i in moved["items"]}
|
||||
assert (by["c1"]["transition"], by["c1"]["cause"]) == ("regressed", "data")
|
||||
assert (by["c2"]["transition"], by["c2"]["cause"]) == ("regressed", "method")
|
||||
assert (by["c3"]["transition"], by["c3"]["cause"]) == ("improved", "params")
|
||||
assert (by["c4"]["transition"], by["c4"]["cause"]) == ("regressed", "path")
|
||||
assert by["c5"]["transition"] == "incomparable" and "题目改过" in by["c5"]["reason"]
|
||||
assert by["c6"]["transition"] == "incomparable" and by["c7"]["transition"] == "incomparable"
|
||||
assert moved["counts"] == {"regressed": 3, "improved": 1, "same": 0, "incomparable": 3}
|
||||
assert compare.sql_change(sql_a, sql_a) == "same"
|
||||
|
||||
|
||||
def test_fingerprint_diff_lists_what_changed():
|
||||
a = {"assistant_model": "m1", "writer_model": None, "catalog": "live:aaa", "prompt": "p1", "schemas": {"s": "x"},
|
||||
"cases": "c", "scorer_version": 1}
|
||||
b = {**a, "assistant_model": "m2", "schemas": {"s": "y"}}
|
||||
diff = compare.fingerprint_diff(a, b)
|
||||
assert [d["key"] for d in diff] == ["assistant_model", "schemas"]
|
||||
assert diff[0] == {"key": "assistant_model", "label": "助手模型", "before": "m1", "after": "m2"}
|
||||
assert "before" not in diff[1], "表结构只说变没变"
|
||||
|
||||
|
||||
def test_the_pair_table():
|
||||
a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")]
|
||||
b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")]
|
||||
table = compare.pair_table([("当前目录", a), ("关目录", b)])
|
||||
[pair] = table["pairs"]
|
||||
assert (pair["both"], pair["neither"], pair["only_a"], pair["only_b"], pair["excluded"]) == (1, 1, 1, 0, 1)
|
||||
assert pair["p_value"] == 1.0 and pair["note"] == compare.NO_DIFFERENCE
|
||||
assert table["arms"][0]["accuracy"]["rate"] == round(2 / 3, 4)
|
||||
|
||||
|
||||
async def test_picking_the_baseline():
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models_eval import EvalRun, EvalSuite
|
||||
|
||||
now = utcnow()
|
||||
async with SessionLocal() as session:
|
||||
suite = EvalSuite(name="虚构题集-基准")
|
||||
session.add(suite)
|
||||
await session.flush()
|
||||
|
||||
def run(minutes: int, key: str, status: str = "succeeded", **extra) -> EvalRun:
|
||||
row = EvalRun(suite_id=suite.id, status=status, fingerprint={"variant_key": key},
|
||||
created_at=now - timedelta(minutes=minutes), **extra)
|
||||
session.add(row)
|
||||
return row
|
||||
|
||||
old = run(50, "k1")
|
||||
pinned = run(40, "k1", pinned=True)
|
||||
latest = run(30, "k1")
|
||||
run(20, "k2")
|
||||
run(15, "k1", status="failed")
|
||||
target = run(10, "k1")
|
||||
later = run(5, "k1")
|
||||
await session.flush()
|
||||
assert (await compare.pick_baseline(session, target)).id == pinned.id, "同一变体里钉住的优先"
|
||||
pinned.pinned = False
|
||||
await session.flush()
|
||||
assert (await compare.pick_baseline(session, target)).id == latest.id, "比这次先发起、已结束的最近一次"
|
||||
assert (await compare.pick_baseline(session, target, explicit=old.id)).id == old.id
|
||||
assert (await compare.pick_baseline(session, target, explicit=later.id)).id == later.id
|
||||
await session.delete(suite)
|
||||
await session.commit()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n, k", [(1, 0), (3, 3)])
|
||||
def test_rates_carry_their_sample_size(n, k):
|
||||
r = stats.rate(k, n)
|
||||
|
||||
Reference in New Issue
Block a user