diff --git a/backend/app/eval/stats.py b/backend/app/eval/stats.py index 744f7d17..395eace6 100644 --- a/backend/app/eval/stats.py +++ b/backend/app/eval/stats.py @@ -1,8 +1,8 @@ """评测的汇总与统计(细案 01 做法 11、12;评测-D10)。所有汇总都在后端算好,界面直接显示(界面层不放逻辑:同一个数不能 有两套算法)。纯计算,输入是一次评测的逐题结果(EvalResult 的列,result_row 的形状)。 -- 准确率和可追溯准确率:分母是能判的题(通过 + 不通过);不能评分、跳过的单列,既不算对也不算错。都带 Wilson 95% 区间和 - 样本数——题集小、模型有随机性,差两三题就可能被读成「变好了」「变差了」(评测-D10:只展示、不卡门)。 +- 准确率和可追溯准确率:分母是能判的题(通过 + 不通过);不能评分、跳过的单列,既不算对也不算错,没开评、停下、出错的 + 另列 not_run。都带 Wilson 95% 区间和样本数——题集小、模型有随机性,差两三题就可能被读成「变好了」「变差了」(评测-D10:只展示、不卡门)。 - 作答路径、出具档位(声明了出具的按 formal / degraded / withheld,没声明的单列「探索问答」)、复核结论(verdict × severity) 和自动重来的次数、复核用时超过问数据页等待(25 秒)的题数。 - 结论裁判(打开了才有):supported / partial / contradicted / insufficient / 未判的句数;文字要点的判断单列。 @@ -79,6 +79,8 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) - from app.eval.score import SCORER_VERSION done = [r for r in results if r.get("status") in ("ok", "skipped", "failed", "cancelled", "partial")] + #: 评完了的(status=ok);没开评、停下、出错的单列成 not_run + evaluated = [r for r in done if r.get("status") == "ok"] verdicts = Counter(r.get("verdict") or "none" for r in done) graded = [r for r in done if r.get("verdict") in ("pass", "fail")] passed = [r for r in graded if r.get("verdict") == "pass"] @@ -116,6 +118,7 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) - "items": len(results), "done": len(done), "cases": len({r.get("case_id") for r in results}), "verdicts": {k: verdicts.get(k, 0) for k in ("pass", "fail", "ungradable", "skipped")}, + "not_run": len(done) - len(evaluated), "accuracy": rate(len(passed), len(graded)), "traced_accuracy": rate(len(traced), len(graded)), "outcomes": dict(outcomes), @@ -141,7 +144,8 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) - "timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()}, "usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}}, "cost_usd": cost, "unpriced_calls": unpriced, - "per_case_cost_usd": round(cost / len(done), 6) if done else None, + # 每题平均花费(下一次开跑前的估算按它):只按评完了的题平均,没开评的不摊 + "per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated else None, "citations": {"numbers": numbers, "numbers_cited": cited, "uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats), "unresolved": sum(int(s.get("unresolved") or 0) for s in stats), diff --git a/backend/tests/test_eval_compare.py b/backend/tests/test_eval_compare.py new file mode 100644 index 00000000..e8cc87de --- /dev/null +++ b/backend/tests/test_eval_compare.py @@ -0,0 +1,95 @@ +"""和上一次比、A/B 对照(app/eval/compare.py;细案 01 做法 12,评测-D10)。纯计算,选基准读库。 + +- 逐题转移(退步 / 进步 / 不变 / 不可比)和归因(数据变、做法变、只参数变、路径变),列出指纹里哪几项变了; +- A/B 配对表:都对 / 都错 / 只 A 对 / 只 B 对,不能判的不进表,精确 McNemar; +- 选基准:点名的 > 同一变体里钉住的 > 同一变体、比这次先发起、已经结束的最近一次。 +题目、模型名都是虚构的。 +""" +from __future__ import annotations + +from datetime import timedelta + +import app.db.models # noqa: F401 - conftest 按已导入的模型建表 +from app.eval import compare + + +def _row(case: str, verdict: str | None, *, path: str = "ran", status: str = "ok", repeat: int = 0, **extra) -> dict: + return {"case_id": case, "case_seq": int(case[-1]) if case[-1].isdigit() else 1, "case_hash": f"h-{case}", + "verdict": verdict, "path": path, "status": status, "repeat": repeat, **extra} + + +def test_transitions_and_their_causes(): + sql_a = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-09-01'"]}] + sql_b = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-08-01'"]}] + sql_c = [{"text": "SELECT SUM(x) FROM t WHERE e = ?", "literals": ["'2026-09-01'"]}] + base = [_row("c1", "pass", gold_key="g1", sql_shapes=sql_a), _row("c2", "pass", gold_key="g2", sql_shapes=sql_a), + _row("c3", "fail", gold_key="g3", sql_shapes=sql_a), _row("c4", "pass", gold_key="g4", path="ran"), + _row("c5", "pass", gold_key="g5"), _row("c6", "fail", gold_key="g6")] + target = [_row("c1", "fail", gold_key="g1-new", sql_shapes=sql_a), _row("c2", "fail", gold_key="g2", sql_shapes=sql_c), + _row("c3", "pass", gold_key="g3", sql_shapes=sql_b), _row("c4", "fail", gold_key="g4", path="reply"), + {**_row("c5", "fail", gold_key="g5"), "case_hash": "changed"}, _row("c6", "ungradable", gold_key="g6"), + _row("c7", "pass")] + moved = compare.transitions(base, target) + by = {i["case_id"]: i for i in moved["items"]} + assert (by["c1"]["transition"], by["c1"]["cause"]) == ("regressed", "data") + assert (by["c2"]["transition"], by["c2"]["cause"]) == ("regressed", "method") + assert (by["c3"]["transition"], by["c3"]["cause"]) == ("improved", "params") + assert (by["c4"]["transition"], by["c4"]["cause"]) == ("regressed", "path") + assert by["c5"]["transition"] == "incomparable" and "题目改过" in by["c5"]["reason"] + assert by["c6"]["transition"] == "incomparable" and by["c7"]["transition"] == "incomparable" + assert moved["counts"] == {"regressed": 3, "improved": 1, "same": 0, "incomparable": 3} + assert compare.sql_change(sql_a, sql_a) == "same" + + +def test_fingerprint_diff_lists_what_changed(): + a = {"assistant_model": "m1", "writer_model": None, "catalog": "live:aaa", "prompt": "p1", "schemas": {"s": "x"}, + "cases": "c", "scorer_version": 1} + b = {**a, "assistant_model": "m2", "schemas": {"s": "y"}} + diff = compare.fingerprint_diff(a, b) + assert [d["key"] for d in diff] == ["assistant_model", "schemas"] + assert diff[0] == {"key": "assistant_model", "label": "助手模型", "before": "m1", "after": "m2"} + assert "before" not in diff[1], "表结构只说变没变" + + +def test_the_pair_table(): + a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")] + b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")] + table = compare.pair_table([("当前目录", a), ("关目录", b)]) + [pair] = table["pairs"] + assert (pair["both"], pair["neither"], pair["only_a"], pair["only_b"], pair["excluded"]) == (1, 1, 1, 0, 1) + assert pair["p_value"] == 1.0 and pair["note"] == compare.NO_DIFFERENCE + assert table["arms"][0]["accuracy"]["rate"] == round(2 / 3, 4) + + +async def test_picking_the_baseline(): + from app.db.base import SessionLocal, utcnow + from app.db.models_eval import EvalRun, EvalSuite + + now = utcnow() + async with SessionLocal() as session: + suite = EvalSuite(name="虚构题集-基准") + session.add(suite) + await session.flush() + + def run(minutes: int, key: str, status: str = "succeeded", **extra) -> EvalRun: + row = EvalRun(suite_id=suite.id, status=status, fingerprint={"variant_key": key}, + created_at=now - timedelta(minutes=minutes), **extra) + session.add(row) + return row + + old = run(50, "k1") + pinned = run(40, "k1", pinned=True) + latest = run(30, "k1") + run(20, "k2") + run(15, "k1", status="failed") + target = run(10, "k1") + later = run(5, "k1") + await session.flush() + assert (await compare.pick_baseline(session, target)).id == pinned.id, "同一变体里钉住的优先" + pinned.pinned = False + await session.flush() + assert (await compare.pick_baseline(session, target)).id == latest.id, "比这次先发起、已结束的最近一次" + assert (await compare.pick_baseline(session, target, explicit=old.id)).id == old.id + assert (await compare.pick_baseline(session, target, explicit=later.id)).id == later.id + await session.delete(suite) + await session.commit() diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index c98df297..6e6899c6 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -230,6 +230,22 @@ async def test_consecutive_assistant_failures_stop_the_evaluation(monkeypatch): assert row.stop_reason.startswith("连续 3 题助手调用失败") and "502" in row.stop_reason +async def test_too_many_timeouts_stop_the_evaluation(monkeypatch): + async def drive(self): + self.turn.path, self.turn.timed_out, self.turn.reason = "build_failed", True, "超过每题时限" + return self.turn + + monkeypatch.setattr(answer_mod.Driver, "drive", drive) + suite_id = await es.make_suite("none", cases=ask_cases(9), budget={"max_parallel": 1}) + [run] = await start(suite_id) + await es.wait_eval(run.id) + row = await get(run.id) + statuses = [r.status for r in await results(run.id)] + assert statuses == ["ok"] * runner.STOP_TIMEOUTS + ["skipped"] * (9 - runner.STOP_TIMEOUTS) + assert row.stop_reason == runner.STOP_TIMEOUT_TEXT.format(window=runner.STOP_WINDOW, n=runner.STOP_TIMEOUTS) + assert row.summary["not_run"] == 9 - runner.STOP_TIMEOUTS + + async def test_a_case_over_its_time_limit_stops_its_run(monkeypatch): stopped: list[str | None] = [] diff --git a/backend/tests/test_eval_stats.py b/backend/tests/test_eval_stats.py index 9f2f290a..7b4ecfe2 100644 --- a/backend/tests/test_eval_stats.py +++ b/backend/tests/test_eval_stats.py @@ -1,11 +1,9 @@ -"""评测的统计与对比(app/eval/stats.py、app/eval/compare.py;细案 01 做法 11、12,评测-D10)。纯计算,和手算对照。 +"""评测的统计(app/eval/stats.py;细案 01 做法 11,评测-D10)。纯计算,和手算对照。 - Wilson 95% 区间、精确 McNemar 双侧 p 值、P50 / P90; - 总表:准确率和可追溯准确率分开报(分母只算能判的)、作答路径、出具档位、复核、look、按角色的用量、引用率、逐题 E1、 - 重复 k 次的题给通过率; -- 和上一次比:逐题转移(退步 / 进步 / 不变 / 不可比)和归因(数据变、做法变、只参数变、路径变),列出指纹里哪几项变了; -- A/B 配对表:都对 / 都错 / 只 A 对 / 只 B 对,不能判的不进表; -- 选基准:点名的 > 同一变体里钉住的 > 同一变体、比这次先发起、已经结束的最近一次。 + 重复 k 次的题给通过率。 +和上一次比、A/B 配对表、选基准在 test_eval_compare.py。 """ from __future__ import annotations @@ -59,7 +57,7 @@ def test_summary_counts_only_gradable_items_and_splits_roles(): summary = stats.summarize(rows) assert summary["accuracy"] == {"passed": 1, "n": 2, "rate": 0.5, "ci": stats.wilson(1, 2)} assert summary["traced_accuracy"]["passed"] == 1 - assert summary["verdicts"] == {"pass": 1, "fail": 1, "ungradable": 1, "skipped": 0} + assert summary["verdicts"] == {"pass": 1, "fail": 1, "ungradable": 1, "skipped": 0} and summary["not_run"] == 1 assert summary["paths"] == {"ran": 3} and summary["tiers"]["formal"] == 1 and summary["tiers"]["explore"] == 2 assert summary["review"]["retries"] == 1 and summary["review"]["slow"] == 1 assert summary["review"]["severity"] == {"degraded": 1} @@ -67,6 +65,7 @@ def test_summary_counts_only_gradable_items_and_splits_roles(): assert summary["timings"]["build_ms"] == {"p50": 1000, "p90": 3000, "n": 2} assert summary["usage"]["assistant"]["calls"] == 6 and summary["usage"]["writer"]["unpriced_calls"] == 2 assert summary["cost_usd"] == 0.08 and summary["unpriced_calls"] == 2 + assert summary["per_case_cost_usd"] == round(0.08 / 3, 6), "按评完了的题平均,没开评的不摊" assert summary["usage"]["gold"] == {"cost_usd": 0.0, "calls": 0}, "期望探针不调模型" assert summary["citations"]["rate"] == 0.5 assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1} @@ -79,83 +78,6 @@ def test_repeats_give_a_pass_rate_per_case(): assert compare.case_outcome(rows)["verdict"] == "pass", "多数决" -def test_transitions_and_their_causes(): - sql_a = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-09-01'"]}] - sql_b = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-08-01'"]}] - sql_c = [{"text": "SELECT SUM(x) FROM t WHERE e = ?", "literals": ["'2026-09-01'"]}] - base = [_row("c1", "pass", gold_key="g1", sql_shapes=sql_a), _row("c2", "pass", gold_key="g2", sql_shapes=sql_a), - _row("c3", "fail", gold_key="g3", sql_shapes=sql_a), _row("c4", "pass", gold_key="g4", path="ran"), - _row("c5", "pass", gold_key="g5"), _row("c6", "fail", gold_key="g6")] - target = [_row("c1", "fail", gold_key="g1-new", sql_shapes=sql_a), _row("c2", "fail", gold_key="g2", sql_shapes=sql_c), - _row("c3", "pass", gold_key="g3", sql_shapes=sql_b), _row("c4", "fail", gold_key="g4", path="reply"), - {**_row("c5", "fail", gold_key="g5"), "case_hash": "changed"}, _row("c6", "ungradable", gold_key="g6"), - _row("c7", "pass")] - moved = compare.transitions(base, target) - by = {i["case_id"]: i for i in moved["items"]} - assert (by["c1"]["transition"], by["c1"]["cause"]) == ("regressed", "data") - assert (by["c2"]["transition"], by["c2"]["cause"]) == ("regressed", "method") - assert (by["c3"]["transition"], by["c3"]["cause"]) == ("improved", "params") - assert (by["c4"]["transition"], by["c4"]["cause"]) == ("regressed", "path") - assert by["c5"]["transition"] == "incomparable" and "题目改过" in by["c5"]["reason"] - assert by["c6"]["transition"] == "incomparable" and by["c7"]["transition"] == "incomparable" - assert moved["counts"] == {"regressed": 3, "improved": 1, "same": 0, "incomparable": 3} - assert compare.sql_change(sql_a, sql_a) == "same" - - -def test_fingerprint_diff_lists_what_changed(): - a = {"assistant_model": "m1", "writer_model": None, "catalog": "live:aaa", "prompt": "p1", "schemas": {"s": "x"}, - "cases": "c", "scorer_version": 1} - b = {**a, "assistant_model": "m2", "schemas": {"s": "y"}} - diff = compare.fingerprint_diff(a, b) - assert [d["key"] for d in diff] == ["assistant_model", "schemas"] - assert diff[0] == {"key": "assistant_model", "label": "助手模型", "before": "m1", "after": "m2"} - assert "before" not in diff[1], "表结构只说变没变" - - -def test_the_pair_table(): - a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")] - b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")] - table = compare.pair_table([("当前目录", a), ("关目录", b)]) - [pair] = table["pairs"] - assert (pair["both"], pair["neither"], pair["only_a"], pair["only_b"], pair["excluded"]) == (1, 1, 1, 0, 1) - assert pair["p_value"] == 1.0 and pair["note"] == compare.NO_DIFFERENCE - assert table["arms"][0]["accuracy"]["rate"] == round(2 / 3, 4) - - -async def test_picking_the_baseline(): - from app.db.base import SessionLocal, utcnow - from app.db.models_eval import EvalRun, EvalSuite - - now = utcnow() - async with SessionLocal() as session: - suite = EvalSuite(name="虚构题集-基准") - session.add(suite) - await session.flush() - - def run(minutes: int, key: str, status: str = "succeeded", **extra) -> EvalRun: - row = EvalRun(suite_id=suite.id, status=status, fingerprint={"variant_key": key}, - created_at=now - timedelta(minutes=minutes), **extra) - session.add(row) - return row - - old = run(50, "k1") - pinned = run(40, "k1", pinned=True) - latest = run(30, "k1") - run(20, "k2") - run(15, "k1", status="failed") - target = run(10, "k1") - later = run(5, "k1") - await session.flush() - assert (await compare.pick_baseline(session, target)).id == pinned.id, "同一变体里钉住的优先" - pinned.pinned = False - await session.flush() - assert (await compare.pick_baseline(session, target)).id == latest.id, "比这次先发起、已结束的最近一次" - assert (await compare.pick_baseline(session, target, explicit=old.id)).id == old.id - assert (await compare.pick_baseline(session, target, explicit=later.id)).id == later.id - await session.delete(suite) - await session.commit() - - @pytest.mark.parametrize("n, k", [(1, 0), (3, 3)]) def test_rates_carry_their_sample_size(n, k): r = stats.rate(k, n)