Files
AgentLab/backend/tests/test_eval_e2e.py
T
ThearchyHelios 2371e75f17 评测执行的主干:后台评测、无头作答、评分、统计、对比与 A/B、清理(下一轮第 3 波 W3-c 第一段)
为什么:第 2 波只有题集和期望答案的来源,评测本身跑不起来。这一段把细案 01 做法 9–12、16、17 的后端接上,
量的是用户在问数据页拿到的同一轮,而不是另抄一份编排。

- eval/answer.py:无头作答,串起 answer_ops → chat/turn 的决定 → launch_run(origin=eval,带目录视图、只读、遮罩)
  → 等终态(停在审批的取消、记「评测不代批」;驳回记路径「被驳回」)→ review_of_run → 按同一规则重来一次;
  工作流题按钉住版本的图跑探索运行(不发正式运行,免得被 pick_base 和应用视图取到)。用量按角色记
- eval/runner.py:用 core/job_runtime,一次评测一个任务,题 × 重复 × 臂一项;A/B 几臂共用期望、同一题同时开跑;
  开评拍目录快照、记上传表格版本和表结构哈希的前后;并发不超过 max_parallel 和全局并发的四分之一;每题时限;
  评测单独一本账(开跑前估算、每题预留后找补、触顶不再开新题);止损;能停、只重试没做完的、重启标中断不续跑
- eval/score.py:六种期望、五种结论(量纲不同单列)、维度配对与弱匹配、路径不对直接不通过、口径 dual 两版都报;
  逐题 E1 用口径账本的 compare_queries(总方案 X2,不另写 signals);SCORER_VERSION 可重评(eval/rescore.py,只读封存)
- eval/judging.py:结论裁判走按需裁判同一条路(evidence._judge_now),文字要点由裁判判、单列不计准确率;花费经
  judge.charged_to 记在评测那本账上(engine/judge.py 只加了一个上下文变量和取账本的一处)
- eval/stats.py、eval/compare.py:准确率与可追溯准确率 + Wilson 区间、各项分布、P50/P90、按角色成本、引用率、逐题 E1;
  选基准、逐题转移与归因、指纹差异、A/B 配对表与精确 McNemar;「目录贡献」预设对照(当前 vs 关,可加只用已核实)
- eval/cleanup.py + core/artifact_store.forget:保留最近 N 次 + 钉住的基线 + A/B 另几臂,删结果、明细工件、底下的运行和
  引用行,工件文件一条引用都不剩才删;总表永久保留
- engine/tolerance.py:双盲复算的 _agree / _delta_pct 原样搬来共用(dual_check 改 import,行为不变)
- 设置加 eval 一组(builder 读、admin 写)和 GET /api/settings/eval/spend;题集预算在全站缺省上覆盖,每日上限只能往下收
- api/evals.py:发起(单臂 / A/B / 目录贡献预设)、估算、进度、逐题、停止、重试、重评、钉基线、和上一次比、配对表、清理
- main.py 的 lifespan:影子起草和评测的启动整理、关停(评测先于运行停,停时取消自己发起的运行)
- 测试:test_eval_e2e(dual_check_shop + 剧本模型:按支付日 330 通过且 SQL 等价,按下单日 310 不通过、差在 time;
  目录对照 1:0;和上一次比找到退步、归因做法变);通知事件、定时接口形状的老测试按新事件和新字段更新
2026-10-16 16:38:09 +08:00

123 lines
6.0 KiB
Python

"""评测从头到尾(细案 01 测试 test_eval_e2e):dual_check_shop 的虚构订单库 + 剧本模型。
题目是「9/1–9/7 按支付日、不含退款的销售额」,期望 SQL 算出 330。助手剧本只在查看结果里看到数据目录写着「按支付日期归属」
时才按支付日写(330),没看到就按下单日写(310)。所以:
- 臂「当前目录」按支付日答 330:通过、可追溯、答案的 SQL 和期望 SQL 等价已证明(逐题 E1);
- 臂「关目录」按下单日答 310:不通过,差异方面是 time;
- 「目录贡献」对照(当前 vs 关)的配对表是 1:0,两臂共用期望(同一份期望运行);
- 再评一次(把目录里的业务日期去掉)和上一次比:这题从通过变成不通过,归因「做法变」,指纹里目录快照变了。
评测运行的来源都是 eval,不进使用统计;运行名「评测:题集名·第 n 题」。
"""
from __future__ import annotations
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select
from app.db.base import SessionLocal
from app.db.models import Run
from app.db.models_eval import EvalResult, EvalRun
from app.engine.origin import counted_runs
from app.engine.runner import run_manager
from app.main import app
from tests import eval_samples as es
from tests.fixtures import dual_check_shop as shop
from tests.fixtures.sources import drop_source
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await es.drop_suites()
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
@pytest.fixture
async def source(tmp_path):
name, sid = await es.make_source(tmp_path)
yield {"name": name, "id": sid}
await drop_source(sid)
async def _results(run_id: str) -> list[EvalResult]:
async with SessionLocal() as session:
return list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == run_id)
.order_by(EvalResult.seq))).scalars())
async def test_catalog_contribution_pair_end_to_end(client, monkeypatch, source):
es.use_copilot(monkeypatch, es.Copilot(source["name"]))
es.use_report(monkeypatch)
suite_id = await es.make_suite(source["id"])
started = await client.post(f"/api/evals/suites/{suite_id}/evaluations",
json={"preset": "catalog", "variant": {"repeats": 1}})
assert started.status_code == 202, started.text
body = started.json()
live, off = body["evaluations"]
assert (live["arm"], off["arm"]) == ("当前目录", "关目录") and body["pair_id"] == live["id"]
await es.wait_eval(live["id"])
got = (await client.get(f"/api/evals/evaluations/{live['id']}")).json()
assert got["status"] == "succeeded", got
[right] = await _results(live["id"])
[wrong] = await _results(off["id"])
assert right.path == "ran" and right.verdict == "pass", (right.error, right.scores)
assert right.scores[0]["outcome"] == "traced" and right.scores[0]["got"] == 330.0
assert right.sql_match["status"] == "proven", right.sql_match
assert wrong.verdict == "fail" and wrong.scores[0]["outcome"] == "wrong" and wrong.scores[0]["got"] == 310.0
assert wrong.sql_match["status"] == "related" and "time" in wrong.sql_match["aspects"], wrong.sql_match
# 两臂共用期望:同一份期望运行
assert right.gold_run_ids == wrong.gold_run_ids and len(right.gold_run_ids) == 2
assert right.gold["status"] == "ok"
pair = (await client.get(f"/api/evals/pairs/{body['pair_id']}")).json()
[table] = pair["table"]["pairs"]
assert (table["only_a"], table["only_b"], table["both"], table["neither"]) == (1, 0, 0, 0)
assert table["p_value"] == 1.0 and table["note"] == "这批题上看不出差别"
summary = got["summary"]
assert summary["accuracy"]["passed"] == 1 and summary["traced_accuracy"]["passed"] == 1
assert summary["accuracy"]["ci"] is not None and summary["e1"]["passed"] == 1
# 评测运行:来源 eval、不进使用统计、名字写题集和题号
async with SessionLocal() as session:
run = await session.get(Run, right.run_id)
assert run.origin_kind == "eval" and run.origin["role"] == "answer"
assert run.workflow_name == "评测:虚构零售题集·第 1 题"
assert run.options["catalog_view"]["mode"] == "snapshot"
offrun = await session.get(Run, wrong.run_id)
assert offrun.options["catalog_view"] == {"mode": "off", "statuses": "all"}
counted = (await session.execute(select(Run.id).where(
Run.id.in_([right.run_id, wrong.run_id, *right.gold_run_ids]), counted_runs()))).all()
assert counted == []
async def test_comparing_with_the_previous_run_finds_the_regression(client, monkeypatch, source):
es.use_copilot(monkeypatch, es.Copilot(source["name"]))
es.use_report(monkeypatch)
suite_id = await es.make_suite(source["id"])
first = (await client.post(f"/api/evals/suites/{suite_id}/evaluations", json={})).json()["evaluations"][0]
await es.wait_eval(first["id"])
# 目录里的业务日期去掉:助手改按下单日写
await es.set_notes(source["id"], {"label": es.NOTES["label"]})
second = (await client.post(f"/api/evals/suites/{suite_id}/evaluations", json={})).json()["evaluations"][0]
await es.wait_eval(second["id"])
async with SessionLocal() as session:
row = await session.get(EvalRun, second["id"])
assert row.baseline_id == first["id"]
assert row.summary["baseline"]["counts"]["regressed"] == 1
compared = (await client.get(f"/api/evals/evaluations/{second['id']}/compare")).json()
assert compared["base"]["id"] == first["id"]
[item] = compared["transitions"]["items"]
assert item["transition"] == "regressed" and item["cause"] == "method", item
assert "catalog" in {d["key"] for d in compared["fingerprint_diff"]}, compared["fingerprint_diff"]
assert shop.RIGHT != shop.BY_ORDER_DATE