From 2371e75f171489110296eeff6bba5e60aa383fe5 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 16:38:09 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E8=AF=84=E6=B5=8B=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E7=9A=84=E4=B8=BB=E5=B9=B2=EF=BC=9A=E5=90=8E=E5=8F=B0=E8=AF=84?= =?UTF-8?q?=E6=B5=8B=E3=80=81=E6=97=A0=E5=A4=B4=E4=BD=9C=E7=AD=94=E3=80=81?= =?UTF-8?q?=E8=AF=84=E5=88=86=E3=80=81=E7=BB=9F=E8=AE=A1=E3=80=81=E5=AF=B9?= =?UTF-8?q?=E6=AF=94=E4=B8=8E=20A/B=E3=80=81=E6=B8=85=E7=90=86=EF=BC=88?= =?UTF-8?q?=E4=B8=8B=E4=B8=80=E8=BD=AE=E7=AC=AC=203=20=E6=B3=A2=20W3-c=20?= =?UTF-8?q?=E7=AC=AC=E4=B8=80=E6=AE=B5=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:第 2 波只有题集和期望答案的来源,评测本身跑不起来。这一段把细案 01 做法 9–12、16、17 的后端接上, 量的是用户在问数据页拿到的同一轮,而不是另抄一份编排。 - eval/answer.py:无头作答,串起 answer_ops → chat/turn 的决定 → launch_run(origin=eval,带目录视图、只读、遮罩) → 等终态(停在审批的取消、记「评测不代批」;驳回记路径「被驳回」)→ review_of_run → 按同一规则重来一次; 工作流题按钉住版本的图跑探索运行(不发正式运行,免得被 pick_base 和应用视图取到)。用量按角色记 - eval/runner.py:用 core/job_runtime,一次评测一个任务,题 × 重复 × 臂一项;A/B 几臂共用期望、同一题同时开跑; 开评拍目录快照、记上传表格版本和表结构哈希的前后;并发不超过 max_parallel 和全局并发的四分之一;每题时限; 评测单独一本账(开跑前估算、每题预留后找补、触顶不再开新题);止损;能停、只重试没做完的、重启标中断不续跑 - eval/score.py:六种期望、五种结论(量纲不同单列)、维度配对与弱匹配、路径不对直接不通过、口径 dual 两版都报; 逐题 E1 用口径账本的 compare_queries(总方案 X2,不另写 signals);SCORER_VERSION 可重评(eval/rescore.py,只读封存) - eval/judging.py:结论裁判走按需裁判同一条路(evidence._judge_now),文字要点由裁判判、单列不计准确率;花费经 judge.charged_to 记在评测那本账上(engine/judge.py 只加了一个上下文变量和取账本的一处) - eval/stats.py、eval/compare.py:准确率与可追溯准确率 + Wilson 区间、各项分布、P50/P90、按角色成本、引用率、逐题 E1; 选基准、逐题转移与归因、指纹差异、A/B 配对表与精确 McNemar;「目录贡献」预设对照(当前 vs 关,可加只用已核实) - eval/cleanup.py + core/artifact_store.forget:保留最近 N 次 + 钉住的基线 + A/B 另几臂,删结果、明细工件、底下的运行和 引用行,工件文件一条引用都不剩才删;总表永久保留 - engine/tolerance.py:双盲复算的 _agree / _delta_pct 原样搬来共用(dual_check 改 import,行为不变) - 设置加 eval 一组(builder 读、admin 写)和 GET /api/settings/eval/spend;题集预算在全站缺省上覆盖,每日上限只能往下收 - api/evals.py:发起(单臂 / A/B / 目录贡献预设)、估算、进度、逐题、停止、重试、重评、钉基线、和上一次比、配对表、清理 - main.py 的 lifespan:影子起草和评测的启动整理、关停(评测先于运行停,停时取消自己发起的运行) - 测试:test_eval_e2e(dual_check_shop + 剧本模型:按支付日 330 通过且 SQL 等价,按下单日 310 不通过、差在 time; 目录对照 1:0;和上一次比找到退步、归因做法变);通知事件、定时接口形状的老测试按新事件和新字段更新 --- backend/app/api/evals.py | 325 +++++++- backend/app/api/schedules.py | 138 +++- backend/app/api/settings.py | 90 ++- backend/app/core/artifact_store.py | 37 + backend/app/engine/judge.py | 34 +- backend/app/engine/nodes/dual_check.py | 20 +- backend/app/engine/tolerance.py | 45 ++ backend/app/eval/__init__.py | 11 +- backend/app/eval/answer.py | 451 +++++++++++ backend/app/eval/cleanup.py | 126 ++++ backend/app/eval/compare.py | 186 +++++ backend/app/eval/judging.py | 173 +++++ backend/app/eval/rescore.py | 74 ++ backend/app/eval/runner.py | 993 +++++++++++++++++++++++++ backend/app/eval/scheduled.py | 67 +- backend/app/eval/score.py | 612 +++++++++++++++ backend/app/eval/snapshots.py | 6 +- backend/app/eval/stats.py | 155 ++++ backend/app/eval/suites.py | 26 +- backend/app/eval/variants.py | 213 ++++++ backend/app/main.py | 15 + backend/app/notify/service.py | 40 +- backend/app/scheduler/service.py | 31 +- backend/tests/eval_samples.py | 156 ++++ backend/tests/test_eval_e2e.py | 122 +++ backend/tests/test_notify_events.py | 20 +- backend/tests/test_schedule_target.py | 13 +- backend/tests/test_schedules_api.py | 8 +- 28 files changed, 4111 insertions(+), 76 deletions(-) create mode 100644 backend/app/engine/tolerance.py create mode 100644 backend/app/eval/answer.py create mode 100644 backend/app/eval/cleanup.py create mode 100644 backend/app/eval/compare.py create mode 100644 backend/app/eval/judging.py create mode 100644 backend/app/eval/rescore.py create mode 100644 backend/app/eval/runner.py create mode 100644 backend/app/eval/score.py create mode 100644 backend/app/eval/stats.py create mode 100644 backend/app/eval/variants.py create mode 100644 backend/tests/eval_samples.py create mode 100644 backend/tests/test_eval_e2e.py diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index 2e3acc95..566ac513 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -1,4 +1,5 @@ -"""评测接口(docs/design/next-round/01-效果度量与回归评测.md):题集、题目、期望答案试算、目录快照;第 3 波加评测、结果、对比。 +"""评测接口(docs/design/next-round/01-效果度量与回归评测.md):题集、题目、期望答案试算、目录快照,评测的发起、进度、结果、 +对比、A/B、清理。 整组只给搭建者及以上(main.py 挂 require_role("builder")):期望值和查询原文只给搭建者。这一波没有界面(第 4 波), 接口的形状就是给界面的契约。请求体自己解析、自己校验,错误一律回中文 detail + 机器码(422 {detail, code, field}, @@ -15,6 +16,14 @@ 为什么不能);GET/POST /suites/{id}/cases/{cid}/upgrade 看、处置口径升版(recompute / dual / incomparable 三选一)。 - 目录快照:GET/POST /catalog-snapshots(当前目录)、POST /catalog-snapshots/shadow/estimate(影子起草的计划和估算)、 POST /catalog-snapshots/shadow(后台起草,202)、GET /catalog-snapshots/{id}、POST /catalog-snapshots/{id}/stop。 +- 评测(app/eval/runner.py,后台任务):POST /suites/{id}/evaluations/estimate(发起前的估算:几项、按上一次每题平均花费估的 + 金额、生效的预算、同时评几题);POST /suites/{id}/evaluations(发起,202;variant 一臂,或 arms 几臂的 A/B,或 + preset=catalog / catalog_reviewed「目录贡献」预设对照);GET /suites/{id}/evaluations(历次);GET /evaluations/{id}(进度、 + 总表、和上一次比的计数);GET /evaluations/{id}/results(逐题)、GET /evaluations/{id}/results/{rid}(一题的明细); + POST /evaluations/{id}/stop、/retry(只重做没做完的)、/rescore(按新评分规则重评,只读封存、不调模型)、/pin(钉住当基线); + GET /evaluations/{id}/compare?base=(和上一次或点名的那一次比:指纹差异、逐题转移和归因);GET /pairs/{pair_id}(A/B 配对表和 + 精确 McNemar);POST /suites/{id}/cleanup(按保留规则清理更早的明细)。「口径命中与重复度」的 E1–E4 直接用口径账本的 + GET /api/ledger/stats(总方案 X2:只有一份实现)。 """ from __future__ import annotations @@ -79,15 +88,23 @@ def _iso(moment: Any) -> str | None: return moment.isoformat() if moment is not None else None +async def _site(session: AsyncSession) -> dict[str, Any]: + """设置里 eval 一组(全站的预算缺省、保留几次、裁判的缺省开关)。""" + from app.api.settings import eval_settings + + return await eval_settings(session) + + # ========================================================================== # 形状 # ========================================================================== -def _suite_out(row: EvalSuite, *, counts: dict[str, int] | None = None) -> dict[str, Any]: +def _suite_out(row: EvalSuite, *, counts: dict[str, int] | None = None, site: dict[str, Any] | None = None + ) -> dict[str, Any]: return {"id": row.id, "name": row.name, "description": row.description, "timezone": row.timezone, "default_variant": dict(row.default_variant or {}), "budget": dict(row.budget or {}), - "effective_budget": effective_budget(row.budget), "archived": bool(row.archived), "version": row.version, + "effective_budget": effective_budget(row.budget, site), "archived": bool(row.archived), "version": row.version, "created_by": row.created_by, "created_at": _iso(row.created_at), "updated_at": _iso(row.updated_at), **({"counts": counts} if counts is not None else {})} @@ -212,6 +229,7 @@ async def list_suites(archived: bool = False, session: AsyncSession = Depends(ge if not archived: query = query.where(EvalSuite.archived.is_(False)) rows = list((await session.execute(query)).scalars()) + site = await _site(session) counts: dict[str, dict[str, int]] = {r.id: {"cases": 0, "enabled": 0} for r in rows} if rows: for suite_id, enabled, n in (await session.execute( @@ -221,7 +239,7 @@ async def list_suites(archived: bool = False, session: AsyncSession = Depends(ge counts[suite_id]["cases"] += n if enabled: counts[suite_id]["enabled"] += n - return {"suites": [_suite_out(r, counts=counts[r.id]) for r in rows]} + return {"suites": [_suite_out(r, counts=counts[r.id], site=site) for r in rows]} @router.post("/suites", status_code=201) @@ -235,7 +253,7 @@ async def create_suite(request: Request, session: AsyncSession = Depends(get_ses session.add(row) await session.commit() await session.refresh(row) - return _suite_out(row, counts={"cases": 0, "enabled": 0}) + return _suite_out(row, counts={"cases": 0, "enabled": 0}, site=await _site(session)) @router.get("/suites/{suite_id}") @@ -246,7 +264,8 @@ async def get_suite(suite_id: str, session: AsyncSession = Depends(get_session)) .order_by(EvalCase.seq))).scalars()) out = await _cases_out(session, rows) active = [c for c in out if not c["archived"]] - return {**_suite_out(suite, counts={"cases": len(active), "enabled": sum(1 for c in active if c["enabled"])}), + return {**_suite_out(suite, counts={"cases": len(active), "enabled": sum(1 for c in active if c["enabled"])}, + site=await _site(session)), "cases": out, "pending_upgrades": sum(1 for c in active if (c["upgrade"] or {}).get("pending"))} @@ -695,4 +714,298 @@ async def stop_snapshot(snapshot_id: str, session: AsyncSession = Depends(get_se return snapshots.snapshot_out(await _snapshot(session, snapshot_id)) +# ========================================================================== +# 评测:发起、进度、结果 +# ========================================================================== + +EVALUATION_NOT_FOUND = "eval_evaluation_not_found" +PAIR_NOT_FOUND = "eval_pair_not_found" +RESULT_NOT_FOUND = "eval_result_not_found" +CLEANED = "eval_details_cleaned" +#: 预设对照 +PRESETS = ("catalog", "catalog_reviewed") + + +def _refused(e: CaseInvalid) -> CodedHTTPException: + return CodedHTTPException(getattr(e, "status", 422), e.message, e.code, field=e.field) + + +async def _arms(session: AsyncSession, suite: EvalSuite, body: dict[str, Any]) -> list[dict[str, Any]]: + """请求体 → 规范化的几臂变体:{variant}(一臂,不写用题集的缺省变体)、{arms: [变体…]}(A/B)或 {preset: catalog | + catalog_reviewed}(目录贡献对照:当前目录 vs 关,可加只用已核实的项;模型照 variant 或题集的缺省)。A/B 缺省每题 3 次。""" + from app.eval import variants as variant_rules + + judge = bool((await _site(session))["judge"]) + preset = body.get("preset") + raw_arms = body.get("arms") + try: + if preset is not None: + if preset not in PRESETS: + raise CaseInvalid(f"预设对照只能是 {'、'.join(PRESETS)}", "preset", variant_rules.VARIANT_INVALID) + base = variant_rules.normalize_variant({**dict(suite.default_variant or {}), **dict(body.get("variant") or {})}, + judge_default=judge, repeats_default=variant_rules.PAIR_REPEATS) + return variant_rules.catalog_preset(base, reviewed=preset == "catalog_reviewed") + if raw_arms is not None: + if not isinstance(raw_arms, list) or not 2 <= len(raw_arms) <= variant_rules.ARMS_MAX: + raise CaseInvalid(f"A/B 要 2 到 {variant_rules.ARMS_MAX} 臂", "arms", variant_rules.VARIANT_INVALID) + arms = [variant_rules.normalize_variant(a, judge_default=judge, repeats_default=variant_rules.PAIR_REPEATS) + for a in raw_arms] + names = [variant_rules.arm_name(a, i) for i, a in enumerate(arms)] + if len(set(names)) != len(names): + raise CaseInvalid("几臂的名字(label)不能重复", "arms", variant_rules.VARIANT_INVALID) + return arms + return [variant_rules.normalize_variant({**dict(suite.default_variant or {}), **dict(body.get("variant") or {})}, + judge_default=judge)] + except CaseInvalid as e: + raise _invalid(e) from e + + +def _evaluation_out(row: EvalRun) -> dict[str, Any]: + from app.eval import runner, variants as variant_rules + + return {"id": row.id, "suite_id": row.suite_id, "status": row.status, "trigger": row.trigger, + "schedule_id": row.schedule_id, "variant": dict(row.variant or {}), + "resolved_models": dict(row.resolved_models or {}), + "variant_text": variant_rules.describe(row.variant or {}, row.resolved_models or {}), + "catalog_snapshot": row.catalog_snapshot, "as_of": _iso(row.as_of), "repeats": row.repeats, + "fingerprint": dict(row.fingerprint or {}), "pair_id": row.pair_id, "arm": row.arm, + "baseline_id": row.baseline_id, "pinned": bool(row.pinned), "budget": dict(row.budget or {}), + "counts": dict(row.counts or {}), "usage": dict(row.usage or {}), "summary": dict(row.summary or {}), + "stop_reason": row.stop_reason, "error": row.error, "created_by": row.created_by, + "created_at": _iso(row.created_at), "started_at": _iso(row.started_at), "finished_at": _iso(row.finished_at), + "cleaned_at": _iso(row.cleaned_at), "running": runner.running(row)} + + +async def _evaluation(session: AsyncSession, evaluation_id: str) -> EvalRun: + from app.eval import runner + + row = await session.get(EvalRun, evaluation_id) + if row is None: + raise CodedHTTPException(404, "评测不存在", EVALUATION_NOT_FOUND) + await runner.reconcile_stale(session, [row]) + return row + + +@router.post("/suites/{suite_id}/evaluations/estimate") +async def estimate_evaluation(suite_id: str, request: Request, session: AsyncSession = Depends(get_session) + ) -> dict[str, Any]: + """发起之前给人看:几臂、各臂实际解析出的模型、几项、估算的花费(按这个题集上一次评测的每题平均花费)、生效的预算、 + 同时评几题、今天评测已经花了多少。不调模型、不查库。""" + from app.core.config import settings + from app.engine.spend import eval_spend + from app.eval import runner, variants as variant_rules + from app.eval.suites import effective_budget + + suite = await _suite(session, suite_id) + arms = await _arms(session, suite, await _body(request)) + out = [] + try: + for i, arm in enumerate(arms): + models = await variant_rules.resolve_models(session, arm) + await variant_rules.snapshot_artifact(session, arm) + out.append({"label": variant_rules.arm_name(arm, i) if len(arms) > 1 else None, "variant": arm, + "models": models, "text": variant_rules.describe(arm, models)}) + except CaseInvalid as e: + raise _refused(e) from e + cases = len(await runner._cases(session, suite_id)) + repeats = max(int(a.get("repeats") or 1) for a in arms) + budget = effective_budget(suite.budget, await _site(session)) + parallel = max(1, min(int(budget["max_parallel"]), int(settings.max_concurrent_runs) // 4 // len(arms) or 1)) + return {"arms": out, "cases": cases, "repeats": repeats, + "estimate": await runner.estimate(session, suite_id, cases * repeats * len(arms)), "budget": budget, + "parallel": parallel, "spend": await eval_spend.daily()} + + +@router.post("/suites/{suite_id}/evaluations", status_code=202) +async def start_evaluation(suite_id: str, request: Request, session: AsyncSession = Depends(get_session), + x_actor: str | None = Header(default=None)) -> dict[str, Any]: + """发起一次评测(后台):{variant?} 一臂,或 {arms: [变体…]} / {preset} 几臂的 A/B(共用期望、同一题同时开跑),可带 + baseline_id 点名和哪一次比。交回 {evaluations: [每一臂], pair_id};进度按 GET /evaluations/{id} 读。""" + from app.eval import runner + + suite = await _suite(session, suite_id) + body = await _body(request) + arms = await _arms(session, suite, body) + baseline = body.get("baseline_id") + if baseline is not None and not isinstance(baseline, str): + raise CodedHTTPException(422, "基线需要是一次评测的 id", runner.EVAL_INVALID, field="baseline_id") + try: + runs = await runner.start(session, suite, arms, actor=actor_of(x_actor), baseline_id=baseline or None) + except CaseInvalid as e: + raise _refused(e) from e + return {"evaluations": [_evaluation_out(r) for r in runs], "pair_id": runs[0].pair_id} + + +@router.get("/suites/{suite_id}/evaluations") +async def list_evaluations(suite_id: str, limit: int = 50, session: AsyncSession = Depends(get_session) + ) -> dict[str, Any]: + """这个题集的历次评测(新的在前;明细清理了的总表还在)。""" + from app.eval import runner + + await _suite(session, suite_id) + rows = list((await session.execute(select(EvalRun).where(EvalRun.suite_id == suite_id) + .order_by(EvalRun.created_at.desc()).limit(max(1, min(limit, 200))))).scalars()) + await runner.reconcile_stale(session, rows) + return {"evaluations": [_evaluation_out(r) for r in rows]} + + +@router.get("/evaluations/{evaluation_id}") +async def get_evaluation(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """一次评测:进度(counts、正在评的几题在做什么)、总表(summary,结束后才有)、和上一次比的计数(summary.baseline)。""" + from app.db.models_eval import EvalResult + + row = await _evaluation(session, evaluation_id) + running = (await session.execute(select(EvalResult.case_seq, EvalResult.repeat, EvalResult.arm, EvalResult.current) + .where(EvalResult.eval_run_id == row.id, EvalResult.status == "running") + .order_by(EvalResult.seq))).all() + return {**_evaluation_out(row), + "current": [{"case_seq": seq, "repeat": rep, "arm": arm, "doing": doing} for seq, rep, arm, doing in running]} + + +@router.get("/evaluations/{evaluation_id}/results") +async def evaluation_results(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """逐题结果:结论、各期望的结论(期望值和答案里对上的数、有没有引用、维度)、作答路径、复核、出具档位、SQL 差在哪、look、 + 耗时、按角色的用量,点 run_id 去运行详情。明细清理了的照实写 cleaned。""" + from app.eval import runner + + row = await _evaluation(session, evaluation_id) + results = [runner.result_dict(r) for r in (await session.execute( + select(runner.EvalResult).where(runner.EvalResult.eval_run_id == row.id) + .order_by(runner.EvalResult.seq))).scalars()] + questions = dict((await session.execute(select(EvalCase.id, EvalCase.question).where( + EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).all()) if results else {} + for r in results: + r["question"] = questions.get(r["case_id"]) + return {"evaluation_id": row.id, "cleaned": row.cleaned_at is not None, "results": results} + + +@router.get("/evaluations/{evaluation_id}/results/{result_id}") +async def evaluation_result(evaluation_id: str, result_id: str, session: AsyncSession = Depends(get_session) + ) -> dict[str, Any]: + """一题的明细:结果的列,加上明细工件(这一轮作答的经过、读出来的答案、裁判的判断、评分的原样)。""" + from app.core import artifact_store + from app.eval import runner + + row = await _evaluation(session, evaluation_id) + result = await session.get(runner.EvalResult, result_id) + if result is None or result.eval_run_id != row.id: + raise CodedHTTPException(404, "这一题的结果不存在(明细可能已清理)", RESULT_NOT_FOUND) + details = None + if result.details_artifact: + try: + details = artifact_store.load(result.details_artifact) + except ValueError: + details = None + return {**runner.result_dict(result), "details": details} + + +@router.post("/evaluations/{evaluation_id}/stop") +async def stop_evaluation(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """停止(A/B 几臂一起停)。已经结束的回 409。正在跑的作答、算期望的运行一并取消。""" + from app.eval import runner + + await _evaluation(session, evaluation_id) + await session.commit() + if not await runner.stop(evaluation_id): + raise CodedHTTPException(409, "这次评测已经结束了", runner.EVAL_FINISHED) + session.expire_all() + return _evaluation_out(await _evaluation(session, evaluation_id)) + + +@router.post("/evaluations/{evaluation_id}/retry", status_code=202) +async def retry_evaluation(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """只把没做完的题接着做(停止、中断、出错、到了上限的那几题;A/B 时同一题几臂一起重做)。""" + from app.eval import runner + + await _evaluation(session, evaluation_id) + await session.commit() + try: + ids = await runner.retry(evaluation_id) + except CaseInvalid as e: + raise _refused(e) from e + session.expire_all() + return {"evaluations": [_evaluation_out(await _evaluation(session, i)) for i in ids]} + + +@router.post("/evaluations/{evaluation_id}/rescore") +async def rescore_evaluation(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """按现在的评分规则(score.SCORER_VERSION)重新评分:只读封存的作答运行和存下的期望、裁判判断,不调模型、不查业务库。 + 总表跟着重算。还在跑、明细已清理的不能重评。""" + from app.eval import rescore, runner + + row = await _evaluation(session, evaluation_id) + if row.status in ("queued", "running"): + raise CodedHTTPException(409, "这次评测还没结束,结束后再重新评分", runner.EVAL_BUSY) + if row.cleaned_at is not None: + raise CodedHTTPException(409, "这次评测的明细已清理,不能重新评分", CLEANED) + await session.commit() + changed = await rescore.rescore(evaluation_id) + session.expire_all() + return {**_evaluation_out(await _evaluation(session, evaluation_id)), "changed": changed} + + +@router.post("/evaluations/{evaluation_id}/pin") +async def pin_evaluation(evaluation_id: str, request: Request, session: AsyncSession = Depends(get_session) + ) -> dict[str, Any]: + """钉住当基线({pinned: true})或取消(false):和上一次比时优先用同一变体里钉住的那一次,清理时也留着它。""" + row = await _evaluation(session, evaluation_id) + body = await _body(request) + pinned = body.get("pinned") + if not isinstance(pinned, bool): + raise CodedHTTPException(422, "pinned 需要是 true 或 false", case_rules.CASE_INVALID, field="pinned") + if pinned and row.status not in ("succeeded", "partial"): + raise CodedHTTPException(409, "只有评完了(或部分评完)的评测能钉成基线", case_rules.CASE_INVALID, field="pinned") + row.pinned = pinned + await session.commit() + await session.refresh(row) + return _evaluation_out(row) + + +@router.get("/evaluations/{evaluation_id}/compare") +async def compare_evaluation(evaluation_id: str, base: str | None = None, session: AsyncSession = Depends(get_session) + ) -> dict[str, Any]: + """和上一次比:base 不给就按规则挑(同一变体钉住的基线,或比这次先发起、已经结束的最近一次)。{base, fingerprint_diff, + transitions: {counts, causes, items}};没有可比的 base 为 null。任何一边的明细已清理,逐题部分为空、照实写 cleaned。""" + from app.eval import compare, runner + + row = await _evaluation(session, evaluation_id) + picked = await compare.pick_baseline(session, row, explicit=base or row.baseline_id) + if picked is None: + return {"base": None, "fingerprint_diff": [], "transitions": None, "cleaned": False} + cleaned = row.cleaned_at is not None or picked.cleaned_at is not None + moved = None if cleaned else compare.transitions(await runner.result_rows(session, picked.id), + await runner.result_rows(session, row.id)) + return {"base": _evaluation_out(picked), + "fingerprint_diff": compare.fingerprint_diff(picked.fingerprint or {}, row.fingerprint or {}), + "transitions": moved, "cleaned": cleaned} + + +@router.get("/pairs/{pair_id}") +async def get_pair(pair_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """一次 A/B:几臂各自的准确率、两两配对表(都对 / 都错 / 只 A 对 / 只 B 对)和精确 McNemar 双侧 p 值;p ≥ 0.05 写 + 「这批题上看不出差别」。几臂共用期望答案。""" + from app.eval import compare, runner + + rows = list((await session.execute(select(EvalRun).where(EvalRun.pair_id == pair_id) + .order_by(EvalRun.created_at))).scalars()) + if not rows: + raise CodedHTTPException(404, "这次 A/B 不存在", PAIR_NOT_FOUND) + await runner.reconcile_stale(session, rows) + cleaned = any(r.cleaned_at is not None for r in rows) + table = None if cleaned else compare.pair_table( + [(r.arm or r.id, await runner.result_rows(session, r.id)) for r in rows]) + return {"pair_id": pair_id, "evaluations": [_evaluation_out(r) for r in rows], "table": table, "cleaned": cleaned} + + +@router.post("/suites/{suite_id}/cleanup") +async def cleanup_suite(suite_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """按保留规则清理这个题集更早的评测明细(最近 keep_runs 次、钉住的基线、A/B 的另几臂留着;总表永久保留)。""" + from app.eval import cleanup + + await _suite(session, suite_id) + await session.commit() + return await cleanup.clean_suite(suite_id) + + __all__ = ["router"] + diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index d7f5b0c3..fca3934f 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -1,6 +1,13 @@ """定时任务:按周期自动发起工作流运行,结束、失败、停在审批时推送 Webhook,运行成功结束后和同一任务上一次成功的运行 自动对比。 +**目标种类**(docs/design/next-round/00-总方案.md X4:定时只有一套):target_kind 是 workflow(缺省,字段和行为一字不变)、 +eval_suite(跑一次评测题集)或 source_maintenance(维护一个数据源)。非工作流目标写 target_id,不写 workflow_id、输入模板、 +运行类别和 Webhook:核对目标交给各自的模块(scheduler/service.check_target → app/eval/scheduled、app/data/source_maintenance); +它们的通知走通知渠道的订阅(评测有退步、评测失败……),不走定时任务的 Webhook——Webhook 的三种事件(运行结束、运行失败、 +等待审批)说的是这一次发起的那次运行,非工作流目标没有那一次运行,配上了也收不到,所以照实拒收。接口返回带 target_kind、 +target_id、target_name(目标现在的名字,不在了用保存时记下的那一份)。 + 形状按前后端约定(Schedule、Fire 两种对象,见 _schedule_out、_fire_outs)。Webhook 地址和签名密钥加密存放,接口 只给主机和末 4 位(url_hint)、有没有密钥(has_secret)。请求体自己解析、自己校验:错误一律回中文 detail,表单 据 field 把报错落到那一格(422 {detail, code, field},同 api/coded.py)。后台的调度循环在 app/scheduler 里。 @@ -24,9 +31,12 @@ from app.scheduler import auto_compare, cron, service, webhook router = APIRouter(prefix="/api/schedules", tags=["schedules"]) -#: 请求不合规(422)的机器码。field 是出错的那一项:name、workflow_id、cron、timezone、run_class、input_template、 -#: enabled、overlap、misfire_grace_s、webhook、version、limit、before +#: 请求不合规(422)的机器码。field 是出错的那一项:name、target_kind、target_id、workflow_id、cron、timezone、run_class、 +#: input_template、enabled、overlap、misfire_grace_s、webhook、version、limit、before SCHEDULE_INVALID = "schedule_invalid" +#: 非工作流目标配了 Webhook(见模块说明) +NO_WEBHOOK = ("目标是「{kind}」的定时任务不推 Webhook:Webhook 的事件说的是定时发起的那次运行。要收这类定时的通知," + "请在「设置 → 通知」的渠道里订阅对应的事件") _GONE = "定时任务不存在,可能已被删除" #: 错过的触发在多少秒内补发:默认 1 小时,最长 7 天 DEFAULT_GRACE_S = 3600 @@ -96,6 +106,35 @@ def _workflow_id(value: Any) -> str: return value.strip() +def _target_kind(value: Any) -> str: + if value not in service.TARGET_KINDS: + labels = "、".join(service.TARGET_LABEL[k] for k in service.TARGET_KINDS) + raise _invalid(f"定时任务的目标需要是{labels}之一{_current(value)}", "target_kind") + return value + + +def _target_id(value: Any, kind: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise _invalid(f"请选择要定时的{service.TARGET_LABEL[kind]}", "target_id") + return value.strip() + + +def _target_extras(body: dict[str, Any], kind: str) -> None: + """非工作流目标不写工作流、输入模板、Webhook(见模块说明);运行类别带了也只认合法的写法,不存。""" + label = service.TARGET_LABEL[kind] + if not _blank(body.get("workflow_id")): + raise _invalid(f"目标是「{label}」的定时任务不指定工作流", "workflow_id") + if body.get("input_template") not in (None, {}): + raise _invalid(f"目标是「{label}」的定时任务没有输入模板", "input_template") + hook = body.get("webhook") + if isinstance(hook, dict) and not _blank(hook.get("url")): + raise _invalid(NO_WEBHOOK.format(kind=label), "webhook") + if hook is not None and not isinstance(hook, dict): + raise _invalid("Webhook 设置需要是一个对象,或者为空表示不推送", "webhook") + if "run_class" in body: + _run_class(body["run_class"]) + + def _cron(value: Any) -> str: try: return cron.parse(value if isinstance(value, str) else "").expr @@ -348,6 +387,8 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d "status": "started" if f.id in launching else f.status, "run_id": f.run_id, "run_status": current.get(f.run_id, (None, None))[0] if f.run_id else None, + # 目标是评测题集时这一次触发建的那次评测(没有运行 id) + "eval_run_id": f.eval_run_id, "reason": None if f.id in launching else f.reason, "notified": notified.get(f.id, []), "compare": compare_of(f), @@ -358,22 +399,35 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d async def _schedule_outs(session: AsyncSession, rows: list[Schedule]) -> list[dict[str, Any]]: ids = [r.id for r in rows] names: dict[str, str] = {} + targets: dict[str, str | None] = {} last: dict[str, dict[str, Any]] = {} if rows: names = dict((await session.execute( - select(Workflow.id, Workflow.name).where(Workflow.id.in_({r.workflow_id for r in rows})))).all()) + select(Workflow.id, Workflow.name).where(Workflow.id.in_({r.workflow_id for r in rows if r.workflow_id})))).all()) + for r in rows: + kind, target_id = service.target_of(r) + if kind != "workflow": + targets[r.id] = await service.target_name(session, kind, target_id) newest = (select(ScheduleFire.schedule_id, func.max(ScheduleFire.created_at).label("at")) .where(ScheduleFire.schedule_id.in_(ids)).group_by(ScheduleFire.schedule_id).subquery()) fires = list((await session.execute(select(ScheduleFire).join(newest, and_( ScheduleFire.schedule_id == newest.c.schedule_id, ScheduleFire.created_at == newest.c.at)))).scalars()) for out, fire in zip(await _fire_outs(session, fires), fires): last.setdefault(fire.schedule_id, out) + def target(r: Schedule) -> dict[str, Any]: + kind, target_id = service.target_of(r) + name = names.get(r.workflow_id) if kind == "workflow" else targets.get(r.id) + # 目标还在就给它现在的名字;删掉之后用保存时记下的那一份 + return {"target_kind": kind, "target_id": target_id, "target_label": service.TARGET_LABEL.get(kind, kind), + "target_name": name or r.workflow_name, "target_exists": name is not None} + return [{ "id": r.id, "name": r.name, + **target(r), "workflow_id": r.workflow_id, - # 工作流还在就给它现在的名字;删掉之后用保存时记下的那一份 - "workflow_name": names.get(r.workflow_id) or r.workflow_name, + # 工作流还在就给它现在的名字;删掉之后用保存时记下的那一份。非工作流目标这一项是目标的名字(同 target_name) + "workflow_name": names.get(r.workflow_id) or targets.get(r.id) or r.workflow_name, "cron": r.cron, "timezone": r.timezone, "run_class": r.run_class, @@ -467,7 +521,18 @@ async def create_schedule(request: Request, session: AsyncSession = Depends(get_ def schedule_values(body: dict[str, Any]) -> dict[str, Any]: """建定时任务的请求体 → 规范化之后的列(不含工作流名、下一次触发),逐项校验,不对抛 422(带 field)。只读, - 不碰库:发布单的预检用它「只检查」,建任务的接口和发布单建任务都先过它。""" + 不碰库:发布单的预检用它「只检查」,建任务的接口和发布单建任务都先过它。 + + target_kind 缺省 workflow:交回的键和以前一字不差。非工作流目标交回 target_kind、target_id,运行类别存探索(不用它)、 + 输入模板为空、没有 Webhook。""" + kind = _target_kind(body.get("target_kind", "workflow")) + if kind != "workflow": + _target_extras(body, kind) + return {"name": _name(body.get("name")), "target_kind": kind, "target_id": _target_id(body.get("target_id"), kind), + "cron": _cron(body.get("cron")), "timezone": _timezone(body.get("timezone")), + "run_class": "exploratory", "input_template": {}, "enabled": _enabled(body.get("enabled", True)), + "overlap": _overlap(body.get("overlap", "skip")), + "misfire_grace_s": _grace(body.get("misfire_grace_s", DEFAULT_GRACE_S)), "webhook": None} values = {"name": _name(body.get("name")), "workflow_id": _workflow_id(body.get("workflow_id")), "cron": _cron(body.get("cron")), "timezone": _timezone(body.get("timezone")), "run_class": _run_class(body.get("run_class", "formal")), @@ -486,10 +551,29 @@ def template_problem(template: dict[str, str], graph: Any, workflow_name: str) - return None +async def _check_target(session: AsyncSession, kind: str, target_id: str) -> str: + """非工作流目标:交给它的模块核对(存在、可用),交回目标现在的名字。不对抛 422(field=target_id)。""" + if problem := await service.check_target(session, kind, target_id): + raise _invalid(problem, "target_id") + return await service.target_name(session, kind, target_id) or "" + + async def create_schedule_row(session: AsyncSession, body: dict[str, Any], *, actor: str | None) -> Schedule: """校验并建一个定时任务(提交)。建任务的接口和发布单(停用着建,应用发布后再启用)共用。""" values = schedule_values(body) hook = values.pop("webhook") + if values.get("target_kind", "workflow") != "workflow": + kind, target_id = values.pop("target_kind"), values.pop("target_id") + name = await _check_target(session, kind, target_id) + # 目标的几列只经 scheduler/service 的一处封装写(非工作流目标:workflow_id 存空串);workflow_name 记保存时的目标名 + values |= service.target_columns(kind, target_id) + row = Schedule(**values, workflow_name=name, + next_run_at=service.next_run(values["cron"], values["timezone"], values["enabled"]), + created_by=actor, **_webhook_columns(None)) + session.add(row) + await session.commit() + await session.refresh(row) + return row workflow = await _check_workflow(session, values["workflow_id"], values["run_class"], values["input_template"]) # 目标的几列只经 scheduler/service 的一处封装写(工作流目标:target_id 同 workflow_id) values |= service.target_columns("workflow", values["workflow_id"]) @@ -527,6 +611,11 @@ async def update_schedule(schedule_id: str, request: Request, raise HTTPException(409, f"这个定时任务已被修改(当前是第 {row.version} 版,提交的是第 {version} 版)," "请刷新后再保存") values: dict[str, Any] = {} + kind, current_target = service.target_of(row) + if "target_kind" in body and _target_kind(body["target_kind"]) != kind: + raise _invalid("定时任务的目标种类建好之后不能改:请新建一个定时任务", "target_kind") + if kind != "workflow": + return await _update_target_schedule(session, row, body, kind, current_target, version) if "name" in body: values["name"] = _name(body["name"]) if "workflow_id" in body: @@ -576,6 +665,40 @@ async def update_schedule(schedule_id: str, request: Request, return await _schedule_out(session, await _get(session, schedule_id)) +async def _update_target_schedule(session: AsyncSession, row: Schedule, body: dict[str, Any], kind: str, + current_target: str, version: int) -> dict[str, Any]: + """改一个非工作流目标的定时任务:名称、目标(同一种)、cron、时区、启用、重叠、补发宽限。工作流、输入模板、运行类别、 + Webhook 不适用(带了照实拒;整份表单带回来的空值不算)。目标真改了才重新核对。""" + probe = {k: body[k] for k in ("workflow_id", "input_template", "webhook", "run_class") if k in body} + if probe.get("webhook") is None: + probe.pop("webhook", None) + _target_extras(probe, kind) + values: dict[str, Any] = {} + if "name" in body: + values["name"] = _name(body["name"]) + if "target_id" in body: + target_id = _target_id(body["target_id"], kind) + if target_id != current_target: + values["workflow_name"] = await _check_target(session, kind, target_id) + values |= service.target_columns(kind, target_id) + for key, check in (("cron", _cron), ("timezone", _timezone), ("enabled", _enabled), ("overlap", _overlap), + ("misfire_grace_s", _grace)): + if key in body: + values[key] = check(body[key]) + if any(k in values and values[k] != getattr(row, k) for k in ("cron", "timezone", "enabled")): + values["next_run_at"] = _next_after_edit(row, values.get("cron", row.cron), values.get("timezone", row.timezone), + values.get("enabled", row.enabled)) + result = await session.execute( + update(Schedule).where(Schedule.id == row.id, Schedule.version == version) + .values(**values, version=version + 1).execution_options(synchronize_session=False)) + if result.rowcount != 1: + await session.rollback() + raise HTTPException(409, "这个定时任务刚被别人修改过,请刷新后再保存") + await session.commit() + session.expire_all() + return await _schedule_out(session, await _get(session, row.id)) + + async def retime(session: AsyncSession, row: Schedule, body: dict[str, Any]) -> Schedule: """按新的设置改一个已有的定时任务(提交):名称、cron、时区、输入模板,校验同建任务;启用状态不动。cron、时区真改了才 按 PATCH 同一个规则重算下一次触发。发布单给已经发布过的蓝图再发一版时用它沿用原来的任务,不另建一个(两个任务会 @@ -644,7 +767,8 @@ async def delete_schedule(schedule_id: str, session: AsyncSession = Depends(get_ async def run_now(schedule_id: str, session: AsyncSession = Depends(get_session), x_actor: str | None = Header(default=None)) -> dict[str, Any]: """立即触发一次(kind=manual),停用的任务也可以。规则和到点触发相同:上一次运行没结束就跳过,正式运行只从当前 - 发布版本发起。运行记在点按钮的人名下,没有署名时记在「定时任务:<名字>」名下。返回 {fire, run_id}。 + 发布版本发起。运行记在点按钮的人名下,没有署名时记在「定时任务:<名字>」名下。返回 {fire, run_id};目标是评测题集的, + 发起的是一次评测,fire.eval_run_id 是它,run_id 为空。 触发时间取到秒:连点两下落在同一秒的,交回同一条触发记录,不会发起两次。""" await _get(session, schedule_id) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index c45a685a..5d7bfc50 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -718,6 +718,76 @@ async def apps_settings(session: AsyncSession) -> dict[str, Any]: return apps_values(row.value if row else None) +# -------------------------------------------------------------------------- +# 评测:并发、每题时限、单次和每日金额上限、保留几次、结论裁判的缺省(docs/design/next-round/01-效果度量与回归评测.md 做法 17) +# -------------------------------------------------------------------------- + +#: 设置里 eval 这一组(评测-D9 拍板的缺省:同时 2 题、每题 900 秒、单次评测 2 美元、评测每日 5 美元)。前四项是全站的预算 +#: 缺省,题集上改过的项盖过它(app/eval/suites.effective_budget);评测每日的金额上限是评测那本账(engine/spend.eval_spend, +#: 单独记账,不挤占裁判、核查的额度)的上限,题集只能往下收。keep_runs:每个题集保留最近几次评测的明细(钉住的基线、A/B +#: 引用的那一对另外留着,总表永久保留,app/eval/cleanup.py)。judge:发起评测时变体里没写的话,结论裁判开不开(缺省关: +#: 裁判按句数花钱)。搭建者只读,管理员写(整个设置接口的规矩) +EVAL_GROUP = "eval" +EVAL_SPEND = "eval_spend" +EVAL_DEFAULTS: dict[str, Any] = {"max_parallel": 2, "per_case_timeout_s": 900, "per_run_max_usd": 2.0, + "daily_max_usd": 5.0, "keep_runs": 20, "judge": False} +EVAL_SETTING_LABEL = {"max_parallel": "评测 · 同时评几道题", "per_case_timeout_s": "评测 · 每题时限(秒)", + "per_run_max_usd": "评测 · 单次评测的金额上限(美元)", "daily_max_usd": "评测 · 每日金额上限(美元)", + "keep_runs": "评测 · 每个题集保留最近几次的明细", "judge": "评测 · 缺省打开结论裁判"} +#: 保留的次数:至少 1(最近一次总要能和下一次比),最多 1000 +EVAL_KEEP_MAX = 1000 + + +def _eval_item_problem(key: str, item: Any) -> str | None: + from app.eval.suites import BUDGET_RULES + + label = EVAL_SETTING_LABEL[key] + if key == "judge": + return None if isinstance(item, bool) else f"「{label}」只能是开或关,当前为「{item}」" + if key == "keep_runs": + ok = isinstance(item, int) and not isinstance(item, bool) and 1 <= item <= EVAL_KEEP_MAX + return None if ok else f"「{label}」需要是 1 到 {EVAL_KEEP_MAX} 之间的整数,当前为「{item}」" + _, kind, low, high = BUDGET_RULES[key] + ok = not isinstance(item, bool) and isinstance(item, (int, float)) and math.isfinite(item) \ + and (kind is float or isinstance(item, int)) and low <= item <= high + unit = "整数" if kind is int else "数" + return None if ok else f"「{label}」需要是 {low:g} 到 {high:g} 之间的{unit},当前为「{item}」" + + +def eval_problem(value: Any) -> str | None: + """设置页写进来的 eval 这一组有什么问题,没问题返回 None。""" + if not isinstance(value, dict): + return f"「评测」设置格式有误,需要是一个对象,当前为「{value}」" + for key, item in value.items(): + if key not in EVAL_DEFAULTS: + return (f"「评测」设置中不支持「{key}」,可设置的项:" + + "、".join(f"「{EVAL_SETTING_LABEL[k]}」" for k in EVAL_DEFAULTS)) + if problem := _eval_item_problem(key, item): + return problem + merged = eval_values(value) + if merged["per_run_max_usd"] > merged["daily_max_usd"]: + return (f"「{EVAL_SETTING_LABEL['per_run_max_usd']}」不能高于「{EVAL_SETTING_LABEL['daily_max_usd']}」," + f"当前为 {merged['per_run_max_usd']:g} 和 {merged['daily_max_usd']:g}") + return None + + +def eval_values(stored: Any) -> dict[str, Any]: + """库里存的(或刚写进来的)eval 一组 → 补齐默认;存坏了的项按默认。""" + stored = stored if isinstance(stored, dict) else {} + out = dict(EVAL_DEFAULTS) + for key in EVAL_DEFAULTS: + if key in stored and _eval_item_problem(key, stored[key]) is None: + value = stored[key] + out[key] = float(value) if key in ("per_run_max_usd", "daily_max_usd") else value + return out + + +async def eval_settings(session: AsyncSession) -> dict[str, Any]: + """设置里的 eval 一组,没存过的补默认。发起评测、清理评测从这里读。""" + row = await session.get(Setting, EVAL_GROUP) + return eval_values(row.value if row else None) + + # -------------------------------------------------------------------------- # 用户设置 # -------------------------------------------------------------------------- @@ -763,6 +833,8 @@ DEFAULT_SETTINGS: dict[str, Any] = { CATALOG_DRAFT_GROUP: catalog_draft_values(None), # 应用:按需视图的每日上限、使用者能不能建个人应用(管理员在设置页改) APPS_GROUP: apps_values(None), + # 评测:预算的全站缺省(题集上可以改)、保留几次明细、结论裁判的缺省开关 + EVAL_GROUP: eval_values(None), } @@ -815,6 +887,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str # 裁判、核查的每日计数是引擎记的账,不是用户设置:设置页整组回写时带着一份旧的,会把今天的花费清零 stored.pop(JUDGE_SPEND, None) stored.pop(VERIFY_SPEND, None) + stored.pop(EVAL_SPEND, None) for key in _hidden_groups(): stored.pop(key, None) merged = {k: {**v, **(stored.get(k) or {})} for k, v in DEFAULT_SETTINGS.items()} @@ -825,6 +898,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str merged[PUBLIC_URL_GROUP] = public_url_values(stored.get(NOTIFY_GROUP)) merged[CATALOG_DRAFT_GROUP] = catalog_draft_values(stored.get(CATALOG_DRAFT_GROUP)) merged[APPS_GROUP] = apps_values(stored.get(APPS_GROUP)) + merged[EVAL_GROUP] = eval_values(stored.get(EVAL_GROUP)) for key, value in stored.items(): merged.setdefault(key, value) return merged @@ -924,10 +998,14 @@ async def put_settings( if problem := apps_problem(payload.values[APPS_GROUP]): raise HTTPException(422, problem) payload.values[APPS_GROUP] = apps_values(payload.values[APPS_GROUP]) + if EVAL_GROUP in payload.values: + if problem := eval_problem(payload.values[EVAL_GROUP]): + raise HTTPException(422, problem) + payload.values[EVAL_GROUP] = eval_values(payload.values[EVAL_GROUP]) if found := await _provider_problem(session, payload.values): raise CodedHTTPException(422, found[1], PROVIDER_INVALID, field=found[0]) for key, value in payload.values.items(): - if key in (TRUST_KEY, "limits", PUBLIC_URL_GROUP, JUDGE_SPEND, VERIFY_SPEND, *_hidden_groups()): + if key in (TRUST_KEY, "limits", PUBLIC_URL_GROUP, JUDGE_SPEND, VERIFY_SPEND, EVAL_SPEND, *_hidden_groups()): # 信任三档只走 PUT /api/tools/trust:设置页整组回写时带着一份旧的, # 会把刚在工具页改的档位冲掉。limits 是环境变量定的、public_url 是算出来的,写进库也不生效。 # 裁判、核查的每日计数是引擎记的账,只有引擎自己写。登录设置只经 /api/auth/settings 改(_hidden_groups) @@ -963,6 +1041,16 @@ async def get_verify_spend(session: AsyncSession = Depends(get_session)) -> dict return {**await verify_daily_spend(), "daily_max_usd": (await verify_settings(session))["daily_max_usd"]} +@router.get("/settings/eval/spend") +async def get_eval_spend(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """今天(本地日期)评测花了多少(评测那本账,engine/spend.eval_spend),放在评测的每日上限旁边。只读:计数只有评测 + 自己记。形状同 /settings/judge/spend:{date, usd, calls, unpriced_calls, daily_max_usd}。calls 是今天开评的题数(每题 + 开评前预留一次)。""" + from app.engine.spend import eval_spend + + return {**await eval_spend.daily(), "daily_max_usd": (await eval_settings(session))["daily_max_usd"]} + + @router.get("/system") async def system_info() -> dict[str, Any]: """给设置页顶部展示的运行环境概览。""" diff --git a/backend/app/core/artifact_store.py b/backend/app/core/artifact_store.py index 27da1f1b..0bbc8d7f 100644 --- a/backend/app/core/artifact_store.py +++ b/backend/app/core/artifact_store.py @@ -115,6 +115,43 @@ def read_bytes(artifact_id: str, *, verify: bool = True) -> bytes | None: return data +async def forget(ids: Any) -> list[str]: + """回收工件文件:这几个 id 里,引用表(artifacts)里**一条引用都不剩**的,删掉文件;还有引用的一个不动。返回删掉了 + 哪几个(docs/design/next-round/01-效果度量与回归评测.md 做法 16)。 + + 为什么按引用数、不按调用方说的:工件按内容寻址,同样的内容只存一份——评测的一次运行和用户的一次运行查到同一份结果、 + 冻结了同一份表结构,文件就是同一个。调用方先删自己那几行引用(删运行时它名下的引用行),再把这些 id 交给这里;别的 + 运行、目录快照、对比结果还引用着的文件照留。put_json 每写一个工件都记一行引用,所以「没有引用」就是没人要了。 + + 先查引用再删文件,两步之间同一份内容又被写一次(put_json 先写文件、后记引用)的话会删掉一个刚有人要的文件:单进程部署, + 回收在评测清理的后台任务里串行做,和写入撞上的窗口极小;真撞上了,读的一方照实报「工件不在」(load 返回 None),不会 + 读到错的内容。""" + from sqlalchemy import select + + from app.db.base import SessionLocal + from app.db.models import Artifact + + wanted = sorted({i for i in ids or () if isinstance(i, str) and i and i.isalnum()}) + if not wanted: + return [] + async with SessionLocal() as session: + kept: set[str] = set() + for start in range(0, len(wanted), 500): + chunk = wanted[start:start + 500] + kept |= set((await session.execute(select(Artifact.id).where(Artifact.id.in_(chunk)))).scalars()) + removed: list[str] = [] + for artifact_id in wanted: + if artifact_id in kept: + continue + path = _path_of(artifact_id) + try: + path.unlink() + except FileNotFoundError: + continue + removed.append(artifact_id) + return removed + + def manifest_hash(rows: list[tuple[int, str, str | None, dict[str, Any]]]) -> str: """把一次运行的全部事件压成一个清单哈希。 diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index 51a73cb5..c34e943b 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -35,7 +35,9 @@ import logging import math import re import time -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import asdict, dataclass from typing import Any @@ -1040,6 +1042,27 @@ async def daily_spend() -> dict[str, Any]: return await _LEDGER.daily() +#: 这一次裁判记在哪本每日账上、按哪个每日上限:缺省(None)是裁判自己那本和裁判设置的每日上限。评测打开结论裁判时 +#: 走的是按需裁判同一条路(app/eval/judging.py),花费却要记在评测那本账上(细案 01 做法 11、17:评测单独记账)—— +#: 记在裁判这本上,评测一跑,当天别的报告就因为到了上限而降档。上下文变量,只管这一段调用 +_CHARGE_TO: ContextVar[tuple[spend.Ledger, float | None] | None] = ContextVar("judge_charge_to", default=None) + + +@contextmanager +def charged_to(ledger: spend.Ledger, daily_max_usd: float | None) -> Iterator[None]: + """这一段里的裁判调用记在 ledger 上、每日上限按 daily_max_usd(None 是不限)。""" + token = _CHARGE_TO.set((ledger, daily_max_usd)) + try: + yield + finally: + _CHARGE_TO.reset(token) + + +def _charging(budget: Budget) -> tuple[spend.Ledger, float | None]: + """这一次记哪本账、按哪个每日上限(见 _CHARGE_TO)。""" + return _CHARGE_TO.get() or (_LEDGER, budget.daily_max_usd) + + # -------------------------------------------------------------------------- # 调用 # -------------------------------------------------------------------------- @@ -1245,6 +1268,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, spent = 0.0 width = 1 cut_limit: int | None = None # 被截断的那次调用的额度:judge_truncated 日志照它写 + ledger, daily_cap = _charging(budget) while pending: remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started) if remaining is not None and remaining <= 0: @@ -1260,8 +1284,8 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, caps = [] if budget.max_cost_usd is not None: caps.append((budget.max_cost_usd - spent - reserved, "max_cost_usd")) - if budget.daily_max_usd is not None: - caps.append((budget.daily_max_usd - (await daily_spend())["usd"], "daily_max_usd")) + if daily_cap is not None: + caps.append((daily_cap - (await ledger.daily())["usd"], "daily_max_usd")) if caps: cap, binding = min(caps, key=lambda x: x[0]) trimmed = False @@ -1277,7 +1301,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, pending = pending[:len(batch)] pending = pending[len(batch):] charged = cost or 0.0 - day = await _LEDGER.charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) + day = await ledger.charge(charged, limit=daily_cap if priced else None, priced=priced) if day is None: # 估算之后、记账之前别的裁判先花掉了今天剩下的钱 stop([*batch, *pending], "daily_max_usd") pending = [] @@ -1312,7 +1336,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, inp, out = sum(t["input_tokens"] for t in tokens), sum(t["output_tokens"] for t in tokens) # 回复没带用量(有的网关不给、调用超时)就按估算记:宁可多记,不能让每日上限漏掉 actual = (pricing.estimate_cost(model_id, inp, out) if inp or out else charged) if priced else 0.0 - await _LEDGER.settle(day, actual - charged) + await ledger.settle(day, actual - charged) spent += actual usage["input_tokens"] += inp usage["output_tokens"] += out diff --git a/backend/app/engine/nodes/dual_check.py b/backend/app/engine/nodes/dual_check.py index 70f31a7a..58428f0f 100644 --- a/backend/app/engine/nodes/dual_check.py +++ b/backend/app/engine/nodes/dual_check.py @@ -81,6 +81,8 @@ from app.engine.labels import DUAL_CHECK_FIELD_LABEL, DUAL_CHECK_ON_REJECT_LABEL from app.engine.masking import derived_columns from app.engine.schema import DUAL_CHECK_ON_REJECT, NodeType, dual_check_card from app.engine.state import GraphState +# 原值和盲算值按相对误差比、相差比例:和评测(app/eval/score.py)同一个函数(engine/tolerance.py) +from app.engine.tolerance import agree as _agree, delta_pct as _delta_pct #: 节点产出的状态:跳过、全部一致、核成了一部分(其余无法复算)、一个都没核成、人已裁决 SKIPPED, AGREED, PARTIAL, NONE_CHECKED, RESOLVED = "skipped", "agreed", "partial", "unchecked", "resolved" @@ -1001,24 +1003,6 @@ def _limit_text(limit: str | None, options: _Options) -> str: return "轮数已用完" -def _agree(original: Any, blind: Any, tolerance: float) -> bool: - """按相对误差比(原值为 0 时按绝对误差)。""" - if original == blind: - return True - if not (_number(original) and _number(blind)): - return False - if original == 0: - return abs(blind) <= tolerance - return abs(blind - original) <= tolerance * abs(original) - - -def _delta_pct(original: Any, blind: Any) -> float | None: - """相差比例,口径同运行对比:(盲算值 − 原流程值) ÷ |原流程值| × 100,两位小数;原流程值为 0 时为 None。""" - if not (_number(original) and _number(blind)) or original == 0: - return None - return round((blind - original) / abs(original) * 100, 2) - - def _compare(row: dict[str, Any], metric: dict[str, Any], results: dict[str, Any], options: _Options, load: Callable[[str], Any], outputs: dict[str, str], masks: dict[str, frozenset[str]], queries: tuple[str, ...]) -> None: diff --git a/backend/app/engine/tolerance.py b/backend/app/engine/tolerance.py new file mode 100644 index 00000000..e7bfa463 --- /dev/null +++ b/backend/app/engine/tolerance.py @@ -0,0 +1,45 @@ +"""两个数算不算对得上:「允许的相对误差」只有这一种比法(docs/design/next-round/01-效果度量与回归评测.md 做法 10)。 + +双盲复算(engine/nodes/dual_check.py)比原流程值和盲算值,评测(app/eval/score.py)比期望值和答案里的数,用的是同一个 +函数、设置里也叫同一个名字「允许的相对误差」:两处各写一份,迟早一边改了另一边没改,同一对数在核查里算一致、在评测里 +算错。所以从双盲复算里原样搬到这里(行为不变),两边都 import。 + +- agree:按相对误差比(|b − a| ≤ tolerance × |a|);原值为 0 时按绝对误差(|b| ≤ tolerance)。值完全相等的直接算对得上 + (文字也行);有一边不是有限的数就对不上。 +- delta_pct:相差比例,口径同运行对比:(b − a) ÷ |a| × 100,两位小数;原值为 0、有一边不是数时为 None。 +""" +from __future__ import annotations + +import math +from typing import Any + + +def is_finite_number(value: Any) -> bool: + """有限的数(布尔不算,NaN、无穷不算)。""" + if not isinstance(value, (int, float)) or isinstance(value, bool): + return False + try: + return math.isfinite(value) + except OverflowError: + return False + + +def agree(original: Any, other: Any, tolerance: float) -> bool: + """按相对误差比(原值为 0 时按绝对误差)。""" + if original == other: + return True + if not (is_finite_number(original) and is_finite_number(other)): + return False + if original == 0: + return abs(other) <= tolerance + return abs(other - original) <= tolerance * abs(original) + + +def delta_pct(original: Any, other: Any) -> float | None: + """相差比例:(other − original) ÷ |original| × 100,两位小数;original 为 0 时为 None。""" + if not (is_finite_number(original) and is_finite_number(other)) or original == 0: + return None + return round((other - original) / abs(original) * 100, 2) + + +__all__ = ["agree", "delta_pct", "is_finite_number"] diff --git a/backend/app/eval/__init__.py b/backend/app/eval/__init__.py index 6ff603b2..a4d8cb01 100644 --- a/backend/app/eval/__init__.py +++ b/backend/app/eval/__init__.py @@ -1,10 +1,15 @@ """评测(docs/design/next-round/01-效果度量与回归评测.md):一把尺——题集 + 由 SQL / 口径算出的期望答案。 -第 2 波(这一包): - suites.py、cases.py:题集和题目的形状与校验(数值期望不许人填,评测-D4;只做单轮,评测-D12)、题目指纹; - gold.py:期望答案的三种来源(SQL 题、口径题、从运行认定)、变量按截至时间渲染、试算、作答前后各算一次、口径升版处置; - snapshots.py:目录快照(当前目录 / 影子起草); -- transfer.py:题集导出导入(按数据源名对上,不含期望值、答案、运行)。 +- transfer.py:题集导出导入(按数据源名对上,不含期望值、答案、运行); +- variants.py:变体(助手、写作模型,目录视图,结论裁判,重复次数)和「同一变体」的认法、目录贡献的预设对照; +- answer.py:无头地走一遍问数据页那一轮(answer_ops → chat/turn 的决定 → 运行 → 复核 → 要的话重来一次); +- runner.py:后台评测(并发、预算、止损、停止、只重试没做完的、重启标中断); +- score.py、judging.py、rescore.py:评分(五种结论、维度配对、逐题 E1)、结论裁判和文字要点(模型判断,单列)、按新规则重评; +- stats.py、compare.py:总表与区间、和上一次比、A/B 配对表与精确 McNemar; +- cleanup.py:按保留规则清理明细、回收工件;scheduled.py:题集的定时。 -第 3 波接着写 runner(后台评测)、score、stats、compare、cleanup;第 4 波是界面。接口在 app/api/evals.py(只给搭建者)。 +界面在第 4 波;接口在 app/api/evals.py(只给搭建者)。 """ diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py new file mode 100644 index 00000000..46d4c667 --- /dev/null +++ b/backend/app/eval/answer.py @@ -0,0 +1,451 @@ +"""评测里的一轮作答:无头地走一遍问数据页走的那一轮(细案 01 做法 1、9;总方案 2.1)。 + +**量的就是用户拿到的那一轮。** 不另写一份 Python 版编排:生成过程是 api/copilot.answer_ops(问数据页经 SSE 收的同一串 +操作),几个「决定」是 chat/turn 的纯函数(问数据页照服务端给的 next、retry_instruction 走),复核是 api/copilot.review_of_run +(问数据页经 /copilot/review 调的同一个函数)。这里只把它们串起来: + + answer_ops → final.next(chat/turn.next_step)→ launch_run(origin=eval) → 等终态 → review_of_run → 要的话带原图重来一次 + +- 生成:问法就是 instruction,intent=answer,限定的数据源同题目;助手模型按变体覆盖(GenerateIn.provider / model,挑表、 + 复核跟着换);读哪份目录按变体(answer_ops 的 catalog_view,查看的表详情、自查的 SQL 检查器都按它)。 +- 结局:reply 直接回答、asked 反问、fail 判失败(建图失败)、ready 只建不跑、retry(老服务端那条路:没自查过的图带着原因 + 重来一次)、run 运行。生成过程交出 error(助手本轮未完成)、开流前就拒绝(模型没配好、数据源不在)都是建图失败,另记 + 「助手调用失败」——连续几题都这样就止损(runner.py)。 +- 运行:fill_input 填入口字段(单轮,history 为空),来源 eval,名字「评测:题集名·第 n 题」,选项带目录视图(Run.options + 的 catalog_view:运行时的 db_schema、冻结快照、自查都按它)、只读和遮罩(同使用者的问数据运行:评测页会显示答案里的数, + 遮罩值不能从这里露出来;评测不写业务库)。写作模型按变体写进图的 defaults:节点自己点了模型的不覆盖,照实记在 + own_models。 +- 终态:停在审批上(危险工具、人工节点)的直接取消、记「评测不代批」;被驳回的(cancelled + end_reason=rejected,总方案 + X9)记路径「被驳回」,不算运行失败;失败、被停止记运行失败。 +- 复核:总是等完(问数据页超过 25 秒不等,评测里照等,复核用时另记,超过 25 秒的单列,见 stats.py)。复核给了 + retry_instruction 就带着这次运行的图重来一次(chat/turn 的规则,最多试 MAX_ATTEMPTS 次)。 + +用量按角色记(做法 3、11):助手 = 生成(answer_ops 末尾的 usage)+ 复核(review_of_run 的 usage);写作 = 答案运行里的 +llm.end(按模型分)。重来的话两次都算——那都是这一题花的钱。 + +时限、停止由调用方管(runner.py 用 asyncio.wait_for 包住 drive):这里随时把正在跑的那次运行记在 current_run,调用方 +超时、停止时据此取消它(stop_run)。 +""" +from __future__ import annotations + +import asyncio +import copy +import logging +import time +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy import func, select + +logger = logging.getLogger(__name__) + +#: 问数据页最多等复核这么久(chat.ts 的 REVIEW_TIMEOUT_MS):评测照等,超过的单列 +REVIEW_PAGE_WAIT_S = 25.0 +#: 取消一次运行之后最多再等它收尾多久 +CANCEL_GRACE_S = 15.0 +#: 运行停在审批上:评测不代批 +APPROVAL_REASON = "运行停在审批上(危险工具、人工节点):评测不代批,已取消这次运行" +#: 每一步在做什么(EvalResult.current) +PHASE_TEXT = {"gold": "正在算期望", "build": "正在建图", "run": "运行中", "review": "正在复核", "score": "正在评分", + "gold_after": "作答后复算期望", "judge": "正在请裁判判断"} + + +@dataclass +class TurnContext: + """这一轮作答的来历(记进运行 origin)和要用的东西。""" + + suite_id: str + suite_name: str + eval_run_id: str + case_id: str | None + case_seq: int | None + actor: str | None + arm: str | None = None + repeat: int = 0 + #: 变体:助手模型 {provider, model}、写作模型 {provider, model} + assistant: Mapping[str, Any] = field(default_factory=dict) + writer: Mapping[str, Any] = field(default_factory=dict) + #: 目录视图(Run.options['catalog_view'] 的形状) + view: Mapping[str, Any] | None = None + + +@dataclass +class Turn: + """一轮作答的结果(也是进行中的状态:current_run、phase 随时更新)。""" + + path: str | None = None + run_id: str | None = None + #: 这一题发起过的全部作答运行(重来的话两次),失败的也在 + runs: list[str] = field(default_factory=list) + attempts: int = 0 + reason: str | None = None + #: 助手调用失败(生成过程交出 error、开流前就被拒):runner 据此止损 + copilot_failed: bool = False + timed_out: bool = False + #: 直接回答的原文、反问的问题 + reply: str | None = None + asked: dict[str, Any] | None = None + review: dict[str, Any] | None = None + looks: list[dict[str, Any]] = field(default_factory=list) + #: 助手这一轮用的模型 id(model 操作) + assistant_model: str | None = None + #: {assistant: usage 汇总(生成,含重来), review: usage 汇总} + usage: dict[str, Any] = field(default_factory=dict) + timings: dict[str, int] = field(default_factory=dict) + #: 写作模型按变体换了,但节点自己点了模型、没被覆盖的节点 id + own_models: list[str] = field(default_factory=list) + #: 正在跑的那次运行(调用方超时、停止时取消它) + current_run: str | None = None + phase: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"path": self.path, "run_id": self.run_id, "runs": list(self.runs), "attempts": self.attempts, + "reason": self.reason, "copilot_failed": self.copilot_failed, "timed_out": self.timed_out, + "reply": self.reply, "asked": self.asked, "review": self.review, "looks": list(self.looks), + "assistant_model": self.assistant_model, "usage": copy.deepcopy(self.usage), + "timings": dict(self.timings), "own_models": list(self.own_models)} + + +def _add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]: + """两份 providers/usage 汇总相加(by_model 逐个模型加)。""" + out = copy.deepcopy(into) if into else {"by_model": {}, "input_tokens": 0, "output_tokens": 0, "calls": 0, + "cost_usd": 0.0, "unpriced_calls": 0} + if not more: + return out + for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"): + out[key] = int(out.get(key) or 0) + int(more.get(key) or 0) + out["cost_usd"] = round(float(out.get("cost_usd") or 0) + float(more.get("cost_usd") or 0), 6) + for model, row in (more.get("by_model") or {}).items(): + mine = out["by_model"].setdefault(model, {"input_tokens": 0, "output_tokens": 0, "calls": 0, "cost_usd": 0.0, + "unpriced_calls": 0}) + for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls"): + mine[key] = int(mine.get(key) or 0) + int(row.get(key) or 0) + mine["cost_usd"] = round(float(mine.get("cost_usd") or 0) + float(row.get("cost_usd") or 0), 6) + return out + + +def add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> dict[str, Any]: + return _add_usage(into, more) + + +async def writer_usage(run_ids: list[str]) -> dict[str, Any]: + """答案运行里的模型调用(llm.end,按模型分):写作这一角色的用量,形状同 providers/usage 的汇总。封存之后追加的 + (按需裁判、质询)不算:那是事后另记的账(裁判记在评测的账上另算,见 judging.py)。""" + from app.core.events import EventType + from app.db.base import SessionLocal + from app.db.models import RunEvent + from app.providers.catalog import unpriced_call + + out = _add_usage(None, None) + if not run_ids: + return out + async with SessionLocal() as session: + rows = (await session.execute(select(RunEvent.data).where( + RunEvent.run_id.in_(run_ids), RunEvent.type == EventType.LLM_END))).scalars().all() + for data in rows: + data = data if isinstance(data, dict) else {} + if data.get("post_seal"): + continue + model = str(data.get("model") or "unknown") + piece = {"input_tokens": int(data.get("input_tokens") or 0), "output_tokens": int(data.get("output_tokens") or 0), + "calls": int(data.get("calls") or 1), "cost_usd": float(data.get("cost_usd") or 0.0), + "unpriced_calls": 1 if unpriced_call(data) else 0} + out = _add_usage(out, {**piece, "by_model": {model: piece}}) + return out + + +def with_writer(graph: dict[str, Any], writer: Mapping[str, Any]) -> tuple[dict[str, Any], list[str]]: + """写作模型按变体写进图的 defaults(只写给了的那几项)。返回 (图, 自己点了模型、不会被覆盖的节点 id)。""" + provider, model = writer.get("provider"), writer.get("model") + if not (provider or model): + return graph, [] + out = copy.deepcopy(graph) + defaults = dict(out.get("defaults") or {}) + if provider: + defaults["provider"] = provider + if model: + defaults["model"] = model + out["defaults"] = defaults + own = [] + for node in out.get("nodes") or []: + config = ((node or {}).get("data") or {}).get("config") or {} + if isinstance(config, Mapping) and (config.get("provider") or config.get("model")): + own.append(str(node.get("id"))) + return out, own + + +def _options(view: Mapping[str, Any] | None) -> dict[str, Any]: + """作答运行带的选项:目录视图、只读、遮罩(见模块说明)。""" + from app.data.catalog_view import OPTION_KEY + from app.tools.datasource import MASK_RESULTS_OPTION, READ_ONLY_OPTION + + out: dict[str, Any] = {MASK_RESULTS_OPTION: True, READ_ONLY_OPTION: True} + if view: + out[OPTION_KEY] = dict(view) + return out + + +def run_name(suite_name: str, case_seq: int | None) -> str: + return f"评测:{suite_name or '题集'}·第 {case_seq or '?'} 题" + + +async def stop_run(run_id: str | None, actor: str | None = None) -> None: + """取消一次还没结束的运行、等它收尾(超时、停止评测、停在审批上时)。已经结束的什么都不做。""" + from app.engine.runner import run_manager + + if not run_id: + return + try: + await run_manager.cancel(run_id, actor=actor) + await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S) + except Exception: # noqa: BLE001 - 取消是收尾,出错只记日志 + logger.exception("评测:取消运行 %s 时出错", run_id) + + +class Driver: + """一轮作答。drive() 跑完一轮,结果在 turn 上;on_phase(阶段) 在每一步开始时调(runner 据此写「正在做什么」)。""" + + def __init__(self, question: str, *, datasource_ids: list[str], ctx: TurnContext, + on_phase: Callable[[str], Awaitable[None]] | None = None) -> None: + self.question = question + self.datasource_ids = list(datasource_ids) + self.ctx = ctx + self.on_phase = on_phase + self.turn = Turn() + + async def _phase(self, phase: str) -> None: + self.turn.phase = phase + if self.on_phase is not None: + try: + await self.on_phase(phase) + except Exception: # noqa: BLE001 - 写进度失败不挡作答 + logger.exception("评测:记进度出错") + + # ------------------------------------------------------------------ 生成 + + async def _generate(self, instruction: str, base_graph: dict[str, Any] | None) -> tuple[list[dict[str, Any]], str | None]: + """跑一次生成过程,收下全部操作。开流前就被拒(HTTPException)交回原因。""" + from fastapi import HTTPException + + from app.api.copilot import GenerateIn, answer_ops + + payload = GenerateIn(instruction=instruction, base_graph=base_graph, intent="answer", + provider=self.ctx.assistant.get("provider"), model=self.ctx.assistant.get("model"), + datasource_ids=self.datasource_ids or None) + ops: list[dict[str, Any]] = [] + try: + async for op in answer_ops(payload, catalog_view=self.ctx.view): + if op.get("op") in ("thinking", "heartbeat"): + continue + ops.append(op) + except HTTPException as e: + return ops, str(e.detail) + return ops, None + + def _take(self, ops: list[dict[str, Any]]) -> dict[str, Any] | None: + """收下这一次生成的用量、查看记录、模型;交回结局(final / reply / error),只有反问的为 None。""" + terminal = None + for op in ops: + kind = op.get("op") + if kind == "usage": + self.turn.usage["assistant"] = _add_usage(self.turn.usage.get("assistant"), + {k: v for k, v in op.items() if k != "op"}) + elif kind == "look": + self.turn.looks.append({k: op.get(k) for k in ("id", "by", "need", "status") if k in op}) + elif kind == "model": + self.turn.assistant_model = str(op.get("model") or "") or None + elif kind in ("final", "reply", "error"): + terminal = op + return terminal + + # ------------------------------------------------------------------ 运行 + + async def _launch(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> str | None: + from app.chat import turn as policy + from app.db.base import SessionLocal + from app.engine.launch import LaunchRefused, launch_run + + graph, own = with_writer(graph, self.ctx.writer) + self.turn.own_models = sorted(set(self.turn.own_models) | set(own)) + origin = {k: v for k, v in { + "suite_id": self.ctx.suite_id, "eval_run_id": self.ctx.eval_run_id, "case_id": self.ctx.case_id, + "role": "answer", "arm": self.ctx.arm, "repeat": self.ctx.repeat, "attempt": self.turn.attempts}.items() + if v is not None} + try: + async with SessionLocal() as session: + inputs = payload if payload is not None else policy.fill_input(graph, self.question, "") + run = await launch_run(session, graph=graph, input_payload=inputs, + actor=self.ctx.actor, options=_options(self.ctx.view), origin_kind="eval", + origin=origin, workflow_name=run_name(self.ctx.suite_name, self.ctx.case_seq)) + except LaunchRefused as e: + self.turn.path, self.turn.reason = "run_failed", f"运行没有发起:{e.message}" + return None + self.turn.runs.append(run.id) + self.turn.run_id = self.turn.current_run = run.id + return run.id + + async def _wait(self, run_id: str) -> Any: + """等运行到终态(或停在审批上)。交回库里的运行。""" + from app.db.base import SessionLocal + from app.db.models import Run + from app.engine.runner import run_manager + + while True: + await run_manager.wait_idle(run_id, timeout=5.0) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is None or row.status in ("succeeded", "failed", "cancelled", "interrupted"): + # 封存在终态之后:等执行任务收完尾(含封存),评分只读封存范围 + await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S) + async with SessionLocal() as session: + return await session.get(Run, run_id) + await asyncio.sleep(0.2) + + async def _pending_approvals(self, run_id: str) -> int: + from app.db.base import SessionLocal + from app.db.models import Approval + + async with SessionLocal() as session: + return int(await session.scalar(select(func.count()).select_from(Approval).where( + Approval.run_id == run_id, Approval.status.in_(("pending", "answered")))) or 0) + + # ------------------------------------------------------------------ 一轮 + + async def _run(self, graph: dict[str, Any], payload: dict[str, Any] | None = None) -> Any | None: + """发起、等到终态。成功结束的交回运行;别的结局记好路径和原因,交回 None。""" + from app.engine.runner import END_REJECTED + + t = self.turn + run_id = await self._launch(graph, payload) + if run_id is None: + return None + row = await self._wait(run_id) + t.current_run = None + usage = dict((row.usage or {}) if row is not None else {}) + t.timings["run_wall_ms"] = t.timings.get("run_wall_ms", 0) + int(usage.get("wall_ms") or 0) + t.timings["run_active_ms"] = t.timings.get("run_active_ms", 0) + int( + usage.get("active_ms") or usage.get("duration_ms") or 0) + if row is None: + t.path, t.reason = "run_failed", "运行记录不见了" + return None + if row.status == "interrupted": + waiting = await self._pending_approvals(run_id) + await stop_run(run_id, self.ctx.actor) + t.path = "run_failed" + t.reason = APPROVAL_REASON if waiting else "运行停在断点上(服务可能重启过):评测不接着跑,已取消这次运行" + return None + if row.status == "cancelled" and row.end_reason == END_REJECTED: + t.path, t.reason = "rejected", "运行被驳回(人的决定,不算运行失败)" + return None + if row.status != "succeeded": + t.path = "run_failed" + t.reason = f"运行{'失败' if row.status == 'failed' else '被停止'}:{row.error or '没有记下原因'}" + return None + return row + + async def drive_workflow(self, graph: dict[str, Any], inputs: dict[str, Any]) -> Turn: + """工作流题:钉住那一版的图按输入跑一次探索运行(来源 eval)。不建图、不复核:考的是这一版本身。不发正式运行—— + 正式运行会被 pick_base 和应用视图的「最近一次封存」取到(同期望探针不发正式运行的理由)。""" + t = self.turn + started = time.monotonic() + try: + await self._phase("run") + if await self._run(graph, inputs) is not None: + t.path = "ran" + return t + finally: + t.timings["total_ms"] = int((time.monotonic() - started) * 1000) + + async def drive(self) -> Turn: + from app.chat import turn as policy + + t = self.turn + started = time.monotonic() + instruction, base_graph = self.question, None + try: + while True: + await self._phase("build") + building = time.monotonic() + ops, refused = await self._generate(instruction, base_graph) + t.timings["build_ms"] = t.timings.get("build_ms", 0) + int((time.monotonic() - building) * 1000) + terminal = self._take(ops) + if refused is not None: + t.path, t.reason, t.copilot_failed = "build_failed", f"助手没有开始:{refused}", True + return t + if terminal is None: + asked = next((o for o in reversed(ops) if o.get("op") == "ask"), None) + t.path, t.asked = "asked", {k: asked.get(k) for k in ("question", "options")} if asked else None + return t + kind = terminal.get("op") + if kind == "error": + t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True + return t + if kind == "reply": + t.path, t.reply = "reply", str(terminal.get("text") or "") + return t + if terminal.get("asked"): + t.path, t.asked = "asked", dict(terminal["asked"]) + return t + self_checked = any(o.get("op") == "check" for o in ops) + step = terminal.get("next") or policy.next_step(terminal, self_checked=self_checked) + graph = terminal.get("graph") or {} + if step == policy.NEXT_RETRY: + if t.attempts + 1 < policy.MAX_ATTEMPTS: + t.attempts += 1 + instruction = policy.retry_instruction(self.question, policy.blocked_reason(terminal), + policy.has_nodes(graph)) + base_graph = graph if policy.has_nodes(graph) else None + continue + t.path, t.reason = "build_failed", policy.blocked_reason(terminal) + return t + if step == policy.NEXT_FAIL: + errors = [str(i.get("message") or "") for i in terminal.get("issues") or [] + if isinstance(i, Mapping) and i.get("level") == "error"] + t.path, t.reason = "build_failed", "生成的工作流没有通过校验:" + ";".join(errors) + return t + if step == policy.NEXT_READY: + t.path, t.reason = "ready", "助手只建了图、没有运行(它判断用户要的是流程本身)" + return t + + await self._phase("run") + row = await self._run(graph) + if row is None: + return t + + await self._phase("review") + reviewing = time.monotonic() + review = await self._review(row) + spent = int((time.monotonic() - reviewing) * 1000) + t.timings["review_ms"] = t.timings.get("review_ms", 0) + spent + t.review = {k: v for k, v in review.items() if k != "usage"} + t.usage["review"] = _add_usage(t.usage.get("review"), review.get("usage")) + if review.get("retry_instruction"): + t.attempts += 1 + instruction = str(review["retry_instruction"]) + base_graph = row.graph if policy.has_nodes(row.graph) else None + continue + t.path = "ran" + return t + finally: + t.timings["total_ms"] = int((time.monotonic() - started) * 1000) + + async def _review(self, run: Any) -> dict[str, Any]: + """复核(review_of_run,问数据页同一个函数)。复核自己出错不挡评分:照实记一条失败的复核。""" + from app.api.copilot import review_of_run + from app.core.errors import describe_exception + from app.db.base import SessionLocal + + try: + async with SessionLocal() as session: + return await review_of_run(session, run, question=self.question, + provider=self.ctx.assistant.get("provider"), + model=self.ctx.assistant.get("model"), attempt=self.turn.attempts) + except Exception as e: # noqa: BLE001 + logger.exception("评测:复核运行 %s 出错", run.id) + return {"verdict": "failed", "note": f"复核出错:{describe_exception(e)}", "signals": [], + "retry_instruction": None} + + +__all__ = ["APPROVAL_REASON", "Driver", "PHASE_TEXT", "REVIEW_PAGE_WAIT_S", "Turn", "TurnContext", "add_usage", + "run_name", "stop_run", "with_writer", "writer_usage"] diff --git a/backend/app/eval/cleanup.py b/backend/app/eval/cleanup.py new file mode 100644 index 00000000..44a22f69 --- /dev/null +++ b/backend/app/eval/cleanup.py @@ -0,0 +1,126 @@ +"""评测的清理与工件回收(细案 01 做法 16)。 + +每次评测有几十次运行(作答、作答前后算期望)和它们的查询快照、报告文档,只增不减,库和工件目录会一直涨。按保留规则清掉更早 +那几次的**明细**,总表(EvalRun.summary:准确率、区间、分布、和上一次比的计数)永久保留,界面上写「明细已清理」(cleaned_at)。 + +**保留哪些**(每个题集):最近 keep_runs 次(设置 eval.keep_runs,缺省 20);钉住的基线;以及这些里面 A/B 的另几臂(同一个 +pair_id,配对表要两边都在)。还没结束的不动。按时间一刀切会把还在当基线的那次删掉,所以钉住的另算。 + +**删什么**:逐题结果行、它们的明细工件、底下的运行(作答、算期望的探索运行,来源都是 eval;事件随运行级联删掉)和这些运行 +在工件引用表里的引用行。**工件文件**要等引用行删完之后一条引用都不剩才删(core/artifact_store.forget):同样的内容可能被别的 +运行引用(同一份表结构快照、同一份查询结果),不能跟着删。评测运行全是探索运行,删正式运行的那道拦截(api/runs.delete_run) +不受影响;这里也只删来源是 eval、已经结束的运行。 +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import SessionLocal, utcnow +from app.db.models_eval import EvalResult, EvalRun + +logger = logging.getLogger(__name__) + +#: 删运行时还在这几种状态的不删(运行在跑、停在审批上) +_LIVE = ("queued", "running", "interrupted") +_background: set[asyncio.Task[Any]] = set() + + +def keep_set(runs: list[EvalRun], keep: int) -> set[str]: + """要留明细的那几次(见模块说明)。runs 是一个题集的全部评测,新的在前。""" + from app.core.job_runtime import ACTIVE + + kept = {r.id for r in runs[:max(1, keep)]} | {r.id for r in runs if r.pinned or r.status in ACTIVE} + pairs = {r.pair_id for r in runs if r.id in kept and r.pair_id} + kept |= {r.id for r in runs if r.pair_id and r.pair_id in pairs} + return kept + + +async def _clean_one(session: AsyncSession, run: EvalRun) -> tuple[int, int, set[str]]: + """清一次评测的明细:(删了几项结果, 删了几次运行, 要回收的工件 id)。不提交。""" + from app.db.models import Artifact, Run + + results = list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == run.id))).scalars()) + run_ids: set[str] = set() + artifacts: set[str] = set() + for r in results: + if r.run_id: + run_ids.add(r.run_id) + run_ids |= {x for x in r.gold_run_ids or [] if isinstance(x, str)} + if r.details_artifact: + artifacts.add(r.details_artifact) + await session.execute(delete(Artifact).where(Artifact.id == r.details_artifact, Artifact.run_id == "") + .execution_options(synchronize_session=False)) + # 这次评测发起的、结果里没记下的运行(重来的前一次、停止时没写进结果的):按来源找 + for rid, origin in (await session.execute(select(Run.id, Run.origin).where(Run.origin_kind == "eval"))).all(): + if (origin or {}).get("eval_run_id") == run.id: + run_ids.add(rid) + removed = 0 + for rid in sorted(run_ids): + row = await session.get(Run, rid) + if row is None or row.origin_kind != "eval" or row.status in _LIVE: + continue + artifacts |= set((await session.execute(select(Artifact.id).where(Artifact.run_id == rid))).scalars()) + await session.execute(delete(Artifact).where(Artifact.run_id == rid).execution_options(synchronize_session=False)) + await session.delete(row) + removed += 1 + for r in results: + await session.delete(r) + run.cleaned_at = utcnow() + return len(results), removed, artifacts + + +async def clean_suite(suite_id: str, *, keep: int | None = None) -> dict[str, Any]: + """按保留规则清一个题集:{runs: 清了几次评测, results, deleted_runs, files: 回收了几个工件文件}。""" + from app.api.settings import eval_settings + from app.core.artifact_store import forget + + async with SessionLocal() as session: + if keep is None: + keep = int((await eval_settings(session))["keep_runs"]) + runs = list((await session.execute(select(EvalRun).where(EvalRun.suite_id == suite_id) + .order_by(EvalRun.created_at.desc()))).scalars()) + kept = keep_set(runs, keep) + targets = [r for r in runs if r.id not in kept and r.cleaned_at is None] + results = removed = 0 + artifacts: set[str] = set() + for run in targets: + n, m, found = await _clean_one(session, run) + results += n + removed += m + artifacts |= found + await session.commit() + files = await forget(artifacts) if artifacts else [] + if targets: + logger.info("评测清理:题集 %s 清了 %d 次评测的明细(%d 项结果、%d 次运行、%d 个工件文件)", suite_id, len(targets), + results, removed, len(files)) + return {"runs": len(targets), "results": results, "deleted_runs": removed, "files": len(files)} + + +def spawn(suite_id: str) -> None: + """在后台清理(评测收尾时调,不挡收尾)。出错只记日志。""" + if not suite_id: + return + + async def go() -> None: + try: + await clean_suite(suite_id) + except Exception: # noqa: BLE001 + logger.exception("评测清理出错(题集 %s)", suite_id) + + task = asyncio.get_running_loop().create_task(go()) + _background.add(task) + task.add_done_callback(_background.discard) + + +async def drain() -> None: + """等后台的清理做完(测试、关停用)。""" + while _background: + await asyncio.gather(*list(_background), return_exceptions=True) + + +__all__ = ["clean_suite", "drain", "keep_set", "spawn"] diff --git a/backend/app/eval/compare.py b/backend/app/eval/compare.py new file mode 100644 index 00000000..35d97996 --- /dev/null +++ b/backend/app/eval/compare.py @@ -0,0 +1,186 @@ +"""和上一次比、A/B 对照(细案 01 做法 12;评测-D10)。纯计算的部分(transitions、pair_table、fingerprint_diff)不读库; +选基准(pick_baseline)读库。 + +**选基准**:发起时点名的基线(baseline_id)> 同题集里钉住的基线(pinned,同一变体)> 同题集、同一变体(variants.variant_key) +里比这次先发起、已经结束(succeeded / partial)的最近一次。按发起先后,不按结束时间(同 api/compare.pick_base)。 + +**逐题转移**:每题按题目 id 配对,k 次重复的按多数(通过的次数多于不通过的算通过)。通过→不通过是退步,不通过→通过是进步, +其余不变;题目改过(case_hash 不同)、一边没有、一边不能判的标「不可比」,不算进步也不算退步。 + +**归因**(只对转移了的题):期望值变了是「数据变」;期望没变、答案执行的 SQL 规范文本不同是「做法变」;只有字面量不同是 +「只参数变」(多半是时间范围理解变了);作答路径变了是「路径变」;其余「说不清」。SQL 的 same / params_only / changed 和运行 +对比(engine/compare)是同一条规则:data/sql_semantics.normalize_sql 的规范文本和字面量。 + +**指纹差异**:助手模型、写作模型、目录快照、提示词指纹、表结构、题目、评分规则,哪几项变了。 + +**A/B**:同一次发起的几臂(共用 pair_id、共用期望)逐题配对,给两臂各自的准确率、配对表(都对 / 都错 / 只 A 对 / 只 B 对)和 +精确 McNemar 双侧 p 值;p ≥ 0.05 时照实写「这批题上看不出差别」。不能判的题不进配对表(单列个数)。 +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +TRANSITIONS = ("regressed", "improved", "same", "incomparable") +TRANSITION_LABEL = {"regressed": "退步", "improved": "进步", "same": "不变", "incomparable": "不可比"} +CAUSES = ("data", "method", "params", "path", "unknown") +CAUSE_LABEL = {"data": "数据变", "method": "做法变", "params": "只参数变", "path": "路径变", "unknown": "说不清"} +#: 指纹里比的几项和叫法 +FINGERPRINT_LABEL = {"assistant_model": "助手模型", "writer_model": "写作模型", "catalog": "目录快照", + "prompt": "提示词", "schemas": "表结构", "cases": "题目", "scorer_version": "评分规则"} +NO_DIFFERENCE = "这批题上看不出差别" +SIGNIFICANT = 0.05 +#: 基准能是哪几种状态 +BASELINE_STATUSES = ("succeeded", "partial") + + +# ========================================================================== +# 选基准 +# ========================================================================== + + +async def pick_baseline(session: AsyncSession, run: Any, *, explicit: str | None = None) -> Any | None: + """这一次和哪一次比(见模块说明)。explicit 是发起时点名的(要同一个题集)。""" + from app.db.models_eval import EvalRun + + if explicit: + row = await session.get(EvalRun, explicit) + return row if row is not None and row.suite_id == run.suite_id and row.id != run.id else None + key = (run.fingerprint or {}).get("variant_key") + rows = (await session.execute( + select(EvalRun).where(EvalRun.suite_id == run.suite_id, EvalRun.id != run.id, + EvalRun.created_at < run.created_at, EvalRun.status.in_(BASELINE_STATUSES)) + .order_by(EvalRun.created_at.desc()).limit(200))).scalars().all() + same = [r for r in rows if (r.fingerprint or {}).get("variant_key") == key] + pinned = next((r for r in same if r.pinned), None) + return pinned or (same[0] if same else None) + + +# ========================================================================== +# 逐题 +# ========================================================================== + + +def case_outcome(results: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """一道题(几次重复)合成一个结论:{verdict: pass|fail|none, passed, graded, path, gold, sql}。多数决;能判的一次都没有 + 为 none。gold 是期望值的指纹(作答前那一份),sql 是答案执行的查询的规范化(第一次能判的那一次)。""" + graded = [r for r in results if r.get("verdict") in ("pass", "fail")] + passed = sum(1 for r in graded if r.get("verdict") == "pass") + verdict = "none" if not graded else "pass" if passed > len(graded) - passed else "fail" + first = graded[0] if graded else (results[0] if results else {}) + return {"verdict": verdict, "passed": passed, "graded": len(graded), "path": first.get("path"), + "gold": first.get("gold_key"), "sql": first.get("sql_shapes") or [], + "case_hash": first.get("case_hash"), "case_seq": first.get("case_seq")} + + +def sql_change(a: Sequence[Mapping[str, Any]], b: Sequence[Mapping[str, Any]]) -> str: + """两次答案执行的查询([{text, literals}],normalize_sql 的 as_dict):same / params_only / changed。""" + if [x.get("text") for x in a] != [x.get("text") for x in b]: + return "changed" + return "params_only" if [x.get("literals") for x in a] != [x.get("literals") for x in b] else "same" + + +def cause(base: Mapping[str, Any], target: Mapping[str, Any]) -> str: + if base.get("gold") != target.get("gold"): + return "data" + if base.get("path") != target.get("path"): + return "path" + change = sql_change(base.get("sql") or [], target.get("sql") or []) + return {"changed": "method", "params_only": "params"}.get(change, "unknown") + + +def transitions(base: Sequence[Mapping[str, Any]], target: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """逐题转移和归因。base、target 是两次评测(同一臂)的逐题结果。""" + def by_case(rows: Sequence[Mapping[str, Any]]) -> dict[str, list[Mapping[str, Any]]]: + out: dict[str, list[Mapping[str, Any]]] = {} + for r in rows: + out.setdefault(str(r.get("case_id")), []).append(r) + return out + + was, now = by_case(base), by_case(target) + items = [] + counts = {k: 0 for k in TRANSITIONS} + causes = {k: 0 for k in CAUSES} + for case_id in list(dict.fromkeys([*now, *was])): + b = case_outcome(was.get(case_id, [])) + t = case_outcome(now.get(case_id, [])) + item: dict[str, Any] = {"case_id": case_id, "case_seq": t.get("case_seq") or b.get("case_seq"), + "before": b["verdict"], "after": t["verdict"]} + if case_id not in was or case_id not in now: + item.update(transition="incomparable", reason="这道题只在一边评过") + elif b.get("case_hash") != t.get("case_hash"): + item.update(transition="incomparable", reason="题目改过,不可比") + elif "none" in (b["verdict"], t["verdict"]): + item.update(transition="incomparable", reason="有一边不能判") + elif b["verdict"] == t["verdict"]: + item.update(transition="same") + else: + item.update(transition="regressed" if b["verdict"] == "pass" else "improved") + item["cause"] = cause(b, t) + causes[item["cause"]] += 1 + counts[item["transition"]] += 1 + items.append(item) + return {"counts": counts, "causes": causes, "items": items} + + +def fingerprint_diff(base: Mapping[str, Any], target: Mapping[str, Any]) -> list[dict[str, Any]]: + """两次之间指纹里哪几项变了:[{key, label, before, after}](表结构、题目只说变没变,不展开)。""" + out = [] + for key, label in FINGERPRINT_LABEL.items(): + a, b = base.get(key), target.get(key) + if a == b: + continue + item: dict[str, Any] = {"key": key, "label": label} + if key in ("assistant_model", "writer_model", "catalog", "scorer_version"): + item.update(before=a, after=b) + out.append(item) + return out + + +# ========================================================================== +# A/B +# ========================================================================== + + +def pair_table(arms: Sequence[tuple[str, Sequence[Mapping[str, Any]]]]) -> dict[str, Any]: + """几臂([(名字, 逐题结果)])两两配对:{arms: [{arm, accuracy}], pairs: [{a, b, both, neither, only_a, only_b, + excluded, p_value, note}]}。""" + from app.eval.stats import rate + + outcomes: dict[str, dict[str, Mapping[str, Any]]] = {} + summary = [] + for name, rows in arms: + cases: dict[str, list[Mapping[str, Any]]] = {} + for r in rows: + cases.setdefault(str(r.get("case_id")), []).append(r) + outcomes[name] = {cid: case_outcome(rs) for cid, rs in cases.items()} + graded = [r for r in rows if r.get("verdict") in ("pass", "fail")] + summary.append({"arm": name, "accuracy": rate(sum(1 for r in graded if r.get("verdict") == "pass"), len(graded))}) + pairs = [] + names = [n for n, _ in arms] + for i, a in enumerate(names): + for b in names[i + 1:]: + both = neither = only_a = only_b = excluded = 0 + for cid in dict.fromkeys([*outcomes[a], *outcomes[b]]): + x, y = outcomes[a].get(cid), outcomes[b].get(cid) + if x is None or y is None or "none" in (x["verdict"], y["verdict"]): + excluded += 1 + continue + pa, pb = x["verdict"] == "pass", y["verdict"] == "pass" + both += pa and pb + neither += not pa and not pb + only_a += pa and not pb + only_b += pb and not pa + from app.eval.stats import mcnemar_exact + + p = mcnemar_exact(only_a, only_b) + pairs.append({"a": a, "b": b, "both": both, "neither": neither, "only_a": only_a, "only_b": only_b, + "excluded": excluded, "p_value": p, "note": NO_DIFFERENCE if p >= SIGNIFICANT else None}) + return {"arms": summary, "pairs": pairs} + + +__all__ = ["CAUSES", "CAUSE_LABEL", "FINGERPRINT_LABEL", "NO_DIFFERENCE", "TRANSITIONS", "TRANSITION_LABEL", + "case_outcome", "cause", "fingerprint_diff", "pair_table", "pick_baseline", "sql_change", "transitions"] diff --git a/backend/app/eval/judging.py b/backend/app/eval/judging.py new file mode 100644 index 00000000..371c28c9 --- /dev/null +++ b/backend/app/eval/judging.py @@ -0,0 +1,173 @@ +"""评测里的两种模型判断:结论裁判、文字要点(细案 01 做法 10、11;评测-D4)。都只在变体打开了裁判时做,都是概率性的判断, +单列「模型判断」,不进准确率;花费都记在评测那本账上(engine/spend.eval_spend),不占裁判自己的每日额度。 + +- 结论裁判(judge_claims):走按需裁判同一条路——api/evidence 的 _judge_now(证据面板上点「请模型判断」就是它):只取封存 + 范围内的工件、遮罩列不给裁判、判定作为 evidence.judged 追加在封存之后(post_seal)、报告文档一个字不改。送哪几句由裁判 + 自己的预筛定(judge.candidates:结论句,挂了依据的在前)。花费经 judge.charged_to 记在评测那本账上、按评测的每日上限。 + 结果按 supported / partial / contradicted / insufficient / 未判 数句数。 +- 文字要点(judge_points):问法、答案原文和要点交给裁判模型(设置里的证据裁判模型,和按需裁判同一个取值链),逐条说答到 + 了(covered)、答到一部分(partial)、没答到(missing),附一句理由。答案原文只是这次运行的成果(遮过的),要点是人写的。 + 不放查询结果、不放 SQL:要点问的是「说没说到」,不是「数对不对」(数由确定性的评分判)。 + +判断出错(模型没配好、超时、格式不对)一律照实记「没有判」和原因,不挡评分。 +""" +from __future__ import annotations + +import json +import logging +import re +from collections.abc import Mapping, Sequence +from typing import Any + +logger = logging.getLogger(__name__) + +#: 结论裁判按句数分(judge.COUNT_KEYS 里评测关心的几种,unsupported 是旧数据,并进 contradicted) +CLAIM_KEYS = ("supported", "partial", "contradicted", "insufficient", "unjudged") +#: 文字要点一次最多交给裁判多少字的答案 +ANSWER_CHARS = 6000 +#: 文字要点的输出额度、时限 +POINTS_MAX_TOKENS = 2048 +POINTS_TIMEOUT_S = 90 + +POINTS_SYSTEM_PROMPT = """\ +你是评测的裁判。下面给出一个问题、对它的回答,以及期望回答里应当说到的几条要点。逐条判断回答有没有说到这条要点: +- covered:说到了(意思一致即可,不要求字面相同) +- partial:说到了一部分,或者说得含糊 +- missing:没有说到,或者说的和要点相反 +只看回答里写了什么,不要自己补充事实,也不要判断回答里的数对不对(数另有确定性的核对)。 +只输出一个 JSON 数组,每条要点一项:[{"point": 要点序号(从 1 起), "verdict": "covered|partial|missing", "reason": "一句话理由"}]""" +POINTS_HUMAN_PROMPT = "问题:{question}\n\n回答:\n{answer}\n\n要点:\n{points}" + + +async def judge_claims(run_id: str, *, daily_max_usd: float | None) -> dict[str, Any]: + """按需裁判这次作答的报告(结论句)。{counts, model, cost_usd, calls, reason};判不了的照实写 reason。""" + from fastapi import HTTPException + + from app.api import evidence + from app.engine import judge as judging + from app.engine.spend import eval_spend + + empty = {k: 0 for k in CLAIM_KEYS} + try: + async with evidence._JUDGING.hold(run_id): + sealed = await evidence._sealed(run_id) + if not sealed.sealed or not sealed.trusted or not sealed.ledger_on: + return {"counts": empty, "reason": "作答的运行没有可信的封存,不判"} + reports = evidence._reports(sealed) + if not reports: + return {"counts": empty, "reason": "答案没有报告文档,没有结论句可判"} + chosen = evidence._choose(reports, None, sealed) + if chosen.doc is None or chosen.hash_ok is False: + return {"counts": empty, "reason": "报告文档读不出来或被改过,不判"} + units = [c.unit for c in judging.candidates(chosen.doc)][:evidence.JUDGE_MAX_UNITS] + if not units: + return {"counts": empty, "reason": "报告里没有要判的结论句"} + with judging.charged_to(eval_spend, daily_max_usd): + result = await evidence._judge_now(sealed, chosen, units) + except HTTPException as e: + return {"counts": empty, "reason": str(e.detail)} + except Exception as e: # noqa: BLE001 - 裁判是附加的一步:出错照实记,不挡评分 + from app.core.errors import describe_exception + + logger.exception("评测:结论裁判出错(运行 %s)", run_id) + return {"counts": empty, "reason": f"结论裁判出错:{describe_exception(e)}"} + counts = dict(empty) + for verdict in (result.get("verdicts") or {}).values(): + status = (verdict or {}).get("status") + status = "contradicted" if status == judging.UNSUPPORTED else status + if status in counts: + counts[status] += 1 + elif status != "not_a_claim": + counts["unjudged"] += 1 + return {"counts": counts, "units": len(units), "model": result.get("model"), "priced": result.get("priced"), + "cost_usd": float(result.get("cost_usd") or 0.0), "calls": int(result.get("calls") or 0), + "limited": bool(result.get("limited")), "reason": result.get("message")} + + +def _parse_points(text: str, n: int) -> list[dict[str, Any]] | None: + match = re.search(r"\[.*\]", text or "", re.DOTALL) + if match is None: + return None + try: + items = json.loads(match.group(0)) + except ValueError: + return None + if not isinstance(items, list): + return None + out: dict[int, dict[str, Any]] = {} + for item in items: + if not isinstance(item, Mapping): + continue + index = item.get("point") + verdict = item.get("verdict") + if isinstance(index, int) and 1 <= index <= n and verdict in ("covered", "partial", "missing"): + out[index] = {"point": index, "verdict": verdict, "reason": str(item.get("reason") or "")[:200]} + if len(out) != n: + return None + return [out[i] for i in range(1, n + 1)] + + +async def judge_points(question: str, answer: str, expectations: Sequence[Mapping[str, Any]], *, + daily_max_usd: float | None, per_run_left: float | None = None) -> dict[str, Any]: + """文字要点:{expectations: {期望 id: {points: [{point, verdict, reason}], reason?}}, model, cost_usd, calls, reason?}。 + 先按目录价估一次、在评测那本账上预留,加上会超过每日上限(或这次评测剩下的钱)就不判、照实写。""" + from langchain_core.messages import HumanMessage, SystemMessage + + from app.core.errors import describe_exception + from app.db.base import SessionLocal + from app.engine.judge import judge_model_spec + from app.engine.spend import eval_spend + from app.providers import usage as usage_meter + from app.providers.catalog import estimate_cost, find_pricing + from app.providers.factory import get_chat_model + + texts = [e for e in expectations if e.get("kind") == "text" and e.get("points")] + if not texts: + return {"expectations": {}} + if not (answer or "").strip(): + return {"expectations": {}, "reason": "答案是空的,文字要点没有判"} + try: + async with SessionLocal() as session: + spec = await judge_model_spec(session, None) + model, model_id = await get_chat_model(session, spec.model_copy(update={ + "max_tokens": POINTS_MAX_TOKENS, "timeout": POINTS_TIMEOUT_S})) + except Exception as e: # noqa: BLE001 - 裁判模型没配好:照实写没有判 + return {"expectations": {}, "reason": f"裁判模型用不了:{describe_exception(e)}"} + priced = find_pricing(model_id) is not None + out: dict[str, Any] = {} + meter = usage_meter.meter() + reason = None + for exp in texts: + points = [str(p) for p in exp.get("points") or []] + human = POINTS_HUMAN_PROMPT.format(question=question, answer=answer[:ANSWER_CHARS], + points="\n".join(f"{i}. {p}" for i, p in enumerate(points, start=1))) + estimate = estimate_cost(model_id, (len(POINTS_SYSTEM_PROMPT) + len(human)) // 2, 60 * len(points)) if priced \ + else 0.0 + if per_run_left is not None and priced and estimate > per_run_left: + out[exp["id"]] = {"reason": "这次评测的金额上限不够再判文字要点"} + continue + day = await eval_spend.charge(estimate, limit=daily_max_usd if priced else None, priced=priced) + if day is None: + out[exp["id"]] = {"reason": "评测今天的金额已到上限,文字要点没有判"} + continue + before = meter.summary()["cost_usd"] + try: + with usage_meter.metering(meter): + reply = await model.ainvoke([SystemMessage(POINTS_SYSTEM_PROMPT), HumanMessage(human)]) + text = reply.content if isinstance(reply.content, str) else json.dumps(reply.content, ensure_ascii=False) + parsed = _parse_points(text, len(points)) + out[exp["id"]] = {"points": parsed} if parsed else {"reason": "裁判的回复格式不对,文字要点没有判"} + except Exception as e: # noqa: BLE001 + logger.exception("评测:判文字要点出错") + reason = f"判文字要点出错:{describe_exception(e)}" + out[exp["id"]] = {"reason": reason} + spent = meter.summary()["cost_usd"] - before + await eval_spend.settle(day, (spent if priced else 0.0) - estimate) + if per_run_left is not None: + per_run_left -= spent + summary = meter.summary() + return {"expectations": out, "model": model_id, "usage": summary, "cost_usd": summary["cost_usd"], + "calls": summary["calls"], **({"reason": reason} if reason else {})} + + +__all__ = ["CLAIM_KEYS", "POINTS_HUMAN_PROMPT", "POINTS_SYSTEM_PROMPT", "judge_claims", "judge_points"] diff --git a/backend/app/eval/rescore.py b/backend/app/eval/rescore.py new file mode 100644 index 00000000..f1fda1f3 --- /dev/null +++ b/backend/app/eval/rescore.py @@ -0,0 +1,74 @@ +"""按新规则重新评分(细案 01 做法 10):评分规则改了(score.SCORER_VERSION 加一),已经评完的评测不必重跑一遍。 + +**只读封存,不调模型、不查业务库。** 每一项:期望用存下的那一份(EvalResult.gold:作答前后两次和结论),答案从作答那次运行的 +封存里重新读(score.read_answer),文字要点用当时裁判给的判断(EvalResult.judge,不再请裁判),逐题 SQL 差异按同一个比对函数 +再比一次(纯计算)。作答前就算不出期望、没作答的,照旧按存下的期望结论。总表跟着重算(runner.refresh_summary)。 + +明细(作答经过、直接回答的原文)在明细工件里:清理掉了的评测不能重评(接口先拦)。 +""" +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import select + +from app.db.base import SessionLocal +from app.db.models_eval import EvalCase, EvalResult, EvalRun + +logger = logging.getLogger(__name__) + + +async def rescore(eval_run_id: str) -> int: + """重评一次评测(只这一臂)。交回结论变了的项数。""" + from app.core import artifact_store + from app.eval import runner, score + from app.eval.cases import case_fields + + async with SessionLocal() as session: + run = await session.get(EvalRun, eval_run_id) + if run is None: + return 0 + rows = list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == eval_run_id) + .order_by(EvalResult.seq))).scalars()) + cases = {c.id: case_fields(c) for c in (await session.execute( + select(EvalCase).where(EvalCase.id.in_({r.case_id for r in rows if r.case_id})))).scalars()} + changed = 0 + updates: dict[str, dict[str, Any]] = {} + for row in rows: + case = cases.get(row.case_id or "") + if row.status != "ok" or case is None: + continue + details: dict[str, Any] = {} + if row.details_artifact: + try: + details = artifact_store.load(row.details_artifact) or {} + except ValueError: + details = {} + turn = details.get("turn") or {} + if row.path is None: + graded = score.score_case(case, row.gold, score.Answer(run_id=None), path=None) + match = row.sql_match + else: + answer = await score.read_answer(row.run_id if row.path == "ran" else None, reply=turn.get("reply")) + graded = score.score_case(case, row.gold, answer, path=row.path, path_reason=turn.get("reason"), + judgement=(row.judge or {}).get("text")) + match = await score.sql_match(case, row.gold, answer) + if graded.verdict != row.verdict: + changed += 1 + updates[row.id] = {"verdict": graded.verdict, "scores": graded.scores, "sql_match": match, + "error": graded.reason if graded.verdict != "pass" else None} + async with SessionLocal() as session: + for result_id, values in updates.items(): + row = await session.get(EvalResult, result_id) + if row is not None: + for key, value in values.items(): + setattr(row, key, value) + await session.flush() + run = await session.get(EvalRun, eval_run_id) + await runner.refresh_summary(session, run) + await session.commit() + return changed + + +__all__ = ["rescore"] diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py new file mode 100644 index 00000000..1f24e540 --- /dev/null +++ b/backend/app/eval/runner.py @@ -0,0 +1,993 @@ +"""评测执行:一次评测一个后台任务,每题 × 重复 × 臂一项(细案 01 做法 9;评测-D9、D10;总方案 X5)。 + +**后台任务。** 用 core/job_runtime(和数据目录的起草、剖析,影子起草同一套生命周期):进程内一个 asyncio 任务;能停; +写库经 SerialWriter 一条一条来、停止时不被打断;服务重启时排队、进行中的标中断,**不自动续跑**——复制出来的数据目录 +(检查脚本、验收副本)启动时不能自己去调模型、查业务库,界面上点「重试」只把没做完的题接着做。 + +**一次评测**是 EvalRun 一行;A/B(几臂一次发起)是几行,共用 pair_id(就是第一臂的 id,它也是后台任务的编号):两臂共用 +期望答案、同一题同时开跑(gold.bracket 在作答那一步里同时跑几臂),所以一个任务跑全部几臂。每题 × 重复 × 臂是一项 +EvalResult:建评测时就全部建好(pending),进度、「正在做什么」(current)随时落库。 + +**不占数据源的队。** 开评时拍一份当前目录的快照(目录选「当前目录」「只用已核实的项」的都按它读,data/catalog_view 的 +snapshot 视图):起草、剖析照常进行,也不影响这次评测的可比性。各数据源的上传表格版本、表结构哈希开始和结束各记一次, +变了在指纹里标出(sources_changed)。 + +**并发。** 同时最多 max_parallel 题,且所有臂合起来不超过全局运行并发的四分之一(全局只有一个信号量、没有优先级,评测不能 +挤占真实用户的运行)。A/B 的同一题几臂同时开跑。每题时限 per_case_timeout_s:作答一臂超时就取消它正在跑的运行、记超时; +期望探针每次也按这个时限。 + +**预算**(评测-D9;单独一本账 engine/spend.eval_spend,不挤占裁判、核查的额度):开跑前按上一次的每题平均花费估算(estimate)。 +每题开评前按估算预留——加上会超过这次评测的金额上限、或评测每日上限就不再开新题,剩下的记「已到上限」;评完按实际找补。 +结论裁判、文字要点的花费在判的时候直接记在这本账上(judging.py),不再找补一次。估不出金额的模型(不在价格目录里)只计 +次数,金额上限对它不起作用,照实记在用量里。 + +**止损**:连续 STOP_COPILOT 题助手调用失败(模型没配好、助手本轮未完成),或最近 STOP_WINDOW 题里有 STOP_TIMEOUTS 题超时, +就不再开新题、原因照实写。 + +**收尾。** 每一臂按逐题结果算总表(stats.summarize,永久保留)、挑基准、和上一次比(compare.py);定时发起的,有一道题从通过 +变成不通过就推「评测有退步」,没评完(失败、到了上限、止损)推「评测失败」——消息里只有题集名、通过数、退步题数和链接,不放 +期望值也不放答案里的数(notify/service.notify_eval)。最后按保留规则清理更早的评测明细(cleanup.py,在后台做)。 +""" +from __future__ import annotations + +import asyncio +import copy +import logging +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import job_runtime as runtime +from app.db.base import SessionLocal, utcnow +from app.db.models_eval import EvalCase, EvalResult, EvalRun, EvalSuite +from app.eval.cases import CaseInvalid + +logger = logging.getLogger(__name__) + +KIND = "eval_run" +#: 机器码 +EVAL_INVALID = "eval_run_invalid" +EVAL_BUSY = "eval_run_busy" +EVAL_FINISHED = "eval_run_finished" +NOTHING_TO_RETRY = "eval_nothing_to_retry" +#: 止损(细案 01 做法 9) +STOP_COPILOT = 3 +STOP_WINDOW = 10 +STOP_TIMEOUTS = 7 +#: 说法 +INTERRUPTED = "服务重启时评测还没做完,没有自动续跑。可以点「重试」只把没做完的题接着做" +STOPPED = "评测已停止" +NOT_STARTED = "评测停下时这道题还没开评" +LIMIT_RUN = "已到这次评测的金额上限(${limit:g}),没有再开新题" +LIMIT_DAILY = "评测今天的金额已到上限(${limit:g}),没有再开新题" +STOP_COPILOT_TEXT = "连续 {n} 题助手调用失败,评测停下了:{reason}" +STOP_TIMEOUT_TEXT = "最近 {window} 题里有 {n} 题超时,评测停下了" +CASE_GONE = "这道题已被删除" +TIMED_OUT = "超过每题时限({s:g} 秒),已停止这一题正在跑的运行" +#: 结束后多久做清理(在后台、不挡收尾) +DETAILS_KIND = "eval_result" +DETAILS_FORMAT = 1 + + +class EvalRefused(CaseInvalid): + """发起、停止、重试不了:message 给人看,code 给程序认,status 是 HTTP 状态码。""" + + def __init__(self, message: str, code: str = EVAL_INVALID, status: int = 422, field_name: str | None = None) -> None: + super().__init__(message, field_name, code) + self.status = status + + +def leader_of(run: Any) -> str: + """跑这一次评测的后台任务的编号:A/B 是第一臂的 id(pair_id),单臂就是自己。""" + return run.pair_id or run.id + + +# ========================================================================== +# 指纹、估算 +# ========================================================================== + + +def prompt_fingerprint() -> str: + """助手提示词的指纹:节点手册、输出协议、问数据的请求包法、重来的两种措辞、复核的规则。改了任何一段,和上一次比时 + 就标「提示词变了」。""" + from app.api import copilot + from app.chat import turn + from app.core.artifact_store import canonical_json, content_hash + from app.engine import review + + parts = [copilot.NODE_REFERENCE, copilot._STREAM_PROTOCOL, + copilot._user_message(copilot.GenerateIn(instruction="{question}", intent="answer"), patch=False), + turn.RETRY_WITH_BASE, turn.RETRY_FROM_SCRATCH, + str(getattr(review, "REVIEW_SYSTEM", "") or getattr(review, "SYSTEM_PROMPT", ""))] + return content_hash(canonical_json(parts))[:16] + + +async def source_versions(session: AsyncSession, source_ids: Sequence[str]) -> dict[str, dict[str, Any]]: + """各数据源此刻的上传表格版本(当前快照)和表结构哈希:{源 id: {name, snapshot, schema}}。""" + from app.core.artifact_store import canonical_json, content_hash + from app.db.models import DataSource + + out: dict[str, dict[str, Any]] = {} + for sid in dict.fromkeys(source_ids): + row = await session.get(DataSource, sid) + if row is None: + out[sid] = {"name": None, "snapshot": None, "schema": None, "gone": True} + continue + out[sid] = {"name": row.name, "snapshot": getattr(row, "current_snapshot_id", None), + "schema": content_hash(canonical_json(row.schema_cache or {}))[:16]} + return out + + +async def estimate(session: AsyncSession, suite_id: str, items: int) -> dict[str, Any]: + """开跑前的估算:按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出,照实写。""" + row = (await session.execute(select(EvalRun).where( + EvalRun.suite_id == suite_id, EvalRun.status.in_(("succeeded", "partial"))) + .order_by(EvalRun.created_at.desc()).limit(1))).scalar_one_or_none() + per_item = ((row.summary or {}).get("per_case_cost_usd") if row is not None else None) + if per_item is None: + return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None, + "note": "这个题集还没有评完过,估不出花费:按每题开评前的实际花费累计,到了上限就停"} + return {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4), + "basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算(不含估不出金额的模型)"} + + +# ========================================================================== +# 发起 +# ========================================================================== + + +async def _cases(session: AsyncSession, suite_id: str) -> list[EvalCase]: + return list((await session.execute(select(EvalCase).where( + EvalCase.suite_id == suite_id, EvalCase.enabled.is_(True), EvalCase.archived.is_(False)) + .order_by(EvalCase.seq))).scalars()) + + +async def active_run(session: AsyncSession, suite_id: str) -> EvalRun | None: + """这个题集还没结束的评测(排队、进行中,而且真在这个进程里跑着的)。记着进行中、却没在跑的(服务重启过)先照实标中断。""" + rows = list((await session.execute(select(EvalRun).where( + EvalRun.suite_id == suite_id, EvalRun.status.in_(runtime.ACTIVE)))).scalars()) + await reconcile_stale(session, rows) + return next((r for r in rows if r.status in runtime.ACTIVE), None) + + +async def start(session: AsyncSession, suite: EvalSuite, variants: Sequence[Mapping[str, Any]], *, actor: str | None, + trigger: str = "manual", baseline_id: str | None = None, schedule_id: str | None = None, + as_of: datetime | None = None, cron: str | None = None) -> list[EvalRun]: + """建一次评测(几臂就几行)、逐项建好、交给后台。variants 是规范化过的变体(variants.normalize_variant)。不发起抛 + EvalRefused / CaseInvalid。""" + from app.api.settings import eval_settings + from app.eval import variants as variant_rules + from app.eval.suites import effective_budget + + if suite.archived: + raise EvalRefused("题集已归档,不能发起评测", status=409) + if not variants or len(variants) > variant_rules.ARMS_MAX: + raise EvalRefused(f"一次评测要有 1 到 {variant_rules.ARMS_MAX} 臂", field_name="arms") + busy = await active_run(session, suite.id) + if busy is not None: + raise EvalRefused(f"这个题集上一次评测 #{busy.id[:6]} 还没结束,结束后再发起", EVAL_BUSY, 409) + cases = await _cases(session, suite.id) + if not cases: + raise EvalRefused("题集里没有启用的题目", status=409) + repeats = max(int(v.get("repeats") or 1) for v in variants) + resolved: list[tuple[dict[str, Any], dict[str, Any], str | None]] = [] + for variant in variants: + models = await variant_rules.resolve_models(session, variant) + snapshot = await variant_rules.snapshot_artifact(session, variant) + resolved.append((dict(variant), models, snapshot)) + site = await eval_settings(session) + budget = effective_budget(suite.budget, site) + items = len(cases) * repeats * len(variants) + budget["estimate"] = await estimate(session, suite.id, items) + if baseline_id is not None: + base = await session.get(EvalRun, baseline_id) + if base is None or base.suite_id != suite.id: + raise EvalRefused("点名的基线不是这个题集的评测", field_name="baseline_id") + now = utcnow() + moment = as_of or now + runs: list[EvalRun] = [] + leader_id = None + for index, (variant, models, snapshot) in enumerate(resolved): + run = EvalRun(suite_id=suite.id, status="queued", + trigger="pair" if len(resolved) > 1 and trigger == "manual" else trigger, + schedule_id=schedule_id, variant={**variant, "repeats": repeats}, resolved_models=models, + as_of=moment, repeats=repeats, baseline_id=baseline_id, budget=dict(budget), created_by=actor, + arm=variant_rules.arm_name(variant, index) if len(resolved) > 1 else None, + fingerprint={"variant_key": variant_rules.variant_key(variant, models), + "assistant_model": models.get("assistant"), "writer_model": models.get("writer"), + "catalog_mode": (variant.get("catalog") or {}).get("mode"), + "snapshot_artifact": snapshot, "cron": cron}, + counts={"total": len(cases) * repeats, "done": 0}) + session.add(run) + await session.flush() + if leader_id is None: + leader_id = run.id + if len(resolved) > 1: + run.pair_id = leader_id + seq = 0 + for case in cases: + for repeat in range(repeats): + seq += 1 + session.add(EvalResult(eval_run_id=run.id, seq=seq, case_id=case.id, case_seq=case.seq, + case_hash=case.case_hash, repeat=repeat, arm=run.arm, status="pending")) + runs.append(run) + await session.commit() + for run in runs: + await session.refresh(run) + manager.start(leader_id) + return runs + + +# ========================================================================== +# 执行 +# ========================================================================== + + +@dataclass +class _Arm: + run_id: str + variant: dict[str, Any] + models: dict[str, Any] + view: dict[str, Any] | None = None + #: 这一臂的名字(A/B 时) + name: str | None = None + #: 这一臂各项:{(题目 id, 第几次): 结果 id} + items: dict[tuple[str, int], str] = field(default_factory=dict) + + +@dataclass +class _Unit: + case_id: str + repeat: int + #: {臂的运行 id: 结果 id} + results: dict[str, str] + + +class EvalExecution(runtime.JobRun): + """一次评测(A/B 是几臂一起)在后台的执行。不占数据源的队。""" + + kind = KIND + model = EvalRun + + def __init__(self, job_id: str) -> None: + super().__init__(job_id) + self.arms: list[_Arm] = [] + self.suite: dict[str, Any] = {} + self.cases: dict[str, dict[str, Any]] = {} + self.budget: dict[str, Any] = {} + self.as_of: datetime | None = None + self.cron: str | None = None + self.actor: str | None = None + self.spent = 0.0 + self.reserved = 0.0 + self.halt: str | None = None + self.copilot_streak = 0 + self.copilot_reason = "" + self.timeouts: list[bool] = [] + self.tasks: set[asyncio.Task[Any]] = set() + + def lane_of(self, job: Any) -> str | None: + return None + + # ---------------------------------------------------------------- 认领、准备 + + async def _claim(self, session: AsyncSession) -> bool: + claimed = await runtime.claim(session, EvalRun, self.job_id) + if claimed: + await session.execute(update(EvalRun).where( + EvalRun.pair_id == self.job_id, EvalRun.id != self.job_id, EvalRun.status == "queued") + .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + return claimed + + async def _prepare(self) -> list[_Unit]: + from app.eval import variants as variant_rules + from app.eval.cases import case_fields + from app.eval.snapshots import take_live + + async with SessionLocal() as session: + leader = await session.get(EvalRun, self.job_id) + runs = [leader] if not leader.pair_id else list((await session.execute( + select(EvalRun).where(EvalRun.pair_id == leader.pair_id).order_by(EvalRun.created_at))).scalars()) + suite = await session.get(EvalSuite, leader.suite_id) + if suite is None: + raise runtime.JobFailed("题集不存在,可能已被删除") + self.suite = {"id": suite.id, "name": suite.name, "timezone": suite.timezone} + self.budget = dict(leader.budget or {}) + self.as_of = leader.as_of or utcnow() + self.cron = (leader.fingerprint or {}).get("cron") + self.actor = leader.created_by + results = list((await session.execute(select(EvalResult).where( + EvalResult.eval_run_id.in_([r.id for r in runs])).order_by(EvalResult.seq))).scalars()) + case_ids = {r.case_id for r in results if r.case_id} + for row in (await session.execute(select(EvalCase).where(EvalCase.id.in_(case_ids)))).scalars(): + self.cases[row.id] = case_fields(row) + # 拍一份当前目录(目录选当前目录、只用已核实的项的都按它读)。已经拍过的(重试)沿用那一份:同一次评测 + # 前后读的是同一份目录 + snapshot = leader.catalog_snapshot + wants_live = any((r.variant.get("catalog") or {}).get("mode") in ("live", "reviewed") for r in runs) + sources = await self._sources(session) + if wants_live and not snapshot: + from app.eval.snapshots import MAX_SOURCES, SnapshotInvalid + + if len(sources) > MAX_SOURCES: + raise runtime.JobFailed(f"这次评测用到的数据源超过 {MAX_SOURCES} 个,拍不了目录快照:请在题目里限定数据源") + if sources: + try: + snap = await take_live(session, sources, actor=self.actor) + except SnapshotInvalid as e: + raise runtime.JobFailed(f"拍目录快照没有成功:{e.message}") from e + snapshot = snap.artifact_id + versions = await source_versions(session, sources) + for run in runs: + fp = dict(run.fingerprint or {}) + if "sources_before" not in fp: + from app.eval.score import SCORER_VERSION + + mode = (run.variant.get("catalog") or {}).get("mode") or "live" + used = snapshot if mode in ("live", "reviewed") else fp.get("snapshot_artifact") + fp.update(prompt=prompt_fingerprint(), scorer_version=SCORER_VERSION, + catalog="off" if mode == "off" else f"{mode}:{(used or '')[:16]}", + sources_before=versions, + schemas={sid: v.get("schema") for sid, v in versions.items()}, + cases=_cases_key(results, run.id)) + run.fingerprint = fp + run.catalog_snapshot = snapshot if (run.variant.get("catalog") or {}).get("mode") in ( + "live", "reviewed") else fp.get("snapshot_artifact") + await session.commit() + for run in runs: + mode = (run.variant.get("catalog") or {}).get("mode") or "live" + art = snapshot if mode in ("live", "reviewed") else (run.fingerprint or {}).get("snapshot_artifact") + view = variant_rules.view_of(run.variant, art) if (mode == "off" or art) else None + arm = _Arm(run.id, dict(run.variant or {}), dict(run.resolved_models or {}), view, run.arm) + self.arms.append(arm) + units: dict[tuple[str, int], _Unit] = {} + for r in results: + if r.status != "pending": + continue + key = (str(r.case_id), int(r.repeat or 0)) + units.setdefault(key, _Unit(key[0], key[1], {})).results[r.eval_run_id] = r.id + return list(units.values()) + + async def _sources(self, session: AsyncSession) -> list[str]: + """这次评测用得到的数据源:题目限定的、期望查的;有不限数据源的问数据题时,全部启用的源(助手会在它们里面挑)。""" + from app.db.models import DataSource + + ids: list[str] = [] + unrestricted = False + for case in self.cases.values(): + ids += list(case.get("datasource_ids") or []) + gold = case.get("gold") or {} + if gold.get("datasource_id"): + ids.append(gold["datasource_id"]) + if case.get("kind") == "ask" and not case.get("datasource_ids"): + unrestricted = True + if unrestricted: + ids += list((await session.execute(select(DataSource.id).where(DataSource.enabled.is_(True)) + .order_by(DataSource.created_at))).scalars()) + return list(dict.fromkeys(ids)) + + # ---------------------------------------------------------------- 主循环 + + async def _run(self) -> None: + if not await self._write(self._claim): + return + units = await self._prepare() + parallel = self._parallel() + gate = asyncio.Semaphore(parallel) + for unit in units: + await gate.acquire() + charge = None + if self.halt is None: + self.halt, charge = await self._reserve() + if self.halt is not None: + gate.release() + break + task = asyncio.create_task(self._guarded(unit, gate, charge)) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + if self.tasks: + await asyncio.gather(*list(self.tasks), return_exceptions=True) + await self._settle() + await self._finish() + + def _parallel(self) -> int: + from app.core.config import settings + + arms = max(1, len(self.arms)) + quarter = max(1, int(settings.max_concurrent_runs) // 4 // arms) + return max(1, min(int(self.budget.get("max_parallel") or 2), quarter)) + + async def _reserve(self) -> tuple[str | None, tuple[str, float] | None]: + """开一题之前在评测那本账上预留估算的花费(几臂一起)。触顶交回 (原因, None),不再开新题;否则 (None, (记在哪天, + 预留了多少)),评完按实际找补。""" + from app.engine.spend import eval_spend + + per_item = float(((self.budget.get("estimate") or {}).get("per_item_usd")) or 0.0) * max(1, len(self.arms)) + cap = self.budget.get("per_run_max_usd") + if cap is not None and self.spent + self.reserved + per_item > float(cap) + 1e-9: + return LIMIT_RUN.format(limit=float(cap)), None + daily = self.budget.get("daily_max_usd") + day = await eval_spend.charge(per_item, limit=float(daily) if daily is not None else None, priced=True) + if day is None: + return LIMIT_DAILY.format(limit=float(daily)), None + self.reserved += per_item + return None, (day, per_item) + + async def _guarded(self, unit: _Unit, gate: asyncio.Semaphore, charge: tuple[str, float] | None) -> None: + actual = 0.0 + try: + actual = await self._unit(unit) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 一题出错不让整次评测停下:这一题照实记失败 + from app.core.errors import describe_exception + + logger.exception("评测 %s:第 %s 题出错", self.job_id, unit.case_id) + await self._write(self._fail_unit, unit, f"评这道题时出错:{describe_exception(e)}") + finally: + gate.release() + if charge is not None: + from app.engine.spend import eval_spend + + day, reserved = charge + self.reserved -= reserved + try: + await eval_spend.settle(day, actual - reserved) + except Exception: # noqa: BLE001 + logger.exception("评测:找补花费出错") + + # ---------------------------------------------------------------- 一题(几臂一起) + + async def _unit(self, unit: _Unit) -> float: + """评一题(几臂同时)。返回这一题按预留找补的那部分实际花费(助手 + 写作;裁判在判的时候已经记过账)。""" + from app.eval import gold as gold_rules + from app.eval.cases import render, variable_values + + case = self.cases.get(unit.case_id) + if case is None: + await self._write(self._fail_unit, unit, CASE_GONE) + return 0.0 + arms = [a for a in self.arms if a.run_id in unit.results] + await self._write(self._mark, unit, "running", "gold") + values = variable_values(self.as_of, self.suite["timezone"], self.cron) + question = render(case.get("question") or "", values) + timeout = float(self.budget.get("per_case_timeout_s") or 900) + ctx = gold_rules.Context(as_of=self.as_of, tz=self.suite["timezone"], cron=self.cron, actor=self.actor, + suite_id=self.suite["id"], suite_name=self.suite["name"], eval_run_id=self.job_id, + role="gold", extra={"repeat": unit.repeat}, timeout_s=timeout) + + async def answer(_before: Any) -> list[Any]: + return await asyncio.gather(*(self._answer(arm, unit, case, question, values, timeout) for arm in arms)) + + turns, settled = await gold_rules.bracket(case, ctx, answer) + actual = 0.0 + for index, arm in enumerate(arms): + turn = turns[index] if turns is not None else None + actual += await self._score(arm, unit, case, question, turn, settled) + return actual + + async def _answer(self, arm: _Arm, unit: _Unit, case: Mapping[str, Any], question: str, + values: Mapping[str, str], timeout: float) -> Any: + from app.eval.answer import Driver, TurnContext, stop_run + + result_id = unit.results[arm.run_id] + + async def on_phase(phase: str) -> None: + await self._write(self._current, result_id, phase) + + ctx = TurnContext(suite_id=self.suite["id"], suite_name=self.suite["name"], eval_run_id=arm.run_id, + case_id=unit.case_id, case_seq=case.get("seq"), actor=self.actor, + arm=arm.name, repeat=unit.repeat, + assistant=arm.variant.get("assistant") or {}, writer=arm.variant.get("writer") or {}, + view=arm.view) + driver = Driver(question, datasource_ids=list(case.get("datasource_ids") or []), ctx=ctx, on_phase=on_phase) + try: + if case.get("kind") == "workflow": + graph, inputs = await self._workflow_graph(case, values) + await asyncio.wait_for(driver.drive_workflow(graph, inputs), timeout=timeout) + else: + await asyncio.wait_for(driver.drive(), timeout=timeout) + except asyncio.TimeoutError: + turn = driver.turn + turn.timed_out = True + await stop_run(turn.current_run, self.actor) + turn.current_run = None + if turn.path is None: + turn.path = {"build": "build_failed", "run": "run_failed", "review": "ran"}.get(turn.phase or "", + "build_failed") + turn.reason = TIMED_OUT.format(s=timeout) + except _WorkflowRefused as e: + driver.turn.path, driver.turn.reason = "run_failed", str(e) + return driver.turn + + async def _workflow_graph(self, case: Mapping[str, Any], values: Mapping[str, str]) -> tuple[dict[str, Any], dict]: + from app.apps import versions as app_versions + from app.scheduler import service + + async with SessionLocal() as session: + row = await app_versions.version_row(session, str(case.get("workflow_id")), int(case.get("workflow_version"))) + if row is None: + raise _WorkflowRefused(f"工作流 v{case.get('workflow_version')} 不在了") + return copy.deepcopy(row.graph), service.render_inputs(dict(case.get("input_template") or {}), dict(values)) + + async def _score(self, arm: _Arm, unit: _Unit, case: Mapping[str, Any], question: str, turn: Any, + settled: Any) -> float: + """评一臂:读答案、(打开了的话)请裁判判断、评分、比 SQL、写结果。返回要找补的实际花费。""" + from app.core.artifact_store import put_json + from app.eval import judging, score + from app.eval.answer import add_usage, writer_usage + + result_id = unit.results[arm.run_id] + await self._write(self._current, result_id, "score") + gold_dict = settled.to_dict() + if turn is None: + # 作答前就算不出期望(升版待处置、拒收、失败):不作答,结论照搬原因 + graded = score.score_case(case, gold_dict, score.Answer(run_id=None), path=None) + await self._write(self._save, result_id, arm, { + "gold": gold_dict, "gold_run_ids": list(settled.before.run_ids), "verdict": graded.verdict, + "scores": graded.scores, "error": graded.reason}, None, unit) + return 0.0 + answer = await score.read_answer(turn.run_id if turn.path == "ran" else None, reply=turn.reply) + judge: dict[str, Any] = {} + judge_cost = 0.0 + if arm.variant.get("judge") and turn.path == "ran" and turn.run_id: + await self._write(self._current, result_id, "judge") + daily = self.budget.get("daily_max_usd") + judge["claims"] = await judging.judge_claims(turn.run_id, daily_max_usd=daily) + judge_cost += float(judge["claims"].get("cost_usd") or 0) + cap = self.budget.get("per_run_max_usd") + left = None if cap is None else float(cap) - self.spent - judge_cost + judge["text"] = await judging.judge_points(question, answer.text, case.get("expectations") or [], + daily_max_usd=daily, per_run_left=left) + judge_cost += float(judge["text"].get("cost_usd") or 0) + graded = score.score_case(case, gold_dict, answer, path=turn.path, path_reason=turn.reason, + judgement=judge.get("text")) + match = await score.sql_match(case, gold_dict, answer) + writer = await writer_usage(turn.runs) + assistant = add_usage(turn.usage.get("assistant"), turn.usage.get("review")) + usage = {"assistant": assistant, "writer": writer, "review": turn.usage.get("review"), + "judge": {"cost_usd": round(judge_cost, 6), + "calls": int((judge.get("claims") or {}).get("calls") or 0) + + int((judge.get("text") or {}).get("calls") or 0)}} + cost = float(assistant.get("cost_usd") or 0) + float(writer.get("cost_usd") or 0) + self.spent += cost + judge_cost + looks = {"model": sum(1 for x in turn.looks if x.get("by") == "model"), + "system": sum(1 for x in turn.looks if x.get("by") == "system")} + from app.look.base import LOOK_LIMIT + + looks["limit_hit"] = looks["model"] >= LOOK_LIMIT + shapes = _sql_shapes(answer) + details = {"format": DETAILS_FORMAT, "eval_run_id": arm.run_id, "result_id": result_id, + "case_id": unit.case_id, "repeat": unit.repeat, "turn": turn.to_dict(), "answer": answer.to_dict(), + "judge": judge, "sql_shapes": shapes, "gold_key": gold_key(gold_dict), + "scored": graded.to_dict()} + artifact = await put_json(details, kind=DETAILS_KIND, meta={"eval_run_id": arm.run_id, "result_id": result_id}) + columns = {"run_id": turn.run_id, "gold": gold_dict, + "gold_run_ids": list(settled.before.run_ids) + list((settled.after.run_ids if settled.after else [])), + "path": turn.path, "verdict": graded.verdict, "scores": graded.scores, "review": turn.review, + "tier": answer.tier, "judge": judge or None, "looks": looks, + "calls": int(assistant.get("calls") or 0), "timings": dict(turn.timings), "usage": usage, + "sql_match": match, "details_artifact": artifact, + "error": graded.reason if graded.verdict != "pass" else None} + await self._write(self._save, result_id, arm, columns, turn, unit) + self._watch(turn) + return cost + + def _watch(self, turn: Any) -> None: + """止损:连续几题助手调用失败、最近几题超时太多,就不再开新题。""" + if turn.copilot_failed: + self.copilot_streak += 1 + self.copilot_reason = turn.reason or "" + else: + self.copilot_streak = 0 + self.timeouts = [*self.timeouts, bool(turn.timed_out)][-STOP_WINDOW:] + if self.halt is None and self.copilot_streak >= STOP_COPILOT: + self.halt = STOP_COPILOT_TEXT.format(n=STOP_COPILOT, reason=self.copilot_reason) + elif self.halt is None and sum(self.timeouts) >= STOP_TIMEOUTS: + self.halt = STOP_TIMEOUT_TEXT.format(window=STOP_WINDOW, n=STOP_TIMEOUTS) + + # ---------------------------------------------------------------- 库操作(经 SerialWriter) + + async def _mark(self, session: AsyncSession, unit: _Unit, status: str, phase: str) -> None: + from app.eval.answer import PHASE_TEXT + + await session.execute(update(EvalResult).where(EvalResult.id.in_(list(unit.results.values()))) + .values(status=status, started_at=utcnow(), current=PHASE_TEXT.get(phase, phase)) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _current(self, session: AsyncSession, result_id: str, phase: str) -> None: + from app.eval.answer import PHASE_TEXT + + await session.execute(update(EvalResult).where(EvalResult.id == result_id, EvalResult.status == "running") + .values(current=PHASE_TEXT.get(phase, phase)).execution_options(synchronize_session=False)) + await session.commit() + + async def _save(self, session: AsyncSession, result_id: str, arm: _Arm, columns: dict[str, Any], turn: Any, + unit: _Unit) -> None: + row = await session.get(EvalResult, result_id) + if row is None: + return + for key, value in columns.items(): + setattr(row, key, value) + row.status, row.current, row.finished_at = "ok", None, utcnow() + await session.flush() + await _recount(session, arm.run_id) + await session.commit() + + async def _fail_unit(self, session: AsyncSession, unit: _Unit, reason: str) -> None: + await session.execute(update(EvalResult).where(EvalResult.id.in_(list(unit.results.values())), + EvalResult.status.in_(runtime.OPEN)) + .values(status="failed", current=None, error=reason, finished_at=utcnow()) + .execution_options(synchronize_session=False)) + for run_id in unit.results: + await _recount(session, run_id) + await session.commit() + + # ---------------------------------------------------------------- 收尾 + + async def _finish(self) -> None: + reason = self.halt + skipped = reason or NOT_STARTED + + async def close(session: AsyncSession) -> None: + ids = [a.run_id for a in self.arms] + await session.execute(update(EvalResult).where(EvalResult.eval_run_id.in_(ids), + EvalResult.status.in_(runtime.OPEN)) + .values(status="skipped", current=None, error=skipped, finished_at=utcnow()) + .execution_options(synchronize_session=False)) + await session.commit() + + await self._write(close) + await self._conclude(None, reason) + + async def _conclude(self, forced: str | None, reason: str | None, error: str | None = None) -> None: + """每一臂:算总表、定状态、挑基准、比一次;收尾之后推通知、在后台清理。forced 是停止、出错时的状态。""" + notices: list[tuple[str, str, dict[str, Any]]] = [] + + async def write(session: AsyncSession) -> None: + sources_after = None + for arm in self.arms: + run = await session.get(EvalRun, arm.run_id) + if run is None: + continue + rows = await result_rows(session, run.id) + ok = [r for r in rows if r["status"] == "ok"] + status = forced or ("succeeded" if len(ok) == len(rows) else "partial" if ok else "failed") + fp = dict(run.fingerprint or {}) + if sources_after is None and fp.get("sources_before"): + sources_after = await source_versions(session, list(fp["sources_before"])) + if sources_after is not None and fp.get("sources_before"): + fp["sources_after"] = sources_after + fp["sources_changed"] = [v.get("name") or sid for sid, v in sources_after.items() + if fp["sources_before"].get(sid) != v] + run.fingerprint = fp + summary = await refresh_summary(session, run, rows) + run.status, run.finished_at = status, utcnow() + run.stop_reason = reason + if error: + run.error = error + if run.trigger == "schedule": + regressed = ((summary.get("baseline") or {}).get("counts") or {}).get("regressed", 0) + counts = {"passed": summary["verdicts"]["pass"], "graded": summary["accuracy"]["n"], + "regressed": int(regressed or 0)} + if regressed: + notices.append(("eval.regressed", run.id, counts)) + if status in ("failed", "partial"): + notices.append(("eval.failed", run.id, counts)) + await session.commit() + + await self._write(write) + if forced in (None, "failed"): + from app.notify import service as notify + + for event, run_id, counts in notices: + notify.spawn(notify.notify_eval(run_id, event, **counts)) + from app.eval import cleanup + + cleanup.spawn(self.suite.get("id") or "") + + async def _close(self, status: str, error: str) -> None: + """停止、服务关停、整体出错:没结束的项记下原因,每一臂收尾(总表照算已经评完的那些)。""" + await self._settle() + + async def write(session: AsyncSession) -> None: + ids = [a.run_id for a in self.arms] or [self.job_id] + item_status = "cancelled" if status in ("cancelled", "interrupted") else "failed" + await session.execute(update(EvalResult).where(EvalResult.eval_run_id.in_(ids), + EvalResult.status.in_(runtime.OPEN)) + .values(status=item_status, current=None, error=error, finished_at=utcnow()) + .execution_options(synchronize_session=False)) + await session.commit() + + try: + await self._write(write) + if not self.arms: + async def bare(session: AsyncSession) -> None: + await session.execute(update(EvalRun).where( + (EvalRun.id == self.job_id) | (EvalRun.pair_id == self.job_id), + EvalRun.status.in_(runtime.ACTIVE)) + .values(status=status, error=error, finished_at=utcnow()) + .execution_options(synchronize_session=False)) + await session.commit() + + await self._write(bare) + return + await self._conclude(status, None, error) + except Exception: # noqa: BLE001 + logger.exception("评测 %s 收尾时写库出错", self.job_id) + + async def _stopped(self) -> None: + for task in list(self.tasks): + task.cancel() + if self.tasks: + await asyncio.gather(*list(self.tasks), return_exceptions=True) + for arm_run in await self._running_runs(): + from app.eval.answer import stop_run + + await stop_run(arm_run, self.actor) + if self.stop_reason == "shutdown": + await self._close("interrupted", INTERRUPTED) + else: + await self._close("cancelled", STOPPED) + + async def _running_runs(self) -> list[str]: + """这次评测发起的、还没结束的运行(停止时一并取消:作答、算期望的都算)。""" + from app.db.models import Run + + ids = [a.run_id for a in self.arms] or [self.job_id] + async with SessionLocal() as session: + rows = (await session.execute(select(Run.id, Run.origin).where( + Run.origin_kind == "eval", Run.status.in_(("queued", "running", "interrupted"))))).all() + return [rid for rid, origin in rows if (origin or {}).get("eval_run_id") in ids] + + async def _failed(self, reason: str) -> None: + await self._close("failed", reason) + + async def _crashed(self, e: Exception) -> None: + from app.core.errors import describe_exception + + logger.exception("评测 %s 出错", self.job_id) + await self._close("failed", f"评测出错:{describe_exception(e)}") + + +class _WorkflowRefused(Exception): + pass + + +def _cases_key(results: Sequence[EvalResult], run_id: str) -> str: + """这一臂评的题目:题目 id 和指纹(改过一道题指纹就变)。""" + from app.core.artifact_store import canonical_json, content_hash + + pairs = sorted({(str(r.case_id), r.case_hash) for r in results if r.eval_run_id == run_id}) + return content_hash(canonical_json(pairs))[:16] + + +def gold_key(gold: Mapping[str, Any] | None) -> str | None: + """期望值的指纹(作答前那一份):和上一次比时认「数据变了」。""" + from app.core.artifact_store import canonical_json, content_hash + + before = (gold or {}).get("before") or {} + if (gold or {}).get("status") not in ("ok", "drifted"): + return None + return content_hash(canonical_json([[s.get("version"), s.get("values")] for s in before.get("sets") or []]))[:16] + + +def _sql_shapes(answer: Any) -> list[dict[str, Any]]: + """答案执行的查询的规范化(核查类的不算):和上一次比时认「做法变」「只参数变」。""" + from app.data.sql_semantics import normalize_sql + + out = [] + for q in answer.queries: + if q.get("role") == "verify" or not q.get("sql"): + continue + out.append(normalize_sql(str(q["sql"])).as_dict()) + return out + + +def _counts(rows: Sequence[Mapping[str, Any]]) -> dict[str, int]: + out = {"total": len(rows), "done": 0, "running": 0, "pending": 0, "passed": 0, "failed": 0, "ungradable": 0, + "skipped": 0, "errors": 0} + for r in rows: + status = r.get("status") + if status in runtime.TASK_DONE: + out["done"] += 1 + elif status in ("running", "pending"): + out[status] += 1 + if status in ("failed", "cancelled"): + out["errors"] += 1 + verdict = r.get("verdict") + if verdict == "pass": + out["passed"] += 1 + elif verdict == "fail": + out["failed"] += 1 + elif verdict == "ungradable": + out["ungradable"] += 1 + elif verdict == "skipped" or status == "skipped": + out["skipped"] += 1 + return out + + +async def _recount(session: AsyncSession, run_id: str) -> None: + rows = (await session.execute(select(EvalResult.status, EvalResult.verdict).where( + EvalResult.eval_run_id == run_id))).all() + run = await session.get(EvalRun, run_id) + if run is not None: + run.counts = _counts([{"status": s, "verdict": v} for s, v in rows]) + + +def result_dict(row: EvalResult) -> dict[str, Any]: + """逐题结果的列(统计、对比的输入;接口也按它给)。traced 从各期望的结论算。""" + scores = list(row.scores or []) + counted = [s for s in scores if s.get("required") and s.get("kind") != "text"] + return {"id": row.id, "eval_run_id": row.eval_run_id, "seq": row.seq, "case_id": row.case_id, + "case_seq": row.case_seq, "case_hash": row.case_hash, "repeat": row.repeat, "arm": row.arm, + "status": row.status, "run_id": row.run_id, "gold_run_ids": list(row.gold_run_ids or []), + "gold": row.gold, "path": row.path, "verdict": row.verdict, "scores": scores, + "traced": row.verdict == "pass" and bool(counted) and all(s.get("outcome") == "traced" for s in counted), + "review": row.review, "tier": row.tier, "judge": row.judge, "looks": row.looks, "calls": row.calls, + "timings": dict(row.timings or {}), "usage": dict(row.usage or {}), "sql_match": row.sql_match, + "details_artifact": row.details_artifact, "current": row.current, "error": row.error, + "started_at": row.started_at.isoformat() if row.started_at else None, + "finished_at": row.finished_at.isoformat() if row.finished_at else None} + + +async def result_rows(session: AsyncSession, run_id: str) -> list[dict[str, Any]]: + """一次评测的逐题结果,加上明细工件里统计、对比要用的几样(期望值的指纹、答案查询的规范化、报告的引用统计、重来了几次)。 + 明细清理掉了的,这几样为空。""" + from app.core import artifact_store + + rows = list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == run_id) + .order_by(EvalResult.seq))).scalars()) + out = [] + for row in rows: + item = result_dict(row) + details = None + if row.details_artifact: + try: + details = artifact_store.load(row.details_artifact) + except ValueError: + details = None + details = details if isinstance(details, dict) else {} + item["gold_key"] = details.get("gold_key") or gold_key(row.gold) + item["sql_shapes"] = details.get("sql_shapes") or [] + item["answer_stats"] = ((details.get("answer") or {}).get("stats")) or {} + item["attempts"] = int(((details.get("turn") or {}).get("attempts")) or 0) + out.append(item) + return out + + +async def refresh_summary(session: AsyncSession, run: EvalRun, rows: list[dict[str, Any]] | None = None + ) -> dict[str, Any]: + """按逐题结果(重)算一臂的总表、计数、按角色的用量,挑基准、和它比一次(计数、指纹差异记进 summary.baseline)。 + 收尾、重新评分都调它。不提交。""" + from app.eval import compare, stats + + rows = rows if rows is not None else await result_rows(session, run.id) + summary = stats.summarize(rows, judge_on=bool((run.variant or {}).get("judge"))) + base = await compare.pick_baseline(session, run, explicit=run.baseline_id) + if base is not None: + base_rows = await result_rows(session, base.id) + moved = compare.transitions(base_rows, rows) if base_rows else None + summary["baseline"] = {"id": base.id, "pinned": bool(base.pinned), + "created_at": base.created_at.isoformat() if base.created_at else None, + "fingerprint_diff": compare.fingerprint_diff(base.fingerprint or {}, run.fingerprint or {}), + **({"counts": moved["counts"], "causes": moved["causes"]} if moved else {"cleaned": True})} + run.baseline_id = base.id + run.summary = summary + run.usage = {k: summary["usage"].get(k) for k in ("assistant", "writer", "judge")} | { + "cost_usd": summary["cost_usd"], "unpriced_calls": summary["unpriced_calls"]} + run.counts = _counts(rows) + return summary + + +class EvalManager(runtime.JobManager): + def __init__(self) -> None: + super().__init__(EvalExecution, label="评测任务", name="eval-run") + + +manager = EvalManager() + + +# ========================================================================== +# 停止、重试、启动整理 +# ========================================================================== + + +def running(run: Any) -> bool: + return manager.running(leader_of(run)) + + +async def reconcile_stale(session: AsyncSession, rows: Sequence[EvalRun]) -> None: + """读接口当场整理:记着排队、进行中,这个进程里却没在跑(服务重启过)的,照实标中断。""" + stale = [r for r in rows if r.status in runtime.ACTIVE and not running(r)] + for row in stale: + await runtime.close_job(session, EvalRun, EvalResult, row.id, "interrupted", INTERRUPTED, "cancelled", + INTERRUPTED) + await session.refresh(row) + + +async def startup() -> int: + """服务启动时:没结束的评测标中断(没结束的项记停止),不自动续跑。""" + async def close(session: AsyncSession, job_id: str) -> None: + await runtime.close_job(session, EvalRun, EvalResult, job_id, "interrupted", INTERRUPTED, "cancelled", + INTERRUPTED) + + return await runtime.interrupt_on_startup(EvalRun, close, running=manager.running) + + +async def stop(run_id: str) -> bool: + """停止一次评测(A/B 的话几臂一起停)。排队中的直接记 cancelled;进行中的停掉、等它收尾。已经结束的返回 False。""" + async with SessionLocal() as session: + row = await session.get(EvalRun, run_id) + if row is None or row.status not in runtime.ACTIVE: + return False + leader = leader_of(row) + if await manager.stop(leader, "user"): + return True + async with SessionLocal() as session: + arms = list((await session.execute(select(EvalRun.id).where( + (EvalRun.id == leader) | (EvalRun.pair_id == leader)))).scalars()) + done = False + for arm in arms: + done = await runtime.cancel_queued(session, EvalRun, EvalResult, arm) or done + return done + + +async def retry(run_id: str) -> list[str]: + """只把没做完的题接着做(停止、中断、出错、到了上限的那几题):这几项改回待评、整次评测重新排队。A/B 的话同一题几臂一起 + 重做(几臂共用期望,要在同一份数据上一起答)。交回重排的几臂的 id。没有可重试的抛 EvalRefused。""" + async with SessionLocal() as session: + row = await session.get(EvalRun, run_id) + if row is None: + raise EvalRefused("评测不存在", status=404) + if row.status in runtime.ACTIVE: + raise EvalRefused("这次评测还没结束", EVAL_BUSY, 409) + if row.cleaned_at is not None: + raise EvalRefused("这次评测的明细已清理,不能重试", NOTHING_TO_RETRY, 409) + leader = leader_of(row) + arms = list((await session.execute(select(EvalRun).where( + (EvalRun.id == leader) | (EvalRun.pair_id == leader)))).scalars()) + suite = await session.get(EvalSuite, row.suite_id) + if suite is None or suite.archived: + raise EvalRefused("题集已归档或不存在,不能重试", status=409) + busy = await active_run(session, row.suite_id) + if busy is not None and leader_of(busy) != leader: + raise EvalRefused(f"这个题集另一次评测 #{busy.id[:6]} 还没结束", EVAL_BUSY, 409) + results = list((await session.execute(select(EvalResult).where( + EvalResult.eval_run_id.in_([a.id for a in arms])))).scalars()) + # 评完了的(status=ok,含升版待处置、结论是跳过的)不重做;停止、中断、出错、到了上限没开评的重做 + open_units = {(r.case_id, r.repeat) for r in results if r.status != "ok"} + if not open_units: + raise EvalRefused("这次评测没有没做完的题", NOTHING_TO_RETRY, 409) + blank = dict(status="pending", run_id=None, gold_run_ids=[], gold=None, path=None, verdict=None, scores=[], + review=None, tier=None, judge=None, looks=None, calls=0, timings={}, usage={}, sql_match=None, + details_artifact=None, current=None, error=None, started_at=None, finished_at=None) + for r in results: + if (r.case_id, r.repeat) in open_units: + for key, value in blank.items(): + setattr(r, key, value) + for arm in arms: + arm.status, arm.error, arm.stop_reason, arm.finished_at = "queued", None, None, None + await session.flush() + for arm in arms: + await _recount(session, arm.id) + await session.commit() + manager.start(leader) + return [a.id for a in arms] + + +async def shutdown() -> None: + await manager.shutdown() + + +__all__ = ["EVAL_BUSY", "EVAL_FINISHED", "EVAL_INVALID", "EvalExecution", "EvalRefused", "INTERRUPTED", + "NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "leader_of", "manager", + "prompt_fingerprint", "reconcile_stale", "refresh_summary", "result_dict", "result_rows", "retry", "running", + "shutdown", "source_versions", "start", "startup", "stop"] diff --git a/backend/app/eval/scheduled.py b/backend/app/eval/scheduled.py index f2ba0d43..143a5937 100644 --- a/backend/app/eval/scheduled.py +++ b/backend/app/eval/scheduled.py @@ -1,7 +1,16 @@ -"""评测题集的定时(docs/design/next-round/01-效果度量与回归评测.md 做法 15)。 +"""评测题集的定时(细案 01 做法 15;总方案 X4:定时只有一套)。 -scheduler/service.py 按目标种类导入本模块(_TARGET_MODULES):fire 到点发起一次,check_target 建任务时核对目标。 -第 3 波 W3-c 填;填之前照实记「当前版本还不能按时发起这类目标」,不发起。 +scheduler/service.py 按目标种类导入本模块(_TARGET_MODULES):认领、补跑(错过的按任务的宽限补一次)、重叠跳过、触发记录都是 +定时那一套,这里只管两件事: + +- fire:到点(或立即运行)发起一次评测。变体用题集的缺省变体,定时跑 1 次(评测-D10:对照才跑 3 次),截至时间就是这次 + 计划触发的时间(题目里的 {{ schedule.* }} 按它、按这条定时的 cron 渲染,和工作流的定时同一份变量);基准是同一变体的 + 上一次(runner 收尾时按 compare.pick_baseline 挑)。这个题集上一次评测还没结束就记 skipped_overlap、写清是哪一次;题集 + 不在、归档了、没有启用的题、变体里的模型用不了,记 failed_to_start 和原因。发起了记 started,ScheduleFire.eval_run_id 指向 + 这一次评测(用 service.record_fire 写,和工作流同一套去重:同一次触发撞上唯一约束就交回已有的那一条)。 +- check_target:建任务、改目标时核对题集在、没归档、有启用的题。 + +有退步、没评完的通知在评测收尾时推(runner.py → notify/service.notify_eval),只推定时发起的。 """ from __future__ import annotations @@ -11,11 +20,57 @@ from typing import Any async def fire(*, schedule_id: str, fire_at: datetime, kind: str, actor: str | None, target_id: str, name: str, tz: str, cron_expr: str) -> Any: + from app.db.base import SessionLocal + from app.db.models_eval import EvalSuite + from app.eval import runner, variants + from app.eval.cases import CaseInvalid from app.scheduler import service - reason = service.TARGET_UNSUPPORTED.format(kind=service.TARGET_LABEL["eval_suite"]) - return await service.record_fire(schedule_id, fire_at, kind, "failed_to_start", reason) + async with SessionLocal() as session: + suite = await session.get(EvalSuite, target_id) + if suite is None: + return await service.record_fire(schedule_id, fire_at, kind, "failed_to_start", "评测题集不存在,可能已被删除") + busy = await runner.active_run(session, suite.id) + if busy is not None: + reason = f"题集「{suite.name}」上一次评测 #{busy.id[:6]} 还没结束,本次触发已跳过" + return await service.record_fire(schedule_id, fire_at, kind, "skipped_overlap", reason) + existing = await service._existing(schedule_id, fire_at, kind) + if existing is not None: + return existing + try: + from app.api.settings import eval_settings + + judge = bool((await eval_settings(session))["judge"]) + variant = variants.normalize_variant(dict(suite.default_variant or {}) | {"repeats": variants.SCHEDULE_REPEATS}, + judge_default=judge) + [run] = await runner.start(session, suite, [variant], actor=actor or service.actor_label(name), + trigger="schedule", schedule_id=schedule_id, as_of=fire_at, cron=cron_expr) + except CaseInvalid as e: + return await service.record_fire(schedule_id, fire_at, kind, "failed_to_start", f"没有发起评测:{e.message}") + return await service.record_fire(schedule_id, fire_at, kind, "started", None, eval_run_id=run.id) async def check_target(session: Any, target_id: str) -> str | None: - return "当前版本还不能按时发起「评测题集」这类目标" + """题集在、没归档、有启用的题。不合规返回给人看的一句。""" + from sqlalchemy import func, select + + from app.db.models_eval import EvalCase, EvalSuite + + suite = await session.get(EvalSuite, target_id) + if suite is None: + return "评测题集不存在,可能已被删除" + if suite.archived: + return f"题集「{suite.name}」已归档,不能定时评测" + enabled = await session.scalar(select(func.count()).select_from(EvalCase).where( + EvalCase.suite_id == target_id, EvalCase.enabled.is_(True), EvalCase.archived.is_(False))) + if not enabled: + return f"题集「{suite.name}」里没有启用的题目" + return None + + +async def target_name(session: Any, target_id: str) -> str | None: + """接口里给人看的目标名(题集名);不在了为 None。""" + from app.db.models_eval import EvalSuite + + suite = await session.get(EvalSuite, target_id) + return suite.name if suite is not None else None diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py new file mode 100644 index 00000000..6e4d8fcd --- /dev/null +++ b/backend/app/eval/score.py @@ -0,0 +1,612 @@ +"""评分:拿期望和答案比,给出每个期望的结论和这道题过没过(细案 01 做法 10;评测-D4)。 + +**只读封存,不调模型、不查库。** 答案从作答那次运行封存范围内的报告文档、查询快照里取(api/evidence 的封存读法,和证据 +面板、口径账本同一份);期望是作答前后算好的那一份(EvalResult.gold,gold.settle 的形状);文字要点的判断是作答时裁判给的 +(EvalResult.judge,judging.py)。所以可以「按新规则重新评分」:规则改了 SCORER_VERSION 就加一,重评只把这几样再读一遍。 + +**从答案里取数。** +- 报告文档里引用解析成的数(segment.cite.role=value、status=resolved):取 cite.value 原值(显示时的换算、取整不影响)。 + 引用的是查询的一格时,读那份快照里这一行的非数值列当维度;引用的是口径卡的指标时没有维度。 +- 正文裸写的数(uncited_number)、没有报告时成果里的文字、直接回答的原文:按出具校验同一套认法取(engine/issuance 的 + extract_numbers:千分位、万 / 亿后缀、百分号、中文数字),容差按写出的精度推。叙述里自己算出来的数也在这里——只能算 + 「对但没引用」,不算可追溯(叙述层无算术)。 + +**每个期望五种结论**(OUTCOMES):traced 对且可追溯(引用的格或指标对上了)、untraced 对但没引用(只有正文裸写的对上了)、 +wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、比率和百分数写反了:单列,也不算对)、ungradable 不能评分(期望 +没算出来、作答期间数据变了、答案引用的是遮罩格、封存核对不上)。 +- scalar:数值按「允许的相对误差」比(engine/tolerance.agree,和双盲复算同一个函数);期望没写容差时,引用的原值按 + DEFAULT_TOLERANCE,裸写的数按它写出的精度(3.5 万 ±500)。维度配对:期望写了 where(或期望那一行本身有维度)时,答案 + 那一格所在行的非数值列要有这个取值;答案那一格没有维度(按条件筛过、只剩一个数)不算对不上。期望值是 0–10 的小整数 + 又没有维度可配时标「弱匹配」(巧合的数:3 家门店正好等于期望的 3)。 +- table:逐行按 keys 的取值配对,每个要比的值同 scalar;ordered 时按答案里各行键值出现的先后核对顺序。 +- set:名单里每一项都要在答案里出现(ordered 时按顺序);每一项都出现在答案查过的结果里算可追溯。 +- none:期望答「没有数据」:答案里没有数,而且查过的查询里有返回 0 行的(可追溯);没有数、但也没有一条查询返回 0 行的 + 算「对但没引用」。 +- ask:期望反问:作答路径是反问才对。 +- text:文字要点,由裁判判(covered / partial / missing / unjudged),单列「模型判断」,不进准确率(评测-D4)。 + +**题目结论**(verdict):路径不对(要答出数的题直接回答了、反问了、只建不跑、建图或运行失败)直接不通过、写明原因; +必需的期望全是 traced / untraced 就通过,另报「可追溯通过」(全是 traced);有 wrong / scale 的不通过;剩下的(有不能评分的、 +没有错的)不能评分。期望没算出来按 gold 的结论:口径升版待处置是跳过,别的是不能评分——不混进「答错」。被驳回(总方案 X9) +记路径「被驳回」,不算失败:不能评分。只有文字要点的题不能评分(单列,不计入准确率)。 + +**逐题的 E1**(sql_match):答案里的查询(先看报告引用了的)和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点) +用口径账本同一个比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在 +哪个方面(总方案 X2:E1 只有一份实现)。 +""" +from __future__ import annotations + +import logging +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from app.engine.tolerance import agree, is_finite_number + +logger = logging.getLogger(__name__) + +#: 评分规则的版本:规则改了就加一,结果里记着按哪一版评的,可以按新规则重评 +SCORER_VERSION = 1 + +OUTCOMES = ("traced", "untraced", "wrong", "scale", "ungradable") +OUTCOME_LABEL = {"traced": "对且可追溯", "untraced": "对但没引用", "wrong": "错", "scale": "量纲不同", + "ungradable": "不能评分"} +#: 文字要点的判断(裁判给的) +TEXT_OUTCOMES = ("covered", "partial", "missing", "unjudged") +TEXT_LABEL = {"covered": "都答到了", "partial": "答到一部分", "missing": "没答到", "unjudged": "没有判"} +VERDICT_LABEL = {"pass": "通过", "fail": "不通过", "ungradable": "不能评分", "skipped": "跳过"} +PATH_LABEL = {"reply": "直接回答", "asked": "反问", "ready": "只建不跑", "build_failed": "建图失败", + "run_failed": "运行失败", "ran": "跑完了", "rejected": "被驳回"} +#: 期望没写容差时,引用的原值按这个相对误差比(同双盲复算的缺省) +DEFAULT_TOLERANCE = 1e-6 +#: 量纲不同:差这几个倍数(元 / 万元、比率 / 百分数……) +SCALE_FACTORS = (10.0, 100.0, 1000.0, 1e4, 1e5, 1e6, 1e8) +#: 弱匹配:期望是这么小的整数、又没有维度可配 +WEAK_MAX = 10 + +#: 路径不对时给人看的原因 +WRONG_PATH = { + "reply": "该查不查:助手直接回答了,没有查数", + "asked": "该答不答:助手反问了", + "ready": "只建了图、没有运行", + "build_failed": "建图失败", + "run_failed": "运行失败", +} + + +# ========================================================================== +# 从封存的运行里读答案 +# ========================================================================== + + +def _numbers(text: str) -> list[dict[str, Any]]: + from app.engine.issuance import _tolerance, extract_numbers + + out = [] + for token in extract_numbers(text or ""): + out.append({"raw": token.raw, "value": token.value, "decimals": token.decimals, "percent": token.is_percent, + "tolerance": _tolerance(token)}) + return out + + +def _snapshot(artifact: str | None, cache: dict[str, Any]) -> dict[str, Any] | None: + from app.core import artifact_store + + if not artifact: + return None + if artifact not in cache: + try: + content = artifact_store.load(artifact) + except ValueError: + content = None + cache[artifact] = content if isinstance(content, dict) else None + return cache[artifact] + + +def _masked_columns(snapshot: Mapping[str, Any]) -> set[str]: + return {str(c) for c in snapshot.get("masked_at_query") or []} + + +def _cell_facts(cite: Mapping[str, Any], catalog: Mapping[str, Any], cache: dict[str, Any]) -> dict[str, Any]: + """引用的一格:哪份快照、第几行哪一列、这一行的维度(非数值列)、是不是遮罩格。""" + entry = catalog.get(cite.get("alias") or "") or {} + locator = cite.get("locator") or {} + artifact = entry.get("artifact") + out: dict[str, Any] = {"artifact": artifact, "row": locator.get("row"), "column": locator.get("column"), "dims": {}} + snapshot = _snapshot(artifact, cache) + if snapshot is None: + return out + columns = [str(c) for c in snapshot.get("columns") or []] + masked = _masked_columns(snapshot) + out["masked"] = bool(snapshot.get("masked_whole")) or str(locator.get("column")) in masked + row_index = locator.get("row") + rows = snapshot.get("rows") or [] + if isinstance(row_index, int) and 0 <= row_index < len(rows) and isinstance(rows[row_index], (list, tuple)): + row = rows[row_index] + out["dims"] = {c: row[i] for i, c in enumerate(columns) + if i < len(row) and c != locator.get("column") and c not in masked and row[i] is not None + and not is_finite_number(row[i]) and not isinstance(row[i], bool)} + return out + + +@dataclass +class Answer: + """从封存的作答运行里读出来的答案(评分只看这些)。problem 不为空时整份答案读不了(封存核对不上、没有封存)。""" + + run_id: str | None + text: str = "" + values: list[dict[str, Any]] = field(default_factory=list) + bare: list[dict[str, Any]] = field(default_factory=list) + queries: list[dict[str, Any]] = field(default_factory=list) + cited: list[str] = field(default_factory=list) + stats: dict[str, Any] = field(default_factory=dict) + tier: str | None = None + report: str | None = None + problem: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"run_id": self.run_id, "text": self.text, "values": self.values, "bare": self.bare, + "queries": self.queries, "cited": self.cited, "stats": self.stats, "tier": self.tier, + "report": self.report, "problem": self.problem} + + @classmethod + def from_dict(cls, raw: Mapping[str, Any]) -> Answer: + return cls(run_id=raw.get("run_id"), text=str(raw.get("text") or ""), values=list(raw.get("values") or []), + bare=list(raw.get("bare") or []), queries=list(raw.get("queries") or []), + cited=list(raw.get("cited") or []), stats=dict(raw.get("stats") or {}), tier=raw.get("tier"), + report=raw.get("report"), problem=raw.get("problem")) + + +def _output_text(output: Any) -> str: + """成果里给人看的文字(没有报告文档时从这里取数):字段值里的文字,_ 开头的元数据不算。""" + if not isinstance(output, Mapping): + return str(output or "") + parts = [] + for key, value in output.items(): + if str(key).startswith("_"): + continue + if isinstance(value, str): + parts.append(value) + elif isinstance(value, (int, float)) and not isinstance(value, bool): + parts.append(str(value)) + return "\n".join(parts) + + +async def read_answer(run_id: str | None, *, reply: str | None = None) -> Answer: + """读一次作答运行的答案。没有运行(直接回答、反问)的只有原文:reply。""" + from fastapi import HTTPException + + from app.api.evidence import _choose, _reports, _sealed + + if not run_id: + text = reply or "" + return Answer(run_id=None, text=text, bare=_numbers(text)) + try: + sealed = await _sealed(run_id) + except HTTPException as e: + return Answer(run_id=run_id, problem=str(e.detail)) + if not sealed.sealed: + return Answer(run_id=run_id, problem="作答的运行没有封存,答案无从核对") + if not sealed.trusted: + return Answer(run_id=run_id, problem="作答的运行封存核对不一致,答案不能当证据") + cache: dict[str, Any] = {} + answer = Answer(run_id=run_id) + for artifact, info in sealed.queries.items(): + snap = _snapshot(artifact, cache) + if snap is None: + continue + rows = snap.get("rows") or [] + answer.queries.append({"artifact": artifact, "node_id": (info or {}).get("node_id"), + "source": snap.get("source") or (info or {}).get("source"), "sql": snap.get("sql"), + "columns": [str(c) for c in snap.get("columns") or []], + "row_count": snap.get("row_count", len(rows)), "truncated": bool(snap.get("truncated")), + "role": snap.get("role")}) + from app.db.base import SessionLocal + from app.db.models import Run + + async with SessionLocal() as session: + row = await session.get(Run, run_id) + final = row.output if row is not None else None + issuance = (final or {}).get("_issuance") if isinstance(final, Mapping) else None + answer.tier = (issuance or {}).get("tier") if isinstance(issuance, Mapping) else None + reports = _reports(sealed) + chosen = _choose(reports, None, sealed) if reports else None + if chosen is None or chosen.doc is None or chosen.hash_ok is False: + answer.text = _output_text(final) + answer.bare = _numbers(answer.text) + if chosen is not None and chosen.hash_ok is False: + answer.problem = "报告文档与哈希不一致,疑似被修改,答案不能当证据" + return answer + doc = chosen.doc + catalog = doc.get("catalog") or {} + answer.report = chosen.node_id + answer.text = str(doc.get("markdown") or "") + answer.stats = dict(doc.get("stats") or {}) + from app.engine.evidence import iter_segments + + cited: list[str] = [] + for seg in iter_segments(doc): + cite = seg.get("cite") or {} + if cite.get("role") == "value" and cite.get("status") == "resolved": + item: dict[str, Any] = {"value": cite.get("value"), "kind": cite.get("kind"), "text": seg.get("text"), + "segment": seg.get("id"), "alias": cite.get("alias")} + if cite.get("kind") == "metric": + entry = catalog.get(cite.get("alias") or "") or {} + item.update(metric=(cite.get("locator") or {}).get("metric"), artifact=entry.get("artifact"), dims={}) + else: + item.update(_cell_facts(cite, catalog, cache)) + answer.values.append(item) + if item.get("artifact"): + cited.append(item["artifact"]) + elif seg.get("issue") == "uncited_number" or (seg.get("kind") == "number" and not cite): + answer.bare += _numbers(str(seg.get("text") or "")) + answer.cited = list(dict.fromkeys(cited)) + return answer + + +# ========================================================================== +# 比 +# ========================================================================== + + +def _close(expected: Any, got: Any, tolerance: float) -> bool: + if is_finite_number(expected) and is_finite_number(got): + return agree(float(expected), float(got), tolerance) + return expected is not None and got is not None and str(expected).strip() == str(got).strip() + + +def _bare_close(expected: Any, token: Mapping[str, Any], tolerance: float | None) -> bool: + """裸写的数对不对得上:容差取期望写的、或者这个数写出的精度(大的那个)。百分数和比率互认。""" + if not is_finite_number(expected): + return False + value = float(token["value"]) + allowed = max(float(token.get("tolerance") or 0), abs(float(expected)) * (tolerance or 0)) + if abs(value - float(expected)) <= allowed + 1e-9: + return True + return bool(token.get("percent")) and abs(value - float(expected) * 100) <= allowed + 1e-9 + + +def _scaled(expected: Any, got: float) -> bool: + """差 10 的整数次幂(量纲不同)。""" + if not is_finite_number(expected) or expected == 0 or got == 0: + return False + ratio = abs(got / float(expected)) + return any(math.isclose(ratio, f, rel_tol=1e-6) or math.isclose(ratio, 1 / f, rel_tol=1e-6) for f in SCALE_FACTORS) + + +def _dims_ok(wanted: Mapping[str, Any], dims: Mapping[str, Any]) -> bool: + """维度配对:答案那一格所在行有维度时,每个期望的键值都要出现在这一行的非数值列里(按取值认,列名可以不同)。""" + if not wanted or not dims: + return True + shown = {str(v).strip() for v in dims.values()} + return all(str(v).strip() in shown for v in wanted.values()) + + +def _key_of(exp: Mapping[str, Any], gold: Mapping[str, Any]) -> dict[str, Any]: + return dict(exp.get("where") or gold.get("dims") or {}) + + +def _score_scalar(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answer) -> dict[str, Any]: + expected = gold.get("value") + tolerance = exp.get("tolerance") + key = _key_of(exp, gold) + out: dict[str, Any] = {"expected": expected} + if key: + out["key"] = key + if is_finite_number(expected) and float(expected).is_integer() and 0 <= float(expected) <= WEAK_MAX and not key: + out["weak"] = True + usable = [v for v in answer.values if _dims_ok(key, v.get("dims") or {})] + masked = [v for v in usable if v.get("masked")] + clean = [v for v in usable if not v.get("masked")] + for v in clean: + if _close(expected, v.get("value"), tolerance if tolerance is not None else DEFAULT_TOLERANCE): + return {**out, "outcome": "traced", "got": v.get("value"), "cell": _cell_ref(v)} + for token in answer.bare: + if _bare_close(expected, token, tolerance): + return {**out, "outcome": "untraced", "got": token["value"], "raw": token["raw"]} + if not is_finite_number(expected) and expected is not None and str(expected).strip() and \ + str(expected).strip() in answer.text: + return {**out, "outcome": "untraced", "got": str(expected)} + if masked and not clean: + return {**out, "outcome": "ungradable", "reason": "答案引用的是遮罩的格,不评分"} + for v in clean: + if is_finite_number(v.get("value")) and _scaled(expected, float(v["value"])): + return {**out, "outcome": "scale", "got": v.get("value"), "cell": _cell_ref(v)} + for token in answer.bare: + if _scaled(expected, float(token["value"])): + return {**out, "outcome": "scale", "got": token["value"], "raw": token["raw"]} + got = next((v.get("value") for v in clean), None) + return {**out, "outcome": "wrong", **({"got": got} if got is not None else {}), + "reason": "答案里没有对得上的数" if not answer.values and not answer.bare else "答案里的数和期望对不上"} + + +def _cell_ref(value: Mapping[str, Any]) -> dict[str, Any]: + return {k: value.get(k) for k in ("artifact", "row", "column", "metric", "segment") if value.get(k) is not None} + + +def _first_position(text: str, word: Any) -> int: + return text.find(str(word).strip()) if word is not None and str(word).strip() else -1 + + +def _score_table(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answer) -> dict[str, Any]: + keys = list(gold.get("keys") or []) + columns = [c for c in gold.get("columns") or [] if c not in keys] + rows = list(gold.get("rows") or []) + tolerance = exp.get("tolerance") + cells: list[str] = [] + for row in rows: + key = {k: row.get(k) for k in keys} + for column in columns: + sub = _score_scalar({"tolerance": tolerance, "where": key}, {"value": row.get(column)}, answer) + cells.append(sub["outcome"]) + outcome = _combine(cells) + out: dict[str, Any] = {"rows": len(rows), "cells": dict(_count(cells)), "outcome": outcome} + if outcome in ("traced", "untraced") and gold.get("ordered"): + positions = [_first_position(answer.text, row.get(keys[0])) if keys else -1 for row in rows] + if any(p < 0 for p in positions) or positions != sorted(positions): + out.update(outcome="wrong", reason="行的先后和期望不一致") + return out + + +def _score_set(exp: Mapping[str, Any], gold: Mapping[str, Any], answer: Answer, cache: dict[str, Any]) -> dict[str, Any]: + values = [v for v in gold.get("values") or [] if v is not None] + positions = [_first_position(answer.text, v) for v in values] + missing = [v for v, p in zip(values, positions) if p < 0] + if missing: + return {"outcome": "wrong", "missing": missing[:20], "reason": f"答案里少了 {len(missing)} 项"} + if gold.get("ordered") and positions != sorted(positions): + return {"outcome": "wrong", "reason": "名单的先后和期望不一致"} + seen: set[str] = set() + for query in answer.queries: + snap = _snapshot(query.get("artifact"), cache) or {} + masked = _masked_columns(snap) + columns = [str(c) for c in snap.get("columns") or []] + for row in snap.get("rows") or []: + if isinstance(row, (list, tuple)): + seen |= {str(c).strip() for i, c in enumerate(row) if c is not None and + (i >= len(columns) or columns[i] not in masked)} + traced = all(str(v).strip() in seen for v in values) + return {"outcome": "traced" if traced else "untraced", "items": len(values)} + + +def _score_none(answer: Answer) -> dict[str, Any]: + if answer.values or answer.bare: + return {"outcome": "wrong", "reason": "期望答「没有数据」,答案里却给了数"} + if any(int(q.get("row_count") or 0) == 0 and q.get("role") != "verify" for q in answer.queries): + return {"outcome": "traced"} + return {"outcome": "untraced", "reason": "答案没有给数,但没有一条查询返回 0 行可以说明「没有数据」"} + + +def _combine(outcomes: Sequence[str]) -> str: + """几个结论合成一个:有错就错(全是量纲不同的算量纲不同),有不能评分的不能评分,全可追溯才可追溯。""" + if not outcomes: + return "ungradable" + bad = [o for o in outcomes if o in ("wrong", "scale")] + if bad: + return "scale" if all(o == "scale" for o in bad) else "wrong" + if "ungradable" in outcomes: + return "ungradable" + return "traced" if all(o == "traced" for o in outcomes) else "untraced" + + +def _count(outcomes: Sequence[str]) -> dict[str, int]: + out: dict[str, int] = {} + for o in outcomes: + out[o] = out.get(o, 0) + 1 + return out + + +def _text_outcome(exp: Mapping[str, Any], judgement: Mapping[str, Any] | None) -> dict[str, Any]: + """文字要点:裁判对每条要点的判断(judging.judge_points 的结果),合成一个。""" + points = list(exp.get("points") or []) + found = ((judgement or {}).get("expectations") or {}).get(exp["id"]) if judgement else None + if not found: + reason = (judgement or {}).get("reason") or "这次评测没有打开结论裁判,文字要点没有判" + return {"outcome": "unjudged", "points": len(points), "reason": reason} + verdicts = [p.get("verdict") for p in found.get("points") or []] + if not verdicts or any(v not in ("covered", "partial", "missing") for v in verdicts): + return {"outcome": "unjudged", "points": len(points), "reason": found.get("reason") or "裁判没有给出判断"} + outcome = "covered" if all(v == "covered" for v in verdicts) else \ + "missing" if all(v == "missing" for v in verdicts) else "partial" + return {"outcome": outcome, "points": len(points), "detail": found.get("points")} + + +# ========================================================================== +# 一道题 +# ========================================================================== + + +@dataclass +class CaseScore: + verdict: str + traced: bool + reason: str | None + scores: list[dict[str, Any]] + #: 口径题选了 dual 时新版那一份期望的结论(旧的那一份就是 scores) + by_version: list[dict[str, Any]] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return {"verdict": self.verdict, "traced": self.traced, "reason": self.reason, "scores": self.scores, + "by_version": self.by_version, "scorer_version": SCORER_VERSION} + + +def _expectation_scores(expectations: Sequence[Mapping[str, Any]], values: Mapping[str, Any], answer: Answer, + path: str | None, judgement: Mapping[str, Any] | None, cache: dict[str, Any] + ) -> list[dict[str, Any]]: + out = [] + for exp in expectations: + kind = exp.get("kind") + base = {"id": exp["id"], "kind": kind, "required": bool(exp.get("required", True))} + if exp.get("label"): + base["label"] = exp["label"] + if kind == "text": + out.append({**base, **_text_outcome(exp, judgement)}) + continue + if kind == "ask": + ok = path == "asked" + out.append({**base, "outcome": "traced" if ok else "wrong", + **({} if ok else {"reason": "期望反问(问法含糊),助手却没有反问"})}) + continue + gold = values.get(exp["id"]) + if gold is None: + out.append({**base, "outcome": "ungradable", "reason": "期望没有算出这一项"}) + continue + if kind == "scalar": + out.append({**base, **_score_scalar(exp, gold, answer)}) + elif kind == "table": + out.append({**base, **_score_table(exp, gold, answer)}) + elif kind == "set": + out.append({**base, **_score_set(exp, gold, answer, cache)}) + elif kind == "none": + out.append({**base, **_score_none(answer)}) + return out + + +def _verdict(scores: Sequence[Mapping[str, Any]]) -> tuple[str, bool, str | None]: + counted = [s for s in scores if s.get("required") and s.get("kind") != "text"] + if not counted: + return "ungradable", False, "这道题只有文字要点:由裁判判、单列,不计入准确率" + outcomes = [s["outcome"] for s in counted] + if any(o in ("wrong", "scale") for o in outcomes): + first = next(s for s in counted if s["outcome"] in ("wrong", "scale")) + return "fail", False, first.get("reason") or OUTCOME_LABEL[first["outcome"]] + if any(o == "ungradable" for o in outcomes): + first = next(s for s in counted if s["outcome"] == "ungradable") + return "ungradable", False, first.get("reason") or OUTCOME_LABEL["ungradable"] + return "pass", all(o == "traced" for o in outcomes), None + + +def score_case(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *, path: str | None, + path_reason: str | None = None, judgement: Mapping[str, Any] | None = None) -> CaseScore: + """一道题的评分。gold 是 gold.Settled.to_dict() 的形状;answer 是 read_answer 读出来的。纯计算。""" + from app.eval.cases import NEEDS_GOLD + + expectations = list(case.get("expectations") or []) + needs_numbers = any(e.get("kind") in NEEDS_GOLD for e in expectations) + gold = dict(gold or {}) + if needs_numbers and gold.get("status") != "ok": + verdict = "skipped" if gold.get("status") == "skipped" else "ungradable" + reason = gold.get("reason") or "期望没有算出来" + scores = [{"id": e["id"], "kind": e.get("kind"), "required": bool(e.get("required", True)), + "outcome": "ungradable", "reason": reason} for e in expectations if e.get("kind") != "text"] + return CaseScore(verdict, False, reason, scores) + if path == "rejected": + return CaseScore("ungradable", False, path_reason or "运行被驳回(人的决定,不算失败)", []) + ask_only = any(e.get("kind") == "ask" for e in expectations) + if needs_numbers and path != "ran": + reason = WRONG_PATH.get(path or "", "没有跑出答案") + if path_reason: + reason = f"{reason}:{path_reason}" + return CaseScore("fail", False, reason, [{"id": e["id"], "kind": e.get("kind"), + "required": bool(e.get("required", True)), "outcome": "wrong", + "reason": reason} for e in expectations if e.get("kind") != "text"]) + if answer.problem and needs_numbers: + return CaseScore("ungradable", False, answer.problem, []) + cache: dict[str, Any] = {} + sets = list(((gold.get("before") or {}).get("sets") or [])) if needs_numbers else [{}] + first = sets[0] if sets else {} + scores = _expectation_scores(expectations, (first or {}).get("values") or {}, answer, path, judgement, cache) + verdict, traced, reason = _verdict(scores) + if ask_only and path != "asked": + verdict, traced, reason = "fail", False, "期望反问(问法含糊),助手却没有反问" + by_version = [] + for extra in sets[1:]: + more = _expectation_scores(expectations, (extra or {}).get("values") or {}, answer, path, judgement, cache) + v, t, r = _verdict(more) + by_version.append({"version": (extra or {}).get("version"), "verdict": v, "traced": t, "reason": r, + "scores": more}) + return CaseScore(verdict, traced, reason, scores, by_version) + + +# ========================================================================== +# 逐题的 E1:答案的查询和期望的取数比一次(ledger/match 的比对函数) +# ========================================================================== + + +async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None) -> list[Any]: + """期望的取数,包成口径账本比对用的 Caliber:SQL 题是一条期望 SQL(按截至时间渲染好的那一条,不带参数),口径题是 + 钉住版本里口径卡读的取数节点(带模板参数,由 match_template 绑定)。拿不到的为空。""" + from app.db.base import SessionLocal + from app.db.models import DataSource, WorkflowVersion + from app.eval.gold import QUERY_NODE + from app.ledger.match import Caliber, Fetch, _table_keys, caliber_of + + spec = (case.get("gold") or {}) + if not spec or (gold or {}).get("status") not in ("ok", "drifted"): + return [] + before = (gold or {}).get("before") or {} + if spec.get("kind") in ("sql", "from_run"): + sql = str((before.get("rendered") or {}).get("sql") or "") + async with SessionLocal() as session: + source = await session.get(DataSource, spec.get("datasource_id")) + if not sql or source is None: + return [] + return [Caliber(workflow_id="gold", version=0, level="", graph_hash=None, + fetches=(Fetch(QUERY_NODE, source.name, sql, _table_keys(sql)),))] + if spec.get("kind") == "caliber": + from sqlalchemy import select + + async with SessionLocal() as session: + row = (await session.execute(select(WorkflowVersion).where( + WorkflowVersion.workflow_id == spec.get("workflow_id"), + WorkflowVersion.version == spec.get("version")))).scalar_one_or_none() + found = caliber_of(str(spec.get("workflow_id")), row) if row is not None else None + return [found] if found is not None else [] + return [] + + +async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *, + timeout: float = 8.0) -> dict[str, Any] | None: + """逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。 + 比对在线程里跑(纯计算,和账本一样限时)。""" + import asyncio + import time + + from app.api.evidence import _sealed + from app.ledger.match import _kinds, compare_queries, read_queries + + if not answer.run_id or answer.problem: + return None + calibers = await gold_calibers(case, gold) + if not calibers: + return None + try: + sealed = await _sealed(answer.run_id) + except Exception: # noqa: BLE001 - 读不出封存就不比 + return None + from app.db.base import SessionLocal + + async with SessionLocal() as session: + kinds = await _kinds(session) + + def work() -> list[dict[str, Any]]: + queries = [q for q in read_queries(sealed, kinds)] + return compare_queries(queries, calibers, deadline=time.monotonic() + timeout) + + try: + rows = await asyncio.wait_for(asyncio.to_thread(work), timeout=timeout + 2) + except asyncio.TimeoutError: + return {"status": "skipped", "reason": "比对超过时限,没有做完"} + except Exception: # noqa: BLE001 - 比对是附加的一步,出错照实记没比 + logger.exception("评测:逐题比 SQL 出错") + return {"status": "skipped", "reason": "比对出错"} + cited = [r for r in rows if r.get("artifact") in set(answer.cited)] + pool = cited or rows + proven = next((r for r in pool if r.get("status") == "proven"), None) + chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \ + next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None) + if chosen is None: + return {"status": "none", "reason": "答案没有查询"} + candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \ + next(iter(chosen.get("candidates") or []), None) or {} + diff = candidate.get("diff") or {} + aspects = sorted({i.get("aspect") for i in diff.get("items") or [] if i.get("aspect")}) + sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None) + return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects, + "diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"), + "reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)} + + +__all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION", + "TEXT_LABEL", "TEXT_OUTCOMES", "VERDICT_LABEL", "gold_calibers", "read_answer", "score_case", "sql_match"] diff --git a/backend/app/eval/snapshots.py b/backend/app/eval/snapshots.py index 183a754b..3aa29f8c 100644 --- a/backend/app/eval/snapshots.py +++ b/backend/app/eval/snapshots.py @@ -21,9 +21,9 @@ - 起草按表计费,大库很贵:只起草题集用得到的表(从期望 SQL、口径的取数 SQL 推,plan_tables),开始前先估算花费 (estimate,照实写是按经验估的、不含思考)。 -进程内的后台任务(core/job_runtime 的 JobManager):服务重启时没结束的标中断、不自动续跑。startup() 是给 main.py 启动 -整理调的那一步(接进 lifespan 和评测执行一起,第 3 波);在那之前读接口按 reconcile_stale 当场整理:记着进行中、这个进程 -里却没在跑的,照实标中断。 +进程内的后台任务(core/job_runtime 的 JobManager):服务重启时没结束的标中断、不自动续跑。startup() 在 main.py 的启动整理里 +调(和评测执行一起),关停时 manager.shutdown();读接口另按 reconcile_stale 当场整理:记着进行中、这个进程里却没在跑的, +照实标中断。 """ from __future__ import annotations diff --git a/backend/app/eval/stats.py b/backend/app/eval/stats.py new file mode 100644 index 00000000..744f7d17 --- /dev/null +++ b/backend/app/eval/stats.py @@ -0,0 +1,155 @@ +"""评测的汇总与统计(细案 01 做法 11、12;评测-D10)。所有汇总都在后端算好,界面直接显示(界面层不放逻辑:同一个数不能 +有两套算法)。纯计算,输入是一次评测的逐题结果(EvalResult 的列,result_row 的形状)。 + +- 准确率和可追溯准确率:分母是能判的题(通过 + 不通过);不能评分、跳过的单列,既不算对也不算错。都带 Wilson 95% 区间和 + 样本数——题集小、模型有随机性,差两三题就可能被读成「变好了」「变差了」(评测-D10:只展示、不卡门)。 +- 作答路径、出具档位(声明了出具的按 formal / degraded / withheld,没声明的单列「探索问答」)、复核结论(verdict × severity) + 和自动重来的次数、复核用时超过问数据页等待(25 秒)的题数。 +- 结论裁判(打开了才有):supported / partial / contradicted / insufficient / 未判的句数;文字要点的判断单列。 +- look:每题平均几次(分模型和系统),用满这一轮上限的比例;主模型(助手)调用次数。 +- 耗时:建图、运行(墙钟、执行)、复核、整题的 P50 / P90。 +- token 和成本按角色:助手 = 建图 + 挑表 + 复核;写作 = 答案运行里的模型调用(按模型分);裁判;期望探针不调模型,为 0。 +- 报告引用率:numbers_cited / numbers,另给 uncited_numbers、unresolved、unknown_entities。 +- 逐题 E1:答案的查询和期望的取数等价的比例、差异方面的分布(score.sql_match,口径账本同一个比对函数)。 +- 重复跑 k 次的题:每题的通过率。 + +McNemar(精确、双侧)给 A/B 的配对表用(compare.py):只看「只 A 对」「只 B 对」的那几题,二项分布 p=0.5 的双侧概率。 +""" +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +#: 95% 区间的 z +Z95 = 1.959963984540054 +#: 问数据页等复核最多这么久(毫秒,同 answer.REVIEW_PAGE_WAIT_S) +REVIEW_PAGE_WAIT_MS = 25_000 +TIERS = ("formal", "degraded", "withheld", "explore") +TIMING_KEYS = ("build_ms", "run_wall_ms", "run_active_ms", "review_ms", "total_ms") + + +def wilson(passed: int, n: int, z: float = Z95) -> tuple[float, float] | None: + """Wilson 区间(缺省 95%)。n 为 0 时没有区间。""" + if n <= 0: + return None + p = passed / n + denom = 1 + z * z / n + centre = (p + z * z / (2 * n)) / denom + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom + return round(max(0.0, centre - half), 4), round(min(1.0, centre + half), 4) + + +def mcnemar_exact(only_a: int, only_b: int) -> float: + """精确 McNemar 双侧 p 值:不一致的那几题里,「只 A 对」服从 Binomial(b + c, 0.5)。没有不一致的题时为 1。""" + n = only_a + only_b + if n == 0: + return 1.0 + k = min(only_a, only_b) + tail = sum(math.comb(n, i) for i in range(k + 1)) / (2 ** n) + return round(min(1.0, 2 * tail), 6) + + +def percentile(values: Sequence[float], q: float) -> float | None: + """最近秩法的百分位(P50、P90)。没有值时为 None。""" + data = sorted(v for v in values if v is not None) + if not data: + return None + rank = max(1, math.ceil(q / 100 * len(data))) + return data[min(rank, len(data)) - 1] + + +def rate(part: int, whole: int) -> dict[str, Any]: + """{passed, n, rate, ci}:比例和它的 Wilson 区间。""" + return {"passed": part, "n": whole, "rate": round(part / whole, 4) if whole else None, "ci": wilson(part, whole)} + + +def _usage_total(rows: Iterable[Mapping[str, Any] | None]) -> dict[str, Any]: + from app.eval.answer import add_usage + + out = add_usage(None, None) + for row in rows: + out = add_usage(out, row) + return out + + +def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) -> dict[str, Any]: + """一次评测(一臂)的总表。results 是逐题结果(result_row 的形状),只算已经结束的那些。""" + from app.eval.score import SCORER_VERSION + + done = [r for r in results if r.get("status") in ("ok", "skipped", "failed", "cancelled", "partial")] + verdicts = Counter(r.get("verdict") or "none" for r in done) + graded = [r for r in done if r.get("verdict") in ("pass", "fail")] + passed = [r for r in graded if r.get("verdict") == "pass"] + traced = [r for r in passed if r.get("traced")] + outcomes = Counter(s.get("outcome") for r in done for s in r.get("scores") or [] + if s.get("required") and s.get("kind") != "text") + text = Counter(s.get("outcome") for r in done for s in r.get("scores") or [] if s.get("kind") == "text") + answered = [r for r in done if r.get("path")] + tiers = Counter((r.get("tier") or "explore") for r in answered if r.get("path") == "ran") + reviews = [r.get("review") or {} for r in answered if r.get("review")] + looks = [r.get("looks") or {} for r in answered] + timings = {k: [float((r.get("timings") or {}).get(k)) for r in answered if (r.get("timings") or {}).get(k) is not None] + for k in TIMING_KEYS} + usage = {role: _usage_total((r.get("usage") or {}).get(role) for r in done) + for role in ("assistant", "writer", "judge")} + cost = round(sum(u["cost_usd"] for u in usage.values()), 6) + unpriced = sum(int(u.get("unpriced_calls") or 0) for u in usage.values()) + stats = [((r.get("answer_stats") or {})) for r in answered] + numbers = sum(int(s.get("numbers") or 0) for s in stats) + cited = sum(int(s.get("numbers_cited") or 0) for s in stats) + e1 = [r.get("sql_match") for r in done if isinstance(r.get("sql_match"), Mapping) + and (r["sql_match"].get("status") in ("proven", "related", "none"))] + aspects = Counter(a for m in e1 if m.get("status") == "related" for a in m.get("aspects") or []) + claims = Counter() + for r in done: + for key, n in (((r.get("judge") or {}).get("claims") or {}).get("counts") or {}).items(): + claims[key] += int(n or 0) + by_case: dict[str, list[Mapping[str, Any]]] = {} + for r in graded: + by_case.setdefault(str(r.get("case_id")), []).append(r) + repeated = {cid: rate(sum(1 for r in rows if r.get("verdict") == "pass"), len(rows)) + for cid, rows in by_case.items() if len(rows) > 1} + return { + "scorer_version": SCORER_VERSION, + "items": len(results), "done": len(done), + "cases": len({r.get("case_id") for r in results}), + "verdicts": {k: verdicts.get(k, 0) for k in ("pass", "fail", "ungradable", "skipped")}, + "accuracy": rate(len(passed), len(graded)), + "traced_accuracy": rate(len(traced), len(graded)), + "outcomes": dict(outcomes), + "text": dict(text), + "paths": dict(Counter(r.get("path") for r in answered)), + "tiers": {k: tiers.get(k, 0) for k in TIERS}, + "review": { + "verdicts": dict(Counter(str(v.get("verdict") or "ok") for v in reviews)), + "severity": dict(Counter(str(v.get("severity")) for v in reviews if v.get("severity"))), + "retries": sum(1 for r in answered if int(r.get("attempts") or 0) > 0), + "slow": sum(1 for r in answered if ((r.get("timings") or {}).get("review_ms") or 0) > REVIEW_PAGE_WAIT_MS), + }, + "judge": {"on": judge_on, "claims": dict(claims) if judge_on else None}, + "looks": { + "model": sum(int(x.get("model") or 0) for x in looks), + "system": sum(int(x.get("system") or 0) for x in looks), + "per_case": round(sum(int(x.get("model") or 0) + int(x.get("system") or 0) for x in looks) / len(looks), 2) + if looks else None, + "limit_hit": rate(sum(1 for x in looks if x.get("limit_hit")), len(looks))["rate"] if looks else None, + }, + "calls": {"assistant": int(usage["assistant"].get("calls") or 0), + "per_case": round(int(usage["assistant"].get("calls") or 0) / len(answered), 2) if answered else None}, + "timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()}, + "usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}}, + "cost_usd": cost, "unpriced_calls": unpriced, + "per_case_cost_usd": round(cost / len(done), 6) if done else None, + "citations": {"numbers": numbers, "numbers_cited": cited, + "uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats), + "unresolved": sum(int(s.get("unresolved") or 0) for s in stats), + "unknown_entities": sum(int(s.get("unknown_entities") or 0) for s in stats), + "rate": round(cited / numbers, 4) if numbers else None}, + "e1": {**rate(sum(1 for m in e1 if m.get("status") == "proven"), len(e1)), "aspects": dict(aspects)}, + "pass_rates": repeated, + } + + +__all__ = ["REVIEW_PAGE_WAIT_MS", "TIERS", "Z95", "mcnemar_exact", "percentile", "rate", "summarize", "wilson"] diff --git a/backend/app/eval/suites.py b/backend/app/eval/suites.py index ac545036..9393390b 100644 --- a/backend/app/eval/suites.py +++ b/backend/app/eval/suites.py @@ -1,11 +1,11 @@ """题集的形状与校验(细案 01 做法 7、17;评测-D9)。 题集:name(必填)、description、timezone(题目里的 {{ schedule.* }} 按它算)、default_variant(发起评测时缺省用的变体, -第 3 波解析;这里只要求是对象)、budget(只存改过的项)、archived。改题集带上读到的 version(乐观锁,接口回 409)。 +形状按 app/eval/variants.normalize_variant 核对)、budget(只存改过的项)、archived。改题集带上读到的 version(乐观锁,接口回 409)。 **预算**(评测-D9,用户拍板):缺省见 db/models_eval.DEFAULT_BUDGET——同时 2 题、每题 900 秒、单次评测 2 美元、评测每日 -5 美元。daily_max_usd 是评测那本账(engine/spend 的第三本,第 3 波接上)的当日上限:评测单独记账,不挤占裁判、核查的额度。 -这一波只定字段和取值范围,执行(预留、找补、触顶不再开新题)在第 3 波。 +5 美元。全站的缺省在设置 eval 一组(管理员改),题集上改过的项盖过它;daily_max_usd 是评测那本账(engine/spend.eval_spend) +的当日上限:评测单独记账,不挤占裁判、核查的额度,题集只能往下收。执行(预留、找补、触顶不再开新题)见 app/eval/runner.py。 """ from __future__ import annotations @@ -63,9 +63,16 @@ def normalize_budget(raw: Any) -> dict[str, Any]: return out -def effective_budget(stored: Mapping[str, Any] | None) -> dict[str, Any]: - """生效的预算:缺省加上题集改过的项。""" - return {**DEFAULT_BUDGET, **{k: v for k, v in (stored or {}).items() if k in DEFAULT_BUDGET}} +def effective_budget(stored: Mapping[str, Any] | None, site: Mapping[str, Any] | None = None) -> dict[str, Any]: + """生效的预算:评测-D9 的缺省 ← 设置里 eval 一组(全站,管理员定,api/settings.eval_settings)← 题集改过的项。 + + 评测每日的金额上限是全站那一本账(engine/spend.eval_spend)的上限:题集只能往下收(取两者小的),搭建者改题集不能 + 越过管理员定的每日上限。""" + base = {**DEFAULT_BUDGET, **{k: v for k, v in (site or {}).items() if k in DEFAULT_BUDGET}} + own = {k: v for k, v in (stored or {}).items() if k in DEFAULT_BUDGET} + out = {**base, **own} + out["daily_max_usd"] = min(float(base["daily_max_usd"]), float(own.get("daily_max_usd", base["daily_max_usd"]))) + return out def normalize_suite(body: Mapping[str, Any], *, current: Mapping[str, Any] | None = None) -> dict[str, Any]: @@ -96,6 +103,13 @@ def normalize_suite(body: Mapping[str, Any], *, current: Mapping[str, Any] | Non variant = pick("default_variant", {}) or {} if not isinstance(variant, Mapping) or len(canonical_json(variant)) > VARIANT_MAX: raise _invalid("默认变体需要是一个对象", "default_variant") + # 形状按变体的规矩核对(app/eval/variants.py);存的是写进来的样子,没写的项发起时按设置补 + from app.eval.variants import normalize_variant + + try: + normalize_variant(variant) + except CaseInvalid as e: + raise _invalid(f"默认变体:{e.message}", "default_variant") from e archived = pick("archived", False) if not isinstance(archived, bool): raise _invalid("是否归档需要是 true 或 false", "archived") diff --git a/backend/app/eval/variants.py b/backend/app/eval/variants.py new file mode 100644 index 00000000..f4c1ad0a --- /dev/null +++ b/backend/app/eval/variants.py @@ -0,0 +1,213 @@ +"""评测的变体:这一次用哪几个模型、读哪一份目录、重复几次(细案 01 做法 9、12;评测-D5、D10)。 + +变体原样存进 EvalRun.variant(发起时规范化过),实际解析出的模型 id 另存 resolved_models。几项: + +- assistant {provider, model}:助手(建图、挑表、复核)用哪个模型;不写按「编排」页助手栏的设置(copilot 那一项),和 + 问数据页同一条取值链(GenerateIn.provider / model 按请求覆盖,挑表、复核跟着换)。 +- writer {provider, model}:写作模型——答案运行里模型节点的缺省(写进图的 defaults)。节点自己点了模型的不覆盖,结果里 + 照实标出(answer.py 的 own_models)。不写就是各节点自己的、再不然按接入的缺省(和问数据页一样)。 +- judge:结论裁判开不开(缺省按设置 eval.judge)。打开了才判,走按需裁判同一条路,花费记在评测那本账上;模型按设置里的 + 证据裁判模型(按需裁判就是这么取的,变体不另换裁判模型)。文字要点的期望也只在打开时判。 +- catalog {mode, snapshot}:读哪一份目录(评测-D5)——live 当前目录、reviewed 只用已核实和已确认的项、off 关目录、snapshot + 指定一份目录快照(EvalCatalogSnapshot 的 id,影子起草的或拍好的)。live、reviewed 在开评时都拍一份当前目录的快照,按 + 快照读:评测一跑几十分钟,期间起草、剖析照常改目录,也不影响这次的可比性(做法 9「不占数据源的队」)。 +- repeats:每题重复几次(1–5)。定时跑 1 次,对照缺省 3 次(评测-D10)。 +- label:这一臂叫什么(A/B 时给人看,「当前目录」「关目录」……)。 + +**同一变体**(和上一次比时挑基准用,做法 12):助手、写作两项实际解析出的模型、目录的 mode(指定快照的连同快照 id); +重复次数、裁判开不开不算——它们不改变答案本身。variant_key 是它的指纹。 + +**目录贡献对照**(做法 12):一组预设的 A/B——当前目录 vs 关,可以再加第三臂「只用已核实和已确认的项」(catalog_preset)。 +""" +from __future__ import annotations + +import copy +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.eval.cases import CaseInvalid + +#: 变体不合规(422)的机器码 +VARIANT_INVALID = "eval_variant_invalid" +#: 变体里点名的模型、快照用不了(422) +VARIANT_UNAVAILABLE = "eval_variant_unavailable" + +CATALOG_MODES = ("live", "reviewed", "off", "snapshot") +CATALOG_LABEL = {"live": "当前目录", "reviewed": "只用已核实和已确认的项", "off": "关目录", "snapshot": "目录快照"} +REPEATS_MAX = 5 +#: 定时跑 1 次,A/B 对照缺省 3 次(评测-D10) +SCHEDULE_REPEATS = 1 +PAIR_REPEATS = 3 +#: 一次 A/B 最多几臂(目录贡献对照最多三臂) +ARMS_MAX = 3 +LABEL_MAX = 40 +_KEYS = ("assistant", "writer", "judge", "catalog", "repeats", "label") + + +def _invalid(message: str, field_name: str) -> CaseInvalid: + return CaseInvalid(message, field_name, VARIANT_INVALID) + + +def _model_pair(raw: Any, where: str, field_name: str) -> dict[str, str | None]: + if raw in (None, {}): + return {"provider": None, "model": None} + if not isinstance(raw, Mapping) or set(raw) - {"provider", "model"}: + raise _invalid(f"{where}需要是 {{provider, model}} 的对象(都可以留空)", field_name) + out: dict[str, str | None] = {} + for key in ("provider", "model"): + value = raw.get(key) + if value is not None and not isinstance(value, str): + raise _invalid(f"{where}的「{key}」需要是文字", f"{field_name}.{key}") + out[key] = value.strip()[:200] or None if isinstance(value, str) else None + return out + + +def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default: int = 1) -> dict[str, Any]: + """请求里的变体 → 规范化的形状(全部键都在)。不合规抛 CaseInvalid(code=eval_variant_invalid)。""" + if raw in (None, {}): + raw = {} + if not isinstance(raw, Mapping): + raise _invalid("变体需要是一个对象", "variant") + unknown = sorted(set(raw) - set(_KEYS)) + if unknown: + raise _invalid(f"变体里不认识这些项:{'、'.join(unknown)}(认识的:{'、'.join(_KEYS)})", f"variant.{unknown[0]}") + judge = raw.get("judge", judge_default) + if not isinstance(judge, bool): + raise _invalid("变体的「结论裁判」需要是 true 或 false", "variant.judge") + catalog = raw.get("catalog") or {"mode": "live"} + if isinstance(catalog, str): + catalog = {"mode": catalog} + if not isinstance(catalog, Mapping) or catalog.get("mode") not in CATALOG_MODES \ + or set(catalog) - {"mode", "snapshot"}: + raise _invalid(f"变体的目录需要是 {{mode, snapshot}},mode 只能是 {'、'.join(CATALOG_MODES)}", "variant.catalog") + snapshot = catalog.get("snapshot") + if catalog["mode"] == "snapshot": + if not isinstance(snapshot, str) or not snapshot.strip(): + raise _invalid("目录选「目录快照」时要指定是哪一份快照(snapshot)", "variant.catalog.snapshot") + snapshot = snapshot.strip() + elif snapshot not in (None, ""): + raise _invalid("只有目录选「目录快照」时才指定快照", "variant.catalog.snapshot") + else: + snapshot = None + repeats = raw.get("repeats", repeats_default) + if isinstance(repeats, bool) or not isinstance(repeats, int) or not 1 <= repeats <= REPEATS_MAX: + raise _invalid(f"每题重复的次数需要是 1 到 {REPEATS_MAX} 的整数", "variant.repeats") + label = raw.get("label") + if label is not None and not isinstance(label, str): + raise _invalid("这一臂的名字需要是文字", "variant.label") + return {"assistant": _model_pair(raw.get("assistant"), "助手模型", "variant.assistant"), + "writer": _model_pair(raw.get("writer"), "写作模型", "variant.writer"), + "judge": judge, "catalog": {"mode": catalog["mode"], "snapshot": snapshot}, "repeats": repeats, + "label": (label or "").strip()[:LABEL_MAX] or None} + + +def catalog_preset(base: Mapping[str, Any], *, reviewed: bool = False) -> list[dict[str, Any]]: + """「目录的贡献」预设对照:当前目录 vs 关(reviewed 时再加「只用已核实和已确认的项」)。模型、裁判、重复次数照 base。""" + arms = [("live", CATALOG_LABEL["live"]), ("off", CATALOG_LABEL["off"])] + if reviewed: + arms.insert(1, ("reviewed", CATALOG_LABEL["reviewed"])) + return [{**copy.deepcopy(dict(base)), "catalog": {"mode": mode, "snapshot": None}, "label": label} + for mode, label in arms] + + +def arm_name(variant: Mapping[str, Any], index: int) -> str: + """这一臂的名字:变体里写了就用,否则按目录(只有目录不同时一看就懂),再不然 A、B、C。""" + return str(variant.get("label") or "") or "ABC"[index] + + +# ========================================================================== +# 解析:实际用哪个模型、读哪份目录 +# ========================================================================== + + +@dataclass +class Resolved: + """发起时解析出来的:各角色实际的模型 id、目录快照(工件 id)、目录视图(Run.options['catalog_view'] 的形状)。""" + + models: dict[str, Any] = field(default_factory=dict) + snapshot_artifact: str | None = None + view: dict[str, Any] | None = None + + +async def _model_id(session: AsyncSession, pair: Mapping[str, Any], *, assistant: bool) -> str: + """这一对 {provider, model} 实际是哪个模型 id。助手没点名的按助手栏的设置;用不了照实拒(VARIANT_UNAVAILABLE)。""" + from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model + + try: + if assistant: + from app.api.copilot import GenerateIn, copilot_model_spec + + spec = await copilot_model_spec(session, GenerateIn(instruction="eval", provider=pair.get("provider"), + model=pair.get("model"))) + else: + spec = ModelSpec(provider=pair.get("provider"), model=pair.get("model")) + _model, model_id = await get_chat_model(session, spec) + except ProviderNotConfigured as e: + what = "助手模型" if assistant else "写作模型" + raise CaseInvalid(f"{what}用不了:{e}", "variant.assistant" if assistant else "variant.writer", + VARIANT_UNAVAILABLE) from e + return model_id + + +async def resolve_models(session: AsyncSession, variant: Mapping[str, Any]) -> dict[str, Any]: + """{assistant: 模型 id, writer: 模型 id 或 None(没换写作模型:各节点自己的), judge: 开不开}。""" + writer = variant.get("writer") or {} + return {"assistant": await _model_id(session, variant.get("assistant") or {}, assistant=True), + "writer": await _model_id(session, writer, assistant=False) if (writer.get("provider") or writer.get("model")) + else None, + "judge": bool(variant.get("judge"))} + + +def view_of(variant: Mapping[str, Any], snapshot_artifact: str | None) -> dict[str, Any]: + """这一臂的目录视图(data/catalog_view.CatalogView 的形状)。live、reviewed 按开评时拍的快照读。""" + mode = (variant.get("catalog") or {}).get("mode") or "live" + if mode == "off": + return {"mode": "off", "statuses": "all"} + if not snapshot_artifact: + raise ValueError("目录视图要读快照,却没有快照工件") + return {"mode": "snapshot", "statuses": "reviewed" if mode == "reviewed" else "all", "snapshot": snapshot_artifact} + + +async def snapshot_artifact(session: AsyncSession, variant: Mapping[str, Any]) -> str | None: + """变体指定的那份目录快照的工件 id(mode=snapshot);别的模式为 None。快照不在、还没起草完的照实拒。""" + from app.db.models_eval import EvalCatalogSnapshot + + catalog = variant.get("catalog") or {} + if catalog.get("mode") != "snapshot": + return None + row = await session.get(EvalCatalogSnapshot, str(catalog.get("snapshot") or "")) + if row is None: + raise CaseInvalid("变体指定的目录快照不存在", "variant.catalog.snapshot", VARIANT_UNAVAILABLE) + if not row.artifact_id or row.status not in ("succeeded", "partial"): + raise CaseInvalid("变体指定的目录快照还没有出来(影子起草没做完、停止了或失败了)", "variant.catalog.snapshot", + VARIANT_UNAVAILABLE) + return row.artifact_id + + +def variant_key(variant: Mapping[str, Any], models: Mapping[str, Any]) -> str: + """「同一变体」的指纹:助手、写作实际解析出的模型,目录的 mode(指定快照的连同快照 id)。""" + from app.core.artifact_store import canonical_json, content_hash + + catalog = variant.get("catalog") or {} + body = {"assistant": models.get("assistant"), "writer": models.get("writer"), "catalog": catalog.get("mode"), + "snapshot": catalog.get("snapshot") if catalog.get("mode") == "snapshot" else None} + return content_hash(canonical_json(body))[:16] + + +def describe(variant: Mapping[str, Any], models: Mapping[str, Any] | None = None) -> str: + """给人看的一句:「助手 x · 写作 y · 当前目录 · 裁判关 · 每题 1 次」。""" + models = models or {} + catalog = variant.get("catalog") or {} + parts = [f"助手 {models.get('assistant') or (variant.get('assistant') or {}).get('model') or '缺省'}", + f"写作 {models.get('writer') or (variant.get('writer') or {}).get('model') or '各节点自己的'}", + CATALOG_LABEL.get(catalog.get("mode") or "live", "当前目录"), + f"结论裁判{'开' if variant.get('judge') else '关'}", f"每题 {variant.get('repeats') or 1} 次"] + return " · ".join(parts) + + +__all__ = ["ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", "PAIR_REPEATS", "REPEATS_MAX", "Resolved", "SCHEDULE_REPEATS", + "VARIANT_INVALID", "VARIANT_UNAVAILABLE", "arm_name", "catalog_preset", "describe", "normalize_variant", + "resolve_models", "snapshot_artifact", "variant_key", "view_of"] diff --git a/backend/app/main.py b/backend/app/main.py index 60d2f860..7343ab82 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -118,6 +118,18 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await introspect_jobs.startup() except Exception: # noqa: BLE001 logger.exception("探查任务的启动整理失败,服务照常启动") + # 评测(next-round 01 做法 6、9):上次没做完的评测、影子起草同一个规矩——标中断(没做完的题记停止),不自动续跑: + # 拷来的数据目录启动时不能替人去调模型、查业务库。界面上点「重试」只把没做完的题接着做 + from app.eval import runner as eval_runner, snapshots as eval_snapshots + + try: + await eval_snapshots.startup() + except Exception: # noqa: BLE001 + logger.exception("影子起草的启动整理失败,服务照常启动") + try: + await eval_runner.startup() + except Exception: # noqa: BLE001 + logger.exception("评测任务的启动整理失败,服务照常启动") await run_manager.setup() # 流程:存量的待审批补建待办(派给任何人,幂等);上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实 @@ -151,6 +163,9 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await catalog_jobs.manager.shutdown() await catalog_profile_jobs.manager.shutdown() await introspect_jobs.manager.shutdown() + # 评测要在运行之前停:它停下时会取消自己发起的作答、算期望的运行,再标中断 + await eval_runner.shutdown() + await eval_snapshots.manager.shutdown() await run_manager.shutdown() from app.sandbox.manager import sandbox_manager diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index b2b1f218..dc3ac48e 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -71,9 +71,10 @@ EVENTS: dict[str, Event] = { "source.changed": Event("数据源有变化要处理", ()), # 口径升版后召回重算,数真变了(02 第 7 项):答案的主人有 recall 待办;口径名、版本、链接,不放新旧值和差值 "ledger.recall": Event("口径升版,答案的数变了", ()), - # 定时评测有题从通过变成不通过 / 评测没跑完(01 第 15 项):题集名、通过数、退步题数、链接;不放期望值和答案里的数 - "eval.regressed": Event("评测有退步", ()), - "eval.failed": Event("评测失败", ()), + # 定时评测有题从通过变成不通过 / 评测没跑完(01 第 15 项):题集名、通过数、退步题数、链接;不放期望值和答案里的数。 + # 只推全局渠道:评测是搭建者们共用的一把尺,不归哪个人(notify_eval) + "eval.regressed": Event("评测有退步", ("global",)), + "eval.failed": Event("评测失败", ("global",)), # 每天一条待办汇总(04 B3):只推个人渠道;条数、最早的时限、最多 5 条(节点在图上的名字)、去待办箱的链接 "task.digest": Event("每天的待办汇总", ()), # 访问密钥到期前 7 天、1 天各一次(04 B3):个人密钥推本人,服务账号的推全局渠道;密钥名、到期日、链接 @@ -357,3 +358,36 @@ async def notify_action(action_id: str, event: str) -> int: "subject": {"action_id": action_id, "run_id": rec.run_id, "task_id": task_id}} return await _push(channels, message, subject_kind="action", subject_id=action_id, event=event, run_id=rec.run_id, task_id=task_id) + + +async def notify_eval(eval_run_id: str, event: str, *, passed: int, graded: int, regressed: int) -> int: + """定时评测的通知(eval.regressed 有一道题从通过变成不通过、eval.failed 没评完:失败、到了上限、止损)。只推订阅了它的 + 全局渠道。消息里只有题集名、状态、通过几题、退步几题、评测编号和链接(细案 01 做法 15、总方案 X6):不放期望值、答案里的数、 + 题目原文。链接去发起它的那条定时任务的触发历史(评测页在第 4 波)。""" + from app.db.models_eval import EvalRun, EvalSuite + + async with SessionLocal() as session: + run = await session.get(EvalRun, eval_run_id) + if run is None: + return 0 + suite = await session.get(EvalSuite, run.suite_id) + channels = await _channels(session, event, None, []) + if not channels: + return 0 + base = await _base_url(session) + name = (suite.name if suite is not None else "题集")[:80] + regressed_event = event == "eval.regressed" + status = {"succeeded": "评完了", "partial": "没评完", "failed": "失败", "cancelled": "已停止", + "interrupted": "中断"}.get(run.status, run.status) + lines = [("状态", status), ("通过", f"{passed} / {graded} 题")] + if regressed_event or regressed: + lines.append(("退步", f"{regressed} 题")) + if run.arm: + lines.append(("对照", run.arm[:40])) + lines.append(("评测", f"#{eval_run_id[:6]}")) + links = [("查看定时任务", f"{base}/schedules/{run.schedule_id}?tab=history")] if base and run.schedule_id else [] + message = {"event": event, "status": run.status, "ok": False, + "title": f"评测「{name}」{'有退步' if regressed_event else '没有评完'}", "lines": lines, "links": links, + "subject": {"eval_run_id": eval_run_id, "suite_id": run.suite_id, "schedule_id": run.schedule_id}} + return await _push(channels, message, subject_kind="eval_run", subject_id=eval_run_id, event=event, run_id=None, + task_id=None) diff --git a/backend/app/scheduler/service.py b/backend/app/scheduler/service.py index 76bc5056..94435419 100644 --- a/backend/app/scheduler/service.py +++ b/backend/app/scheduler/service.py @@ -5,7 +5,8 @@ 发起,这一条不会在定时任务这边被绕过。 **定时的目标**(docs/design/next-round/00-总方案.md X4:定时只有一套):一个定时任务指向一个目标——工作流、评测题集 -(第 3 波)或数据源维护(第 3 波)。认领、补跑、重叠跳过、触发记录、通知都只有这一套,fire_once 按目标种类分派。 +(app/eval/scheduled.py)或数据源维护(app/data/source_maintenance.py)。认领、补跑、重叠跳过、触发记录、通知都只有这一套, +fire_once 按目标种类分派;建任务、改目标时核对目标也按种类交给各自的模块(check_target)。 老表的 workflow_id 是非空约束,只加不改的迁移放宽不了:目标不是工作流时它存空串。这条规矩只在 target_of / target_columns 这一处,别处读写目标都经它们,不直接碰这两列。 """ @@ -69,7 +70,7 @@ INTERRUPTED_REASON = "服务在发起运行前停止,这次触发没有发起 GENERIC_REASON = "发起运行时出错,详细信息见服务日志" -#: 定时的目标种类(Schedule.target_kind),顺序即以后界面上的顺序。这一波 fire_once 只会发起 workflow +#: 定时的目标种类(Schedule.target_kind),顺序即界面上的顺序。workflow 由本模块发起,另两种交给各自的模块(_TARGET_MODULES) TARGET_KINDS = ("workflow", "eval_suite", "source_maintenance") TARGET_LABEL = {"workflow": "工作流", "eval_suite": "评测题集", "source_maintenance": "数据源维护"} #: 目标种类这个版本还发起不了(库里有别的版本建的任务、手改过库):照实记「未能发起」,不当成工作流去跑 @@ -103,6 +104,29 @@ def target_handler(kind: str) -> Any | None: return importlib.import_module(module) if module else None +async def check_target(session: Any, kind: str, target_id: str) -> str | None: + """建任务、改目标时核对一个非工作流目标(存在、可用):交给这种目标的模块(check_target)。不合规返回给人看的一句。 + 工作流目标由接口自己核对(api/schedules._check_workflow:还要对照入口字段)。""" + handler = target_handler(kind) + if handler is None: + return f"不认识的定时目标种类:{kind}" + return await handler.check_target(session, target_id) + + +async def target_name(session: Any, kind: str, target_id: str) -> str | None: + """非工作流目标现在叫什么(题集名、数据源名),给接口和定时页列表显示;不在了为 None。模块里有 target_name 就用它, + 数据源维护的目标直接读数据源名。""" + handler = target_handler(kind) + if handler is not None and hasattr(handler, "target_name"): + return await handler.target_name(session, target_id) + if kind == "source_maintenance": + from app.db.models import DataSource + + source = await session.get(DataSource, target_id) + return source.name if source is not None else None + return None + + async def record_fire(schedule_id: str, fire_at: datetime, kind: str, status: str, reason: str | None, inputs: dict[str, Any] | None = None, **columns: Any) -> ScheduleFire | None: """非工作流目标写触发记录(同 _record:同一次触发已经记过就交回已有的那一条)。columns 写额外的列,比如评测的 @@ -348,7 +372,8 @@ async def fire_once(schedule_id: str, fire_at: datetime, kind: str, *, actor: st """处理一次触发,返回记下的触发记录。同一 (任务, fire_at, kind) 已经有记录(两拍撞在一起、立即运行连点两下) 时不再发起,返回已有的那一条;任务不存在返回 None。 - 按目标种类分派(target_of):这一波只有工作流(_fire_workflow),别的种类记 failed_to_start、写明原因,不发起。 + 按目标种类分派(target_of):工作流在这里发起(_fire_workflow),评测题集、数据源维护交给各自模块的 fire(同样持着 + launch_lock,触发记录用 record_fire 写);不认识的种类记 failed_to_start、写明原因,不发起。 """ async with launch_lock(): async with SessionLocal() as session: diff --git a/backend/tests/eval_samples.py b/backend/tests/eval_samples.py new file mode 100644 index 00000000..d06f4904 --- /dev/null +++ b/backend/tests/eval_samples.py @@ -0,0 +1,156 @@ +"""评测执行测试共用的虚构样本:dual_check_shop 的订单库(tests/fixtures/dual_check_shop.py)、助手和报告撰写的剧本、建题集和 +题目、等评测跑完。库、表、数字、问法都是编的。 + +助手剧本(Copilot):先 look 一眼订单表,查看结果里看到数据目录写着「按支付日期归属」就按支付日写 SQL(330),没看到就按 +下单日写(310)——目录开着和关着答得不一样,「目录的贡献」对照才量得出来。图是入口 → 调用工具(查库)→ 报告撰写 → 出口; +报告撰写的剧本引用查到的那一格([[v:Q1.r0.gmv]])。 +""" +from __future__ import annotations + +import json +from typing import Any + +from langchain_core.messages import AIMessage, AIMessageChunk +from sqlalchemy import delete + +from app.data import catalog +from app.data.catalog import make_item +from app.db.base import SessionLocal +from tests.fixtures import dual_check_shop as shop + +#: 题目:9/1–9/7 按支付日、不含退款的销售额(期望 330) +QUESTION = "2026-09-01 至 2026-09-07 按支付日统计、不含退款的销售额是多少?" +PAID_RULE = "按支付日期归属" +NOTES = {"label": make_item("订单", "human"), + "business_date": make_item({"column": "paid_date", "rule": PAID_RULE}, "human"), + "valid_filter": make_item("status <> 'refunded'", "human")} +REPORT = "按支付日统计,销售额 [[v:Q1.r0.gmv]]。" + + +def _line(op: dict[str, Any]) -> AIMessageChunk: + return AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n") + + +def graph_ops(source: str, sql: str) -> list[dict[str, Any]]: + def node(nid: str, ntype: str, label: str, **config: Any) -> dict[str, Any]: + return {"op": "add_node", "node": {"id": nid, "type": ntype, "label": label, "config": config}} + + def edge(a: str, b: str) -> dict[str, Any]: + return {"op": "add_edge", "edge": {"source": a, "target": b}} + + return [node("start", "input", "输入"), node("q", "tool", "查订单", tool=f"db_query__{source}", args={"sql": sql}), + edge("start", "q"), node("write", "report", "写答案", instructions="回答:{{ input.question }}"), + edge("q", "write"), + node("out", "output", "答案", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]), + edge("write", "out"), {"op": "done", "explanation": "查一下", "run": True}] + + +class Copilot: + """助手剧本(见模块说明)。mode:answer 照上面答;fail 每次都抛错(助手调用失败);reply 直接回答;ask 反问; + slow 每一步都慢(测超时)。calls 记下每次收到的整串消息。""" + + def __init__(self, source: str, *, mode: str = "answer", delay: float = 0.0) -> None: + self.source = source + self.mode = mode + self.delay = delay + self.calls: list[list] = [] + + async def astream(self, messages): + import asyncio + + self.calls.append(list(messages)) + if self.delay: + await asyncio.sleep(self.delay) + if self.mode == "fail": + raise RuntimeError("接入返回了 502") + if "上线前自查" in messages[1][1]: + yield _line({"op": "done"}) + return + if self.mode == "reply": + yield _line({"op": "reply", "text": "销售额是 330。"}) + return + if self.mode == "ask": + yield _line({"op": "plan", "summary": "先问清楚"}) + yield _line({"op": "ask", "question": "要按下单日还是支付日?", "options": ["下单日", "支付日"]}) + return + if len(messages) == 2: + yield _line({"op": "plan", "summary": "查销售额"}) + yield _line({"op": "look", "need": "订单", "targets": [{"kind": "table", "name": "orders"}]}) + return + looked = str(messages[-1][1]) + sql = shop.RIGHT if PAID_RULE in looked else shop.BY_ORDER_DATE + for op in graph_ops(self.source, sql): + yield _line(op) + + +def use_copilot(monkeypatch, model: Any) -> None: + import app.api.copilot as copilot + + async def _model(*_a, **_k): + return model, "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + monkeypatch.setattr(copilot, "_HEARTBEAT_SECONDS", 3600.0) + + +def use_report(monkeypatch, text: str = REPORT) -> None: + """运行里的模型调用(报告撰写)一律答 text。""" + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text)) + + +async def make_source(tmp_path, *, notes: dict[str, Any] | None = NOTES) -> tuple[str, str]: + name, sid = await shop.add_source(shop.build(tmp_path / "shop.db")) + if notes is not None: + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", notes, if_version=0, actor=None) + return name, sid + + +async def set_notes(source_id: str, notes: dict[str, Any]) -> None: + async with SessionLocal() as session: + current = (await catalog.read_catalogs(session, [source_id])).get(source_id, {}).get("orders") + await catalog.write_entry(session, source_id, "orders", notes, if_version=current.version if current else 0, + actor=None) + + +async def make_suite(source_id: str, *, name: str = "虚构零售题集", cases: list[dict[str, Any]] | None = None, + default_variant: dict[str, Any] | None = None, budget: dict[str, Any] | None = None) -> str: + """直接写库建题集和题目(不走接口的试算:期望在评测里每次都重算)。cases 不给就是一道 330 的题。""" + from app.db.models_eval import EvalCase, EvalSuite + from app.eval.cases import case_hash, normalize_case + + cases = cases if cases is not None else [paid_case(source_id)] + async with SessionLocal() as session: + suite = EvalSuite(name=name, default_variant=default_variant or {}, budget=budget or {}) + session.add(suite) + await session.flush() + for seq, body in enumerate(cases, start=1): + fields = normalize_case(body) + session.add(EvalCase(suite_id=suite.id, seq=seq, case_hash=case_hash(fields), **fields)) + await session.commit() + return suite.id + + +def paid_case(source_id: str, *, question: str = QUESTION, **extra: Any) -> dict[str, Any]: + return {"question": question, "datasource_ids": [source_id], + "gold": {"kind": "sql", "datasource_id": source_id, "sql": shop.RIGHT}, + "expectations": [{"kind": "scalar", "column": "gmv"}], **extra} + + +async def wait_eval(leader_id: str, timeout: float = 120.0) -> None: + from app.eval import cleanup, runner + from app.notify import service as notify + + await runner.manager.wait(leader_id, timeout=timeout) + await cleanup.drain() + await notify.drain() + + +async def drop_suites() -> None: + from app.db.models_eval import EvalSuite + + async with SessionLocal() as session: + await session.execute(delete(EvalSuite)) + await session.commit() diff --git a/backend/tests/test_eval_e2e.py b/backend/tests/test_eval_e2e.py new file mode 100644 index 00000000..3a017db2 --- /dev/null +++ b/backend/tests/test_eval_e2e.py @@ -0,0 +1,122 @@ +"""评测从头到尾(细案 01 测试 test_eval_e2e):dual_check_shop 的虚构订单库 + 剧本模型。 + +题目是「9/1–9/7 按支付日、不含退款的销售额」,期望 SQL 算出 330。助手剧本只在查看结果里看到数据目录写着「按支付日期归属」 +时才按支付日写(330),没看到就按下单日写(310)。所以: + +- 臂「当前目录」按支付日答 330:通过、可追溯、答案的 SQL 和期望 SQL 等价已证明(逐题 E1); +- 臂「关目录」按下单日答 310:不通过,差异方面是 time; +- 「目录贡献」对照(当前 vs 关)的配对表是 1:0,两臂共用期望(同一份期望运行); +- 再评一次(把目录里的业务日期去掉)和上一次比:这题从通过变成不通过,归因「做法变」,指纹里目录快照变了。 + +评测运行的来源都是 eval,不进使用统计;运行名「评测:题集名·第 n 题」。 +""" +from __future__ import annotations + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.db.base import SessionLocal +from app.db.models import Run +from app.db.models_eval import EvalResult, EvalRun +from app.engine.origin import counted_runs +from app.engine.runner import run_manager +from app.main import app +from tests import eval_samples as es +from tests.fixtures import dual_check_shop as shop +from tests.fixtures.sources import drop_source + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await es.drop_suites() + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def source(tmp_path): + name, sid = await es.make_source(tmp_path) + yield {"name": name, "id": sid} + await drop_source(sid) + + +async def _results(run_id: str) -> list[EvalResult]: + async with SessionLocal() as session: + return list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == run_id) + .order_by(EvalResult.seq))).scalars()) + + +async def test_catalog_contribution_pair_end_to_end(client, monkeypatch, source): + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + suite_id = await es.make_suite(source["id"]) + started = await client.post(f"/api/evals/suites/{suite_id}/evaluations", + json={"preset": "catalog", "variant": {"repeats": 1}}) + assert started.status_code == 202, started.text + body = started.json() + live, off = body["evaluations"] + assert (live["arm"], off["arm"]) == ("当前目录", "关目录") and body["pair_id"] == live["id"] + await es.wait_eval(live["id"]) + + got = (await client.get(f"/api/evals/evaluations/{live['id']}")).json() + assert got["status"] == "succeeded", got + [right] = await _results(live["id"]) + [wrong] = await _results(off["id"]) + assert right.path == "ran" and right.verdict == "pass", (right.error, right.scores) + assert right.scores[0]["outcome"] == "traced" and right.scores[0]["got"] == 330.0 + assert right.sql_match["status"] == "proven", right.sql_match + assert wrong.verdict == "fail" and wrong.scores[0]["outcome"] == "wrong" and wrong.scores[0]["got"] == 310.0 + assert wrong.sql_match["status"] == "related" and "time" in wrong.sql_match["aspects"], wrong.sql_match + # 两臂共用期望:同一份期望运行 + assert right.gold_run_ids == wrong.gold_run_ids and len(right.gold_run_ids) == 2 + assert right.gold["status"] == "ok" + + pair = (await client.get(f"/api/evals/pairs/{body['pair_id']}")).json() + [table] = pair["table"]["pairs"] + assert (table["only_a"], table["only_b"], table["both"], table["neither"]) == (1, 0, 0, 0) + assert table["p_value"] == 1.0 and table["note"] == "这批题上看不出差别" + summary = got["summary"] + assert summary["accuracy"]["passed"] == 1 and summary["traced_accuracy"]["passed"] == 1 + assert summary["accuracy"]["ci"] is not None and summary["e1"]["passed"] == 1 + + # 评测运行:来源 eval、不进使用统计、名字写题集和题号 + async with SessionLocal() as session: + run = await session.get(Run, right.run_id) + assert run.origin_kind == "eval" and run.origin["role"] == "answer" + assert run.workflow_name == "评测:虚构零售题集·第 1 题" + assert run.options["catalog_view"]["mode"] == "snapshot" + offrun = await session.get(Run, wrong.run_id) + assert offrun.options["catalog_view"] == {"mode": "off", "statuses": "all"} + counted = (await session.execute(select(Run.id).where( + Run.id.in_([right.run_id, wrong.run_id, *right.gold_run_ids]), counted_runs()))).all() + assert counted == [] + + +async def test_comparing_with_the_previous_run_finds_the_regression(client, monkeypatch, source): + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + suite_id = await es.make_suite(source["id"]) + first = (await client.post(f"/api/evals/suites/{suite_id}/evaluations", json={})).json()["evaluations"][0] + await es.wait_eval(first["id"]) + # 目录里的业务日期去掉:助手改按下单日写 + await es.set_notes(source["id"], {"label": es.NOTES["label"]}) + second = (await client.post(f"/api/evals/suites/{suite_id}/evaluations", json={})).json()["evaluations"][0] + await es.wait_eval(second["id"]) + async with SessionLocal() as session: + row = await session.get(EvalRun, second["id"]) + assert row.baseline_id == first["id"] + assert row.summary["baseline"]["counts"]["regressed"] == 1 + compared = (await client.get(f"/api/evals/evaluations/{second['id']}/compare")).json() + assert compared["base"]["id"] == first["id"] + [item] = compared["transitions"]["items"] + assert item["transition"] == "regressed" and item["cause"] == "method", item + assert "catalog" in {d["key"] for d in compared["fingerprint_diff"]}, compared["fingerprint_diff"] + assert shop.RIGHT != shop.BY_ORDER_DATE diff --git a/backend/tests/test_notify_events.py b/backend/tests/test_notify_events.py index 211d4dc1..73805af5 100644 --- a/backend/tests/test_notify_events.py +++ b/backend/tests/test_notify_events.py @@ -30,8 +30,10 @@ from app.notify import service TERMS_TS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts" URL = "https://hooks.example.com/agentlab/notify-events-5d1c" -#: 本轮登记的六个(总方案 X6):都还没有推送 +#: 本轮登记的六个(总方案 X6) THIS_ROUND = ("source.changed", "ledger.recall", "eval.regressed", "eval.failed", "task.digest", "key.expiring") +#: 其中已经推送的(第 3 波:定时评测的通知只推全局渠道,notify_eval),其余还没有 +PUSHED_THIS_ROUND = {"eval.regressed": {"global"}, "eval.failed": {"global"}} @pytest.fixture @@ -54,6 +56,8 @@ def test_this_rounds_events_are_registered_once_and_not_pushed_yet(): for name in THIS_ROUND: assert name in service.EVENTS, f"{name} 没有登记" assert service.EVENTS[name].label, name + if name in PUSHED_THIS_ROUND: + continue assert service.EVENTS[name].scopes == (), f"{name} 还没有推送:不该有人订阅得到" assert len({e.label for e in service.EVENTS.values()}) == len(service.EVENTS), "两个事件叫成了同一句" assert service.EVENT_LABEL == {k: v.label for k, v in service.EVENTS.items()} @@ -66,6 +70,7 @@ def test_events_that_are_pushed_today_keep_their_audience(): "task.opened": every, "run.finished": every, "run.failed": every, "action.unknown": every, # 发给提交人自己(和订阅了它的全局渠道),不归哪个应用 "review.done": {"global", "user"}, + **PUSHED_THIS_ROUND, } for event in service.EVENTS.values(): assert set(event.scopes) <= every @@ -104,19 +109,20 @@ async def test_events_endpoint_says_who_receives_each_event(client): rows = (await client.get("/api/notify/events")).json() assert [r["event"] for r in rows] == list(service.EVENTS) by = {r["event"]: r for r in rows} - assert by["eval.regressed"] == {"event": "eval.regressed", "label": "评测有退步", "scopes": []} + assert by["eval.regressed"] == {"event": "eval.regressed", "label": "评测有退步", "scopes": ["global"]} + assert by["ledger.recall"]["scopes"] == [] assert by["review.done"]["scopes"] == ["global", "user"] async def test_events_not_pushed_yet_cannot_be_subscribed(client): resp = await client.post("/api/notify/channels", json={ - "name": "事件登记-评测", "url": URL, "events": ["task.opened", "eval.regressed"]}) + "name": "事件登记-评测", "url": URL, "events": ["task.opened", "ledger.recall"]}) assert resp.status_code == 422, resp.text detail = resp.json()["detail"] - assert "评测有退步" in detail and "还没有推送" in detail + assert "口径升版,答案的数变了" in detail and "还没有推送" in detail unknown = await client.post("/api/notify/channels", json={"name": "事件登记-未知", "url": URL, "events": ["nope"]}) assert unknown.status_code == 422 - assert "评测有退步" not in unknown.json()["detail"], "可选的里面不列还没有推送的" + assert "口径升版,答案的数变了" not in unknown.json()["detail"], "可选的里面不列还没有推送的" assert "待办打开" in unknown.json()["detail"] @@ -149,14 +155,14 @@ async def test_channels_only_walk_the_paths_the_event_is_registered_for(monkeypa monkeypatch.setitem(service.EVENTS, "demo.personal", service.Event("只推个人的演示事件", ("user",))) async with SessionLocal() as session: row = NotifyChannel(name="事件登记-全局", scope="global", url=encrypt(URL), - events=["demo.personal", "task.opened", "eval.failed"]) + events=["demo.personal", "task.opened", "ledger.recall"]) session.add(row) await session.commit() try: ids = lambda found: [c.id for c in found] # noqa: E731 assert row.id not in ids(await service._channels(session, "demo.personal", None, [])), "全局渠道这条路没登记" assert row.id in ids(await service._channels(session, "task.opened", None, [])) - assert await service._channels(session, "eval.failed", None, []) == [], "还没有推送的:谁都不推" + assert await service._channels(session, "ledger.recall", None, []) == [], "还没有推送的:谁都不推" with pytest.raises(ValueError, match="没有登记"): await service._channels(session, "demo.unregistered", None, []) finally: diff --git a/backend/tests/test_schedule_target.py b/backend/tests/test_schedule_target.py index ad717c8d..74587e40 100644 --- a/backend/tests/test_schedule_target.py +++ b/backend/tests/test_schedule_target.py @@ -6,9 +6,9 @@ scheduler/service.py 的一处封装里:workflow 目标的 target_id 同 workf - 存量任务(target_kind 是缺省的 workflow、target_id 为空)照旧按 workflow_id 发起,行为一字不变; - 建任务、改工作流都经同一处封装写目标,两列对得上; -- fire_once 按目标种类分派。这一波只实现 workflow,别的种类照实记「未能发起」并写明原因,不发起运行、不悄悄当成 - 工作流去跑空串; -- 接口这一波不暴露新种类(接口返回的形状不变,测试见 test_schedules_api.py 的 SCHEDULE_KEYS)。 +- fire_once 按目标种类分派:评测题集、数据源维护交给各自的模块(目标不在的照实记「未能发起」并写明原因,不发起运行、 + 不悄悄当成工作流去跑空串;评测题集的发起见 test_eval_schedule.py); +- 接口返回带 target_kind / target_id / 目标名(测试见 test_schedules_api.py 的 SCHEDULE_KEYS)。 工作流、任务名都是虚构的。 """ @@ -92,7 +92,8 @@ async def test_creating_and_retargeting_through_the_api_write_both_columns(): "input_template": {"stat_date": "{{ schedule.yesterday }}"}}) assert made.status_code == 201, made.text body = made.json() - assert "target_kind" not in body and "target_id" not in body, "这一波接口不暴露新种类" + assert (body["target_kind"], body["target_id"]) == ("workflow", wf_a) + assert body["target_name"] == body["workflow_name"] and body["target_label"] == "工作流" async with SessionLocal() as session: row = await session.get(Schedule, body["id"]) assert (row.target_kind, row.target_id, row.workflow_id) == ("workflow", wf_a, wf_a) @@ -118,7 +119,7 @@ async def test_a_legacy_schedule_still_fires_its_workflow(): assert run.workflow_id == wf and run.input == {"stat_date": "2026-10-04"} -async def test_a_target_kind_this_wave_cannot_fire_is_recorded_not_run(): +async def test_a_missing_eval_suite_is_recorded_not_run(): async with SessionLocal() as session: row = Schedule(name="题集回归", cron="0 9 * * *", timezone="Asia/Shanghai", **service.target_columns("eval_suite", "suite-x"), next_run_at=NINE) @@ -128,7 +129,7 @@ async def test_a_target_kind_this_wave_cannot_fire_is_recorded_not_run(): runs_before = await session.scalar(select(func.count()).select_from(Run)) fire = await service.fire_once(schedule_id, NINE, "manual") assert fire is not None and fire.status == "failed_to_start" and fire.run_id is None - assert fire.reason == service.TARGET_UNSUPPORTED.format(kind=service.TARGET_LABEL["eval_suite"]) + assert fire.reason == "评测题集不存在,可能已被删除" async with SessionLocal() as session: assert await session.scalar(select(func.count()).select_from(Run)) == runs_before, "不能发起任何运行" assert await session.scalar(select(func.count()).select_from(ScheduleFire) diff --git a/backend/tests/test_schedules_api.py b/backend/tests/test_schedules_api.py index bd9a5ce1..10b623f1 100644 --- a/backend/tests/test_schedules_api.py +++ b/backend/tests/test_schedules_api.py @@ -39,8 +39,12 @@ HOOK = "https://open.feishu.cn/open-apis/bot/v2/hook/7c6b5a49-dddd-eeee-ffff-123 SECRET = "SECdemo-api-not-real-7" SCHEDULE_KEYS = {"id", "name", "workflow_id", "workflow_name", "cron", "timezone", "run_class", "input_template", "enabled", "overlap", "misfire_grace_s", "webhook", "next_run_at", "last_fire", "description_text", - "created_by", "created_at", "updated_at", "version"} -FIRE_KEYS = {"id", "fire_at", "kind", "status", "run_id", "run_status", "reason", "notified", "compare", "created_at"} + "created_by", "created_at", "updated_at", "version", + # 定时的目标(next-round X4):种类、id、叫法、现在的名字、还在不在 + "target_kind", "target_id", "target_label", "target_name", "target_exists"} +FIRE_KEYS = {"id", "fire_at", "kind", "status", "run_id", "run_status", "reason", "notified", "compare", "created_at", + # 目标是评测题集时这一次触发建的那次评测 + "eval_run_id"} @pytest.fixture(autouse=True) From e29adf80d0b9cf05112342a4a33caa44c3bcc824 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 16:49:47 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E8=AF=84=E6=B5=8B=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E7=9A=84=E6=B5=8B=E8=AF=95=EF=BC=9A=E6=89=A7=E8=A1=8C=E3=80=81?= =?UTF-8?q?=E8=AF=84=E5=88=86=E3=80=81=E7=BB=9F=E8=AE=A1=E4=B8=8E=E5=AF=B9?= =?UTF-8?q?=E6=AF=94=E3=80=81=E6=B8=85=E7=90=86=E3=80=81=E5=AE=9A=E6=97=B6?= =?UTF-8?q?=E4=B8=8E=E9=80=9A=E7=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:照细案 01 的测试清单把这一波的行为钉住,顺带修了跑测试时撞出来的两处。 - test_eval_runner:后台排队、并发不超过 max_parallel 也不超过全局并发的四分之一、同一题集同时只能一次、停止后只重做 没做完的(评完的不动)、重启标中断不续跑、单次金额上限触顶不再开新题、评测每日账单独记(不占裁判、核查的额度)、 连续 3 题助手调用失败止损、每题超时取消正在跑的运行、停在审批上的作答取消掉(评测不代批)、接口的估算与进度、设置里 eval 一组和 /settings/eval/spend - test_eval_score:五种结论、写出精度和百分数、维度配对与弱匹配、多行 / 名单 / 没有数据 / 反问、路径和期望先于数字、 dual 两版、允许的相对误差和双盲复算是同一个函数;读封存的答案、逐题 E1 用账本的比对函数;重评只读封存、不调模型 - test_eval_stats:Wilson、精确 McNemar、P50/P90 和手算对照;总表的各项;转移与四种归因;指纹差异;配对表;选基准 - test_eval_cleanup:保留规则;forget 只删一条引用都不剩的文件;清理后总表还在、结果和底下的运行删掉、共用的工件留着 - test_eval_schedule:定时接口收题集(拒工作流、输入模板、Webhook)、到点建一次评测并写 eval_run_id、重叠跳过、错过的按 宽限补;有退步推「评测有退步」、没评完推「评测失败」,消息里没有期望值和答案里的数 - test_eval_e2e 补:开评拍目录快照、记表结构前后;打开裁判时结论裁判走按需裁判那条路、花费记在评测账上,文字要点单列 - 修:期望反问的题在建图、运行失败(含超时)时把原因写全;改非工作流目标的定时任务时提交后不再读过期对象 --- backend/app/api/schedules.py | 3 +- backend/app/eval/score.py | 3 + backend/tests/test_eval_cleanup.py | 143 ++++++++++++ backend/tests/test_eval_e2e.py | 44 ++++ backend/tests/test_eval_runner.py | 333 ++++++++++++++++++++++++++++ backend/tests/test_eval_schedule.py | 227 +++++++++++++++++++ backend/tests/test_eval_score.py | 233 +++++++++++++++++++ backend/tests/test_eval_stats.py | 162 ++++++++++++++ 8 files changed, 1147 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_eval_cleanup.py create mode 100644 backend/tests/test_eval_runner.py create mode 100644 backend/tests/test_eval_schedule.py create mode 100644 backend/tests/test_eval_score.py create mode 100644 backend/tests/test_eval_stats.py diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index fca3934f..ad115aa1 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -694,9 +694,10 @@ async def _update_target_schedule(session: AsyncSession, row: Schedule, body: di if result.rowcount != 1: await session.rollback() raise HTTPException(409, "这个定时任务刚被别人修改过,请刷新后再保存") + schedule_id = row.id await session.commit() session.expire_all() - return await _schedule_out(session, await _get(session, row.id)) + return await _schedule_out(session, await _get(session, schedule_id)) async def retime(session: AsyncSession, row: Schedule, body: dict[str, Any]) -> Schedule: diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py index 6e4d8fcd..be673a75 100644 --- a/backend/app/eval/score.py +++ b/backend/app/eval/score.py @@ -510,6 +510,9 @@ def score_case(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: verdict, traced, reason = _verdict(scores) if ask_only and path != "asked": verdict, traced, reason = "fail", False, "期望反问(问法含糊),助手却没有反问" + if path in ("build_failed", "run_failed"): + # 没走到能不能反问那一步(建图、运行失败、超时):照实写是哪一步、为什么 + reason = WRONG_PATH[path] + (f":{path_reason}" if path_reason else "") by_version = [] for extra in sets[1:]: more = _expectation_scores(expectations, (extra or {}).get("values") or {}, answer, path, judgement, cache) diff --git a/backend/tests/test_eval_cleanup.py b/backend/tests/test_eval_cleanup.py new file mode 100644 index 00000000..ec7b8e7c --- /dev/null +++ b/backend/tests/test_eval_cleanup.py @@ -0,0 +1,143 @@ +"""评测的清理与工件回收(app/eval/cleanup.py、core/artifact_store.forget;细案 01 做法 16)。 + +- 保留规则:最近 N 次、钉住的基线、A/B 的另几臂,还没结束的不动; +- 清掉的:逐题结果、明细工件、底下的运行(作答、算期望的)和它们的工件引用行;总表永久保留,cleaned_at 写明已清理; +- 工件文件一条引用都不剩才删:被别的运行引用着的同一份内容照留; +- 清理过的评测不能重试、不能重评。 +题集、问法都是虚构的。 +""" +from __future__ import annotations + +from datetime import timedelta + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import func, select + +from app.core import artifact_store +from app.db.base import SessionLocal, utcnow +from app.db.models import Artifact, Run +from app.db.models_eval import EvalResult, EvalRun +from app.engine.runner import run_manager +from app.eval import answer as answer_mod +from app.eval import cleanup, runner +from app.main import app +from tests import eval_samples as es +from tests.fixtures.sources import drop_source + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await runner.manager.shutdown() + await es.drop_suites() + await run_manager.shutdown() + + +def test_the_keep_rule(): + now = utcnow() + + def run(i: int, **extra) -> EvalRun: + return EvalRun(id=f"r{i}", suite_id="s", status=extra.pop("status", "succeeded"), + created_at=now - timedelta(minutes=i), **extra) + + runs = [run(1), run(2, pair_id="r2"), run(3, pair_id="r2"), run(4, pinned=True), run(5), run(6, status="running")] + kept = cleanup.keep_set(runs, 2) + assert kept == {"r1", "r2", "r3", "r4", "r6"}, "最近 2 次 + A/B 另一臂 + 钉住的 + 还没结束的" + + +async def test_forget_only_removes_files_nobody_refers_to(): + lonely = await artifact_store.put_json({"评测": "只有一处引用", "n": 1}, kind="test_forget") + shared = await artifact_store.put_json({"评测": "两处引用", "n": 2}, kind="test_forget") + async with SessionLocal() as session: + session.add(Artifact(id=shared, run_id="x" * 32, kind="test_forget")) + await session.execute(Artifact.__table__.delete().where(Artifact.id.in_([lonely, shared]), + Artifact.run_id == "")) + await session.commit() + removed = await artifact_store.forget([lonely, shared, "not-a-hash", lonely]) + assert removed == [lonely] + assert artifact_store.load(lonely) is None and artifact_store.load(shared)["n"] == 2 + async with SessionLocal() as session: + await session.execute(Artifact.__table__.delete().where(Artifact.id == shared)) + await session.commit() + assert await artifact_store.forget([shared]) == [shared] + + +async def test_cleaning_keeps_the_summary_and_shared_files(monkeypatch, tmp_path): + async def drive(self): + self.turn.path, self.turn.reply = "reply", "不查了" + return self.turn + + monkeypatch.setattr(answer_mod.Driver, "drive", drive) + name, sid = await es.make_source(tmp_path) + try: + suite_id = await es.make_suite(sid) + made: list[str] = [] + for i in range(3): + # 中间那一次换一下数:它的期望查询结果只有它自己有(别的两次查到的是同一份,文件共用) + _amount(tmp_path, 101.0 if i == 1 else 100.0) + async with SessionLocal() as session: + from app.db.models_eval import EvalSuite + from app.eval.variants import normalize_variant + + suite = await session.get(EvalSuite, suite_id) + [run] = await runner.start(session, suite, [normalize_variant({})], actor="评测测试") + await es.wait_eval(run.id) + made.append(run.id) + first, middle, last = made + async with SessionLocal() as session: + (await session.get(EvalRun, first)).pinned = True + gold_runs = (await session.execute(select(EvalResult.gold_run_ids).where( + EvalResult.eval_run_id == middle))).scalar_one() + owned = list((await session.execute(select(Artifact.id).where(Artifact.run_id.in_(gold_runs)))).scalars()) + details = (await session.execute(select(EvalResult.details_artifact).where( + EvalResult.eval_run_id == middle))).scalar_one() + # 同一份内容另有一次运行(不是这次评测的)也引用着:文件要留 + shared = owned[0] + session.add(Artifact(id=shared, run_id="y" * 32, kind="query_snapshot")) + await session.commit() + summary_before = (await _run(middle)).summary + out = await cleanup.clean_suite(suite_id, keep=1) + assert out["runs"] == 1 and out["results"] == 1 and out["deleted_runs"] == 2 and out["files"] >= 1 + cleaned = await _run(middle) + assert cleaned.cleaned_at is not None and cleaned.summary == summary_before, "总表永久保留" + assert (await _run(first)).cleaned_at is None and (await _run(last)).cleaned_at is None + async with SessionLocal() as session: + assert await session.scalar(select(func.count()).select_from(EvalResult).where( + EvalResult.eval_run_id == middle)) == 0 + assert await session.scalar(select(func.count()).select_from(Run).where(Run.id.in_(gold_runs))) == 0 + assert await session.scalar(select(func.count()).select_from(Artifact).where( + Artifact.run_id.in_(gold_runs))) == 0 + assert artifact_store.load(shared) is not None, "别的运行还引用着的文件不删" + async with SessionLocal() as session: + still = set((await session.execute(select(Artifact.id).where(Artifact.id.in_(owned)))).scalars()) + # 同样的内容被另外两次评测的运行引用着的(同一条期望 SQL 查到同一份结果)也留着;一条引用都不剩的才删 + for artifact in owned: + assert (artifact_store.load(artifact) is not None) == (artifact in still), artifact + assert any(a not in still for a in owned), "这次评测独有的工件文件删掉了" + assert artifact_store.load(details) is None, "明细工件跟着删" + with pytest.raises(runner.EvalRefused): + await runner.retry(middle) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + refused = await client.post(f"/api/evals/evaluations/{middle}/rescore") + assert refused.status_code == 409 and refused.json()["code"] == "eval_details_cleaned" + compared = (await client.get(f"/api/evals/evaluations/{last}/compare", + params={"base": middle})).json() + assert compared["cleaned"] is True and compared["transitions"] is None + again = (await client.post(f"/api/evals/suites/{suite_id}/cleanup")).json() + assert again["runs"] == 0, "已经清过的不再清" + finally: + await drop_source(sid) + + +def _amount(tmp_path, value: float) -> None: + import sqlite3 + + with sqlite3.connect(tmp_path / "shop.db") as db: + db.execute("UPDATE orders SET amount = ? WHERE id = 1", (value,)) + + +async def _run(run_id: str) -> EvalRun: + async with SessionLocal() as session: + return await session.get(EvalRun, run_id) diff --git a/backend/tests/test_eval_e2e.py b/backend/tests/test_eval_e2e.py index 3a017db2..3b72ac3d 100644 --- a/backend/tests/test_eval_e2e.py +++ b/backend/tests/test_eval_e2e.py @@ -87,6 +87,13 @@ async def test_catalog_contribution_pair_end_to_end(client, monkeypatch, source) assert summary["accuracy"]["passed"] == 1 and summary["traced_accuracy"]["passed"] == 1 assert summary["accuracy"]["ci"] is not None and summary["e1"]["passed"] == 1 + # 开评时拍了一份当前目录(不占数据源的队),上传表格版本和表结构哈希前后各记一次 + async with SessionLocal() as session: + ev = await session.get(EvalRun, live["id"]) + assert ev.catalog_snapshot and ev.fingerprint["catalog"].startswith("live:") + assert ev.fingerprint["sources_before"] == ev.fingerprint["sources_after"] + assert ev.fingerprint["sources_changed"] == [] and ev.fingerprint["prompt"] + # 评测运行:来源 eval、不进使用统计、名字写题集和题号 async with SessionLocal() as session: run = await session.get(Run, right.run_id) @@ -120,3 +127,40 @@ async def test_comparing_with_the_previous_run_finds_the_regression(client, monk assert item["transition"] == "regressed" and item["cause"] == "method", item assert "catalog" in {d["key"] for d in compared["fingerprint_diff"]}, compared["fingerprint_diff"] assert shop.RIGHT != shop.BY_ORDER_DATE + + +async def test_judge_on_charges_the_eval_ledger_and_text_points_stay_apart(client, monkeypatch, source): + from langchain_core.messages import AIMessage + + from app.engine import judge + from app.engine.spend import eval_spend + from app.eval.judging import POINTS_SYSTEM_PROMPT + from app.providers import mock_model + + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + + def decide(self, messages): + text = "\n".join(str(m.content) for m in messages) + if POINTS_SYSTEM_PROMPT[:20] in text: + return AIMessage(content='[{"point": 1, "verdict": "covered", "reason": "说了按支付日"}]') + return AIMessage(content=es.REPORT) + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + case = es.paid_case(source["id"]) + case["expectations"].append({"kind": "text", "points": ["说明按支付日统计"]}) + suite_id = await es.make_suite(source["id"], cases=[case]) + judge_before = await judge.judge_spend.daily() + ledger_before = (await eval_spend.daily())["calls"] + [ev] = (await client.post(f"/api/evals/suites/{suite_id}/evaluations", + json={"variant": {"judge": True}})).json()["evaluations"] + await es.wait_eval(ev["id"]) + [item] = await _results(ev["id"]) + assert item.verdict == "pass", item.error + text = next(s for s in item.scores if s["kind"] == "text") + assert text["outcome"] == "covered" and text["required"] is False, "文字要点单列" + assert item.judge["claims"]["calls"] >= 1, "走了按需裁判那条路" + assert set(item.judge["claims"]["counts"]) == {"supported", "partial", "contradicted", "insufficient", "unjudged"} + assert await judge.judge_spend.daily() == judge_before, "裁判的花费记在评测那本账上" + assert (await eval_spend.daily())["calls"] > ledger_before + got = (await client.get(f"/api/evals/evaluations/{ev['id']}")).json() + assert got["summary"]["judge"]["on"] is True and got["summary"]["text"] == {"covered": 1} diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py new file mode 100644 index 00000000..06a2c233 --- /dev/null +++ b/backend/tests/test_eval_runner.py @@ -0,0 +1,333 @@ +"""评测执行(app/eval/runner.py;细案 01 做法 9,评测-D9)。 + +作答这一步多半换成桩(Driver.drive):这里守的是后台任务本身——排队、并发上限(不超过 max_parallel,也不超过全局运行 +并发的四分之一)、停止、只重试没做完的、重启标中断不续跑、单次和每日金额上限(评测单独一本账,不占裁判、核查的额度)、 +连续几题助手调用失败止损、每题超时会取消正在跑的运行;停在审批上的作答走真的运行(评测不代批,取消掉)。 +题目多半是「期望反问」的题:没有期望来源,不用跑期望探针。问法、题集名都是虚构的。 +""" +from __future__ import annotations + +import asyncio + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.db.base import SessionLocal, utcnow +from app.db.models import Run, Setting +from app.db.models_eval import EvalResult, EvalRun +from app.engine.runner import run_manager +from app.eval import answer as answer_mod +from app.eval import runner +from app.main import app +from tests import eval_samples as es +from tests.fixtures.sources import drop_source + + +async def _clear_ledger() -> None: + async with SessionLocal() as session: + row = await session.get(Setting, "eval_spend") + if row is not None: + await session.delete(row) + await session.commit() + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + await _clear_ledger() + yield + await _clear_ledger() + await runner.manager.shutdown() + await es.drop_suites() + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def ask_cases(n: int) -> list[dict]: + return [{"question": f"第 {i} 个含糊的问题", "expectations": [{"kind": "ask"}]} for i in range(1, n + 1)] + + +def stub(monkeypatch, *, delay: float = 0.0, path: str = "asked", cost: float = 0.0, copilot_failed: bool = False, + current_run: str | None = None, live: list | None = None): + """作答换成桩:等 delay 秒,交回 path;cost 记在助手的用量上。live 记下同时在答的有几题。""" + peak = {"now": 0, "max": 0} + + async def drive(self): + peak["now"] += 1 + peak["max"] = max(peak["max"], peak["now"]) + try: + self.turn.current_run = current_run + await self._phase("build") + await asyncio.sleep(delay) + self.turn.path = path + self.turn.copilot_failed = copilot_failed + if copilot_failed: + self.turn.reason = "助手本轮未完成:接入返回了 502" + self.turn.usage["assistant"] = {"by_model": {}, "input_tokens": 0, "output_tokens": 0, "calls": 1, + "cost_usd": cost, "unpriced_calls": 0} + return self.turn + finally: + peak["now"] -= 1 + + monkeypatch.setattr(answer_mod.Driver, "drive", drive) + return peak + + +async def start(suite_id: str, variant: dict | None = None, **kw) -> list[EvalRun]: + from app.db.models_eval import EvalSuite + from app.eval.variants import normalize_variant + + async with SessionLocal() as session: + suite = await session.get(EvalSuite, suite_id) + return await runner.start(session, suite, [normalize_variant(variant or {})], actor="评测测试", **kw) + + +async def results(run_id: str) -> list[EvalResult]: + async with SessionLocal() as session: + return list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == run_id) + .order_by(EvalResult.seq))).scalars()) + + +async def get(run_id: str) -> EvalRun: + async with SessionLocal() as session: + return await session.get(EvalRun, run_id) + + +async def test_items_run_in_the_background_within_the_parallel_cap(monkeypatch): + peak = stub(monkeypatch, delay=0.15) + suite_id = await es.make_suite("none", cases=ask_cases(5), budget={"max_parallel": 3}) + [run] = await start(suite_id) + assert run.status == "queued" and run.counts["total"] == 5 + await es.wait_eval(run.id) + row = await get(run.id) + assert row.status == "succeeded" and row.counts["done"] == 5 and row.counts["passed"] == 5 + assert peak["max"] == 3, "同时最多 max_parallel 题" + assert all(r.status == "ok" and r.verdict == "pass" and r.current is None for r in await results(run.id)) + assert row.summary["accuracy"]["passed"] == 5 and row.summary["verdicts"]["pass"] == 5 + + +async def test_the_cap_never_exceeds_a_quarter_of_the_global_concurrency(monkeypatch): + from app.core.config import settings + + monkeypatch.setattr(settings, "max_concurrent_runs", 4) + peak = stub(monkeypatch, delay=0.1) + suite_id = await es.make_suite("none", cases=ask_cases(3), budget={"max_parallel": 3}) + [run] = await start(suite_id) + await es.wait_eval(run.id) + assert peak["max"] == 1, "全局并发 4 的四分之一是 1" + + +async def test_a_second_evaluation_of_the_same_suite_waits(monkeypatch): + stub(monkeypatch, delay=0.3) + suite_id = await es.make_suite("none", cases=ask_cases(1)) + [run] = await start(suite_id) + with pytest.raises(runner.EvalRefused) as refused: + await start(suite_id) + assert refused.value.code == runner.EVAL_BUSY and refused.value.status == 409 + await es.wait_eval(run.id) + + +async def test_stop_then_retry_only_the_unfinished(monkeypatch): + stub(monkeypatch, delay=0.05) + suite_id = await es.make_suite("none", cases=ask_cases(4), budget={"max_parallel": 1}) + [run] = await start(suite_id) + # 等第一题评完再停 + for _ in range(100): + if (await get(run.id)).counts.get("done", 0) >= 1: + break + await asyncio.sleep(0.02) + stub(monkeypatch, delay=5.0) + await asyncio.sleep(0.1) + assert await runner.stop(run.id) + row = await get(run.id) + assert row.status == "cancelled" and row.error == runner.STOPPED + before = {r.id: (r.status, r.finished_at) for r in await results(run.id)} + done = [rid for rid, (status, _) in before.items() if status == "ok"] + assert done and any(status == "cancelled" for status, _ in before.values()) + assert not await runner.stop(run.id), "已经结束的停不了" + + stub(monkeypatch, delay=0.01) + await runner.retry(run.id) + await es.wait_eval(run.id) + after = {r.id: (r.status, r.finished_at) for r in await results(run.id)} + assert all(status == "ok" for status, _ in after.values()) + assert all(after[rid] == before[rid] for rid in done), "评完了的不重做" + assert (await get(run.id)).status == "succeeded" + with pytest.raises(runner.EvalRefused) as nothing: + await runner.retry(run.id) + assert nothing.value.code == runner.NOTHING_TO_RETRY + + +async def test_a_restart_marks_unfinished_evaluations_interrupted_and_does_not_resume(): + suite_id = await es.make_suite("none", cases=ask_cases(1)) + async with SessionLocal() as session: + row = EvalRun(suite_id=suite_id, status="running", started_at=utcnow()) + session.add(row) + await session.flush() + session.add(EvalResult(eval_run_id=row.id, status="running")) + await session.commit() + run_id = row.id + assert await runner.startup() >= 1 + row = await get(run_id) + assert row.status == "interrupted" and row.error == runner.INTERRUPTED + [item] = await results(run_id) + assert item.status == "cancelled" + assert not runner.manager.running(run_id), "不自动续跑" + + +async def test_the_per_run_cap_stops_new_items(monkeypatch): + stub(monkeypatch, cost=1.0) + suite_id = await es.make_suite("none", cases=ask_cases(3), budget={"max_parallel": 1, "per_run_max_usd": 1.5}) + [run] = await start(suite_id) + await es.wait_eval(run.id) + row = await get(run.id) + items = await results(run.id) + assert [r.status for r in items] == ["ok", "ok", "skipped"] + assert row.status == "partial" and row.stop_reason == runner.LIMIT_RUN.format(limit=1.5) + assert "金额上限" in items[2].error and row.summary["cost_usd"] == 2.0 + + +async def test_the_daily_cap_is_its_own_ledger(monkeypatch): + from app.engine import judge + from app.engine.spend import _today, eval_spend, verify_spend + + stub(monkeypatch, cost=0.4) + async with SessionLocal() as session: + for key in ("eval_spend", "judge_spend", "verify_spend"): + row = await session.get(Setting, key) + if row is not None: + await session.delete(row) + session.add(Setting(key="eval_spend", value={"date": _today(), "usd": 4.9, "calls": 3, "unpriced_calls": 0})) + await session.commit() + judge_before, verify_before = await judge.judge_spend.daily(), await verify_spend.daily() + suite_id = await es.make_suite("none", cases=ask_cases(3), budget={"max_parallel": 1}) + [first] = await start(suite_id) + await es.wait_eval(first.id) + # 第一次没有可估的每题花费:按实际累计,越过每日上限之后不再开新题 + spent = await eval_spend.daily() + assert spent["usd"] > 5.0 - 1e-9 and spent["calls"] == 3 + 1, spent + row = await get(first.id) + assert [r.status for r in await results(first.id)] == ["ok", "skipped", "skipped"] + assert row.stop_reason == runner.LIMIT_DAILY.format(limit=5.0) + assert await judge.judge_spend.daily() == judge_before and await verify_spend.daily() == verify_before, \ + "评测不占裁判、核查的额度" + + +async def test_consecutive_assistant_failures_stop_the_evaluation(monkeypatch): + stub(monkeypatch, path="build_failed", copilot_failed=True) + suite_id = await es.make_suite("none", cases=ask_cases(5), budget={"max_parallel": 1}) + [run] = await start(suite_id) + await es.wait_eval(run.id) + row = await get(run.id) + statuses = [r.status for r in await results(run.id)] + assert statuses[:3] == ["ok", "ok", "ok"] and statuses[3:] == ["skipped", "skipped"] + assert row.stop_reason.startswith("连续 3 题助手调用失败") and "502" in row.stop_reason + + +async def test_a_case_over_its_time_limit_stops_its_run(monkeypatch): + stopped: list[str | None] = [] + + async def fake_stop(run_id, actor=None): + stopped.append(run_id) + + monkeypatch.setattr(answer_mod, "stop_run", fake_stop) + stub(monkeypatch, delay=5.0, current_run="run-in-flight") + original = runner.EvalExecution._prepare + + async def prepare(self): + units = await original(self) + self.budget["per_case_timeout_s"] = 0.2 + return units + + monkeypatch.setattr(runner.EvalExecution, "_prepare", prepare) + suite_id = await es.make_suite("none", cases=ask_cases(1)) + [run] = await start(suite_id) + await es.wait_eval(run.id) + [item] = await results(run.id) + assert stopped == ["run-in-flight"], "超时取消正在跑的那次运行" + assert item.status == "ok" and item.path == "build_failed" and "每题时限" in item.error + + +async def test_a_run_stopped_at_an_approval_is_cancelled_not_approved(client, monkeypatch, tmp_path): + from app.eval.answer import APPROVAL_REASON + + class Approving(es.Copilot): + async def astream(self, messages): + if "上线前自查" in messages[1][1]: + yield es._line({"op": "done"}) + return + for op in [{"op": "add_node", "node": {"id": "start", "type": "input", "label": "输入", "config": {}}}, + {"op": "add_node", "node": {"id": "ok", "type": "human", "label": "确认", + "config": {"mode": "approve", "title": "请确认"}}}, + {"op": "add_edge", "edge": {"source": "start", "target": "ok"}}, + {"op": "add_node", "node": {"id": "out", "type": "output", "label": "答案", + "config": {"fields": [{"name": "答案", "value": "好"}]}}}, + {"op": "add_edge", "edge": {"source": "ok", "target": "out", "sourceHandle": "approved"}}, + {"op": "done", "explanation": "确认一下", "run": True}]: + yield es._line(op) + + name, sid = await es.make_source(tmp_path) + try: + es.use_copilot(monkeypatch, Approving(name)) + suite_id = await es.make_suite(sid, cases=[{"question": "要人确认的问题", "expectations": [{"kind": "ask"}]}]) + [run] = await start(suite_id) + await es.wait_eval(run.id) + [item] = await results(run.id) + assert item.path == "run_failed" and "评测不代批" in item.error + async with SessionLocal() as session: + answered = await session.get(Run, item.run_id) + assert answered.status == "cancelled", "评测不代批:取消掉" + details = (await client.get(f"/api/evals/evaluations/{run.id}/results/{item.id}")).json() + assert details["details"]["turn"]["reason"] == APPROVAL_REASON + finally: + await drop_source(sid) + + +async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch): + stub(monkeypatch, delay=0.05) + suite_id = await es.make_suite("none", cases=ask_cases(2)) + estimate = (await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate", json={})).json() + assert estimate["cases"] == 2 and estimate["estimate"]["total_usd"] is None and estimate["parallel"] == 2 + assert estimate["budget"]["daily_max_usd"] == 5.0 and estimate["arms"][0]["models"]["assistant"] == "mock-fast" + started = await client.post(f"/api/evals/suites/{suite_id}/evaluations", json={"variant": {"repeats": 2}}) + assert started.status_code == 202, started.text + [ev] = started.json()["evaluations"] + assert ev["counts"]["total"] == 4 and ev["variant"]["repeats"] == 2 + await es.wait_eval(ev["id"]) + got = (await client.get(f"/api/evals/evaluations/{ev['id']}")).json() + assert got["status"] == "succeeded" and got["current"] == [] and got["summary"]["pass_rates"] + listed = (await client.get(f"/api/evals/suites/{suite_id}/evaluations")).json()["evaluations"] + assert [e["id"] for e in listed] == [ev["id"]] + rows = (await client.get(f"/api/evals/evaluations/{ev['id']}/results")).json()["results"] + assert len(rows) == 4 and rows[0]["question"] == "第 1 个含糊的问题" + second = (await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate", json={})).json() + assert second["estimate"]["basis"] == ev["id"], "第二次按上一次的每题平均花费估" + bad = await client.post(f"/api/evals/suites/{suite_id}/evaluations", json={"variant": {"catalog": "nope"}}) + assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid" + + +async def test_eval_settings_group_and_spend_endpoint(client): + got = (await client.get("/api/settings")).json()["eval"] + assert got == {"max_parallel": 2, "per_case_timeout_s": 900, "per_run_max_usd": 2.0, "daily_max_usd": 5.0, + "keep_runs": 20, "judge": False} + bad = await client.put("/api/settings", json={"values": {"eval": {"keep_runs": 0}}}) + assert bad.status_code == 422 and "保留" in bad.json()["detail"] + bad = await client.put("/api/settings", json={"values": {"eval": {"per_run_max_usd": 9.0}}}) + assert bad.status_code == 422, "单次上限不能高于每日上限" + ok = await client.put("/api/settings", json={"values": {"eval": {"daily_max_usd": 3.0, "judge": True}, + "eval_spend": {"usd": 0}}}) + assert ok.status_code == 200 and ok.json()["eval"]["daily_max_usd"] == 3.0 and "eval_spend" not in ok.json() + spend = (await client.get("/api/settings/eval/spend")).json() + assert set(spend) == {"date", "usd", "calls", "unpriced_calls", "daily_max_usd"} and spend["daily_max_usd"] == 3.0 + # 题集只能把每日上限往下收 + from app.eval.suites import effective_budget + + assert effective_budget({"daily_max_usd": 9.0}, {"daily_max_usd": 3.0})["daily_max_usd"] == 3.0 + assert effective_budget({"daily_max_usd": 1.0}, {"daily_max_usd": 3.0})["daily_max_usd"] == 1.0 + await client.put("/api/settings", json={"values": {"eval": {}}}) diff --git a/backend/tests/test_eval_schedule.py b/backend/tests/test_eval_schedule.py new file mode 100644 index 00000000..cab8b309 --- /dev/null +++ b/backend/tests/test_eval_schedule.py @@ -0,0 +1,227 @@ +"""评测题集的定时与通知(app/eval/scheduled.py、notify/service.notify_eval;细案 01 做法 15,总方案 X4、X6)。 + +- 定时接口按 target_kind 收评测题集:核对目标交给 app/eval/scheduled.check_target(题集在、没归档、有启用的题);非工作流 + 目标不收工作流、输入模板、Webhook(Webhook 的事件说的是那一次运行,非工作流目标收不到,照实拒);接口返回目标的种类、id、 + 名字;工作流的定时行为一字不变(test_schedules_api.py); +- 到点建一次评测:定时跑 1 次,截至时间是计划触发时间,ScheduleFire.eval_run_id 指向它;上一次还没结束就 skipped_overlap; + 错过的按宽限补(同一套调度循环); +- 有一道题从通过变成不通过推「评测有退步」,没评完推「评测失败」:只推订阅了的全局渠道,消息里只有题集名、通过数、退步题数、 + 链接,不放期望值和答案里的数。题集名、渠道都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import json +import uuid +from datetime import datetime, timedelta, timezone + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete, select + +from app.core.crypto import encrypt +from app.db.base import SessionLocal, utcnow +from app.db.models import NotifyChannel, Schedule, ScheduleFire +from app.db.models_eval import EvalResult, EvalRun +from app.engine.runner import run_manager +from app.eval import answer as answer_mod +from app.eval import runner +from app.main import app +from app.notify import core +from app.notify import service as notify +from app.scheduler import loop, service +from tests import eval_samples as es + +UTC = timezone.utc +URL = "https://hooks.example.com/agentlab/eval-7c2a" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Schedule)) + await session.commit() + yield + await runner.manager.shutdown() + await notify.drain() + async with SessionLocal() as session: + await session.execute(delete(Schedule)) + await session.execute(delete(NotifyChannel).where(NotifyChannel.name.like("评测通知-%"))) + await session.commit() + await es.drop_suites() + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +def received(monkeypatch): + got: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + got.append(request) + return httpx.Response(200, json={"ok": True}) + + monkeypatch.setattr(core, "_transport", httpx.MockTransport(handler)) + monkeypatch.setattr(core, "RETRY_DELAYS", (0, 0, 0)) + return got + + +def stub(monkeypatch, *, delay: float = 0.0, path: str = "asked"): + async def drive(self): + await asyncio.sleep(delay) + self.turn.path = path + return self.turn + + monkeypatch.setattr(answer_mod.Driver, "drive", drive) + + +ASK = [{"question": "含糊的问题", "expectations": [{"kind": "ask"}]}] + + +async def _schedule(client, suite_id: str, **extra) -> dict: + made = await client.post("/api/schedules", json={"name": "每周回归", "target_kind": "eval_suite", + "target_id": suite_id, "cron": "0 9 * * 1", + "timezone": "Asia/Shanghai", **extra}) + assert made.status_code == 201, made.text + return made.json() + + +async def test_the_schedule_api_takes_an_eval_suite(client): + suite_id = await es.make_suite("none", name="虚构回归题集", cases=ASK) + body = await _schedule(client, suite_id) + assert (body["target_kind"], body["target_id"], body["target_name"]) == ("eval_suite", suite_id, "虚构回归题集") + assert body["target_label"] == "评测题集" and body["workflow_id"] == "" and body["webhook"] is None + assert body["input_template"] == {} and body["target_exists"] is True + listed = (await client.get("/api/schedules")).json()["items"] + assert [s["target_name"] for s in listed] == ["虚构回归题集"] + for bad, field in (({"target_id": "nope"}, "target_id"), ({"target_id": None}, "target_id"), + ({"webhook": {"url": URL}}, "webhook"), ({"input_template": {"a": "b"}}, "input_template"), + ({"workflow_id": "wf"}, "workflow_id"), ({"target_kind": "nope"}, "target_kind")): + refused = await client.post("/api/schedules", json={"name": "x", "target_kind": "eval_suite", + "target_id": suite_id, "cron": "0 9 * * 1", + "timezone": "Asia/Shanghai", **bad}) + assert refused.status_code == 422 and refused.json()["field"] == field, (bad, refused.text) + # 改名、改时间、停用都照常;目标种类建好不能改 + patched = await client.patch(f"/api/schedules/{body['id']}", json={"name": "每天回归", "cron": "0 9 * * *", + "enabled": False, "webhook": None, + "version": body["version"]}) + assert patched.status_code == 200, patched.text + assert patched.json()["name"] == "每天回归" and patched.json()["next_run_at"] is None + moved = await client.patch(f"/api/schedules/{body['id']}", json={"target_kind": "workflow", + "version": patched.json()["version"]}) + assert moved.status_code == 422 and moved.json()["field"] == "target_kind" + # 题集归档了:照实拒 + archived = await es.make_suite("none", name="归档的题集", cases=ASK) + async with SessionLocal() as session: + from app.db.models_eval import EvalSuite + + (await session.get(EvalSuite, archived)).archived = True + await session.commit() + refused = await client.post("/api/schedules", json={"name": "x", "target_kind": "eval_suite", "target_id": archived, + "cron": "0 9 * * 1", "timezone": "Asia/Shanghai"}) + assert refused.status_code == 422 and "已归档" in refused.json()["detail"] + + +async def test_firing_starts_one_evaluation_and_skips_on_overlap(client, monkeypatch): + stub(monkeypatch, delay=0.5) + suite_id = await es.make_suite("none", cases=ASK, default_variant={"repeats": 3}) + body = await _schedule(client, suite_id) + now = utcnow().replace(microsecond=0) + fire = await service.fire_once(body["id"], now, "manual", actor="王芳") + assert fire.status == "started" and fire.eval_run_id and fire.run_id is None + async with SessionLocal() as session: + run = await session.get(EvalRun, fire.eval_run_id) + assert run.trigger == "schedule" and run.schedule_id == body["id"] and run.repeats == 1, "定时只跑 1 次" + assert run.as_of.replace(tzinfo=UTC) == now and run.created_by == "王芳" + again = await service.fire_once(body["id"], now + timedelta(seconds=1), "manual") + assert again.status == "skipped_overlap" and f"#{run.id[:6]}" in again.reason + same = await service.fire_once(body["id"], now, "manual") + assert same.id == fire.id, "同一次触发只发起一次" + await es.wait_eval(run.id) + history = (await client.get(f"/api/schedules/{body['id']}/fires")).json()["items"] + assert {f["eval_run_id"] for f in history if f["status"] == "started"} == {run.id} + + +async def test_a_missed_fire_within_grace_is_made_up(client, monkeypatch): + stub(monkeypatch) + suite_id = await es.make_suite("none", cases=ASK) + body = await _schedule(client, suite_id) + due = utcnow().replace(microsecond=0) - timedelta(minutes=20) + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == body["id"]).values(next_run_at=due)) + await session.commit() + fires = await loop.fire_due(due + timedelta(minutes=20)) + [made_up] = [f for f in fires if f.status == "started"] + assert made_up.kind == "misfire" and made_up.eval_run_id + await es.wait_eval(made_up.eval_run_id) + + +async def _channel(events: list[str]) -> str: + async with SessionLocal() as session: + row = NotifyChannel(name=f"评测通知-{uuid.uuid4().hex[:6]}", scope="global", format="generic", url=encrypt(URL), + events=events) + session.add(row) + await session.commit() + return row.id + + +async def test_a_regression_is_pushed_without_numbers(client, monkeypatch, received, tmp_path): + name, sid = await es.make_source(tmp_path) + es.use_copilot(monkeypatch, es.Copilot(name)) + es.use_report(monkeypatch) + suite_id = await es.make_suite(sid, name="虚构周回归") + body = await _schedule(client, suite_id) + await _channel(["eval.regressed", "eval.failed"]) + first = await service.fire_once(body["id"], utcnow().replace(microsecond=0) - timedelta(minutes=5), "manual") + await es.wait_eval(first.eval_run_id) + assert received == [], "第一次没有可比的、也评完了:不推" + await es.set_notes(sid, {"label": es.NOTES["label"]}) + second = await service.fire_once(body["id"], utcnow().replace(microsecond=0), "manual") + await es.wait_eval(second.eval_run_id) + [request] = received + message = json.loads(request.content) + assert message["event"] == "eval.regressed" and message["title"] == "评测「虚构周回归」有退步" + text = message["text"] + assert "通过:0 / 1 题" in text and "退步:1 题" in text + for secret in ("330", "310", "gmv", "orders", "paid_date"): + assert secret not in json.dumps(message, ensure_ascii=False), f"通知里不该有 {secret}" + from app.db.models import NotifyDelivery + + async with SessionLocal() as session: + [delivery] = (await session.execute(select(NotifyDelivery).where( + NotifyDelivery.subject_id == second.eval_run_id))).scalars().all() + assert delivery.subject_kind == "eval_run" and delivery.status == "ok" + from tests.fixtures.sources import drop_source + + await drop_source(sid) + + +async def test_an_unfinished_scheduled_evaluation_pushes_failed(client, monkeypatch, received): + async def drive(self): + self.turn.path, self.turn.copilot_failed, self.turn.reason = "build_failed", True, "助手本轮未完成" + return self.turn + + monkeypatch.setattr(answer_mod.Driver, "drive", drive) + suite_id = await es.make_suite("none", cases=ASK * 4, name="虚构失败题集", budget={"max_parallel": 1}) + body = await _schedule(client, suite_id) + await _channel(["eval.failed"]) + fire = await service.fire_once(body["id"], utcnow().replace(microsecond=0), "manual") + await es.wait_eval(fire.eval_run_id) + async with SessionLocal() as session: + run = await session.get(EvalRun, fire.eval_run_id) + skipped = (await session.execute(select(EvalResult).where( + EvalResult.eval_run_id == run.id, EvalResult.status == "skipped"))).scalars().all() + assert run.status == "partial" and run.stop_reason.startswith("连续 3 题") and len(skipped) == 1 + [request] = received + message = json.loads(request.content) + assert message["event"] == "eval.failed" and message["title"] == "评测「虚构失败题集」没有评完" + assert "通过:0 / 3 题" in message["text"] + # 链接去那条定时任务的触发历史(设了对外访问地址才有链接) + assert all("/schedules/" in link["url"] for link in message["links"]) diff --git a/backend/tests/test_eval_score.py b/backend/tests/test_eval_score.py new file mode 100644 index 00000000..645f92b4 --- /dev/null +++ b/backend/tests/test_eval_score.py @@ -0,0 +1,233 @@ +"""评分(app/eval/score.py;细案 01 做法 10,评测-D4)。纯计算的部分直接喂构造好的答案;读封存、逐题 E1、重评走一次真的 +运行(tests/ledger_samples 的店铺库)。数字、问法、地区都是虚构的。 + +- scalar:对且可追溯(引用的格对上)、对但没引用(只有正文裸写的数对上,含叙述里自己算的比率)、错、量纲不同(差 10 的 + 整数次幂)、答案引用遮罩格的不能评分;维度配对(华东的那一格才算)、小整数又没维度的标弱匹配; +- table:按键配对、要求顺序时核对顺序;set:名单齐不齐、顺序;none:没给数 + 有查询返回 0 行;ask:反问才对;text:单列不计; +- 题目结论:路径不对直接不通过;期望没算出来不能评分 / 升版待处置跳过;只有文字要点的不能评分; +- dual:新旧两版各给一份结论; +- 允许的相对误差和双盲复算是同一个函数(engine/tolerance); +- 重评只读封存、不调模型。 +""" +from __future__ import annotations + +import pytest + +import app.db.models # noqa: F401 - conftest 按已导入的模型建表:只跑这一个文件时也要有表 +from app.engine import tolerance +from app.eval.score import Answer, score_case + + +def _case(*expectations: dict, gold_kind: str | None = "sql") -> dict: + return {"kind": "ask", "question": "虚构问题", "expectations": list(expectations), + "gold": {"kind": gold_kind, "datasource_id": "s", "sql": "SELECT 1"} if gold_kind else None} + + +def _gold(values: dict, *more: dict) -> dict: + sets = [{"run_id": "g", "snapshot": "x", "version": None, "values": values}] + sets += [{"run_id": "g2", "snapshot": "y", "version": m.pop("_version", 2), "values": m} for m in more] + return {"status": "ok", "reason": None, "code": None, "before": {"status": "ok", "sets": sets}, + "after": {"status": "ok", "sets": sets}} + + +def _scalar(value, **extra) -> dict: + return {"kind": "scalar", "value": value, "column": "gmv", "row": 0, "dims": {}, **extra} + + +def _cell(value, **dims) -> dict: + return {"value": value, "kind": "cell", "artifact": "a1", "row": 0, "column": "gmv", "dims": dims} + + +def _bare(text: str) -> list[dict]: + from app.eval.score import _numbers + + return _numbers(text) + + +EXP = {"id": "e1", "kind": "scalar", "column": "gmv", "required": True} + + +def test_scalar_outcomes(): + case = _case(EXP) + gold = _gold({"e1": _scalar(330.0)}) + traced = score_case(case, gold, Answer(run_id="r", values=[_cell(330.0)]), path="ran") + assert traced.verdict == "pass" and traced.traced and traced.scores[0]["outcome"] == "traced" + untraced = score_case(case, gold, Answer(run_id="r", text="销售额 330 元", bare=_bare("销售额 330 元")), path="ran") + assert untraced.verdict == "pass" and not untraced.traced and untraced.scores[0]["outcome"] == "untraced" + wrong = score_case(case, gold, Answer(run_id="r", values=[_cell(310.0)]), path="ran") + assert wrong.verdict == "fail" and wrong.scores[0]["outcome"] == "wrong" and wrong.scores[0]["got"] == 310.0 + # 差 100 倍:量纲不同,单列,也不算对 + scale = score_case(case, gold, Answer(run_id="r", values=[_cell(3.3)]), path="ran") + assert scale.verdict == "fail" and scale.scores[0]["outcome"] == "scale" + masked = score_case(case, gold, Answer(run_id="r", values=[{**_cell(330.0), "masked": True}]), path="ran") + assert masked.verdict == "ungradable" and "遮罩" in masked.scores[0]["reason"] + + +def test_bare_numbers_follow_their_written_precision_and_percent(): + case = _case(EXP) + # 3.3 万写到千位:33,000 ± 500 都算对 + gold = _gold({"e1": _scalar(33280.0)}) + assert score_case(case, gold, Answer(run_id="r", bare=_bare("约 3.3 万")), path="ran").verdict == "pass" + # 比率写成百分数 + ratio = _gold({"e1": _scalar(0.125)}) + got = score_case(case, ratio, Answer(run_id="r", bare=_bare("占比 12.5%")), path="ran") + assert got.verdict == "pass" and got.scores[0]["outcome"] == "untraced", "叙述里写的比率只算对但没引用" + + +def test_dimensions_pair_and_small_integers_are_weak(): + case = _case({**EXP, "where": {"region": "华东"}}) + gold = _gold({"e1": _scalar(300.0, where={"region": "华东"}, dims={"region": "华东"})}) + other = score_case(case, gold, Answer(run_id="r", values=[_cell(300.0, region="华南")]), path="ran") + assert other.verdict == "fail", "华南那一格碰巧也是 300,不算" + same = score_case(case, gold, Answer(run_id="r", values=[_cell(300.0, 地区="华东")]), path="ran") + assert same.verdict == "pass" and same.scores[0]["key"] == {"region": "华东"} + # 按条件筛过、只剩一个数(没有维度列)不算对不上 + assert score_case(case, gold, Answer(run_id="r", values=[_cell(300.0)]), path="ran").verdict == "pass" + weak = score_case(_case(EXP), _gold({"e1": _scalar(3)}), Answer(run_id="r", values=[_cell(3)]), path="ran") + assert weak.verdict == "pass" and weak.scores[0]["weak"] is True + + +def test_tables_sets_none_and_ask(): + table = {"id": "t", "kind": "table", "keys": ["region"], "ordered": True, "required": True} + gold = _gold({"t": {"kind": "table", "keys": ["region"], "columns": ["region", "gmv"], "ordered": True, + "rows": [{"region": "华东", "gmv": 300.0}, {"region": "华北", "gmv": 30.0}]}}) + cells = [_cell(300.0, region="华东"), _cell(30.0, region="华北")] + ok = score_case(_case(table), gold, Answer(run_id="r", text="华东 300,华北 30", values=cells), path="ran") + assert ok.verdict == "pass" and ok.traced + flipped = score_case(_case(table), gold, Answer(run_id="r", text="华北 30,华东 300", values=cells), path="ran") + assert flipped.verdict == "fail" and "先后" in flipped.scores[0]["reason"] + + names = {"id": "s", "kind": "set", "column": "region", "required": True} + gold = _gold({"s": {"kind": "set", "column": "region", "values": ["华东", "华北"], "ordered": False}}) + assert score_case(_case(names), gold, Answer(run_id="r", text="有华东、华北两个"), path="ran").verdict == "pass" + assert score_case(_case(names), gold, Answer(run_id="r", text="只有华东"), path="ran").verdict == "fail" + + none = {"id": "n", "kind": "none", "required": True} + gold = _gold({"n": {"kind": "none", "rows": 0}}) + empty = [{"artifact": "q", "row_count": 0, "sql": "SELECT 1"}] + traced = score_case(_case(none), gold, Answer(run_id="r", text="华中区没有数据", queries=empty), path="ran") + assert traced.verdict == "pass" and traced.traced + with_number = score_case(_case(none), gold, Answer(run_id="r", text="华中区 12 元", bare=_bare("华中区 12 元"), + queries=empty), path="ran") + assert with_number.verdict == "fail" + + ask = _case({"id": "a", "kind": "ask", "required": True}, gold_kind=None) + assert score_case(ask, {"status": "ok"}, Answer(run_id=None), path="asked").verdict == "pass" + assert score_case(ask, {"status": "ok"}, Answer(run_id="r"), path="ran").verdict == "fail" + + +def test_the_path_and_the_gold_decide_before_numbers(): + case = _case(EXP) + gold = _gold({"e1": _scalar(330.0)}) + replied = score_case(case, gold, Answer(run_id=None, text="330", bare=_bare("330")), path="reply") + assert replied.verdict == "fail" and "该查不查" in replied.reason, "直接回答的数是没查过的" + skipped = score_case(case, {"status": "skipped", "reason": "口径已升版到 v2,待处置"}, Answer(run_id=None), path=None) + assert skipped.verdict == "skipped" and "待处置" in skipped.reason + drifted = score_case(case, {"status": "drifted", "reason": "作答期间数据变了"}, Answer(run_id="r"), path="ran") + assert drifted.verdict == "ungradable", "数据变了不判,不算答错" + rejected = score_case(case, gold, Answer(run_id="r"), path="rejected") + assert rejected.verdict == "ungradable", "驳回是人的决定,不算失败" + text_only = _case({"id": "p", "kind": "text", "points": ["说明口径"], "required": False}, gold_kind=None) + judged = {"expectations": {"p": {"points": [{"point": 1, "verdict": "covered", "reason": ""}]}}} + scored = score_case(text_only, {"status": "ok"}, Answer(run_id="r", text="按支付日"), path="ran", judgement=judged) + assert scored.verdict == "ungradable" and scored.scores[0]["outcome"] == "covered", "文字要点单列、不计准确率" + unjudged = score_case(text_only, {"status": "ok"}, Answer(run_id="r"), path="ran") + assert unjudged.scores[0]["outcome"] == "unjudged" + + +def test_dual_reports_both_versions(): + case = _case(EXP, gold_kind="caliber") + case["gold"] = {"kind": "caliber", "workflow_id": "w", "version": 1, "node_id": "card", "inputs": {}} + case["expectations"] = [{"id": "e1", "kind": "scalar", "metric": "gmv", "required": True}] + gold = _gold({"e1": _scalar(330.0)}, {"e1": _scalar(380.0), "_version": 2}) + got = score_case(case, gold, Answer(run_id="r", values=[_cell(330.0)]), path="ran") + assert got.verdict == "pass" and got.by_version[0]["version"] == 2 and got.by_version[0]["verdict"] == "fail" + + +def test_the_tolerance_is_the_dual_check_function(): + from app.engine.nodes import dual_check + + assert dual_check._agree is tolerance.agree and dual_check._delta_pct is tolerance.delta_pct + assert tolerance.agree(100.0, 100.001, 1e-6) is False and tolerance.agree(100.0, 100.001, 1e-3) + assert tolerance.agree(0, 0.0000001, 1e-6) and tolerance.delta_pct(200, 180) == -10.0 + assert tolerance.delta_pct(0, 1) is None and tolerance.agree("华东", "华东", 0) + + +# -------------------------------------------------------------------------- +# 读封存的答案、逐题 E1、重评(走一次真的运行) +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine_on(): + from app.engine.runner import run_manager + + await run_manager.setup() + yield + await run_manager.shutdown() + + +async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, monkeypatch, tmp_path): + from app.eval import score + from tests import ledger_samples as ls + + await ls.fresh_source(tmp_path / "shop.db") + run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]],同比增长 12%。") + answer = await score.read_answer(run.id) + assert answer.problem is None and answer.report == "write" + [value] = answer.values + assert value["value"] == 170.5 and value["kind"] == "cell" and value["column"] == "销售额" + assert [b["raw"] for b in answer.bare] == ["12%"] and answer.stats["numbers_cited"] == 1 + assert len(answer.queries) == 1 and answer.cited == [answer.queries[0]["artifact"]] + + from app.db.base import SessionLocal + from app.db.models import DataSource + from sqlalchemy import select + + async with SessionLocal() as session: + sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar() + case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": ls.WITH_REFUNDS}, "expectations": []} + gold = {"status": "ok", "before": {"rendered": {"sql": ls.WITH_REFUNDS}, "sets": []}} + match = await score.sql_match(case, gold, answer) + assert match["status"] == "related" and match["cited"] and "filter" in match["aspects"], match + gold["before"]["rendered"]["sql"] = ("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' " + "AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'") + assert (await score.sql_match(case, gold, answer))["status"] == "proven" + + +async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path): + from app.db.base import SessionLocal + from app.db.models_eval import EvalResult, EvalRun + from app.eval import rescore + from app.providers import mock_model + from tests import eval_samples as es + from tests import ledger_samples as ls + + await ls.fresh_source(tmp_path / "shop.db") + run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 [[v:Q1.r0.销售额]]。") + suite_id = await es.make_suite("none", cases=[{"question": "本周销售额?", "expectations": [ + {"kind": "scalar", "column": "gmv"}], "gold": {"kind": "sql", "datasource_id": "none", "sql": "SELECT 1"}}]) + + def boom(*_a, **_k): + raise AssertionError("重评不该调模型") + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", boom) + async with SessionLocal() as session: + from app.db.models_eval import EvalCase + from sqlalchemy import select + + case = (await session.execute(select(EvalCase).where(EvalCase.suite_id == suite_id))).scalar_one() + er = EvalRun(suite_id=suite_id, status="succeeded") + session.add(er) + await session.flush() + session.add(EvalResult(eval_run_id=er.id, case_id=case.id, case_hash=case.case_hash, status="ok", path="ran", + run_id=run.id, verdict="fail", scores=[], + gold=_gold({"e1": _scalar(170.5)}))) + await session.commit() + er_id = er.id + assert await rescore.rescore(er_id) == 1 + async with SessionLocal() as session: + row = (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == er_id))).scalar_one() + assert row.verdict == "pass" and row.scores[0]["outcome"] == "traced" + assert (await session.get(EvalRun, er_id)).summary["accuracy"]["passed"] == 1 + await es.drop_suites() diff --git a/backend/tests/test_eval_stats.py b/backend/tests/test_eval_stats.py new file mode 100644 index 00000000..9f2f290a --- /dev/null +++ b/backend/tests/test_eval_stats.py @@ -0,0 +1,162 @@ +"""评测的统计与对比(app/eval/stats.py、app/eval/compare.py;细案 01 做法 11、12,评测-D10)。纯计算,和手算对照。 + +- Wilson 95% 区间、精确 McNemar 双侧 p 值、P50 / P90; +- 总表:准确率和可追溯准确率分开报(分母只算能判的)、作答路径、出具档位、复核、look、按角色的用量、引用率、逐题 E1、 + 重复 k 次的题给通过率; +- 和上一次比:逐题转移(退步 / 进步 / 不变 / 不可比)和归因(数据变、做法变、只参数变、路径变),列出指纹里哪几项变了; +- A/B 配对表:都对 / 都错 / 只 A 对 / 只 B 对,不能判的不进表; +- 选基准:点名的 > 同一变体里钉住的 > 同一变体、比这次先发起、已经结束的最近一次。 +""" +from __future__ import annotations + +from datetime import timedelta + +import pytest + +import app.db.models # noqa: F401 - conftest 按已导入的模型建表 +from app.eval import compare, stats + + +def test_wilson_and_mcnemar_match_hand_computation(): + assert stats.wilson(8, 10) == (0.4902, 0.9433) + assert stats.wilson(0, 0) is None + low, high = stats.wilson(0, 5) + assert low == 0.0 and 0.43 < high < 0.44 + assert stats.mcnemar_exact(1, 0) == 1.0 + assert stats.mcnemar_exact(6, 0) == 0.03125 + assert stats.mcnemar_exact(7, 2) == 0.179688 + assert stats.mcnemar_exact(0, 0) == 1.0 + assert stats.percentile([5, 1, 3, 2, 4], 50) == 3 and stats.percentile([5, 1, 3, 2, 4], 90) == 5 + assert stats.percentile([], 50) is None + + +def _row(case: str, verdict: str | None, *, path: str = "ran", status: str = "ok", repeat: int = 0, **extra) -> dict: + scores = [{"id": "e1", "kind": "scalar", "required": True, + "outcome": {"pass": "traced", "fail": "wrong"}.get(verdict or "", "ungradable")}] + return {"case_id": case, "case_seq": int(case[-1]) if case[-1].isdigit() else 1, "case_hash": f"h-{case}", + "verdict": verdict, "path": path, "status": status, "repeat": repeat, "scores": scores, + "traced": verdict == "pass", **extra} + + +def test_summary_counts_only_gradable_items_and_splits_roles(): + usage = {"assistant": {"by_model": {"m": {"calls": 3, "cost_usd": 0.03}}, "calls": 3, "cost_usd": 0.03, + "input_tokens": 10, "output_tokens": 5, "unpriced_calls": 0}, + "writer": {"by_model": {}, "calls": 1, "cost_usd": 0.01, "input_tokens": 1, "output_tokens": 1, + "unpriced_calls": 1}, + "judge": {"cost_usd": 0.0, "calls": 0}} + rows = [ + _row("c1", "pass", tier="formal", usage=usage, looks={"model": 3, "system": 0, "limit_hit": True}, + timings={"build_ms": 1000, "total_ms": 4000, "review_ms": 30_000}, review={"verdict": "annotated", + "severity": "degraded"}, + answer_stats={"numbers": 2, "numbers_cited": 1, "uncited_numbers": 1}, attempts=1, + sql_match={"status": "proven", "aspects": []}), + _row("c2", "fail", usage=usage, timings={"build_ms": 3000, "total_ms": 6000}, + sql_match={"status": "related", "aspects": ["time"]}), + _row("c3", "ungradable"), + _row("c4", None, path=None, status="skipped"), + ] + rows[0]["scores"][0]["outcome"] = "traced" + summary = stats.summarize(rows) + assert summary["accuracy"] == {"passed": 1, "n": 2, "rate": 0.5, "ci": stats.wilson(1, 2)} + assert summary["traced_accuracy"]["passed"] == 1 + assert summary["verdicts"] == {"pass": 1, "fail": 1, "ungradable": 1, "skipped": 0} + assert summary["paths"] == {"ran": 3} and summary["tiers"]["formal"] == 1 and summary["tiers"]["explore"] == 2 + assert summary["review"]["retries"] == 1 and summary["review"]["slow"] == 1 + assert summary["review"]["severity"] == {"degraded": 1} + assert summary["looks"]["model"] == 3 and summary["looks"]["limit_hit"] == round(1 / 3, 4) + assert summary["timings"]["build_ms"] == {"p50": 1000, "p90": 3000, "n": 2} + assert summary["usage"]["assistant"]["calls"] == 6 and summary["usage"]["writer"]["unpriced_calls"] == 2 + assert summary["cost_usd"] == 0.08 and summary["unpriced_calls"] == 2 + assert summary["usage"]["gold"] == {"cost_usd": 0.0, "calls": 0}, "期望探针不调模型" + assert summary["citations"]["rate"] == 0.5 + assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1} + + +def test_repeats_give_a_pass_rate_per_case(): + rows = [_row("c1", "pass", repeat=0), _row("c1", "fail", repeat=1), _row("c1", "pass", repeat=2)] + summary = stats.summarize(rows) + assert summary["pass_rates"]["c1"]["passed"] == 2 and summary["pass_rates"]["c1"]["n"] == 3 + assert compare.case_outcome(rows)["verdict"] == "pass", "多数决" + + +def test_transitions_and_their_causes(): + sql_a = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-09-01'"]}] + sql_b = [{"text": "SELECT SUM(x) FROM t WHERE d = ?", "literals": ["'2026-08-01'"]}] + sql_c = [{"text": "SELECT SUM(x) FROM t WHERE e = ?", "literals": ["'2026-09-01'"]}] + base = [_row("c1", "pass", gold_key="g1", sql_shapes=sql_a), _row("c2", "pass", gold_key="g2", sql_shapes=sql_a), + _row("c3", "fail", gold_key="g3", sql_shapes=sql_a), _row("c4", "pass", gold_key="g4", path="ran"), + _row("c5", "pass", gold_key="g5"), _row("c6", "fail", gold_key="g6")] + target = [_row("c1", "fail", gold_key="g1-new", sql_shapes=sql_a), _row("c2", "fail", gold_key="g2", sql_shapes=sql_c), + _row("c3", "pass", gold_key="g3", sql_shapes=sql_b), _row("c4", "fail", gold_key="g4", path="reply"), + {**_row("c5", "fail", gold_key="g5"), "case_hash": "changed"}, _row("c6", "ungradable", gold_key="g6"), + _row("c7", "pass")] + moved = compare.transitions(base, target) + by = {i["case_id"]: i for i in moved["items"]} + assert (by["c1"]["transition"], by["c1"]["cause"]) == ("regressed", "data") + assert (by["c2"]["transition"], by["c2"]["cause"]) == ("regressed", "method") + assert (by["c3"]["transition"], by["c3"]["cause"]) == ("improved", "params") + assert (by["c4"]["transition"], by["c4"]["cause"]) == ("regressed", "path") + assert by["c5"]["transition"] == "incomparable" and "题目改过" in by["c5"]["reason"] + assert by["c6"]["transition"] == "incomparable" and by["c7"]["transition"] == "incomparable" + assert moved["counts"] == {"regressed": 3, "improved": 1, "same": 0, "incomparable": 3} + assert compare.sql_change(sql_a, sql_a) == "same" + + +def test_fingerprint_diff_lists_what_changed(): + a = {"assistant_model": "m1", "writer_model": None, "catalog": "live:aaa", "prompt": "p1", "schemas": {"s": "x"}, + "cases": "c", "scorer_version": 1} + b = {**a, "assistant_model": "m2", "schemas": {"s": "y"}} + diff = compare.fingerprint_diff(a, b) + assert [d["key"] for d in diff] == ["assistant_model", "schemas"] + assert diff[0] == {"key": "assistant_model", "label": "助手模型", "before": "m1", "after": "m2"} + assert "before" not in diff[1], "表结构只说变没变" + + +def test_the_pair_table(): + a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")] + b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")] + table = compare.pair_table([("当前目录", a), ("关目录", b)]) + [pair] = table["pairs"] + assert (pair["both"], pair["neither"], pair["only_a"], pair["only_b"], pair["excluded"]) == (1, 1, 1, 0, 1) + assert pair["p_value"] == 1.0 and pair["note"] == compare.NO_DIFFERENCE + assert table["arms"][0]["accuracy"]["rate"] == round(2 / 3, 4) + + +async def test_picking_the_baseline(): + from app.db.base import SessionLocal, utcnow + from app.db.models_eval import EvalRun, EvalSuite + + now = utcnow() + async with SessionLocal() as session: + suite = EvalSuite(name="虚构题集-基准") + session.add(suite) + await session.flush() + + def run(minutes: int, key: str, status: str = "succeeded", **extra) -> EvalRun: + row = EvalRun(suite_id=suite.id, status=status, fingerprint={"variant_key": key}, + created_at=now - timedelta(minutes=minutes), **extra) + session.add(row) + return row + + old = run(50, "k1") + pinned = run(40, "k1", pinned=True) + latest = run(30, "k1") + run(20, "k2") + run(15, "k1", status="failed") + target = run(10, "k1") + later = run(5, "k1") + await session.flush() + assert (await compare.pick_baseline(session, target)).id == pinned.id, "同一变体里钉住的优先" + pinned.pinned = False + await session.flush() + assert (await compare.pick_baseline(session, target)).id == latest.id, "比这次先发起、已结束的最近一次" + assert (await compare.pick_baseline(session, target, explicit=old.id)).id == old.id + assert (await compare.pick_baseline(session, target, explicit=later.id)).id == later.id + await session.delete(suite) + await session.commit() + + +@pytest.mark.parametrize("n, k", [(1, 0), (3, 3)]) +def test_rates_carry_their_sample_size(n, k): + r = stats.rate(k, n) + assert r["n"] == n and r["passed"] == k and r["ci"] == stats.wilson(k, n) From df6c3ce10d89f47b72d27d29163962ab04954e2f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 17:04:05 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E9=A1=B5=E7=85=A7?= =?UTF-8?q?=E5=AE=9E=E6=98=BE=E7=A4=BA=E9=9D=9E=E5=B7=A5=E4=BD=9C=E6=B5=81?= =?UTF-8?q?=E7=9B=AE=E6=A0=87=EF=BC=9B=E8=AF=84=E6=B5=8B=E5=81=9C=E6=AD=A2?= =?UTF-8?q?=E6=97=B6=E5=87=A0=E8=87=82=E4=B8=80=E8=B5=B7=E6=94=B6=E5=B0=BE?= =?UTF-8?q?=EF=BC=9B=E5=B7=A5=E4=BD=9C=E6=B5=81=E9=A2=98=E7=9A=84=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:这一波后端开始接受评测题集、数据源维护这两种定时目标(总方案 X4),定时页要能把它们照实列出来、点进去 不报错;界面上不能有点了没用的入口。 - 定时页列表:非工作流目标写种类(评测题集 / 数据源维护)和目标名,不标运行类别,「编辑」写成「查看」 - 详情:非工作流目标的设置只读地列出名称、目标、频率、时区、补发宽限,不出工作流那张表单(选工作流、输入模板、 Webhook 对它们不适用);页头的启用、立即运行、删除和触发历史照常 - 立即运行发起的是一次评测时说「已发起一次评测」(以前会落到「未能发起运行」);触发历史写评测的编号 - types.ts:Schedule 加 target_kind / target_id / target_label / target_name / target_exists,触发记录加 eval_run_id; 文案进 SCHEDULE_TEXT - check-schedules 加一段「目标不是工作流的定时任务」 - runner:还没准备好就被停下、出错时,按 pair_id 找全几臂一起收尾 - 测试:工作流题按钉住版本跑探索运行(不发正式运行、不建图);钉住基线的接口 - 后端几句给人看的话按文案规范改(不用「跑」「钉」) --- backend/app/api/evals.py | 2 +- backend/app/eval/answer.py | 2 +- backend/app/eval/runner.py | 8 +-- backend/app/eval/score.py | 6 +-- backend/tests/test_eval_runner.py | 48 +++++++++++++++++ frontend/src/lib/terms.ts | 17 +++++++ frontend/src/pages/schedules/FireHistory.tsx | 3 ++ .../src/pages/schedules/ScheduleDetail.tsx | 41 ++++++++++++++- .../src/pages/schedules/SchedulesPage.tsx | 27 ++++++---- frontend/src/pages/schedules/parts.tsx | 15 +++++- frontend/src/types.ts | 14 +++++ scripts/check-schedules.mjs | 51 +++++++++++++++++++ 12 files changed, 214 insertions(+), 20 deletions(-) diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index 566ac513..dc5e9419 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -954,7 +954,7 @@ async def pin_evaluation(evaluation_id: str, request: Request, session: AsyncSes if not isinstance(pinned, bool): raise CodedHTTPException(422, "pinned 需要是 true 或 false", case_rules.CASE_INVALID, field="pinned") if pinned and row.status not in ("succeeded", "partial"): - raise CodedHTTPException(409, "只有评完了(或部分评完)的评测能钉成基线", case_rules.CASE_INVALID, field="pinned") + raise CodedHTTPException(409, "只有评完了(或部分评完)的评测能设为基线", case_rules.CASE_INVALID, field="pinned") row.pinned = pinned await session.commit() await session.refresh(row) diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py index 46d4c667..9d281947 100644 --- a/backend/app/eval/answer.py +++ b/backend/app/eval/answer.py @@ -333,7 +333,7 @@ class Driver: waiting = await self._pending_approvals(run_id) await stop_run(run_id, self.ctx.actor) t.path = "run_failed" - t.reason = APPROVAL_REASON if waiting else "运行停在断点上(服务可能重启过):评测不接着跑,已取消这次运行" + t.reason = APPROVAL_REASON if waiting else "运行停在断点上(服务可能重启过):评测不继续这次运行,已取消" return None if row.status == "cancelled" and row.end_reason == END_REJECTED: t.path, t.reason = "rejected", "运行被驳回(人的决定,不算运行失败)" diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index 1f24e540..c4c89be5 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -60,7 +60,7 @@ STOP_COPILOT = 3 STOP_WINDOW = 10 STOP_TIMEOUTS = 7 #: 说法 -INTERRUPTED = "服务重启时评测还没做完,没有自动续跑。可以点「重试」只把没做完的题接着做" +INTERRUPTED = "服务重启时评测还没做完,没有自动继续。可以点「重试」只把没做完的题接着做" STOPPED = "评测已停止" NOT_STARTED = "评测停下时这道题还没开评" LIMIT_RUN = "已到这次评测的金额上限(${limit:g}),没有再开新题" @@ -68,7 +68,7 @@ LIMIT_DAILY = "评测今天的金额已到上限(${limit:g}),没有再开 STOP_COPILOT_TEXT = "连续 {n} 题助手调用失败,评测停下了:{reason}" STOP_TIMEOUT_TEXT = "最近 {window} 题里有 {n} 题超时,评测停下了" CASE_GONE = "这道题已被删除" -TIMED_OUT = "超过每题时限({s:g} 秒),已停止这一题正在跑的运行" +TIMED_OUT = "超过每题时限({s:g} 秒),已停止这一题正在执行的运行" #: 结束后多久做清理(在后台、不挡收尾) DETAILS_KIND = "eval_result" DETAILS_FORMAT = 1 @@ -698,7 +698,9 @@ class EvalExecution(runtime.JobRun): await self._settle() async def write(session: AsyncSession) -> None: - ids = [a.run_id for a in self.arms] or [self.job_id] + # 还没准备好就停下、出错的(arms 还是空的):按 pair_id 找全几臂 + ids = [a.run_id for a in self.arms] or list((await session.execute(select(EvalRun.id).where( + (EvalRun.id == self.job_id) | (EvalRun.pair_id == self.job_id)))).scalars()) item_status = "cancelled" if status in ("cancelled", "interrupted") else "failed" await session.execute(update(EvalResult).where(EvalResult.eval_run_id.in_(ids), EvalResult.status.in_(runtime.OPEN)) diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py index be673a75..f471c750 100644 --- a/backend/app/eval/score.py +++ b/backend/app/eval/score.py @@ -56,8 +56,8 @@ OUTCOME_LABEL = {"traced": "对且可追溯", "untraced": "对但没引用", "wr TEXT_OUTCOMES = ("covered", "partial", "missing", "unjudged") TEXT_LABEL = {"covered": "都答到了", "partial": "答到一部分", "missing": "没答到", "unjudged": "没有判"} VERDICT_LABEL = {"pass": "通过", "fail": "不通过", "ungradable": "不能评分", "skipped": "跳过"} -PATH_LABEL = {"reply": "直接回答", "asked": "反问", "ready": "只建不跑", "build_failed": "建图失败", - "run_failed": "运行失败", "ran": "跑完了", "rejected": "被驳回"} +PATH_LABEL = {"reply": "直接回答", "asked": "反问", "ready": "只建图未运行", "build_failed": "建图失败", + "run_failed": "运行失败", "ran": "运行完成", "rejected": "被驳回"} #: 期望没写容差时,引用的原值按这个相对误差比(同双盲复算的缺省) DEFAULT_TOLERANCE = 1e-6 #: 量纲不同:差这几个倍数(元 / 万元、比率 / 百分数……) @@ -495,7 +495,7 @@ def score_case(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: return CaseScore("ungradable", False, path_reason or "运行被驳回(人的决定,不算失败)", []) ask_only = any(e.get("kind") == "ask" for e in expectations) if needs_numbers and path != "ran": - reason = WRONG_PATH.get(path or "", "没有跑出答案") + reason = WRONG_PATH.get(path or "", "没有得到答案") if path_reason: reason = f"{reason}:{path_reason}" return CaseScore("fail", False, reason, [{"id": e["id"], "kind": e.get("kind"), diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index 06a2c233..c98df297 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -331,3 +331,51 @@ async def test_eval_settings_group_and_spend_endpoint(client): assert effective_budget({"daily_max_usd": 9.0}, {"daily_max_usd": 3.0})["daily_max_usd"] == 3.0 assert effective_budget({"daily_max_usd": 1.0}, {"daily_max_usd": 3.0})["daily_max_usd"] == 1.0 await client.put("/api/settings", json={"values": {"eval": {}}}) + + +async def test_a_workflow_case_runs_its_pinned_version_as_an_exploratory_run(client, monkeypatch, tmp_path): + """工作流题:钉住那一版的图按输入模板跑一次探索运行(来源 eval),不发正式运行、不建图、不复核。""" + from app.core.artifact_store import graph_hash + from app.db.models import Workflow, WorkflowVersion + from tests.fixtures import dual_check_shop as shop + + name, sid = await es.make_source(tmp_path) + try: + es.use_report(monkeypatch) + + def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + nodes = [node("start", "input", fields=[{"name": "week", "required": True}]), + node("q", "tool", tool=f"db_query__{name}", args={"sql": shop.RIGHT}), + node("write", "report", instructions="写周销售"), + node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}])] + graph = {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + async with SessionLocal() as session: + wf = Workflow(name="虚构周销售", graph=graph, status="published", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph, graph_hash=graph_hash(graph), + level="published")) + await session.commit() + wf_id = wf.id + suite_id = await es.make_suite(sid, cases=[{ + "kind": "workflow", "workflow_id": wf_id, "workflow_version": 1, + "input_template": {"week": "{{ schedule.last_week }}"}, + "gold": {"kind": "sql", "datasource_id": sid, "sql": shop.RIGHT}, + "expectations": [{"kind": "scalar", "column": "gmv"}]}]) + [run] = await start(suite_id) + await es.wait_eval(run.id) + [item] = await results(run.id) + assert item.path == "ran" and item.verdict == "pass", (item.error, item.scores) + async with SessionLocal() as session: + answered = await session.get(Run, item.run_id) + assert answered.run_class == "exploratory" and answered.origin_kind == "eval" and answered.workflow_id is None + assert set(answered.input) == {"week"} and answered.input["week"].count("-W") == 1 + # 钉住当基线、取消 + pinned = await client.post(f"/api/evals/evaluations/{run.id}/pin", json={"pinned": True}) + assert pinned.status_code == 200 and pinned.json()["pinned"] is True + assert (await client.post(f"/api/evals/evaluations/{run.id}/pin", json={"pinned": "yes"})).status_code == 422 + assert (await client.get("/api/evals/pairs/nope")).status_code == 404 + finally: + await drop_source(sid) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index e93f53d7..9f7489df 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -3884,7 +3884,24 @@ export const SCHEDULE_TEXT = { /** 启用开关遇到版本不符(409):服务端那句是写给表单保存的,开关用这句,原话放进详情 */ toggleConflict: (name: string) => `「${name}」已在别处被修改,已重新载入最新状态,请确认后再切换`, + // ---- 目标(定时只有一套:目标可以是工作流、评测题集或数据源维护) + targetKind: { workflow: '工作流', eval_suite: '评测题集', source_maintenance: '数据源维护' } as Record< + import('../types').ScheduleTargetKind, string>, + /** 目标已经不在了:名字是保存时记下的 */ + targetGone: (name: string) => `${name}(已不存在)`, + view: '查看', + viewLabel: (name: string) => `查看定时任务「${name}」`, + /** 非工作流目标的设置是只读的:启用、暂停、立即运行、触发历史照常可用 */ + targetSettingsTitle: '设置', + targetSettingsNote: (kind: string) => `这个定时任务的目标是${kind}:可以启用、暂停、立即运行和查看触发历史;名称、频率和目标在这里只能查看,不能修改`, + targetField: '目标', + graceField: '补发宽限', + // ---- 立即运行的结果 + /** 目标是评测题集:发起的是一次评测,没有运行可看 */ + evalStarted: '已发起一次评测', + /** 触发历史里评测题集那一次:评测的编号 */ + evalFire: (short: string) => `评测 #${short}`, runStarted: '已发起运行', runSkipped: '上一次运行尚未结束,本次已按重叠策略跳过', runFailed: (reason: string | null) => (reason ? `未能发起运行:${reason}` : '未能发起运行'), diff --git a/frontend/src/pages/schedules/FireHistory.tsx b/frontend/src/pages/schedules/FireHistory.tsx index e69ba212..50f311a0 100644 --- a/frontend/src/pages/schedules/FireHistory.tsx +++ b/frontend/src/pages/schedules/FireHistory.tsx @@ -143,6 +143,9 @@ function FireRow({ fire: f, tz, now }: { fire: ScheduleFire; tz: string; now: nu {T.openRun} + ) : f.eval_run_id ? ( + // 目标是评测题集:这一次发起的是一次评测(评测页在后续版本),这里只写编号 + {T.evalFire(f.eval_run_id.slice(0, 6))} ) : {T.noRun}} {f.reason && ( diff --git a/frontend/src/pages/schedules/ScheduleDetail.tsx b/frontend/src/pages/schedules/ScheduleDetail.tsx index 70d917e4..e42c0f0d 100644 --- a/frontend/src/pages/schedules/ScheduleDetail.tsx +++ b/frontend/src/pages/schedules/ScheduleDetail.tsx @@ -8,7 +8,11 @@ import { SCHEDULE_TEXT as T } from '../../lib/terms' import { useOnReconnect } from '../../store/catalog' import type { Schedule, ScheduleFire } from '../../types' import { FireHistory } from './FireHistory' -import { EnableSwitch, LastFire, NextRun, isUnsupported, rememberScheduleNames, removeSchedule, runNow, setEnabled } from './parts' +import { + EnableSwitch, LastFire, NextRun, isUnsupported, rememberScheduleNames, removeSchedule, runNow, setEnabled, targetKindOf, + targetText, +} from './parts' +import { graceLabel } from './model' import { ScheduleEditor } from './ScheduleEditor' const BACK = ( @@ -234,7 +238,12 @@ export function ScheduleDetailPage({ id }: { id: string }) {
- {settingsSeen && ( + {settingsSeen && targetKindOf(schedule) !== 'workflow' && ( + + )} + {settingsSeen && targetKindOf(schedule) === 'workflow' && (