From 4aa3e6f01fc99701da46c866039975e87cc1fe9a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 11:40:27 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=84=E6=B5=8B=E4=BC=B0=E7=AE=97=E5=92=8C?= =?UTF-8?q?=E6=AF=8F=E9=A2=98=E5=B9=B3=E5=9D=87=E8=8A=B1=E8=B4=B9=EF=BC=9A?= =?UTF-8?q?=E6=9C=89=E4=BC=B0=E4=B8=8D=E5=87=BA=E9=87=91=E9=A2=9D=E7=9A=84?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=97=B6=E4=B8=8D=E5=86=8D=E5=86=99=E6=88=90?= =?UTF-8?q?=200=20=E7=BE=8E=E5=85=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 价格目录里没有助手模型时（真实模型验收的接入就是这样），第一次评测估不出、照实写了；第二次起估算按上一次的每题平均 花费算，而每题平均是「估得出的那部分」除以题数 = 0，开跑对话框写「约 $0」、评测总览写「每题约 $0」，像不花钱。 - eval/stats.summarize：每题平均花费一分都没估出来（有估不出的调用、估得出的部分是 0）时为 None，估出了一部分的照给； - eval/runner.estimate：上一次的调用金额都估不出时照实写估不出（带次数和原因，金额上限只管得住估得出的调用）；估出了一部分 的标 at_least、带上另有几次估不出，说明写「实际至少这么多」；老总表里每题平均记成 0、却有估不出的调用的也按估不出算； - 按角色用量里裁判那一份（runner.judge_usage）补上估不出的次数：以前只有金额和次数，裁判模型不在价格目录里时写成 $0； - 评测页：开跑对话框估出一部分时写「至少约 …（每项至少约 …）」，总览的每题平均有估不出的写「每题至少约 …」，一分都没估出来 的不写（金额那一格已是「金额无法估算」）。 providers/usage 的汇总形状不变：cost_usd 是估得出的那部分、unpriced_calls 是估不出的次数，和运行的 usage 是全系统一个口径 （细案 01 做法 3 定的），前端的金额写法（runCostText）一直按这两项一起写「≥」或「金额无法估算」；漏掉的是只读 cost_usd 去算 平均、做估算的这几处。模块说明里把这条写明了。 测试：test_eval_stats.py::test_the_per_case_cost_is_not_zero_when_calls_could_not_be_priced、 test_eval_runner.py::test_the_estimate_says_it_cannot_price_instead_of_zero（全估不出、估出一部分、目录价是 0 的才是真 不花钱）、test_eval_runner.py::test_the_judge_role_keeps_its_unpriced_calls。去掉修复回跑三条都失败。 --- backend/app/eval/runner.py | 44 ++++++++++++++++----- backend/app/eval/stats.py | 6 ++- backend/app/providers/usage.py | 5 ++- backend/tests/test_eval_runner.py | 38 ++++++++++++++++++ backend/tests/test_eval_stats.py | 12 ++++++ frontend/src/lib/termsEval.ts | 4 ++ frontend/src/pages/evals/EvaluationPage.tsx | 14 ++++++- frontend/src/pages/evals/StartDialog.tsx | 5 +-- frontend/src/typesEval.ts | 6 ++- 9 files changed, 116 insertions(+), 18 deletions(-) diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index 3412c005..7e19065c 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -124,16 +124,44 @@ async def source_versions(session: AsyncSession, source_ids: Sequence[str]) -> d async def estimate(session: AsyncSession, suite_id: str, items: int) -> dict[str, Any]: - """开跑前的估算：按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出，照实写。""" + """开跑前的估算：按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出，照实写。 + + 上一次有估不出金额的调用（模型不在价格目录里、用量没报全）时，平均花费只是估得出的那部分：一分都没估出来的照实写估不出 + （以前写「约 $0」，像不花钱），估出了一部分的写「至少」（at_least）、带上另有几次估不出（unpriced_calls）。老总表里每题 + 平均记成 0、却有估不出的调用的，也按估不出算。""" row = (await session.execute(select(EvalRun).where( EvalRun.suite_id == suite_id, EvalRun.status.in_(("succeeded", "partial"))) .order_by(EvalRun.created_at.desc()).limit(1))).scalar_one_or_none() - per_item = ((row.summary or {}).get("per_case_cost_usd") if row is not None else None) - if per_item is None: + if row is None: return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None, "note": "这个题集还没有评完过，估不出花费：按每题开评前的实际花费累计，到了上限就停"} - return {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4), - "basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算（不含估不出金额的模型）"} + summary = row.summary or {} + per_item = summary.get("per_case_cost_usd") + unpriced = int(summary.get("unpriced_calls") or 0) + if per_item is None or (unpriced and not float(per_item)): + if not unpriced: + return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None, + "note": "这个题集上一次评测没有评完一道题，估不出花费：按每题开评前的实际花费累计，到了上限就停"} + return {"items": items, "per_item_usd": None, "total_usd": None, "basis": row.id, "unpriced_calls": unpriced, + "note": f"上一次评测的 {unpriced} 次模型调用金额都估不出（模型不在价格目录中，或用量没有报全），估不出花费：" + "金额上限只管得住估得出金额的调用"} + out = {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4), + "basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算"} + if unpriced: + out.update(at_least=True, unpriced_calls=unpriced, + note=f"按这个题集上一次评测的每题平均花费估算，实际至少这么多：上一次另有 {unpriced} 次调用金额估不出，没有算进去") + return out + + +def judge_usage(judge: Mapping[str, Any], cost_usd: float) -> dict[str, Any]: + """按角色的用量里裁判那一份：金额、次数，和估不出金额的次数（结论裁判的模型不在价格目录里时它的每次调用都估不出； + 文字要点按计量汇总）。以前只有金额和次数，裁判模型不在价格目录里时写成 $0。""" + claims = judge.get("claims") or {} + text = judge.get("text") or {} + calls = int(claims.get("calls") or 0) + int(text.get("calls") or 0) + unpriced = (int(claims.get("calls") or 0) if claims.get("priced") is False else 0) \ + + int((text.get("usage") or {}).get("unpriced_calls") or 0) + return {"cost_usd": round(cost_usd, 6), "calls": calls, "unpriced_calls": unpriced} # ========================================================================== @@ -552,9 +580,7 @@ class EvalExecution(runtime.JobRun): writer = await writer_usage(turn.runs) assistant = add_usage(turn.usage.get("assistant"), turn.usage.get("review")) usage = {"assistant": assistant, "writer": writer, "review": turn.usage.get("review"), - "judge": {"cost_usd": round(judge_cost, 6), - "calls": int((judge.get("claims") or {}).get("calls") or 0) - + int((judge.get("text") or {}).get("calls") or 0)}} + "judge": judge_usage(judge, judge_cost)} cost = float(assistant.get("cost_usd") or 0) + float(writer.get("cost_usd") or 0) self.spent += cost + judge_cost looks = {"model": sum(1 for x in turn.looks if x.get("by") == "model"), @@ -1022,6 +1048,6 @@ async def shutdown() -> None: __all__ = ["EVAL_BUSY", "EVAL_FINISHED", "EVAL_INVALID", "EvalExecution", "EvalRefused", "INTERRUPTED", - "NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "leader_of", "manager", + "NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "judge_usage", "leader_of", "manager", "prompt_fingerprint", "reconcile_stale", "refresh_summary", "result_dict", "result_rows", "retry", "running", "shutdown", "source_versions", "start", "startup", "stop"] diff --git a/backend/app/eval/stats.py b/backend/app/eval/stats.py index 7e4f6a8e..77f9dd22 100644 --- a/backend/app/eval/stats.py +++ b/backend/app/eval/stats.py @@ -145,8 +145,10 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) - "timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()}, "usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}}, "cost_usd": cost, "unpriced_calls": unpriced, - # 每题平均花费（下一次开跑前的估算按它）：只按评完了的题平均，没开评的不摊 - "per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated else None, + # 每题平均花费（下一次开跑前的估算按它）：只按评完了的题平均，没开评的不摊。有估不出金额的调用时 cost_usd 只是估得出 + # 的那部分：一分都没估出来（模型都不在价格目录里）的为 None，不写成 0（像不花钱）；估出了一部分的照给，unpriced_calls + # 说明它只是下限 + "per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated and (cost > 0 or not unpriced) else None, "citations": {"numbers": numbers, "numbers_cited": cited, "uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats), "unresolved": sum(int(s.get("unresolved") or 0) for s in stats), diff --git a/backend/app/providers/usage.py b/backend/app/providers/usage.py index 35079236..dbf029d6 100644 --- a/backend/app/providers/usage.py +++ b/backend/app/providers/usage.py @@ -12,7 +12,10 @@ async generator：with 块跨着 yield 的话，变量在两次 yield 之间留 在创建时拷走上下文，照样计到；调用方自己的代码一行都不在计量里。 **金额。** 按 providers/catalog.estimate_cost 算，和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls：模型不在价格 -目录里，或者用量没报全而目录价又不是 0。估不出的不当成 0 元，调用方据此写「≥」或「金额无法估算」。用量没报全有两种： +目录里，或者用量没报全而目录价又不是 0。估不出的不当成 0 元，调用方据此写「≥」或「金额无法估算」。汇总的 cost_usd 永远是 +估得出的那部分（同运行的 usage，全系统一个口径），有 unpriced_calls 时它只是下限：写金额、算平均、做估算的地方都要连着 +unpriced_calls 一起看，一分都没估出来的写估不出、不写 0（评测的开跑估算、每题平均花费以前就漏了这一条，见 eval/runner.estimate、 +eval/stats.summarize）。用量没报全有两种： 回复没带用量（有的网关流式不给）；读到一半就不读了——用量在流的末尾才报全（Anthropic 的输出 token 在最后一条，OpenAI 兼容系开了 stream_usage 也在最后）。生成过程读到 done、look、ask 那一行之后照样把流读完再关（api/copilot_gen._drain）， 流末尾那一段用量收得到；只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流（不为闲话付钱），连接断了也一样， diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index c6e0ee79..a9685e62 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -347,6 +347,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid" +async def test_the_estimate_says_it_cannot_price_instead_of_zero(): + """上一次评测的模型都不在价格目录里（价格目录空）：估算以前写「约 $0」，像不花钱。现在一分都没估出来的写估不出；估出了 + 一部分的写「至少」、另有几次估不出（老总表里每题平均记成 0、却有估不出的调用的，也按估不出算）。""" + from datetime import timedelta + + suite_id = await es.make_suite("none", cases=ask_cases(1)) + now = utcnow() + async with SessionLocal() as session: + session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=2), + summary={"per_case_cost_usd": 0.0, "unpriced_calls": 6})) + await session.commit() + got = await runner.estimate(session, suite_id, 3) + assert got["total_usd"] is None and got["per_item_usd"] is None and got["unpriced_calls"] == 6, got + assert "估不出" in got["note"] and got["basis"], got + async with SessionLocal() as session: + session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=1), + summary={"per_case_cost_usd": 0.02, "unpriced_calls": 2})) + await session.commit() + got = await runner.estimate(session, suite_id, 3) + assert got["total_usd"] == 0.06 and got["at_least"] is True and got["unpriced_calls"] == 2, got + assert "至少" in got["note"] and "2 次" in got["note"], got["note"] + async with SessionLocal() as session: + session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now, + summary={"per_case_cost_usd": 0.0, "unpriced_calls": 0})) + await session.commit() + got = await runner.estimate(session, suite_id, 3) + assert got["total_usd"] == 0.0 and not got.get("at_least") and not got.get("unpriced_calls"), "目录价是 0 的才是真不花钱" + + +def test_the_judge_role_keeps_its_unpriced_calls(): + """按角色的用量里裁判那一份以前只有金额和次数：裁判模型不在价格目录里时写成 $0。""" + usage = runner.judge_usage({"claims": {"cost_usd": 0.0, "calls": 2, "priced": False}, + "text": {"cost_usd": 0.0, "calls": 1, "usage": {"unpriced_calls": 1}}}, 0.0) + assert usage == {"cost_usd": 0.0, "calls": 3, "unpriced_calls": 3} + priced = runner.judge_usage({"claims": {"cost_usd": 0.01, "calls": 2, "priced": True}}, 0.01) + assert priced == {"cost_usd": 0.01, "calls": 2, "unpriced_calls": 0} + + async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch): """逐题结果的 question 是这一题实际问的（变量按这次评测的截至时间渲染好了，和作答时 Driver 收到的一字不差），模板原文 另放 question_template（下一轮验收：以前 question 是带 {{ schedule.last_week_start }} 的原始模板）。没有变量的题两个一样。""" diff --git a/backend/tests/test_eval_stats.py b/backend/tests/test_eval_stats.py index 1b809646..eb997cba 100644 --- a/backend/tests/test_eval_stats.py +++ b/backend/tests/test_eval_stats.py @@ -69,6 +69,18 @@ def test_summary_counts_only_gradable_items_and_splits_roles(): assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1} +def test_the_per_case_cost_is_not_zero_when_calls_could_not_be_priced(): + """模型不在价格目录里（价格目录空）：金额 0 只是「估得出的那部分」，每题平均花费不能写成 0（像不花钱）——一分都没估出来 + 的为 None（估不出），估出了一部分的照给、unpriced_calls 说明它只是下限。""" + unpriced = {"assistant": {"by_model": {}, "calls": 4, "cost_usd": 0.0, "input_tokens": 900, "output_tokens": 90, + "unpriced_calls": 4}} + summary = stats.summarize([_row("c1", "pass", usage=unpriced), _row("c2", "fail", usage=unpriced)]) + assert summary["unpriced_calls"] == 8 and summary["per_case_cost_usd"] is None, summary["per_case_cost_usd"] + partly = {"assistant": {**unpriced["assistant"], "cost_usd": 0.02, "unpriced_calls": 1}} + summary = stats.summarize([_row("c1", "pass", usage=partly), _row("c2", "fail", usage=partly)]) + assert summary["per_case_cost_usd"] == 0.02 and summary["unpriced_calls"] == 2 + + def test_repeats_give_a_pass_rate_per_case(): rows = [_row("c1", "pass", repeat=0), _row("c1", "fail", repeat=1), _row("c1", "pass", repeat=2)] summary = stats.summarize(rows) diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts index 5b9d3acb..91dc7f3e 100644 --- a/frontend/src/lib/termsEval.ts +++ b/frontend/src/lib/termsEval.ts @@ -356,6 +356,8 @@ export const EVAL_TEXT = { estimateText: (items: number, cases: number, repeats: number, arms: number) => `${formatNumber(cases)} 道题 × 每题 ${repeats} 次${arms > 1 ? ` × ${arms} 臂` : ''}，共 ${formatNumber(items)} 项`, cost: (total: string, per: string) => `约 ${total}（每项约 ${per}）`, + /** 上一次有估不出金额的调用：估出来的只是下限 */ + costAtLeast: (total: string, per: string) => `至少约 ${total}（每项至少约 ${per}）`, costNone: '估不出花费', budget: '生效的预算', parallel: (n: number) => `同时评 ${n} 题`, @@ -416,6 +418,8 @@ export const EVAL_TEXT = { cost: '花费', costHint: '助手（建图、挑表、复核）+ 写作（答案运行里的模型调用）+ 结论裁判；算期望不调模型', perCase: (v: string) => `每题约 ${v}`, + /** 有估不出金额的调用：每题平均只是下限 */ + perCaseAtLeast: (v: string) => `每题至少约 ${v}`, e1: '逐题的口径命中（E1）', e1Hint: '答案执行的 SQL 和期望的取数被证明等价的比例（口径账本同一个比对函数）', citations: '报告引用率', diff --git a/frontend/src/pages/evals/EvaluationPage.tsx b/frontend/src/pages/evals/EvaluationPage.tsx index ee2cf788..6d92df03 100644 --- a/frontend/src/pages/evals/EvaluationPage.tsx +++ b/frontend/src/pages/evals/EvaluationPage.tsx @@ -215,8 +215,7 @@ function Overview({ run }: { run: EvalRun }) {
- +
@@ -296,6 +295,17 @@ function Overview({ run }: { run: EvalRun }) { ) } +/** + * 每题平均花费：有估不出金额的调用时只是下限（写「至少」）；一分都没估出来的不写（老总表记成 0 的也一样，金额那一格已经写了 + * 「金额无法估算」） + */ +function perCaseText(s: EvalSummary): string | undefined { + const per = s.per_case_cost_usd + if (per == null) return undefined + if (!s.unpriced_calls) return T.overview.perCase(formatCost(per)) + return per > 0 ? T.overview.perCaseAtLeast(formatCost(per)) : undefined +} + function Section({ title, children, attr }: { title: string; children: ReactNode; attr?: string }) { return (
diff --git a/frontend/src/pages/evals/StartDialog.tsx b/frontend/src/pages/evals/StartDialog.tsx index 62a30944..19386493 100644 --- a/frontend/src/pages/evals/StartDialog.tsx +++ b/frontend/src/pages/evals/StartDialog.tsx @@ -263,9 +263,8 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: { <>

{T.estimateText(est.estimate.items, est.cases, est.repeats, est.arms.length)}

- {est.estimate.total_usd != null - ? T.cost(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd)) - : T.costNone} + {est.estimate.total_usd == null ? T.costNone + : (est.estimate.at_least ? T.costAtLeast : T.cost)(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))} · {est.estimate.note}

{est.arms.map((a, i) => ( diff --git a/frontend/src/typesEval.ts b/frontend/src/typesEval.ts index 1b8a0c9c..77f4b743 100644 --- a/frontend/src/typesEval.ts +++ b/frontend/src/typesEval.ts @@ -365,7 +365,11 @@ export interface EvalEstimate { arms: { label: string | null; variant: EvalVariant; models: EvalResolvedModels; text: string }[] cases: number repeats: number - estimate: { items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string } + /** at_least：上一次另有 unpriced_calls 次调用金额估不出，估算只是下限；total_usd 为空、unpriced_calls 有值：一分都没估出来 */ + estimate: { + items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string + at_least?: boolean; unpriced_calls?: number + } budget: EvalBudget parallel: number spend: EvalSpend