mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
评测估算和每题平均花费：有估不出金额的调用时不再写成 0 美元
价格目录里没有助手模型时（真实模型验收的接入就是这样），第一次评测估不出、照实写了；第二次起估算按上一次的每题平均 花费算，而每题平均是「估得出的那部分」除以题数 = 0，开跑对话框写「约 $0」、评测总览写「每题约 $0」，像不花钱。 - eval/stats.summarize：每题平均花费一分都没估出来（有估不出的调用、估得出的部分是 0）时为 None，估出了一部分的照给； - eval/runner.estimate：上一次的调用金额都估不出时照实写估不出（带次数和原因，金额上限只管得住估得出的调用）；估出了一部分 的标 at_least、带上另有几次估不出，说明写「实际至少这么多」；老总表里每题平均记成 0、却有估不出的调用的也按估不出算； - 按角色用量里裁判那一份（runner.judge_usage）补上估不出的次数：以前只有金额和次数，裁判模型不在价格目录里时写成 $0； - 评测页：开跑对话框估出一部分时写「至少约 …（每项至少约 …）」，总览的每题平均有估不出的写「每题至少约 …」，一分都没估出来 的不写（金额那一格已是「金额无法估算」）。 providers/usage 的汇总形状不变：cost_usd 是估得出的那部分、unpriced_calls 是估不出的次数，和运行的 usage 是全系统一个口径 （细案 01 做法 3 定的），前端的金额写法（runCostText）一直按这两项一起写「≥」或「金额无法估算」；漏掉的是只读 cost_usd 去算 平均、做估算的这几处。模块说明里把这条写明了。 测试：test_eval_stats.py::test_the_per_case_cost_is_not_zero_when_calls_could_not_be_priced、 test_eval_runner.py::test_the_estimate_says_it_cannot_price_instead_of_zero（全估不出、估出一部分、目录价是 0 的才是真 不花钱）、test_eval_runner.py::test_the_judge_role_keeps_its_unpriced_calls。去掉修复回跑三条都失败。
This commit is contained in:
ThearchyHelios
@@ -124,16 +124,44 @@ async def source_versions(session: AsyncSession, source_ids: Sequence[str]) -> d
|
||||
|
||||
|
||||
async def estimate(session: AsyncSession, suite_id: str, items: int) -> dict[str, Any]:
|
||||
"""开跑前的估算：按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出，照实写。"""
|
||||
"""开跑前的估算：按这个题集上一次结束的评测的每题平均花费 × 这次的项数。第一次评测估不出，照实写。
|
||||
|
||||
上一次有估不出金额的调用（模型不在价格目录里、用量没报全）时，平均花费只是估得出的那部分：一分都没估出来的照实写估不出
|
||||
（以前写「约 $0」，像不花钱），估出了一部分的写「至少」（at_least）、带上另有几次估不出（unpriced_calls）。老总表里每题
|
||||
平均记成 0、却有估不出的调用的，也按估不出算。"""
|
||||
row = (await session.execute(select(EvalRun).where(
|
||||
EvalRun.suite_id == suite_id, EvalRun.status.in_(("succeeded", "partial")))
|
||||
.order_by(EvalRun.created_at.desc()).limit(1))).scalar_one_or_none()
|
||||
per_item = ((row.summary or {}).get("per_case_cost_usd") if row is not None else None)
|
||||
if per_item is None:
|
||||
if row is None:
|
||||
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None,
|
||||
"note": "这个题集还没有评完过，估不出花费：按每题开评前的实际花费累计，到了上限就停"}
|
||||
return {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4),
|
||||
"basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算（不含估不出金额的模型）"}
|
||||
summary = row.summary or {}
|
||||
per_item = summary.get("per_case_cost_usd")
|
||||
unpriced = int(summary.get("unpriced_calls") or 0)
|
||||
if per_item is None or (unpriced and not float(per_item)):
|
||||
if not unpriced:
|
||||
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None,
|
||||
"note": "这个题集上一次评测没有评完一道题，估不出花费：按每题开评前的实际花费累计，到了上限就停"}
|
||||
return {"items": items, "per_item_usd": None, "total_usd": None, "basis": row.id, "unpriced_calls": unpriced,
|
||||
"note": f"上一次评测的 {unpriced} 次模型调用金额都估不出（模型不在价格目录中，或用量没有报全），估不出花费："
|
||||
"金额上限只管得住估得出金额的调用"}
|
||||
out = {"items": items, "per_item_usd": round(float(per_item), 6), "total_usd": round(float(per_item) * items, 4),
|
||||
"basis": row.id, "note": "按这个题集上一次评测的每题平均花费估算"}
|
||||
if unpriced:
|
||||
out.update(at_least=True, unpriced_calls=unpriced,
|
||||
note=f"按这个题集上一次评测的每题平均花费估算，实际至少这么多：上一次另有 {unpriced} 次调用金额估不出，没有算进去")
|
||||
return out
|
||||
|
||||
|
||||
def judge_usage(judge: Mapping[str, Any], cost_usd: float) -> dict[str, Any]:
|
||||
"""按角色的用量里裁判那一份：金额、次数，和估不出金额的次数（结论裁判的模型不在价格目录里时它的每次调用都估不出；
|
||||
文字要点按计量汇总）。以前只有金额和次数，裁判模型不在价格目录里时写成 $0。"""
|
||||
claims = judge.get("claims") or {}
|
||||
text = judge.get("text") or {}
|
||||
calls = int(claims.get("calls") or 0) + int(text.get("calls") or 0)
|
||||
unpriced = (int(claims.get("calls") or 0) if claims.get("priced") is False else 0) \
|
||||
+ int((text.get("usage") or {}).get("unpriced_calls") or 0)
|
||||
return {"cost_usd": round(cost_usd, 6), "calls": calls, "unpriced_calls": unpriced}
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
@@ -552,9 +580,7 @@ class EvalExecution(runtime.JobRun):
|
||||
writer = await writer_usage(turn.runs)
|
||||
assistant = add_usage(turn.usage.get("assistant"), turn.usage.get("review"))
|
||||
usage = {"assistant": assistant, "writer": writer, "review": turn.usage.get("review"),
|
||||
"judge": {"cost_usd": round(judge_cost, 6),
|
||||
"calls": int((judge.get("claims") or {}).get("calls") or 0)
|
||||
+ int((judge.get("text") or {}).get("calls") or 0)}}
|
||||
"judge": judge_usage(judge, judge_cost)}
|
||||
cost = float(assistant.get("cost_usd") or 0) + float(writer.get("cost_usd") or 0)
|
||||
self.spent += cost + judge_cost
|
||||
looks = {"model": sum(1 for x in turn.looks if x.get("by") == "model"),
|
||||
@@ -1022,6 +1048,6 @@ async def shutdown() -> None:
|
||||
|
||||
|
||||
__all__ = ["EVAL_BUSY", "EVAL_FINISHED", "EVAL_INVALID", "EvalExecution", "EvalRefused", "INTERRUPTED",
|
||||
"NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "leader_of", "manager",
|
||||
"NOTHING_TO_RETRY", "STOPPED", "active_run", "estimate", "gold_key", "judge_usage", "leader_of", "manager",
|
||||
"prompt_fingerprint", "reconcile_stale", "refresh_summary", "result_dict", "result_rows", "retry", "running",
|
||||
"shutdown", "source_versions", "start", "startup", "stop"]
|
||||
|
||||
@@ -145,8 +145,10 @@ def summarize(results: Sequence[Mapping[str, Any]], *, judge_on: bool = False) -
|
||||
"timings": {k: {"p50": percentile(v, 50), "p90": percentile(v, 90), "n": len(v)} for k, v in timings.items()},
|
||||
"usage": {**usage, "gold": {"cost_usd": 0.0, "calls": 0}},
|
||||
"cost_usd": cost, "unpriced_calls": unpriced,
|
||||
# 每题平均花费（下一次开跑前的估算按它）：只按评完了的题平均，没开评的不摊
|
||||
"per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated else None,
|
||||
# 每题平均花费（下一次开跑前的估算按它）：只按评完了的题平均，没开评的不摊。有估不出金额的调用时 cost_usd 只是估得出
|
||||
# 的那部分：一分都没估出来（模型都不在价格目录里）的为 None，不写成 0（像不花钱）；估出了一部分的照给，unpriced_calls
|
||||
# 说明它只是下限
|
||||
"per_case_cost_usd": round(cost / len(evaluated), 6) if evaluated and (cost > 0 or not unpriced) else None,
|
||||
"citations": {"numbers": numbers, "numbers_cited": cited,
|
||||
"uncited_numbers": sum(int(s.get("uncited_numbers") or 0) for s in stats),
|
||||
"unresolved": sum(int(s.get("unresolved") or 0) for s in stats),
|
||||
|
||||
@@ -12,7 +12,10 @@ async generator：with 块跨着 yield 的话，变量在两次 yield 之间留
|
||||
在创建时拷走上下文，照样计到；调用方自己的代码一行都不在计量里。
|
||||
|
||||
**金额。** 按 providers/catalog.estimate_cost 算，和运行的 llm.end 一个口径。估不出的调用记 unpriced_calls：模型不在价格
|
||||
目录里，或者用量没报全而目录价又不是 0。估不出的不当成 0 元，调用方据此写「≥」或「金额无法估算」。用量没报全有两种：
|
||||
目录里，或者用量没报全而目录价又不是 0。估不出的不当成 0 元，调用方据此写「≥」或「金额无法估算」。汇总的 cost_usd 永远是
|
||||
估得出的那部分（同运行的 usage，全系统一个口径），有 unpriced_calls 时它只是下限：写金额、算平均、做估算的地方都要连着
|
||||
unpriced_calls 一起看，一分都没估出来的写估不出、不写 0（评测的开跑估算、每题平均花费以前就漏了这一条，见 eval/runner.estimate、
|
||||
eval/stats.summarize）。用量没报全有两种：
|
||||
回复没带用量（有的网关流式不给）；读到一半就不读了——用量在流的末尾才报全（Anthropic 的输出 token 在最后一条，OpenAI
|
||||
兼容系开了 stream_usage 也在最后）。生成过程读到 done、look、ask 那一行之后照样把流读完再关（api/copilot_gen._drain），
|
||||
流末尾那一段用量收得到；只有模型在那一行之后还往下写超过 DRAIN_CHARS 字的闲话时才当场关流（不为闲话付钱），连接断了也一样，
|
||||
|
||||
@@ -347,6 +347,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch
|
||||
assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid"
|
||||
|
||||
|
||||
async def test_the_estimate_says_it_cannot_price_instead_of_zero():
|
||||
"""上一次评测的模型都不在价格目录里（价格目录空）：估算以前写「约 $0」，像不花钱。现在一分都没估出来的写估不出；估出了
|
||||
一部分的写「至少」、另有几次估不出（老总表里每题平均记成 0、却有估不出的调用的，也按估不出算）。"""
|
||||
from datetime import timedelta
|
||||
|
||||
suite_id = await es.make_suite("none", cases=ask_cases(1))
|
||||
now = utcnow()
|
||||
async with SessionLocal() as session:
|
||||
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=2),
|
||||
summary={"per_case_cost_usd": 0.0, "unpriced_calls": 6}))
|
||||
await session.commit()
|
||||
got = await runner.estimate(session, suite_id, 3)
|
||||
assert got["total_usd"] is None and got["per_item_usd"] is None and got["unpriced_calls"] == 6, got
|
||||
assert "估不出" in got["note"] and got["basis"], got
|
||||
async with SessionLocal() as session:
|
||||
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now - timedelta(minutes=1),
|
||||
summary={"per_case_cost_usd": 0.02, "unpriced_calls": 2}))
|
||||
await session.commit()
|
||||
got = await runner.estimate(session, suite_id, 3)
|
||||
assert got["total_usd"] == 0.06 and got["at_least"] is True and got["unpriced_calls"] == 2, got
|
||||
assert "至少" in got["note"] and "2 次" in got["note"], got["note"]
|
||||
async with SessionLocal() as session:
|
||||
session.add(EvalRun(suite_id=suite_id, status="succeeded", created_at=now,
|
||||
summary={"per_case_cost_usd": 0.0, "unpriced_calls": 0}))
|
||||
await session.commit()
|
||||
got = await runner.estimate(session, suite_id, 3)
|
||||
assert got["total_usd"] == 0.0 and not got.get("at_least") and not got.get("unpriced_calls"), "目录价是 0 的才是真不花钱"
|
||||
|
||||
|
||||
def test_the_judge_role_keeps_its_unpriced_calls():
|
||||
"""按角色的用量里裁判那一份以前只有金额和次数：裁判模型不在价格目录里时写成 $0。"""
|
||||
usage = runner.judge_usage({"claims": {"cost_usd": 0.0, "calls": 2, "priced": False},
|
||||
"text": {"cost_usd": 0.0, "calls": 1, "usage": {"unpriced_calls": 1}}}, 0.0)
|
||||
assert usage == {"cost_usd": 0.0, "calls": 3, "unpriced_calls": 3}
|
||||
priced = runner.judge_usage({"claims": {"cost_usd": 0.01, "calls": 2, "priced": True}}, 0.01)
|
||||
assert priced == {"cost_usd": 0.01, "calls": 2, "unpriced_calls": 0}
|
||||
|
||||
|
||||
async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch):
|
||||
"""逐题结果的 question 是这一题实际问的（变量按这次评测的截至时间渲染好了，和作答时 Driver 收到的一字不差），模板原文
|
||||
另放 question_template（下一轮验收：以前 question 是带 {{ schedule.last_week_start }} 的原始模板）。没有变量的题两个一样。"""
|
||||
|
||||
@@ -69,6 +69,18 @@ def test_summary_counts_only_gradable_items_and_splits_roles():
|
||||
assert summary["e1"]["passed"] == 1 and summary["e1"]["n"] == 2 and summary["e1"]["aspects"] == {"time": 1}
|
||||
|
||||
|
||||
def test_the_per_case_cost_is_not_zero_when_calls_could_not_be_priced():
|
||||
"""模型不在价格目录里（价格目录空）：金额 0 只是「估得出的那部分」，每题平均花费不能写成 0（像不花钱）——一分都没估出来
|
||||
的为 None（估不出），估出了一部分的照给、unpriced_calls 说明它只是下限。"""
|
||||
unpriced = {"assistant": {"by_model": {}, "calls": 4, "cost_usd": 0.0, "input_tokens": 900, "output_tokens": 90,
|
||||
"unpriced_calls": 4}}
|
||||
summary = stats.summarize([_row("c1", "pass", usage=unpriced), _row("c2", "fail", usage=unpriced)])
|
||||
assert summary["unpriced_calls"] == 8 and summary["per_case_cost_usd"] is None, summary["per_case_cost_usd"]
|
||||
partly = {"assistant": {**unpriced["assistant"], "cost_usd": 0.02, "unpriced_calls": 1}}
|
||||
summary = stats.summarize([_row("c1", "pass", usage=partly), _row("c2", "fail", usage=partly)])
|
||||
assert summary["per_case_cost_usd"] == 0.02 and summary["unpriced_calls"] == 2
|
||||
|
||||
|
||||
def test_repeats_give_a_pass_rate_per_case():
|
||||
rows = [_row("c1", "pass", repeat=0), _row("c1", "fail", repeat=1), _row("c1", "pass", repeat=2)]
|
||||
summary = stats.summarize(rows)
|
||||
|
||||
@@ -356,6 +356,8 @@ export const EVAL_TEXT = {
|
||||
estimateText: (items: number, cases: number, repeats: number, arms: number) =>
|
||||
`${formatNumber(cases)} 道题 × 每题 ${repeats} 次${arms > 1 ? ` × ${arms} 臂` : ''}，共 ${formatNumber(items)} 项`,
|
||||
cost: (total: string, per: string) => `约 ${total}（每项约 ${per}）`,
|
||||
/** 上一次有估不出金额的调用：估出来的只是下限 */
|
||||
costAtLeast: (total: string, per: string) => `至少约 ${total}（每项至少约 ${per}）`,
|
||||
costNone: '估不出花费',
|
||||
budget: '生效的预算',
|
||||
parallel: (n: number) => `同时评 ${n} 题`,
|
||||
@@ -416,6 +418,8 @@ export const EVAL_TEXT = {
|
||||
cost: '花费',
|
||||
costHint: '助手（建图、挑表、复核）+ 写作（答案运行里的模型调用）+ 结论裁判；算期望不调模型',
|
||||
perCase: (v: string) => `每题约 ${v}`,
|
||||
/** 有估不出金额的调用：每题平均只是下限 */
|
||||
perCaseAtLeast: (v: string) => `每题至少约 ${v}`,
|
||||
e1: '逐题的口径命中（E1）',
|
||||
e1Hint: '答案执行的 SQL 和期望的取数被证明等价的比例（口径账本同一个比对函数）',
|
||||
citations: '报告引用率',
|
||||
|
||||
@@ -215,8 +215,7 @@ function Overview({ run }: { run: EvalRun }) {
|
||||
<div className="grid gap-2 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<RateCard title={O.accuracy} hint={O.accuracyHint} rate={s.accuracy} attr="accuracy" />
|
||||
<RateCard title={O.traced} hint={O.tracedHint} rate={s.traced_accuracy} attr="traced" />
|
||||
<StatCard title={O.cost} hint={O.costHint} value={cost} attr="cost"
|
||||
sub={s.per_case_cost_usd != null ? O.perCase(formatCost(s.per_case_cost_usd)) : undefined} />
|
||||
<StatCard title={O.cost} hint={O.costHint} value={cost} attr="cost" sub={perCaseText(s)} />
|
||||
<StatCard title={O.e1} hint={O.e1Hint} value={s.e1?.n ? pct(s.e1.rate) : '—'} sub={rateText(s.e1)} attr="e1" />
|
||||
</div>
|
||||
<Section title={O.verdicts} attr="verdicts">
|
||||
@@ -296,6 +295,17 @@ function Overview({ run }: { run: EvalRun }) {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 每题平均花费：有估不出金额的调用时只是下限（写「至少」）；一分都没估出来的不写（老总表记成 0 的也一样，金额那一格已经写了
|
||||
* 「金额无法估算」）
|
||||
*/
|
||||
function perCaseText(s: EvalSummary): string | undefined {
|
||||
const per = s.per_case_cost_usd
|
||||
if (per == null) return undefined
|
||||
if (!s.unpriced_calls) return T.overview.perCase(formatCost(per))
|
||||
return per > 0 ? T.overview.perCaseAtLeast(formatCost(per)) : undefined
|
||||
}
|
||||
|
||||
function Section({ title, children, attr }: { title: string; children: ReactNode; attr?: string }) {
|
||||
return (
|
||||
<section className="space-y-1 rounded-lg border bg-panel p-3 text-xs" aria-label={title} data-eval-section={attr}>
|
||||
|
||||
@@ -263,9 +263,8 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
<>
|
||||
<p data-eval-estimate-items={est.estimate.items}>{T.estimateText(est.estimate.items, est.cases, est.repeats, est.arms.length)}</p>
|
||||
<p data-eval-estimate-cost="">
|
||||
{est.estimate.total_usd != null
|
||||
? T.cost(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))
|
||||
: T.costNone}
|
||||
{est.estimate.total_usd == null ? T.costNone
|
||||
: (est.estimate.at_least ? T.costAtLeast : T.cost)(formatCost(est.estimate.total_usd), formatCost(est.estimate.per_item_usd))}
|
||||
<span className="text-faint"> · {est.estimate.note}</span>
|
||||
</p>
|
||||
{est.arms.map((a, i) => (
|
||||
|
||||
@@ -365,7 +365,11 @@ export interface EvalEstimate {
|
||||
arms: { label: string | null; variant: EvalVariant; models: EvalResolvedModels; text: string }[]
|
||||
cases: number
|
||||
repeats: number
|
||||
estimate: { items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string }
|
||||
/** at_least：上一次另有 unpriced_calls 次调用金额估不出，估算只是下限；total_usd 为空、unpriced_calls 有值：一分都没估出来 */
|
||||
estimate: {
|
||||
items: number; per_item_usd: number | null; total_usd: number | null; basis: string | null; note: string
|
||||
at_least?: boolean; unpriced_calls?: number
|
||||
}
|
||||
budget: EvalBudget
|
||||
parallel: number
|
||||
spend: EvalSpend
|
||||
|
||||
Reference in New Issue
Block a user