评测逐题结果给出这一题实际问的问题,模板原文另放 question_template

验收:评测结果列表里 question 是原始模板(带 {{ schedule.last_week_start }}),作答运行实际收到的问题已经渲染好了。
results 接口直接取的是 EvalCase.question。

改法:api/evals 的 _asked_questions 按执行时同一个算法、同一组输入渲染:题目模板 + 这次评测的截至时间(as_of,
建评测时就定下、重试不变)+ 题集时区 + 定时的 cron(runner._unit 交给 Driver 的就是这个)。题目在这次评测之后
改过(结果记的 case_hash 对不上现在的)时,现在的模板不一定是当时问的:改取作答运行输入里的 question,也没有
(反问、建图失败没有运行)就给 null,不拿现在的模板编一个。模板原文放 question_template。没有加库表的列:
渲染是确定的,不用另存。

前端:评测页逐题表的问法一格写实际问的问题;模板原文和它不同时(带变量,或题目改过)另起一行「题目原文:…」,
文案进 termsEval。类型补上 question_template。

测试:test_eval_runner 新增一条:带变量的题,结果的 question 和作答时 Driver 收到的一字不差、不含 {{,模板原文在
question_template;没有变量的题两者一样;题目改过、没有作答运行可取时 question 为 null、模板照给。去掉修复回跑挂。
check-evals 的夹具给带变量的那一题配上渲染好的 question 和模板原文,核对逐题表写的是实际问的、原文另起一行,
没有变量的题不多这一行。tsc、check-copy 通过,check-evals 102 项全过(漏到服务端的那个偶发问题另一次提交里修)。
This commit is contained in:
ThearchyHelios
2026-10-17 09:02:57 +08:00
parent f5c9f0c13c
commit eb6abffc61
6 changed files with 95 additions and 7 deletions
+39 -4
View File
@@ -1019,21 +1019,56 @@ async def _actions(session: AsyncSession, row: EvalRun) -> dict[str, Any]:
@router.get("/evaluations/{evaluation_id}/results")
async def evaluation_results(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""逐题结果:结论、各期望的结论(期望值和答案里对上的数、有没有引用、维度)、作答路径、复核、出具档位、SQL 差在哪、look、
耗时、按角色的用量,点 run_id 去运行详情。明细清理了的照实写 cleaned。"""
耗时、按角色的用量,点 run_id 去运行详情。明细清理了的照实写 cleaned。question 是这一题实际问的问题(变量渲染好了,
见 _asked_questions;说不准时为 null),question_template 是题目的原文(带 {{ schedule.… }} 变量的模板)。"""
from app.eval import runner
row = await _evaluation(session, evaluation_id)
results = [runner.result_dict(r) for r in (await session.execute(
select(runner.EvalResult).where(runner.EvalResult.eval_run_id == row.id)
.order_by(runner.EvalResult.seq))).scalars()]
questions = dict((await session.execute(select(EvalCase.id, EvalCase.question).where(
EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).all()) if results else {}
cases = {c.id: (c.question, c.case_hash) for c in (await session.execute(select(EvalCase).where(
EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).scalars()} if results else {}
asked = await _asked_questions(session, row, results, cases)
for r in results:
r["question"] = questions.get(r["case_id"])
template = (cases.get(r["case_id"]) or (None, None))[0]
r["question"], r["question_template"] = asked.get(r["id"]), template
r["cost"] = _result_cost(r.get("usage") or {})
return {"evaluation_id": row.id, "cleaned": row.cleaned_at is not None, "results": results}
async def _asked_questions(session: AsyncSession, row: EvalRun, results: list[dict[str, Any]],
cases: dict[str, tuple[str, str]]) -> dict[str, str | None]:
"""每一题实际问的问题 {结果 id: 问题}:题目的模板按这次评测的截至时间(as_of)、题集的时区、定时的 cron 渲染——和执行时
runner._unit 交给作答的是同一个算法、同一组输入,所以一字不差(as_of 在建评测时就定下、重试也不变)。题目在这次评测之后
改过(结果记的 case_hash 对不上现在的):现在的模板不一定是当时问的,改取作答运行的输入里的 question;也没有(反问、
建图失败没有运行,工作流题的输入里没有问题)就是 None,不拿现在的模板编一个。"""
from app.db.models import Run
from app.eval import runner
from app.eval.cases import render, variable_values
if not results:
return {}
leader = row if runner.leader_of(row) == row.id else await session.get(EvalRun, runner.leader_of(row))
suite = await session.get(EvalSuite, row.suite_id)
as_of = (leader.as_of if leader is not None else None) or row.as_of
values = variable_values(as_of, suite.timezone, ((leader or row).fingerprint or {}).get("cron")) \
if as_of is not None and suite is not None else None
stale = {r["id"] for r in results
if r["case_id"] not in cases or cases[r["case_id"]][1] != r["case_hash"] or values is None}
runs = {r["run_id"] for r in results if r["id"] in stale and r["run_id"]}
inputs = dict((await session.execute(select(Run.id, Run.input).where(Run.id.in_(runs))))
.all()) if runs else {}
out: dict[str, str | None] = {}
for r in results:
if r["id"] in stale:
given = (inputs.get(r["run_id"]) or {}).get("question") if r["run_id"] else None
out[r["id"]] = given if isinstance(given, str) and given else None
else:
out[r["id"]] = render(cases[r["case_id"]][0] or "", values or {})
return out
def _result_cost(usage: dict[str, Any]) -> dict[str, Any]:
"""一题的花费:助手(runner 记的 assistant 已含复核)+ 写作 + 结论裁判,和这次评测记在账上的同一份;估不出金额的调用另计
次数(界面写「≥」或「金额无法估算」)。加在服务端:界面不算数。"""
+38
View File
@@ -328,6 +328,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch
assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid"
async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch):
"""逐题结果的 question 是这一题实际问的(变量按这次评测的截至时间渲染好了,和作答时 Driver 收到的一字不差),模板原文
另放 question_template(下一轮验收:以前 question 是带 {{ schedule.last_week_start }} 的原始模板)。没有变量的题两个一样。"""
from datetime import datetime, timezone
from app.db.models_eval import EvalCase
from app.eval.cases import render, variable_values
asked: list[str] = []
real_init = answer_mod.Driver.__init__
def init(self, question, **kw):
asked.append(question)
real_init(self, question, **kw)
monkeypatch.setattr(answer_mod.Driver, "__init__", init)
stub(monkeypatch)
template = "{{ schedule.last_week_start }} 到 {{ schedule.last_week_end }} 那周的问题"
suite_id = await es.make_suite("none", cases=[{"question": template, "expectations": [{"kind": "ask"}]},
{"question": "没有变量的问题", "expectations": [{"kind": "ask"}]}])
as_of = datetime(2026, 10, 14, 2, 0, tzinfo=timezone.utc)
[run] = await start(suite_id, as_of=as_of)
await es.wait_eval(run.id)
rows = (await client.get(f"/api/evals/evaluations/{run.id}/results")).json()["results"]
expected = render(template, variable_values(as_of, "Asia/Shanghai", None))
assert "{{" not in expected and rows[0]["question"] == expected, rows[0]["question"]
assert rows[0]["question"] in asked, "和作答时实际问的一样"
assert rows[0]["question_template"] == template
assert rows[1]["question"] == rows[1]["question_template"] == "没有变量的问题"
# 题目改过(指纹对不上):现在的模板不一定是当时问的,没有作答运行可取时不编一个,question 为空、模板照给
async with SessionLocal() as session:
case = (await session.execute(select(EvalCase).where(EvalCase.suite_id == suite_id, EvalCase.seq == 1))).scalar_one()
case.question, case.case_hash = "改过的 {{ schedule.today }}", "changed"
await session.commit()
rows = (await client.get(f"/api/evals/evaluations/{run.id}/results")).json()["results"]
assert rows[0]["question"] is None and rows[0]["question_template"] == "改过的 {{ schedule.today }}"
async def test_eval_settings_group_and_spend_endpoint(client):
got = (await client.get("/api/settings")).json()["eval"]
assert got == {"max_parallel": 2, "per_case_timeout_s": 900, "per_run_max_usd": 2.0, "daily_max_usd": 5.0,
+2
View File
@@ -473,6 +473,8 @@ export const EVAL_TEXT = {
cleaned: '明细已清理',
seq: '题',
question: '问法',
/** 逐题表里实际问的问题下面一行:题目原文和实际问的不同(带变量的模板,或者题目之后改过)时写原文 */
questionTemplate: (text: string) => `题目原文:${text}`,
verdict: '结论',
expected: '期望',
got: '答案里的数',
+4 -1
View File
@@ -370,7 +370,10 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
{r.arm && <div className="text-2xs text-faint">{T.run.arm(r.arm)}</div>}
</td>
<td className="max-w-xs px-2 py-1.5">
<div className="line-clamp-2 break-words">{r.question ?? '—'}</div>
<div className="line-clamp-2 break-words" data-eval-result-question="">{r.question ?? '—'}</div>
{r.question_template && r.question_template !== r.question && (
<div className="line-clamp-2 break-words text-2xs text-faint" data-eval-result-template="">{R.questionTemplate(r.question_template)}</div>
)}
{r.path && <div className="text-2xs text-faint">{R.path}:{T.path[r.path]}</div>}
</td>
<td className="px-2 py-1.5">
+3
View File
@@ -558,7 +558,10 @@ export interface EvalResult {
error: string | null
started_at: string | null
finished_at: string | null
/** 这一题实际问的问题(变量按这次评测的截至时间渲染好了);题目之后改过、又没有作答运行可取时为 null */
question?: string | null
/** 题目的原文(可能带 {{ schedule.… }} 变量的模板);题目删了为 null */
question_template?: string | null
/** 这一题的花费:助手(含复核)+ 写作 + 结论裁判,服务端加好的 */
cost?: { cost_usd: number | null; unpriced_calls: number }
}
+9 -2
View File
@@ -161,13 +161,15 @@ const RESULTS = [
scores: [{ id: 'e1', kind: 'scalar', required: true, outcome: 'traced', expected: 330, got: 330 }], traced: true, review: null, tier: null,
judge: null, looks: { model: 1, system: 1 }, calls: 2, timings: { total_ms: 41000 }, usage: {}, sql_match: { status: 'proven', aspects: [] },
details_artifact: 'd-1', current: null, error: null, started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H), question: CASE_SQL.question,
cost: { cost_usd: 0.014, unpriced_calls: 0 } },
question_template: CASE_SQL.question, cost: { cost_usd: 0.014, unpriced_calls: 0 } },
{ id: 'r-2', eval_run_id: 'ev-1', seq: 2, case_id: 'case-2', case_seq: 2, case_hash: 'h2', repeat: 0, arm: null, status: 'ok',
run_id: 'run-answer-2', gold_run_ids: ['run-gold-2'], gold: { status: 'ok' }, path: 'ran', verdict: 'fail',
scores: [{ id: 'e1', kind: 'scalar', required: true, outcome: 'wrong', expected: 330, got: 310, reason: '答案里的数和期望对不上' }], traced: false,
review: null, tier: null, judge: null, looks: { model: 1, system: 0 }, calls: 2, timings: { total_ms: 60000 }, usage: {},
sql_match: { status: 'related', aspects: ['time'] }, details_artifact: 'd-2', current: null, error: '答案里的数和期望对不上',
started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H), question: CASE_CAL.question, cost: { cost_usd: 0.02, unpriced_calls: 0 } },
started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H), cost: { cost_usd: 0.02, unpriced_calls: 0 },
// 带变量的题:question 是实际问的(服务端按这次评测的截至时间渲染好),模板原文另放
question: '10 月 5 日到 10 月 11 日的周销售额?', question_template: '{{ schedule.last_week_start }} 到 {{ schedule.last_week_end }} 的周销售额?' },
{ id: 'r-3', eval_run_id: 'ev-1', seq: 3, case_id: 'case-3', case_seq: 3, case_hash: 'h3', repeat: 0, arm: null, status: 'ok',
run_id: null, gold_run_ids: [], gold: null, path: 'asked', verdict: 'pass', scores: [{ id: 'e1', kind: 'ask', required: true, outcome: 'traced' }],
traced: true, review: null, tier: null, judge: null, looks: { model: 0, system: 0 }, calls: 1, timings: { total_ms: 9000 }, usage: {},
@@ -772,6 +774,11 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => {
check('?case= 打开逐题、定位并展开那道题', await page.locator('[data-eval-result-detail="r-2"]').count() === 1)
check('逐题:结论、期望和答案里的数、SQL 差在哪', t2.includes('不通过') && t2.includes('330') && t2.includes('310') && t2.includes('时间口径'), t2)
check('一题的花费照服务端加好的写', (await page.locator('[data-eval-result="r-1"]').innerText()).includes('$0.014'))
check('逐题的问法写实际问的(变量渲染好了),模板原文另起一行', await text(page, '[data-eval-result="r-2"] [data-eval-result-question]')
=== '10 月 5 日到 10 月 11 日的周销售额?'
&& (await text(page, '[data-eval-result="r-2"] [data-eval-result-template]')).startsWith('题目原文:{{ schedule.last_week_start }}')
&& await page.locator('[data-eval-result="r-1"] [data-eval-result-template]').count() === 0,
await text(page, '[data-eval-result="r-2"] [data-eval-result-question]'))
check('点开去运行详情', await page.locator('[data-eval-result-detail="r-2"] [data-eval-result-run]').getAttribute('href') === '/runs/run-answer-2')
await page.locator('[data-eval-result-details]').waitFor()
check('展开的明细写作答的经过和答案原文(明细工件)', (await text(page, '[data-eval-result-details]')).includes('自动重来了 1 次')