From eb6abffc616b849db741d4024280f9d5756a9bd6 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 09:02:57 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=84=E6=B5=8B=E9=80=90=E9=A2=98=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E7=BB=99=E5=87=BA=E8=BF=99=E4=B8=80=E9=A2=98=E5=AE=9E?= =?UTF-8?q?=E9=99=85=E9=97=AE=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E6=A8=A1?= =?UTF-8?q?=E6=9D=BF=E5=8E=9F=E6=96=87=E5=8F=A6=E6=94=BE=20question=5Ftemp?= =?UTF-8?q?late?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收：评测结果列表里 question 是原始模板（带 {{ schedule.last_week_start }}），作答运行实际收到的问题已经渲染好了。 results 接口直接取的是 EvalCase.question。 改法：api/evals 的 _asked_questions 按执行时同一个算法、同一组输入渲染：题目模板 + 这次评测的截至时间（as_of， 建评测时就定下、重试不变）+ 题集时区 + 定时的 cron（runner._unit 交给 Driver 的就是这个）。题目在这次评测之后 改过（结果记的 case_hash 对不上现在的）时，现在的模板不一定是当时问的：改取作答运行输入里的 question，也没有 （反问、建图失败没有运行）就给 null，不拿现在的模板编一个。模板原文放 question_template。没有加库表的列： 渲染是确定的，不用另存。 前端：评测页逐题表的问法一格写实际问的问题；模板原文和它不同时（带变量，或题目改过）另起一行「题目原文：…」， 文案进 termsEval。类型补上 question_template。 测试：test_eval_runner 新增一条：带变量的题，结果的 question 和作答时 Driver 收到的一字不差、不含 {{，模板原文在 question_template；没有变量的题两者一样；题目改过、没有作答运行可取时 question 为 null、模板照给。去掉修复回跑挂。 check-evals 的夹具给带变量的那一题配上渲染好的 question 和模板原文，核对逐题表写的是实际问的、原文另起一行， 没有变量的题不多这一行。tsc、check-copy 通过，check-evals 102 项全过（漏到服务端的那个偶发问题另一次提交里修）。 --- backend/app/api/evals.py | 43 +++++++++++++++++++-- backend/tests/test_eval_runner.py | 38 ++++++++++++++++++ frontend/src/lib/termsEval.ts | 2 + frontend/src/pages/evals/EvaluationPage.tsx | 5 ++- frontend/src/typesEval.ts | 3 ++ scripts/check-evals.mjs | 11 +++++- 6 files changed, 95 insertions(+), 7 deletions(-) diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index 3c722aaa..37da3987 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -1019,21 +1019,56 @@ async def _actions(session: AsyncSession, row: EvalRun) -> dict[str, Any]: @router.get("/evaluations/{evaluation_id}/results") async def evaluation_results(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """逐题结果：结论、各期望的结论（期望值和答案里对上的数、有没有引用、维度）、作答路径、复核、出具档位、SQL 差在哪、look、 - 耗时、按角色的用量，点 run_id 去运行详情。明细清理了的照实写 cleaned。""" + 耗时、按角色的用量，点 run_id 去运行详情。明细清理了的照实写 cleaned。question 是这一题实际问的问题（变量渲染好了， + 见 _asked_questions；说不准时为 null），question_template 是题目的原文（带 {{ schedule.… }} 变量的模板）。""" from app.eval import runner row = await _evaluation(session, evaluation_id) results = [runner.result_dict(r) for r in (await session.execute( select(runner.EvalResult).where(runner.EvalResult.eval_run_id == row.id) .order_by(runner.EvalResult.seq))).scalars()] - questions = dict((await session.execute(select(EvalCase.id, EvalCase.question).where( - EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).all()) if results else {} + cases = {c.id: (c.question, c.case_hash) for c in (await session.execute(select(EvalCase).where( + EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).scalars()} if results else {} + asked = await _asked_questions(session, row, results, cases) for r in results: - r["question"] = questions.get(r["case_id"]) + template = (cases.get(r["case_id"]) or (None, None))[0] + r["question"], r["question_template"] = asked.get(r["id"]), template r["cost"] = _result_cost(r.get("usage") or {}) return {"evaluation_id": row.id, "cleaned": row.cleaned_at is not None, "results": results} +async def _asked_questions(session: AsyncSession, row: EvalRun, results: list[dict[str, Any]], + cases: dict[str, tuple[str, str]]) -> dict[str, str | None]: + """每一题实际问的问题 {结果 id: 问题}：题目的模板按这次评测的截至时间（as_of）、题集的时区、定时的 cron 渲染——和执行时 + runner._unit 交给作答的是同一个算法、同一组输入，所以一字不差（as_of 在建评测时就定下、重试也不变）。题目在这次评测之后 + 改过（结果记的 case_hash 对不上现在的）：现在的模板不一定是当时问的，改取作答运行的输入里的 question；也没有（反问、 + 建图失败没有运行，工作流题的输入里没有问题）就是 None，不拿现在的模板编一个。""" + from app.db.models import Run + from app.eval import runner + from app.eval.cases import render, variable_values + + if not results: + return {} + leader = row if runner.leader_of(row) == row.id else await session.get(EvalRun, runner.leader_of(row)) + suite = await session.get(EvalSuite, row.suite_id) + as_of = (leader.as_of if leader is not None else None) or row.as_of + values = variable_values(as_of, suite.timezone, ((leader or row).fingerprint or {}).get("cron")) \ + if as_of is not None and suite is not None else None + stale = {r["id"] for r in results + if r["case_id"] not in cases or cases[r["case_id"]][1] != r["case_hash"] or values is None} + runs = {r["run_id"] for r in results if r["id"] in stale and r["run_id"]} + inputs = dict((await session.execute(select(Run.id, Run.input).where(Run.id.in_(runs)))) + .all()) if runs else {} + out: dict[str, str | None] = {} + for r in results: + if r["id"] in stale: + given = (inputs.get(r["run_id"]) or {}).get("question") if r["run_id"] else None + out[r["id"]] = given if isinstance(given, str) and given else None + else: + out[r["id"]] = render(cases[r["case_id"]][0] or "", values or {}) + return out + + def _result_cost(usage: dict[str, Any]) -> dict[str, Any]: """一题的花费：助手（runner 记的 assistant 已含复核）+ 写作 + 结论裁判，和这次评测记在账上的同一份；估不出金额的调用另计 次数（界面写「≥」或「金额无法估算」）。加在服务端：界面不算数。""" diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index 8e348397..61f3beb2 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -328,6 +328,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid" +async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch): + """逐题结果的 question 是这一题实际问的（变量按这次评测的截至时间渲染好了，和作答时 Driver 收到的一字不差），模板原文 + 另放 question_template（下一轮验收：以前 question 是带 {{ schedule.last_week_start }} 的原始模板）。没有变量的题两个一样。""" + from datetime import datetime, timezone + + from app.db.models_eval import EvalCase + from app.eval.cases import render, variable_values + + asked: list[str] = [] + real_init = answer_mod.Driver.__init__ + + def init(self, question, **kw): + asked.append(question) + real_init(self, question, **kw) + + monkeypatch.setattr(answer_mod.Driver, "__init__", init) + stub(monkeypatch) + template = "{{ schedule.last_week_start }} 到 {{ schedule.last_week_end }} 那周的问题" + suite_id = await es.make_suite("none", cases=[{"question": template, "expectations": [{"kind": "ask"}]}, + {"question": "没有变量的问题", "expectations": [{"kind": "ask"}]}]) + as_of = datetime(2026, 10, 14, 2, 0, tzinfo=timezone.utc) + [run] = await start(suite_id, as_of=as_of) + await es.wait_eval(run.id) + rows = (await client.get(f"/api/evals/evaluations/{run.id}/results")).json()["results"] + expected = render(template, variable_values(as_of, "Asia/Shanghai", None)) + assert "{{" not in expected and rows[0]["question"] == expected, rows[0]["question"] + assert rows[0]["question"] in asked, "和作答时实际问的一样" + assert rows[0]["question_template"] == template + assert rows[1]["question"] == rows[1]["question_template"] == "没有变量的问题" + # 题目改过（指纹对不上）：现在的模板不一定是当时问的，没有作答运行可取时不编一个，question 为空、模板照给 + async with SessionLocal() as session: + case = (await session.execute(select(EvalCase).where(EvalCase.suite_id == suite_id, EvalCase.seq == 1))).scalar_one() + case.question, case.case_hash = "改过的 {{ schedule.today }}", "changed" + await session.commit() + rows = (await client.get(f"/api/evals/evaluations/{run.id}/results")).json()["results"] + assert rows[0]["question"] is None and rows[0]["question_template"] == "改过的 {{ schedule.today }}" + + async def test_eval_settings_group_and_spend_endpoint(client): got = (await client.get("/api/settings")).json()["eval"] assert got == {"max_parallel": 2, "per_case_timeout_s": 900, "per_run_max_usd": 2.0, "daily_max_usd": 5.0, diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts index 04e268cd..0086c2bc 100644 --- a/frontend/src/lib/termsEval.ts +++ b/frontend/src/lib/termsEval.ts @@ -473,6 +473,8 @@ export const EVAL_TEXT = { cleaned: '明细已清理', seq: '题', question: '问法', + /** 逐题表里实际问的问题下面一行：题目原文和实际问的不同（带变量的模板，或者题目之后改过）时写原文 */ + questionTemplate: (text: string) => `题目原文：${text}`, verdict: '结论', expected: '期望', got: '答案里的数', diff --git a/frontend/src/pages/evals/EvaluationPage.tsx b/frontend/src/pages/evals/EvaluationPage.tsx index 11d473df..1347302c 100644 --- a/frontend/src/pages/evals/EvaluationPage.tsx +++ b/frontend/src/pages/evals/EvaluationPage.tsx @@ -370,7 +370,10 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase {r.arm &&
{T.run.arm(r.arm)}
} -
{r.question ?? '—'}
+
{r.question ?? '—'}
+ {r.question_template && r.question_template !== r.question && ( +
{R.questionTemplate(r.question_template)}
+ )} {r.path &&
{R.path}：{T.path[r.path]}
} diff --git a/frontend/src/typesEval.ts b/frontend/src/typesEval.ts index 82f851be..4bb426fb 100644 --- a/frontend/src/typesEval.ts +++ b/frontend/src/typesEval.ts @@ -558,7 +558,10 @@ export interface EvalResult { error: string | null started_at: string | null finished_at: string | null + /** 这一题实际问的问题（变量按这次评测的截至时间渲染好了）；题目之后改过、又没有作答运行可取时为 null */ question?: string | null + /** 题目的原文（可能带 {{ schedule.… }} 变量的模板）；题目删了为 null */ + question_template?: string | null /** 这一题的花费：助手（含复核）+ 写作 + 结论裁判，服务端加好的 */ cost?: { cost_usd: number | null; unpriced_calls: number } } diff --git a/scripts/check-evals.mjs b/scripts/check-evals.mjs index 50e238e7..bcc2d66a 100644 --- a/scripts/check-evals.mjs +++ b/scripts/check-evals.mjs @@ -161,13 +161,15 @@ const RESULTS = [ scores: [{ id: 'e1', kind: 'scalar', required: true, outcome: 'traced', expected: 330, got: 330 }], traced: true, review: null, tier: null, judge: null, looks: { model: 1, system: 1 }, calls: 2, timings: { total_ms: 41000 }, usage: {}, sql_match: { status: 'proven', aspects: [] }, details_artifact: 'd-1', current: null, error: null, started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H), question: CASE_SQL.question, - cost: { cost_usd: 0.014, unpriced_calls: 0 } }, + question_template: CASE_SQL.question, cost: { cost_usd: 0.014, unpriced_calls: 0 } }, { id: 'r-2', eval_run_id: 'ev-1', seq: 2, case_id: 'case-2', case_seq: 2, case_hash: 'h2', repeat: 0, arm: null, status: 'ok', run_id: 'run-answer-2', gold_run_ids: ['run-gold-2'], gold: { status: 'ok' }, path: 'ran', verdict: 'fail', scores: [{ id: 'e1', kind: 'scalar', required: true, outcome: 'wrong', expected: 330, got: 310, reason: '答案里的数和期望对不上' }], traced: false, review: null, tier: null, judge: null, looks: { model: 1, system: 0 }, calls: 2, timings: { total_ms: 60000 }, usage: {}, sql_match: { status: 'related', aspects: ['time'] }, details_artifact: 'd-2', current: null, error: '答案里的数和期望对不上', - started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H), question: CASE_CAL.question, cost: { cost_usd: 0.02, unpriced_calls: 0 } }, + started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H), cost: { cost_usd: 0.02, unpriced_calls: 0 }, + // 带变量的题：question 是实际问的（服务端按这次评测的截至时间渲染好），模板原文另放 + question: '10 月 5 日到 10 月 11 日的周销售额？', question_template: '{{ schedule.last_week_start }} 到 {{ schedule.last_week_end }} 的周销售额？' }, { id: 'r-3', eval_run_id: 'ev-1', seq: 3, case_id: 'case-3', case_seq: 3, case_hash: 'h3', repeat: 0, arm: null, status: 'ok', run_id: null, gold_run_ids: [], gold: null, path: 'asked', verdict: 'pass', scores: [{ id: 'e1', kind: 'ask', required: true, outcome: 'traced' }], traced: true, review: null, tier: null, judge: null, looks: { model: 0, system: 0 }, calls: 1, timings: { total_ms: 9000 }, usage: {}, @@ -772,6 +774,11 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => { check('?case= 打开逐题、定位并展开那道题', await page.locator('[data-eval-result-detail="r-2"]').count() === 1) check('逐题：结论、期望和答案里的数、SQL 差在哪', t2.includes('不通过') && t2.includes('330') && t2.includes('310') && t2.includes('时间口径'), t2) check('一题的花费照服务端加好的写', (await page.locator('[data-eval-result="r-1"]').innerText()).includes('$0.014')) + check('逐题的问法写实际问的（变量渲染好了），模板原文另起一行', await text(page, '[data-eval-result="r-2"] [data-eval-result-question]') + === '10 月 5 日到 10 月 11 日的周销售额？' + && (await text(page, '[data-eval-result="r-2"] [data-eval-result-template]')).startsWith('题目原文：{{ schedule.last_week_start }}') + && await page.locator('[data-eval-result="r-1"] [data-eval-result-template]').count() === 0, + await text(page, '[data-eval-result="r-2"] [data-eval-result-question]')) check('点开去运行详情', await page.locator('[data-eval-result-detail="r-2"] [data-eval-result-run]').getAttribute('href') === '/runs/run-answer-2') await page.locator('[data-eval-result-details]').waitFor() check('展开的明细写作答的经过和答案原文（明细工件）', (await text(page, '[data-eval-result-details]')).includes('自动重来了 1 次')