diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index 3c722aaa..37da3987 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -1019,21 +1019,56 @@ async def _actions(session: AsyncSession, row: EvalRun) -> dict[str, Any]: @router.get("/evaluations/{evaluation_id}/results") async def evaluation_results(evaluation_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """逐题结果:结论、各期望的结论(期望值和答案里对上的数、有没有引用、维度)、作答路径、复核、出具档位、SQL 差在哪、look、 - 耗时、按角色的用量,点 run_id 去运行详情。明细清理了的照实写 cleaned。""" + 耗时、按角色的用量,点 run_id 去运行详情。明细清理了的照实写 cleaned。question 是这一题实际问的问题(变量渲染好了, + 见 _asked_questions;说不准时为 null),question_template 是题目的原文(带 {{ schedule.… }} 变量的模板)。""" from app.eval import runner row = await _evaluation(session, evaluation_id) results = [runner.result_dict(r) for r in (await session.execute( select(runner.EvalResult).where(runner.EvalResult.eval_run_id == row.id) .order_by(runner.EvalResult.seq))).scalars()] - questions = dict((await session.execute(select(EvalCase.id, EvalCase.question).where( - EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).all()) if results else {} + cases = {c.id: (c.question, c.case_hash) for c in (await session.execute(select(EvalCase).where( + EvalCase.id.in_({r["case_id"] for r in results if r["case_id"]})))).scalars()} if results else {} + asked = await _asked_questions(session, row, results, cases) for r in results: - r["question"] = questions.get(r["case_id"]) + template = (cases.get(r["case_id"]) or (None, None))[0] + r["question"], r["question_template"] = asked.get(r["id"]), template r["cost"] = _result_cost(r.get("usage") or {}) return {"evaluation_id": row.id, "cleaned": row.cleaned_at is not None, "results": results} +async def _asked_questions(session: AsyncSession, row: EvalRun, results: list[dict[str, Any]], + cases: dict[str, tuple[str, str]]) -> dict[str, str | None]: + """每一题实际问的问题 {结果 id: 问题}:题目的模板按这次评测的截至时间(as_of)、题集的时区、定时的 cron 渲染——和执行时 + runner._unit 交给作答的是同一个算法、同一组输入,所以一字不差(as_of 在建评测时就定下、重试也不变)。题目在这次评测之后 + 改过(结果记的 case_hash 对不上现在的):现在的模板不一定是当时问的,改取作答运行的输入里的 question;也没有(反问、 + 建图失败没有运行,工作流题的输入里没有问题)就是 None,不拿现在的模板编一个。""" + from app.db.models import Run + from app.eval import runner + from app.eval.cases import render, variable_values + + if not results: + return {} + leader = row if runner.leader_of(row) == row.id else await session.get(EvalRun, runner.leader_of(row)) + suite = await session.get(EvalSuite, row.suite_id) + as_of = (leader.as_of if leader is not None else None) or row.as_of + values = variable_values(as_of, suite.timezone, ((leader or row).fingerprint or {}).get("cron")) \ + if as_of is not None and suite is not None else None + stale = {r["id"] for r in results + if r["case_id"] not in cases or cases[r["case_id"]][1] != r["case_hash"] or values is None} + runs = {r["run_id"] for r in results if r["id"] in stale and r["run_id"]} + inputs = dict((await session.execute(select(Run.id, Run.input).where(Run.id.in_(runs)))) + .all()) if runs else {} + out: dict[str, str | None] = {} + for r in results: + if r["id"] in stale: + given = (inputs.get(r["run_id"]) or {}).get("question") if r["run_id"] else None + out[r["id"]] = given if isinstance(given, str) and given else None + else: + out[r["id"]] = render(cases[r["case_id"]][0] or "", values or {}) + return out + + def _result_cost(usage: dict[str, Any]) -> dict[str, Any]: """一题的花费:助手(runner 记的 assistant 已含复核)+ 写作 + 结论裁判,和这次评测记在账上的同一份;估不出金额的调用另计 次数(界面写「≥」或「金额无法估算」)。加在服务端:界面不算数。""" diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index 8e348397..61f3beb2 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -328,6 +328,44 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid" +async def test_results_give_the_question_actually_asked_and_the_template_apart(client, monkeypatch): + """逐题结果的 question 是这一题实际问的(变量按这次评测的截至时间渲染好了,和作答时 Driver 收到的一字不差),模板原文 + 另放 question_template(下一轮验收:以前 question 是带 {{ schedule.last_week_start }} 的原始模板)。没有变量的题两个一样。""" + from datetime import datetime, timezone + + from app.db.models_eval import EvalCase + from app.eval.cases import render, variable_values + + asked: list[str] = [] + real_init = answer_mod.Driver.__init__ + + def init(self, question, **kw): + asked.append(question) + real_init(self, question, **kw) + + monkeypatch.setattr(answer_mod.Driver, "__init__", init) + stub(monkeypatch) + template = "{{ schedule.last_week_start }} 到 {{ schedule.last_week_end }} 那周的问题" + suite_id = await es.make_suite("none", cases=[{"question": template, "expectations": [{"kind": "ask"}]}, + {"question": "没有变量的问题", "expectations": [{"kind": "ask"}]}]) + as_of = datetime(2026, 10, 14, 2, 0, tzinfo=timezone.utc) + [run] = await start(suite_id, as_of=as_of) + await es.wait_eval(run.id) + rows = (await client.get(f"/api/evals/evaluations/{run.id}/results")).json()["results"] + expected = render(template, variable_values(as_of, "Asia/Shanghai", None)) + assert "{{" not in expected and rows[0]["question"] == expected, rows[0]["question"] + assert rows[0]["question"] in asked, "和作答时实际问的一样" + assert rows[0]["question_template"] == template + assert rows[1]["question"] == rows[1]["question_template"] == "没有变量的问题" + # 题目改过(指纹对不上):现在的模板不一定是当时问的,没有作答运行可取时不编一个,question 为空、模板照给 + async with SessionLocal() as session: + case = (await session.execute(select(EvalCase).where(EvalCase.suite_id == suite_id, EvalCase.seq == 1))).scalar_one() + case.question, case.case_hash = "改过的 {{ schedule.today }}", "changed" + await session.commit() + rows = (await client.get(f"/api/evals/evaluations/{run.id}/results")).json()["results"] + assert rows[0]["question"] is None and rows[0]["question_template"] == "改过的 {{ schedule.today }}" + + async def test_eval_settings_group_and_spend_endpoint(client): got = (await client.get("/api/settings")).json()["eval"] assert got == {"max_parallel": 2, "per_case_timeout_s": 900, "per_run_max_usd": 2.0, "daily_max_usd": 5.0, diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts index 04e268cd..0086c2bc 100644 --- a/frontend/src/lib/termsEval.ts +++ b/frontend/src/lib/termsEval.ts @@ -473,6 +473,8 @@ export const EVAL_TEXT = { cleaned: '明细已清理', seq: '题', question: '问法', + /** 逐题表里实际问的问题下面一行:题目原文和实际问的不同(带变量的模板,或者题目之后改过)时写原文 */ + questionTemplate: (text: string) => `题目原文:${text}`, verdict: '结论', expected: '期望', got: '答案里的数', diff --git a/frontend/src/pages/evals/EvaluationPage.tsx b/frontend/src/pages/evals/EvaluationPage.tsx index 11d473df..1347302c 100644 --- a/frontend/src/pages/evals/EvaluationPage.tsx +++ b/frontend/src/pages/evals/EvaluationPage.tsx @@ -370,7 +370,10 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase {r.arm &&