From 06c1e8bd2582208f3f3ed1831c27bd19353c35b3 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 11:33:35 +0800 Subject: [PATCH] =?UTF-8?q?=E9=80=90=E9=A2=98=20E1=20=E6=AF=94=E4=BA=A7?= =?UTF-8?q?=E5=87=BA=E7=AD=94=E6=A1=88=E7=9A=84=E9=82=A3=E6=9D=A1=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=EF=BC=9A=E7=AD=94=E6=A1=88=E6=B2=A1=E5=BC=95=E7=94=A8?= =?UTF-8?q?=E3=80=81=E5=8F=88=E6=9F=A5=E4=BA=86=E5=87=A0=E6=9D=A1=E6=97=B6?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E7=A1=AC=E6=AF=94=E7=AC=AC=E4=B8=80=E6=9D=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 真实模型验收里,答案没有引用查询时逐题 SQL 比对取了第一条相关的查询,取到的往往是探索查询(看看有哪些地区那种), 「差在筛选条件」和答案的数毫无关系。现在 score.sql_match 先定哪条是产出答案的(_answering):答案引用了查询就比被 引用的那几条;没引用、这次运行(核查用的不算)只查了一条,答案的数只可能来自它,按它比、结果里写明依据 (basis=only_query);没引用、查了几条的不比,照实写「答案没有引用查询,这次运行执行了 N 条查询,没法对上是哪条 产出了答案」,记「没比对」、不计入 E1。答案引用的只有合并查询时也照实写没法比。 评测页逐题明细在依据是「只查了这一条」时写一句说明(文案在 termsEval 的 sqlMatchBasis,类型加 basis)。 算第 2 版评分规则的一部分:老评测按新规则重新评分时逐题 E1 跟着重比。 测试:test_eval_score.py::test_the_per_case_sql_match_compares_the_query_behind_the_answer(先探地区再查销售额、 答案没引用:不比;只查一条、答案没引用:按它比并写明依据;去掉修复回跑失败——修复前比的正是探地区的那条)。 --- backend/app/eval/score.py | 45 ++++++++++++++++----- backend/tests/test_eval_score.py | 43 ++++++++++++++++++++ frontend/src/lib/termsEval.ts | 4 ++ frontend/src/pages/evals/EvaluationPage.tsx | 3 ++ frontend/src/typesEval.ts | 2 + 5 files changed, 87 insertions(+), 10 deletions(-) diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py index dd0a1f65..e1c916ec 100644 --- a/backend/app/eval/score.py +++ b/backend/app/eval/score.py @@ -32,9 +32,10 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、 没有错的)不能评分。期望没算出来按 gold 的结论:口径升版待处置是跳过,别的是不能评分——不混进「答错」。被驳回(总方案 X9) 记路径「被驳回」,不算失败:不能评分。只有文字要点的题不能评分(单列,不计入准确率)。 -**逐题的 E1**(sql_match):答案里的查询(先看报告引用了的)和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点) -用口径账本同一个比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在 -哪个方面(总方案 X2:E1 只有一份实现)。 +**逐题的 E1**(sql_match):产出答案的那条查询和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)用口径账本同一个 +比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在哪个方面(总方案 +X2:E1 只有一份实现)。哪条是产出答案的:答案引用了就是被引用的;没引用、只查了一条就是那一条(写明依据);没引用、查了 +几条的不硬比(照实写没法对上是哪条,不计入 E1)。 """ from __future__ import annotations @@ -613,7 +614,8 @@ async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None) async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *, timeout: float = 8.0) -> dict[str, Any] | None: - """逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。 + """逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason, cited, basis}。没有作答运行、没有期望 + 取数的为 None。比的是哪条查询见 _answering:basis 是 cited(答案引用的)或 only_query(答案没引用、运行只查了这一条)。 比对在线程里跑(纯计算,和账本一样限时)。""" import asyncio import time @@ -646,13 +648,12 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans except Exception: # noqa: BLE001 - 比对是附加的一步,出错照实记没比 logger.exception("评测:逐题比 SQL 出错") return {"status": "skipped", "reason": "比对出错"} - cited = [r for r in rows if r.get("artifact") in set(answer.cited)] - pool = cited or rows + pool, basis = _answering(rows, answer) + if isinstance(pool, dict): + return pool proven = next((r for r in pool if r.get("status") == "proven"), None) chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \ - next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None) - if chosen is None: - return {"status": "none", "reason": "答案没有查询"} + next((r for r in pool if r.get("status") == "skipped"), None) or pool[0] candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \ next(iter(chosen.get("candidates") or []), None) or {} diff = candidate.get("diff") or {} @@ -660,7 +661,31 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None) return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects, "diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"), - "reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)} + "reason": chosen.get("reason") or candidate.get("reason"), "cited": basis == "cited", "basis": basis} + + +def _answering(rows: list[dict[str, Any]], answer: Answer) -> tuple[list[dict[str, Any]] | dict[str, Any], str]: + """逐题 E1 拿哪几条查询比:产出答案的那几条。交回 (查询, 依据),确定不了时交回 (照实的结论, "")。 + + - 答案引用了查询(报告里引用的格、指标所在的查询):比被引用的那几条(依据 cited); + - 没引用、这次运行(核查用的不算)只查了一条:答案的数只可能来自它,按它比(依据 only_query); + - 没引用、查了几条:不比——以前取第一条相关的,取到的往往是探索查询(看看有哪些地区那种),说「差在筛选条件」和答案的数 + 毫无关系。照实写没法对上是哪条,不计入 E1。""" + cited = set(answer.cited) + if cited: + pool = [r for r in rows if r.get("artifact") in cited] + if pool: + return pool, "cited" + return {"status": "skipped", "cited": True, + "reason": "答案引用的查询不在可比对的查询里(合并查询不比:它的数都来自它的输入)"}, "" + roles = {q.get("artifact"): q.get("role") for q in answer.queries} + pool = [r for r in rows if roles.get(r.get("artifact")) != "verify"] + if not pool: + return {"status": "none", "reason": "答案没有查询"}, "" + if len(pool) > 1: + return {"status": "skipped", "cited": False, + "reason": f"答案没有引用查询,这次运行执行了 {len(pool)} 条查询,没法对上是哪条产出了答案,不比"}, "" + return pool, "only_query" __all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION", diff --git a/backend/tests/test_eval_score.py b/backend/tests/test_eval_score.py index a97395f2..a65560ab 100644 --- a/backend/tests/test_eval_score.py +++ b/backend/tests/test_eval_score.py @@ -248,6 +248,49 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m assert (await score.sql_match(case, gold, answer))["status"] == "proven" +async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engine_on, monkeypatch, tmp_path): + """逐题 E1 比的是产出答案的那条查询:答案引用了就比被引用的;没引用、这次运行又查了几条时不硬比第一条(往往是探索 + 查询),照实写没法对上是哪条;没引用、只查了一条时它就是唯一可能的那条,按它比、写明依据。""" + from app.db.base import SessionLocal + from app.db.models import DataSource + from app.engine.runner import run_manager + from app.eval import score + from app.ledger import match as ledger_match + from sqlalchemy import select + from tests import ledger_samples as ls + + await ls.fresh_source(tmp_path / "shop.db") + async with SessionLocal() as session: + sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar() + case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []} + gold = {"status": "ok", "before": {"rendered": {"sql": ( + "SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' " + "AND paid_at < '2026-09-08'")}, "sets": []}} + + # 先探了一下有哪些地区,再查销售额;答案没引用查询(数写在叙述里) + graph = ls.chain(ls.node("start", "input"), + ls.node("peek", "tool", tool=ls.tool(), args={"sql": "SELECT DISTINCT region FROM orders"}), + ls.node("q", "tool", tool=ls.tool(), args={"sql": ls.WITH_REFUNDS}), + ls.node("write", "report", instructions="写答案"), + ls.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}])) + ls.script(monkeypatch, "本周销售额约 200 元。") + started = await run_manager.start(graph=graph, input_payload={}, run_class="exploratory", origin_kind="canvas") + await ls.wait(started.id) + await ledger_match.drain() + answer = await score.read_answer(started.id) + assert answer.cited == [] and len(answer.queries) == 2, answer.queries + match = await score.sql_match(case, gold, answer) + assert match["status"] == "skipped" and match["cited"] is False and "没有引用查询" in match["reason"], match + assert match.get("query") is None and match.get("aspects") in (None, []), "没法对上是哪条,就不给差在哪" + + # 只查了一条、答案没引用:唯一的那条就是产出答案的,按它比,依据写明 + run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 170.5 元。") + only = await score.read_answer(run.id) + assert only.cited == [] and len(only.queries) == 1 + match = await score.sql_match(case, gold, only) + assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match + + async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path): from app.db.base import SessionLocal from app.db.models_eval import EvalResult, EvalRun diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts index 348f9404..d09caa4c 100644 --- a/frontend/src/lib/termsEval.ts +++ b/frontend/src/lib/termsEval.ts @@ -464,6 +464,10 @@ export const EVAL_TEXT = { sqlMatch: { proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对', } as Record, + /** 逐题 SQL 比对比的是哪条查询(服务端 score.sql_match 的 basis)。答案引用了查询的不用多说 */ + sqlMatchBasis: { + only_query: '答案没有引用查询;这次运行只执行了这一条查询(核查用的不算),按它比', + } as Record, aspect: { time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联', tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了', diff --git a/frontend/src/pages/evals/EvaluationPage.tsx b/frontend/src/pages/evals/EvaluationPage.tsx index bef39bc5..581725fa 100644 --- a/frontend/src/pages/evals/EvaluationPage.tsx +++ b/frontend/src/pages/evals/EvaluationPage.tsx @@ -417,6 +417,9 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase {s.reason && · {s.reason}}

))} + {r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && ( +

{T.sqlMatchBasis[r.sql_match.basis]}

+ )} {r.sql_match?.reason &&

{r.sql_match.reason}

}
diff --git a/frontend/src/typesEval.ts b/frontend/src/typesEval.ts index 39d3e185..8ed3e80a 100644 --- a/frontend/src/typesEval.ts +++ b/frontend/src/typesEval.ts @@ -531,6 +531,8 @@ export interface EvalSqlMatch { aspects?: string[] reason?: string | null cited?: boolean + /** 比的是哪条查询:cited 答案引用的;only_query 答案没引用、这次运行只查了这一条 */ + basis?: 'cited' | 'only_query' | string } export interface EvalResult {