From 06c1e8bd2582208f3f3ed1831c27bd19353c35b3 Mon Sep 17 00:00:00 2001
From: ThearchyHelios
Date: Sat, 17 Oct 2026 11:33:35 +0800
Subject: [PATCH] =?UTF-8?q?=E9=80=90=E9=A2=98=20E1=20=E6=AF=94=E4=BA=A7?=
=?UTF-8?q?=E5=87=BA=E7=AD=94=E6=A1=88=E7=9A=84=E9=82=A3=E6=9D=A1=E6=9F=A5?=
=?UTF-8?q?=E8=AF=A2=EF=BC=9A=E7=AD=94=E6=A1=88=E6=B2=A1=E5=BC=95=E7=94=A8?=
=?UTF-8?q?=E3=80=81=E5=8F=88=E6=9F=A5=E4=BA=86=E5=87=A0=E6=9D=A1=E6=97=B6?=
=?UTF-8?q?=E4=B8=8D=E5=86=8D=E7=A1=AC=E6=AF=94=E7=AC=AC=E4=B8=80=E6=9D=A1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
真实模型验收里,答案没有引用查询时逐题 SQL 比对取了第一条相关的查询,取到的往往是探索查询(看看有哪些地区那种),
「差在筛选条件」和答案的数毫无关系。现在 score.sql_match 先定哪条是产出答案的(_answering):答案引用了查询就比被
引用的那几条;没引用、这次运行(核查用的不算)只查了一条,答案的数只可能来自它,按它比、结果里写明依据
(basis=only_query);没引用、查了几条的不比,照实写「答案没有引用查询,这次运行执行了 N 条查询,没法对上是哪条
产出了答案」,记「没比对」、不计入 E1。答案引用的只有合并查询时也照实写没法比。
评测页逐题明细在依据是「只查了这一条」时写一句说明(文案在 termsEval 的 sqlMatchBasis,类型加 basis)。
算第 2 版评分规则的一部分:老评测按新规则重新评分时逐题 E1 跟着重比。
测试:test_eval_score.py::test_the_per_case_sql_match_compares_the_query_behind_the_answer(先探地区再查销售额、
答案没引用:不比;只查一条、答案没引用:按它比并写明依据;去掉修复回跑失败——修复前比的正是探地区的那条)。
---
backend/app/eval/score.py | 45 ++++++++++++++++-----
backend/tests/test_eval_score.py | 43 ++++++++++++++++++++
frontend/src/lib/termsEval.ts | 4 ++
frontend/src/pages/evals/EvaluationPage.tsx | 3 ++
frontend/src/typesEval.ts | 2 +
5 files changed, 87 insertions(+), 10 deletions(-)
diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py
index dd0a1f65..e1c916ec 100644
--- a/backend/app/eval/score.py
+++ b/backend/app/eval/score.py
@@ -32,9 +32,10 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、
没有错的)不能评分。期望没算出来按 gold 的结论:口径升版待处置是跳过,别的是不能评分——不混进「答错」。被驳回(总方案 X9)
记路径「被驳回」,不算失败:不能评分。只有文字要点的题不能评分(单列,不计入准确率)。
-**逐题的 E1**(sql_match):答案里的查询(先看报告引用了的)和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)
-用口径账本同一个比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在
-哪个方面(总方案 X2:E1 只有一份实现)。
+**逐题的 E1**(sql_match):产出答案的那条查询和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)用口径账本同一个
+比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在哪个方面(总方案
+X2:E1 只有一份实现)。哪条是产出答案的:答案引用了就是被引用的;没引用、只查了一条就是那一条(写明依据);没引用、查了
+几条的不硬比(照实写没法对上是哪条,不计入 E1)。
"""
from __future__ import annotations
@@ -613,7 +614,8 @@ async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None)
async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *,
timeout: float = 8.0) -> dict[str, Any] | None:
- """逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。
+ """逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason, cited, basis}。没有作答运行、没有期望
+ 取数的为 None。比的是哪条查询见 _answering:basis 是 cited(答案引用的)或 only_query(答案没引用、运行只查了这一条)。
比对在线程里跑(纯计算,和账本一样限时)。"""
import asyncio
import time
@@ -646,13 +648,12 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
except Exception: # noqa: BLE001 - 比对是附加的一步,出错照实记没比
logger.exception("评测:逐题比 SQL 出错")
return {"status": "skipped", "reason": "比对出错"}
- cited = [r for r in rows if r.get("artifact") in set(answer.cited)]
- pool = cited or rows
+ pool, basis = _answering(rows, answer)
+ if isinstance(pool, dict):
+ return pool
proven = next((r for r in pool if r.get("status") == "proven"), None)
chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \
- next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None)
- if chosen is None:
- return {"status": "none", "reason": "答案没有查询"}
+ next((r for r in pool if r.get("status") == "skipped"), None) or pool[0]
candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \
next(iter(chosen.get("candidates") or []), None) or {}
diff = candidate.get("diff") or {}
@@ -660,7 +661,31 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None)
return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects,
"diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"),
- "reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)}
+ "reason": chosen.get("reason") or candidate.get("reason"), "cited": basis == "cited", "basis": basis}
+
+
+def _answering(rows: list[dict[str, Any]], answer: Answer) -> tuple[list[dict[str, Any]] | dict[str, Any], str]:
+ """逐题 E1 拿哪几条查询比:产出答案的那几条。交回 (查询, 依据),确定不了时交回 (照实的结论, "")。
+
+ - 答案引用了查询(报告里引用的格、指标所在的查询):比被引用的那几条(依据 cited);
+ - 没引用、这次运行(核查用的不算)只查了一条:答案的数只可能来自它,按它比(依据 only_query);
+ - 没引用、查了几条:不比——以前取第一条相关的,取到的往往是探索查询(看看有哪些地区那种),说「差在筛选条件」和答案的数
+ 毫无关系。照实写没法对上是哪条,不计入 E1。"""
+ cited = set(answer.cited)
+ if cited:
+ pool = [r for r in rows if r.get("artifact") in cited]
+ if pool:
+ return pool, "cited"
+ return {"status": "skipped", "cited": True,
+ "reason": "答案引用的查询不在可比对的查询里(合并查询不比:它的数都来自它的输入)"}, ""
+ roles = {q.get("artifact"): q.get("role") for q in answer.queries}
+ pool = [r for r in rows if roles.get(r.get("artifact")) != "verify"]
+ if not pool:
+ return {"status": "none", "reason": "答案没有查询"}, ""
+ if len(pool) > 1:
+ return {"status": "skipped", "cited": False,
+ "reason": f"答案没有引用查询,这次运行执行了 {len(pool)} 条查询,没法对上是哪条产出了答案,不比"}, ""
+ return pool, "only_query"
__all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION",
diff --git a/backend/tests/test_eval_score.py b/backend/tests/test_eval_score.py
index a97395f2..a65560ab 100644
--- a/backend/tests/test_eval_score.py
+++ b/backend/tests/test_eval_score.py
@@ -248,6 +248,49 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m
assert (await score.sql_match(case, gold, answer))["status"] == "proven"
+async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engine_on, monkeypatch, tmp_path):
+ """逐题 E1 比的是产出答案的那条查询:答案引用了就比被引用的;没引用、这次运行又查了几条时不硬比第一条(往往是探索
+ 查询),照实写没法对上是哪条;没引用、只查了一条时它就是唯一可能的那条,按它比、写明依据。"""
+ from app.db.base import SessionLocal
+ from app.db.models import DataSource
+ from app.engine.runner import run_manager
+ from app.eval import score
+ from app.ledger import match as ledger_match
+ from sqlalchemy import select
+ from tests import ledger_samples as ls
+
+ await ls.fresh_source(tmp_path / "shop.db")
+ async with SessionLocal() as session:
+ sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar()
+ case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []}
+ gold = {"status": "ok", "before": {"rendered": {"sql": (
+ "SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' "
+ "AND paid_at < '2026-09-08'")}, "sets": []}}
+
+ # 先探了一下有哪些地区,再查销售额;答案没引用查询(数写在叙述里)
+ graph = ls.chain(ls.node("start", "input"),
+ ls.node("peek", "tool", tool=ls.tool(), args={"sql": "SELECT DISTINCT region FROM orders"}),
+ ls.node("q", "tool", tool=ls.tool(), args={"sql": ls.WITH_REFUNDS}),
+ ls.node("write", "report", instructions="写答案"),
+ ls.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
+ ls.script(monkeypatch, "本周销售额约 200 元。")
+ started = await run_manager.start(graph=graph, input_payload={}, run_class="exploratory", origin_kind="canvas")
+ await ls.wait(started.id)
+ await ledger_match.drain()
+ answer = await score.read_answer(started.id)
+ assert answer.cited == [] and len(answer.queries) == 2, answer.queries
+ match = await score.sql_match(case, gold, answer)
+ assert match["status"] == "skipped" and match["cited"] is False and "没有引用查询" in match["reason"], match
+ assert match.get("query") is None and match.get("aspects") in (None, []), "没法对上是哪条,就不给差在哪"
+
+ # 只查了一条、答案没引用:唯一的那条就是产出答案的,按它比,依据写明
+ run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 170.5 元。")
+ only = await score.read_answer(run.id)
+ assert only.cited == [] and len(only.queries) == 1
+ match = await score.sql_match(case, gold, only)
+ assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match
+
+
async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path):
from app.db.base import SessionLocal
from app.db.models_eval import EvalResult, EvalRun
diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts
index 348f9404..d09caa4c 100644
--- a/frontend/src/lib/termsEval.ts
+++ b/frontend/src/lib/termsEval.ts
@@ -464,6 +464,10 @@ export const EVAL_TEXT = {
sqlMatch: {
proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对',
} as Record,
+ /** 逐题 SQL 比对比的是哪条查询(服务端 score.sql_match 的 basis)。答案引用了查询的不用多说 */
+ sqlMatchBasis: {
+ only_query: '答案没有引用查询;这次运行只执行了这一条查询(核查用的不算),按它比',
+ } as Record,
aspect: {
time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联',
tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了',
diff --git a/frontend/src/pages/evals/EvaluationPage.tsx b/frontend/src/pages/evals/EvaluationPage.tsx
index bef39bc5..581725fa 100644
--- a/frontend/src/pages/evals/EvaluationPage.tsx
+++ b/frontend/src/pages/evals/EvaluationPage.tsx
@@ -417,6 +417,9 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
{s.reason && · {s.reason}}
))}
+ {r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && (
+ {T.sqlMatchBasis[r.sql_match.basis]}
+ )}
{r.sql_match?.reason && {r.sql_match.reason}
}
diff --git a/frontend/src/typesEval.ts b/frontend/src/typesEval.ts
index 39d3e185..8ed3e80a 100644
--- a/frontend/src/typesEval.ts
+++ b/frontend/src/typesEval.ts
@@ -531,6 +531,8 @@ export interface EvalSqlMatch {
aspects?: string[]
reason?: string | null
cited?: boolean
+ /** 比的是哪条查询:cited 答案引用的;only_query 答案没引用、这次运行只查了这一条 */
+ basis?: 'cited' | 'only_query' | string
}
export interface EvalResult {