逐题 E1 比产出答案的那条查询:答案没引用、又查了几条时不再硬比第一条

真实模型验收里,答案没有引用查询时逐题 SQL 比对取了第一条相关的查询,取到的往往是探索查询(看看有哪些地区那种),
「差在筛选条件」和答案的数毫无关系。现在 score.sql_match 先定哪条是产出答案的(_answering):答案引用了查询就比被
引用的那几条;没引用、这次运行(核查用的不算)只查了一条,答案的数只可能来自它,按它比、结果里写明依据
(basis=only_query);没引用、查了几条的不比,照实写「答案没有引用查询,这次运行执行了 N 条查询,没法对上是哪条
产出了答案」,记「没比对」、不计入 E1。答案引用的只有合并查询时也照实写没法比。

评测页逐题明细在依据是「只查了这一条」时写一句说明(文案在 termsEval 的 sqlMatchBasis,类型加 basis)。
算第 2 版评分规则的一部分:老评测按新规则重新评分时逐题 E1 跟着重比。

测试:test_eval_score.py::test_the_per_case_sql_match_compares_the_query_behind_the_answer(先探地区再查销售额、
答案没引用:不比;只查一条、答案没引用:按它比并写明依据;去掉修复回跑失败——修复前比的正是探地区的那条)。
This commit is contained in:
ThearchyHelios
2026-10-17 11:33:35 +08:00
parent f5204277aa
commit 06c1e8bd25
5 changed files with 87 additions and 10 deletions
+35 -10
View File
@@ -32,9 +32,10 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、
没有错的)不能评分。期望没算出来按 gold 的结论:口径升版待处置是跳过,别的是不能评分——不混进「答错」。被驳回(总方案 X9)
记路径「被驳回」,不算失败:不能评分。只有文字要点的题不能评分(单列,不计入准确率)。
**逐题的 E1**(sql_match):答案里的查询(先看报告引用了的)和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)
用口径账本同一个比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在
哪个方面(总方案 X2:E1 只有一份实现)。
**逐题的 E1**(sql_match):产出答案的那条查询和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)用口径账本同一个
比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在哪个方面(总方案
X2:E1 只有一份实现)。哪条是产出答案的:答案引用了就是被引用的;没引用、只查了一条就是那一条(写明依据);没引用、查了
几条的不硬比(照实写没法对上是哪条,不计入 E1)。
"""
from __future__ import annotations
@@ -613,7 +614,8 @@ async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None)
async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *,
timeout: float = 8.0) -> dict[str, Any] | None:
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason, cited, basis}。没有作答运行、没有期望
取数的为 None。比的是哪条查询见 _answering:basis 是 cited(答案引用的)或 only_query(答案没引用、运行只查了这一条)。
比对在线程里跑(纯计算,和账本一样限时)。"""
import asyncio
import time
@@ -646,13 +648,12 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
except Exception: # noqa: BLE001 - 比对是附加的一步,出错照实记没比
logger.exception("评测:逐题比 SQL 出错")
return {"status": "skipped", "reason": "比对出错"}
cited = [r for r in rows if r.get("artifact") in set(answer.cited)]
pool = cited or rows
pool, basis = _answering(rows, answer)
if isinstance(pool, dict):
return pool
proven = next((r for r in pool if r.get("status") == "proven"), None)
chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \
next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None)
if chosen is None:
return {"status": "none", "reason": "答案没有查询"}
next((r for r in pool if r.get("status") == "skipped"), None) or pool[0]
candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \
next(iter(chosen.get("candidates") or []), None) or {}
diff = candidate.get("diff") or {}
@@ -660,7 +661,31 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None)
return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects,
"diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"),
"reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)}
"reason": chosen.get("reason") or candidate.get("reason"), "cited": basis == "cited", "basis": basis}
def _answering(rows: list[dict[str, Any]], answer: Answer) -> tuple[list[dict[str, Any]] | dict[str, Any], str]:
"""逐题 E1 拿哪几条查询比:产出答案的那几条。交回 (查询, 依据),确定不了时交回 (照实的结论, "")。
- 答案引用了查询(报告里引用的格、指标所在的查询):比被引用的那几条(依据 cited);
- 没引用、这次运行(核查用的不算)只查了一条:答案的数只可能来自它,按它比(依据 only_query);
- 没引用、查了几条:不比——以前取第一条相关的,取到的往往是探索查询(看看有哪些地区那种),说「差在筛选条件」和答案的数
毫无关系。照实写没法对上是哪条,不计入 E1。"""
cited = set(answer.cited)
if cited:
pool = [r for r in rows if r.get("artifact") in cited]
if pool:
return pool, "cited"
return {"status": "skipped", "cited": True,
"reason": "答案引用的查询不在可比对的查询里(合并查询不比:它的数都来自它的输入)"}, ""
roles = {q.get("artifact"): q.get("role") for q in answer.queries}
pool = [r for r in rows if roles.get(r.get("artifact")) != "verify"]
if not pool:
return {"status": "none", "reason": "答案没有查询"}, ""
if len(pool) > 1:
return {"status": "skipped", "cited": False,
"reason": f"答案没有引用查询,这次运行执行了 {len(pool)} 条查询,没法对上是哪条产出了答案,不比"}, ""
return pool, "only_query"
__all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION",
+43
View File
@@ -248,6 +248,49 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m
assert (await score.sql_match(case, gold, answer))["status"] == "proven"
async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engine_on, monkeypatch, tmp_path):
"""逐题 E1 比的是产出答案的那条查询:答案引用了就比被引用的;没引用、这次运行又查了几条时不硬比第一条(往往是探索
查询),照实写没法对上是哪条;没引用、只查了一条时它就是唯一可能的那条,按它比、写明依据。"""
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.engine.runner import run_manager
from app.eval import score
from app.ledger import match as ledger_match
from sqlalchemy import select
from tests import ledger_samples as ls
await ls.fresh_source(tmp_path / "shop.db")
async with SessionLocal() as session:
sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar()
case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []}
gold = {"status": "ok", "before": {"rendered": {"sql": (
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' "
"AND paid_at < '2026-09-08'")}, "sets": []}}
# 先探了一下有哪些地区,再查销售额;答案没引用查询(数写在叙述里)
graph = ls.chain(ls.node("start", "input"),
ls.node("peek", "tool", tool=ls.tool(), args={"sql": "SELECT DISTINCT region FROM orders"}),
ls.node("q", "tool", tool=ls.tool(), args={"sql": ls.WITH_REFUNDS}),
ls.node("write", "report", instructions="写答案"),
ls.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
ls.script(monkeypatch, "本周销售额约 200 元。")
started = await run_manager.start(graph=graph, input_payload={}, run_class="exploratory", origin_kind="canvas")
await ls.wait(started.id)
await ledger_match.drain()
answer = await score.read_answer(started.id)
assert answer.cited == [] and len(answer.queries) == 2, answer.queries
match = await score.sql_match(case, gold, answer)
assert match["status"] == "skipped" and match["cited"] is False and "没有引用查询" in match["reason"], match
assert match.get("query") is None and match.get("aspects") in (None, []), "没法对上是哪条,就不给差在哪"
# 只查了一条、答案没引用:唯一的那条就是产出答案的,按它比,依据写明
run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 170.5 元。")
only = await score.read_answer(run.id)
assert only.cited == [] and len(only.queries) == 1
match = await score.sql_match(case, gold, only)
assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match
async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path):
from app.db.base import SessionLocal
from app.db.models_eval import EvalResult, EvalRun
+4
View File
@@ -464,6 +464,10 @@ export const EVAL_TEXT = {
sqlMatch: {
proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对',
} as Record<string, string>,
/** 逐题 SQL 比对比的是哪条查询(服务端 score.sql_match 的 basis)。答案引用了查询的不用多说 */
sqlMatchBasis: {
only_query: '答案没有引用查询;这次运行只执行了这一条查询(核查用的不算),按它比',
} as Record<string, string>,
aspect: {
time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联',
tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了',
@@ -417,6 +417,9 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
{s.reason && <span> · {s.reason}</span>}
</p>
))}
{r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && (
<p className="text-faint" data-eval-sql-basis={r.sql_match.basis}>{T.sqlMatchBasis[r.sql_match.basis]}</p>
)}
{r.sql_match?.reason && <p className="text-faint">{r.sql_match.reason}</p>}
<ResultDetail evaluationId={id} resultId={r.id} />
<div className="flex flex-wrap gap-2">
+2
View File
@@ -531,6 +531,8 @@ export interface EvalSqlMatch {
aspects?: string[]
reason?: string | null
cited?: boolean
/** 比的是哪条查询:cited 答案引用的;only_query 答案没引用、这次运行只查了这一条 */
basis?: 'cited' | 'only_query' | string
}
export interface EvalResult {