mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 02:32:01 +00:00
逐题 E1 比产出答案的那条查询:答案没引用、又查了几条时不再硬比第一条
真实模型验收里,答案没有引用查询时逐题 SQL 比对取了第一条相关的查询,取到的往往是探索查询(看看有哪些地区那种), 「差在筛选条件」和答案的数毫无关系。现在 score.sql_match 先定哪条是产出答案的(_answering):答案引用了查询就比被 引用的那几条;没引用、这次运行(核查用的不算)只查了一条,答案的数只可能来自它,按它比、结果里写明依据 (basis=only_query);没引用、查了几条的不比,照实写「答案没有引用查询,这次运行执行了 N 条查询,没法对上是哪条 产出了答案」,记「没比对」、不计入 E1。答案引用的只有合并查询时也照实写没法比。 评测页逐题明细在依据是「只查了这一条」时写一句说明(文案在 termsEval 的 sqlMatchBasis,类型加 basis)。 算第 2 版评分规则的一部分:老评测按新规则重新评分时逐题 E1 跟着重比。 测试:test_eval_score.py::test_the_per_case_sql_match_compares_the_query_behind_the_answer(先探地区再查销售额、 答案没引用:不比;只查一条、答案没引用:按它比并写明依据;去掉修复回跑失败——修复前比的正是探地区的那条)。
This commit is contained in:
+35
-10
@@ -32,9 +32,10 @@ wrong 错、scale 量纲不同(差 10 的整数次幂,比如元和万元、
|
||||
没有错的)不能评分。期望没算出来按 gold 的结论:口径升版待处置是跳过,别的是不能评分——不混进「答错」。被驳回(总方案 X9)
|
||||
记路径「被驳回」,不算失败:不能评分。只有文字要点的题不能评分(单列,不计入准确率)。
|
||||
|
||||
**逐题的 E1**(sql_match):答案里的查询(先看报告引用了的)和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)
|
||||
用口径账本同一个比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在
|
||||
哪个方面(总方案 X2:E1 只有一份实现)。
|
||||
**逐题的 E1**(sql_match):产出答案的那条查询和期望的取数(SQL 题是期望 SQL,口径题是口径卡的取数节点)用口径账本同一个
|
||||
比对函数(ledger/match.compare_queries:先绑参数、再由 explain_sql_diff 判等价)比一次,给出等价与否和差在哪个方面(总方案
|
||||
X2:E1 只有一份实现)。哪条是产出答案的:答案引用了就是被引用的;没引用、只查了一条就是那一条(写明依据);没引用、查了
|
||||
几条的不硬比(照实写没法对上是哪条,不计入 E1)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -613,7 +614,8 @@ async def gold_calibers(case: Mapping[str, Any], gold: Mapping[str, Any] | None)
|
||||
|
||||
async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, answer: Answer, *,
|
||||
timeout: float = 8.0) -> dict[str, Any] | None:
|
||||
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason}。没有作答运行、没有期望取数的为 None。
|
||||
"""逐题的 E1:{status: proven|related|none|skipped, query, aspects, diff, reason, cited, basis}。没有作答运行、没有期望
|
||||
取数的为 None。比的是哪条查询见 _answering:basis 是 cited(答案引用的)或 only_query(答案没引用、运行只查了这一条)。
|
||||
比对在线程里跑(纯计算,和账本一样限时)。"""
|
||||
import asyncio
|
||||
import time
|
||||
@@ -646,13 +648,12 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
|
||||
except Exception: # noqa: BLE001 - 比对是附加的一步,出错照实记没比
|
||||
logger.exception("评测:逐题比 SQL 出错")
|
||||
return {"status": "skipped", "reason": "比对出错"}
|
||||
cited = [r for r in rows if r.get("artifact") in set(answer.cited)]
|
||||
pool = cited or rows
|
||||
pool, basis = _answering(rows, answer)
|
||||
if isinstance(pool, dict):
|
||||
return pool
|
||||
proven = next((r for r in pool if r.get("status") == "proven"), None)
|
||||
chosen = proven or next((r for r in pool if r.get("status") == "related"), None) or \
|
||||
next((r for r in pool if r.get("status") == "skipped"), None) or (pool[0] if pool else None)
|
||||
if chosen is None:
|
||||
return {"status": "none", "reason": "答案没有查询"}
|
||||
next((r for r in pool if r.get("status") == "skipped"), None) or pool[0]
|
||||
candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \
|
||||
next(iter(chosen.get("candidates") or []), None) or {}
|
||||
diff = candidate.get("diff") or {}
|
||||
@@ -660,7 +661,31 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
|
||||
sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None)
|
||||
return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects,
|
||||
"diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"),
|
||||
"reason": chosen.get("reason") or candidate.get("reason"), "cited": bool(cited)}
|
||||
"reason": chosen.get("reason") or candidate.get("reason"), "cited": basis == "cited", "basis": basis}
|
||||
|
||||
|
||||
def _answering(rows: list[dict[str, Any]], answer: Answer) -> tuple[list[dict[str, Any]] | dict[str, Any], str]:
|
||||
"""逐题 E1 拿哪几条查询比:产出答案的那几条。交回 (查询, 依据),确定不了时交回 (照实的结论, "")。
|
||||
|
||||
- 答案引用了查询(报告里引用的格、指标所在的查询):比被引用的那几条(依据 cited);
|
||||
- 没引用、这次运行(核查用的不算)只查了一条:答案的数只可能来自它,按它比(依据 only_query);
|
||||
- 没引用、查了几条:不比——以前取第一条相关的,取到的往往是探索查询(看看有哪些地区那种),说「差在筛选条件」和答案的数
|
||||
毫无关系。照实写没法对上是哪条,不计入 E1。"""
|
||||
cited = set(answer.cited)
|
||||
if cited:
|
||||
pool = [r for r in rows if r.get("artifact") in cited]
|
||||
if pool:
|
||||
return pool, "cited"
|
||||
return {"status": "skipped", "cited": True,
|
||||
"reason": "答案引用的查询不在可比对的查询里(合并查询不比:它的数都来自它的输入)"}, ""
|
||||
roles = {q.get("artifact"): q.get("role") for q in answer.queries}
|
||||
pool = [r for r in rows if roles.get(r.get("artifact")) != "verify"]
|
||||
if not pool:
|
||||
return {"status": "none", "reason": "答案没有查询"}, ""
|
||||
if len(pool) > 1:
|
||||
return {"status": "skipped", "cited": False,
|
||||
"reason": f"答案没有引用查询,这次运行执行了 {len(pool)} 条查询,没法对上是哪条产出了答案,不比"}, ""
|
||||
return pool, "only_query"
|
||||
|
||||
|
||||
__all__ = ["Answer", "CaseScore", "DEFAULT_TOLERANCE", "OUTCOMES", "OUTCOME_LABEL", "PATH_LABEL", "SCORER_VERSION",
|
||||
|
||||
@@ -248,6 +248,49 @@ async def test_reading_the_sealed_answer_and_the_per_case_sql_match(engine_on, m
|
||||
assert (await score.sql_match(case, gold, answer))["status"] == "proven"
|
||||
|
||||
|
||||
async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engine_on, monkeypatch, tmp_path):
|
||||
"""逐题 E1 比的是产出答案的那条查询:答案引用了就比被引用的;没引用、这次运行又查了几条时不硬比第一条(往往是探索
|
||||
查询),照实写没法对上是哪条;没引用、只查了一条时它就是唯一可能的那条,按它比、写明依据。"""
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.engine.runner import run_manager
|
||||
from app.eval import score
|
||||
from app.ledger import match as ledger_match
|
||||
from sqlalchemy import select
|
||||
from tests import ledger_samples as ls
|
||||
|
||||
await ls.fresh_source(tmp_path / "shop.db")
|
||||
async with SessionLocal() as session:
|
||||
sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar()
|
||||
case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []}
|
||||
gold = {"status": "ok", "before": {"rendered": {"sql": (
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' "
|
||||
"AND paid_at < '2026-09-08'")}, "sets": []}}
|
||||
|
||||
# 先探了一下有哪些地区,再查销售额;答案没引用查询(数写在叙述里)
|
||||
graph = ls.chain(ls.node("start", "input"),
|
||||
ls.node("peek", "tool", tool=ls.tool(), args={"sql": "SELECT DISTINCT region FROM orders"}),
|
||||
ls.node("q", "tool", tool=ls.tool(), args={"sql": ls.WITH_REFUNDS}),
|
||||
ls.node("write", "report", instructions="写答案"),
|
||||
ls.node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}]))
|
||||
ls.script(monkeypatch, "本周销售额约 200 元。")
|
||||
started = await run_manager.start(graph=graph, input_payload={}, run_class="exploratory", origin_kind="canvas")
|
||||
await ls.wait(started.id)
|
||||
await ledger_match.drain()
|
||||
answer = await score.read_answer(started.id)
|
||||
assert answer.cited == [] and len(answer.queries) == 2, answer.queries
|
||||
match = await score.sql_match(case, gold, answer)
|
||||
assert match["status"] == "skipped" and match["cited"] is False and "没有引用查询" in match["reason"], match
|
||||
assert match.get("query") is None and match.get("aspects") in (None, []), "没法对上是哪条,就不给差在哪"
|
||||
|
||||
# 只查了一条、答案没引用:唯一的那条就是产出答案的,按它比,依据写明
|
||||
run = await ls.explore(monkeypatch, ls.EQUIVALENT, text="本周销售额 170.5 元。")
|
||||
only = await score.read_answer(run.id)
|
||||
assert only.cited == [] and len(only.queries) == 1
|
||||
match = await score.sql_match(case, gold, only)
|
||||
assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match
|
||||
|
||||
|
||||
async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path):
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models_eval import EvalResult, EvalRun
|
||||
|
||||
@@ -464,6 +464,10 @@ export const EVAL_TEXT = {
|
||||
sqlMatch: {
|
||||
proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对',
|
||||
} as Record<string, string>,
|
||||
/** 逐题 SQL 比对比的是哪条查询(服务端 score.sql_match 的 basis)。答案引用了查询的不用多说 */
|
||||
sqlMatchBasis: {
|
||||
only_query: '答案没有引用查询;这次运行只执行了这一条查询(核查用的不算),按它比',
|
||||
} as Record<string, string>,
|
||||
aspect: {
|
||||
time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联',
|
||||
tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了',
|
||||
|
||||
@@ -417,6 +417,9 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
|
||||
{s.reason && <span> · {s.reason}</span>}
|
||||
</p>
|
||||
))}
|
||||
{r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && (
|
||||
<p className="text-faint" data-eval-sql-basis={r.sql_match.basis}>{T.sqlMatchBasis[r.sql_match.basis]}</p>
|
||||
)}
|
||||
{r.sql_match?.reason && <p className="text-faint">{r.sql_match.reason}</p>}
|
||||
<ResultDetail evaluationId={id} resultId={r.id} />
|
||||
<div className="flex flex-wrap gap-2">
|
||||
|
||||
@@ -531,6 +531,8 @@ export interface EvalSqlMatch {
|
||||
aspects?: string[]
|
||||
reason?: string | null
|
||||
cited?: boolean
|
||||
/** 比的是哪条查询:cited 答案引用的;only_query 答案没引用、这次运行只查了这一条 */
|
||||
basis?: 'cited' | 'only_query' | string
|
||||
}
|
||||
|
||||
export interface EvalResult {
|
||||
|
||||
Reference in New Issue
Block a user