评测逐题比 SQL 和账本同一个口径:传报告引用了哪几列(ledger/match.referenced_of),整条比不等价时按引用的列投影再比;「多查了几列」那一条不算差在哪个方面

This commit is contained in:
ThearchyHelios
2026-10-17 12:29:20 +08:00
parent f8d89e780c
commit 124e1c0f49
2 changed files with 32 additions and 3 deletions
+6 -3
View File
@@ -621,7 +621,7 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
import time
from app.api.evidence import _sealed
from app.ledger.match import _kinds, compare_queries, read_queries
from app.ledger.match import _kinds, compare_queries, read_queries, referenced_of
if not answer.run_id or answer.problem:
return None
@@ -639,7 +639,9 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
def work() -> list[dict[str, Any]]:
queries = [q for q in read_queries(sealed, kinds)]
return compare_queries(queries, calibers, deadline=time.monotonic() + timeout)
# 和账本同一个口径:报告引用了查询的哪几列,整条比不等价时按这几列投影再比(ledger/match._projected)。答案的
# SQL 多查了几列(订单数、日期常量)不该让这一题的 E1 变成「相关」
return compare_queries(queries, calibers, deadline=time.monotonic() + timeout, referenced=referenced_of(sealed))
try:
rows = await asyncio.wait_for(asyncio.to_thread(work), timeout=timeout + 2)
@@ -657,7 +659,8 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans
candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \
next(iter(chosen.get("candidates") or []), None) or {}
diff = candidate.get("diff") or {}
aspects = sorted({i.get("aspect") for i in diff.get("items") or [] if i.get("aspect")})
# projection 那一条只是说「多查了几列,不影响这个数」,不是差在哪个方面(账本的 E1 差异类型也不数它)
aspects = sorted({i.get("aspect") for i in diff.get("items") or [] if i.get("aspect") not in (None, "", "projection")})
sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None)
return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects,
"diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"),
+26
View File
@@ -291,6 +291,32 @@ async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engin
assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match
async def test_the_per_case_sql_match_projects_to_the_cited_columns(engine_on, monkeypatch, tmp_path):
"""逐题 E1 和账本同一个口径:答案的 SQL 多查了一列订单数、报告只引用销售额那一列时,按引用的列投影后和期望 SQL
证明等价;多出来的那列不算差在哪个方面。"""
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.eval import score
from sqlalchemy import select
from tests import ledger_samples as ls
await ls.fresh_source(tmp_path / "shop.db")
async with SessionLocal() as session:
sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar()
case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []}
gold = {"status": "ok", "before": {"rendered": {"sql": (
"SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' "
"AND paid_at < '2026-09-08'")}, "sets": []}}
wider = ("select count(*) as 单数, sum(o.amount) as 销售额 from orders o where o.paid_at >= '2026-09-01' "
"and o.status = 'paid' and o.paid_at < '2026-09-08'")
run = await ls.explore(monkeypatch, wider, text="本周销售额 [[v:Q1.r0.销售额]] 元。")
answer = await score.read_answer(run.id)
assert answer.cited, answer
match = await score.sql_match(case, gold, answer)
assert match["status"] == "proven" and match["cited"] is True, match
assert "projection" not in (match.get("aspects") or []), match
async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path):
from app.db.base import SessionLocal
from app.db.models_eval import EvalResult, EvalRun