From 124e1c0f494c324e3ef1105dbed7b7f42923434d Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 17 Oct 2026 12:29:20 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=84=E6=B5=8B=E9=80=90=E9=A2=98=E6=AF=94?= =?UTF-8?q?=20SQL=20=E5=92=8C=E8=B4=A6=E6=9C=AC=E5=90=8C=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E5=8F=A3=E5=BE=84=EF=BC=9A=E4=BC=A0=E6=8A=A5=E5=91=8A=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E4=BA=86=E5=93=AA=E5=87=A0=E5=88=97=EF=BC=88ledger/ma?= =?UTF-8?q?tch.referenced=5Fof=EF=BC=89=EF=BC=8C=E6=95=B4=E6=9D=A1?= =?UTF-8?q?=E6=AF=94=E4=B8=8D=E7=AD=89=E4=BB=B7=E6=97=B6=E6=8C=89=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E7=9A=84=E5=88=97=E6=8A=95=E5=BD=B1=E5=86=8D=E6=AF=94?= =?UTF-8?q?=EF=BC=9B=E3=80=8C=E5=A4=9A=E6=9F=A5=E4=BA=86=E5=87=A0=E5=88=97?= =?UTF-8?q?=E3=80=8D=E9=82=A3=E4=B8=80=E6=9D=A1=E4=B8=8D=E7=AE=97=E5=B7=AE?= =?UTF-8?q?=E5=9C=A8=E5=93=AA=E4=B8=AA=E6=96=B9=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/eval/score.py | 9 ++++++--- backend/tests/test_eval_score.py | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py index e1c916ec..16fd52ff 100644 --- a/backend/app/eval/score.py +++ b/backend/app/eval/score.py @@ -621,7 +621,7 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans import time from app.api.evidence import _sealed - from app.ledger.match import _kinds, compare_queries, read_queries + from app.ledger.match import _kinds, compare_queries, read_queries, referenced_of if not answer.run_id or answer.problem: return None @@ -639,7 +639,9 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans def work() -> list[dict[str, Any]]: queries = [q for q in read_queries(sealed, kinds)] - return compare_queries(queries, calibers, deadline=time.monotonic() + timeout) + # 和账本同一个口径:报告引用了查询的哪几列,整条比不等价时按这几列投影再比(ledger/match._projected)。答案的 + # SQL 多查了几列(订单数、日期常量)不该让这一题的 E1 变成「相关」 + return compare_queries(queries, calibers, deadline=time.monotonic() + timeout, referenced=referenced_of(sealed)) try: rows = await asyncio.wait_for(asyncio.to_thread(work), timeout=timeout + 2) @@ -657,7 +659,8 @@ async def sql_match(case: Mapping[str, Any], gold: Mapping[str, Any] | None, ans candidate = next((c for c in chosen.get("candidates") or [] if c.get("status") == chosen.get("status")), None) or \ next(iter(chosen.get("candidates") or []), None) or {} diff = candidate.get("diff") or {} - aspects = sorted({i.get("aspect") for i in diff.get("items") or [] if i.get("aspect")}) + # projection 那一条只是说「多查了几列,不影响这个数」,不是差在哪个方面(账本的 E1 差异类型也不数它) + aspects = sorted({i.get("aspect") for i in diff.get("items") or [] if i.get("aspect") not in (None, "", "projection")}) sql = next((q.get("sql") for q in answer.queries if q.get("artifact") == chosen.get("artifact")), None) return {"status": chosen.get("status"), "query": chosen.get("artifact"), "sql": sql, "aspects": aspects, "diff": diff or None, "via": candidate.get("via"), "binding": candidate.get("binding"), diff --git a/backend/tests/test_eval_score.py b/backend/tests/test_eval_score.py index a65560ab..367adbff 100644 --- a/backend/tests/test_eval_score.py +++ b/backend/tests/test_eval_score.py @@ -291,6 +291,32 @@ async def test_the_per_case_sql_match_compares_the_query_behind_the_answer(engin assert match["status"] == "proven" and match["basis"] == "only_query" and not match["cited"], match +async def test_the_per_case_sql_match_projects_to_the_cited_columns(engine_on, monkeypatch, tmp_path): + """逐题 E1 和账本同一个口径:答案的 SQL 多查了一列订单数、报告只引用销售额那一列时,按引用的列投影后和期望 SQL + 证明等价;多出来的那列不算差在哪个方面。""" + from app.db.base import SessionLocal + from app.db.models import DataSource + from app.eval import score + from sqlalchemy import select + from tests import ledger_samples as ls + + await ls.fresh_source(tmp_path / "shop.db") + async with SessionLocal() as session: + sid = (await session.execute(select(DataSource.id).where(DataSource.name == ls.CURRENT["source"]))).scalar() + case = {"gold": {"kind": "sql", "datasource_id": sid, "sql": "x"}, "expectations": []} + gold = {"status": "ok", "before": {"rendered": {"sql": ( + "SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' AND paid_at >= '2026-09-01' " + "AND paid_at < '2026-09-08'")}, "sets": []}} + wider = ("select count(*) as 单数, sum(o.amount) as 销售额 from orders o where o.paid_at >= '2026-09-01' " + "and o.status = 'paid' and o.paid_at < '2026-09-08'") + run = await ls.explore(monkeypatch, wider, text="本周销售额 [[v:Q1.r0.销售额]] 元。") + answer = await score.read_answer(run.id) + assert answer.cited, answer + match = await score.sql_match(case, gold, answer) + assert match["status"] == "proven" and match["cited"] is True, match + assert "projection" not in (match.get("aspects") or []), match + + async def test_rescoring_reads_the_seal_and_never_calls_a_model(engine_on, monkeypatch, tmp_path): from app.db.base import SessionLocal from app.db.models_eval import EvalResult, EvalRun