Files
AgentLab/backend/app/eval/compare.py
T
ThearchyHelios 7794794278 评测的两种分析期望(贡献、判定):期望 SQL 的探针快照上用平台的算子(app.analysis.run,随数据变化的调用错误记成算不了)算出期望的判定,评分沿用五种结论
- 题目(cases.py):contribution 贡献恒为变化拆解(可加、比率、乘积)、另写 top_n(期望点名主要来源的前几段,缺省 1);verdict 判定是波动基线、趋势判断、差异检验、可比性检查。参数同分析工具(不写输入、总数、下钻,不写阈值),文字参数可以带 {{ schedule.* }};来源、方向、判定不许人填(HUMAN_VALUE_KEYS 加 segment、subjects、direction、verdict 等);口径题、期望反问的题配不了
- 算期望(gold.py、judgements.py):遮罩列、目录事实、现查表的日期范围和分析工具同一套取法;阈值用这次评测的分析设置快照(Context.analysis),试算时现读设置;期望 SQL 只返回两期时可以不写基期、本期;算不了的这一项照实记(评分时不能评分),参数和结果对不上的整题没有可用的值;截断的快照照旧拒收。作答前后两次期望、和上一次比的「数据变了」都不比运行当天、现查时刻、输入快照的工件(stable_sets)
- 评分(score.py、judgements.py,SCORER_VERSION 3):判定码归到同一套判定类再比;答案引用了(判定语、具名的数、结果表的格、句末依据)判定对得上的分析结论是对且可追溯,引用它的那句却写反了不算;没引用但正文按说法表(engine/analysis_claims 同一个 scan、同一套绑定规则)写对了是对但没引用;判定、来源、方向对不上或写成相反、根本没做也没说是错。答案的分析结论从作答运行的台账读、引用从报告文档读,有判断词的句子一并读出;逐题 E1 引用分析结论时按它的输入查询比(下钻顺着上一层找);判定语不再当成答案里的数;老结果的明细里没有这两项按空读
- 执行(runner.py、answer.py):有分析期望的题集开评时拍一份分析设置,算期望、各臂作答的运行(Run.options 的 analysis)都按它,记进指纹(compare 的「分析设置」一项),重试沿用;别的题集的指纹、作答运行的选项不变。助手提示词的指纹加上分析方法(guide 的 id 和全文)、写作者的分析规则和说法表版本
- 总表(stats.py):各期望的结论另按期望的种类各一份(outcomes_by_kind)
2026-10-21 18:11:57 +08:00

189 lines
10 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""和上一次比、A/B 对照(细案 01 做法 12;评测-D10)。纯计算的部分(transitions、pair_table、fingerprint_diff)不读库;
选基准(pick_baseline)读库。
**选基准**:发起时点名的基线(baseline_id)> 同题集里钉住的基线(pinned,同一变体)> 同题集、同一变体(variants.variant_key)
里比这次先发起、已经结束(succeeded / partial)的最近一次。按发起先后,不按结束时间(同 api/compare.pick_base)。
**逐题转移**:每题按题目 id 配对,k 次重复的按多数(通过的次数多于不通过的算通过)。通过→不通过是退步,不通过→通过是进步,
其余不变;题目改过(case_hash 不同)、一边没有、一边不能判的标「不可比」,不算进步也不算退步。
**归因**(只对转移了的题):期望值变了是「数据变」;期望没变、答案执行的 SQL 规范文本不同是「做法变」;只有字面量不同是
「只参数变」(多半是时间范围理解变了);作答路径变了是「路径变」;其余「说不清」。SQL 的 same / params_only / changed 和运行
对比(engine/compare)是同一条规则:data/sql_semantics.normalize_sql 的规范文本和字面量。
**指纹差异**:助手模型、写作模型、目录快照、提示词指纹、表结构、题目、评分规则、分析设置(有分析期望的题集开评时拍的
那一份判定阈值),哪几项变了。
**A/B**:同一次发起的几臂(共用 pair_id、共用期望)逐题配对,给两臂各自的准确率、配对表(都对 / 都错 / 只 A 对 / 只 B 对)和
精确 McNemar 双侧 p 值;p ≥ 0.05 时照实写「这批题上看不出差别」。不能判的题不进配对表(单列个数)。
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
TRANSITIONS = ("regressed", "improved", "same", "incomparable")
TRANSITION_LABEL = {"regressed": "退步", "improved": "进步", "same": "不变", "incomparable": "不可比"}
CAUSES = ("data", "method", "params", "path", "unknown")
CAUSE_LABEL = {"data": "数据变", "method": "做法变", "params": "只参数变", "path": "路径变", "unknown": "说不清"}
#: 指纹里比的几项和叫法
FINGERPRINT_LABEL = {"assistant_model": "助手模型", "writer_model": "写作模型", "catalog": "目录快照",
"prompt": "提示词", "schemas": "表结构", "cases": "题目", "scorer_version": "评分规则",
"analysis": "分析设置"}
NO_DIFFERENCE = "这批题上看不出差别"
SIGNIFICANT = 0.05
#: 基准能是哪几种状态
BASELINE_STATUSES = ("succeeded", "partial")
# ==========================================================================
# 选基准
# ==========================================================================
async def pick_baseline(session: AsyncSession, run: Any, *, explicit: str | None = None) -> Any | None:
"""这一次和哪一次比(见模块说明)。explicit 是发起时点名的(要同一个题集)。"""
from app.db.models_eval import EvalRun
if explicit:
row = await session.get(EvalRun, explicit)
return row if row is not None and row.suite_id == run.suite_id and row.id != run.id else None
key = (run.fingerprint or {}).get("variant_key")
rows = (await session.execute(
select(EvalRun).where(EvalRun.suite_id == run.suite_id, EvalRun.id != run.id,
EvalRun.created_at < run.created_at, EvalRun.status.in_(BASELINE_STATUSES))
.order_by(EvalRun.created_at.desc(), EvalRun.id.desc()).limit(200))).scalars().all()
same = [r for r in rows if (r.fingerprint or {}).get("variant_key") == key]
pinned = next((r for r in same if r.pinned), None)
return pinned or (same[0] if same else None)
# ==========================================================================
# 逐题
# ==========================================================================
def case_outcome(results: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
"""一道题(几次重复)合成一个结论:{verdict: pass|fail|none, passed, graded, path, gold, sql}。多数决;能判的一次都没有
为 none。gold 是期望值的指纹(作答前那一份),sql 是答案执行的查询的规范化(第一次能判的那一次)。"""
graded = [r for r in results if r.get("verdict") in ("pass", "fail")]
passed = sum(1 for r in graded if r.get("verdict") == "pass")
verdict = "none" if not graded else "pass" if passed > len(graded) - passed else "fail"
first = graded[0] if graded else (results[0] if results else {})
return {"verdict": verdict, "passed": passed, "graded": len(graded), "path": first.get("path"),
"gold": first.get("gold_key"), "sql": first.get("sql_shapes") or [],
"case_hash": first.get("case_hash"), "case_seq": first.get("case_seq")}
def sql_change(a: Sequence[Mapping[str, Any]], b: Sequence[Mapping[str, Any]]) -> str:
"""两次答案执行的查询([{text, literals}],normalize_sql 的 as_dict):same / params_only / changed。"""
if [x.get("text") for x in a] != [x.get("text") for x in b]:
return "changed"
return "params_only" if [x.get("literals") for x in a] != [x.get("literals") for x in b] else "same"
def cause(base: Mapping[str, Any], target: Mapping[str, Any]) -> str:
if base.get("gold") != target.get("gold"):
return "data"
if base.get("path") != target.get("path"):
return "path"
change = sql_change(base.get("sql") or [], target.get("sql") or [])
return {"changed": "method", "params_only": "params"}.get(change, "unknown")
def transitions(base: Sequence[Mapping[str, Any]], target: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
"""逐题转移和归因。base、target 是两次评测(同一臂)的逐题结果。"""
def by_case(rows: Sequence[Mapping[str, Any]]) -> dict[str, list[Mapping[str, Any]]]:
out: dict[str, list[Mapping[str, Any]]] = {}
for r in rows:
out.setdefault(str(r.get("case_id")), []).append(r)
return out
was, now = by_case(base), by_case(target)
items = []
counts = {k: 0 for k in TRANSITIONS}
causes = {k: 0 for k in CAUSES}
for case_id in list(dict.fromkeys([*now, *was])):
b = case_outcome(was.get(case_id, []))
t = case_outcome(now.get(case_id, []))
item: dict[str, Any] = {"case_id": case_id, "case_seq": t.get("case_seq") or b.get("case_seq"),
"before": b["verdict"], "after": t["verdict"]}
if case_id not in was or case_id not in now:
item.update(transition="incomparable", reason="这道题只在一边评过")
elif b.get("case_hash") != t.get("case_hash"):
item.update(transition="incomparable", reason="题目改过,不可比")
elif "none" in (b["verdict"], t["verdict"]):
item.update(transition="incomparable", reason="有一边不能判")
elif b["verdict"] == t["verdict"]:
item.update(transition="same")
else:
item.update(transition="regressed" if b["verdict"] == "pass" else "improved")
item["cause"] = cause(b, t)
causes[item["cause"]] += 1
counts[item["transition"]] += 1
items.append(item)
return {"counts": counts, "causes": causes, "items": items}
def fingerprint_diff(base: Mapping[str, Any], target: Mapping[str, Any]) -> list[dict[str, Any]]:
"""两次之间指纹里哪几项变了:[{key, label, before, after}](表结构、题目只说变没变,不展开)。"""
out = []
for key, label in FINGERPRINT_LABEL.items():
a, b = base.get(key), target.get(key)
if a == b:
continue
item: dict[str, Any] = {"key": key, "label": label}
if key in ("assistant_model", "writer_model", "catalog", "scorer_version"):
item.update(before=a, after=b)
out.append(item)
return out
# ==========================================================================
# A/B
# ==========================================================================
def pair_table(arms: Sequence[tuple[str, Sequence[Mapping[str, Any]]]]) -> dict[str, Any]:
"""几臂([(名字, 逐题结果)])两两配对:{arms: [{arm, accuracy}], pairs: [{a, b, both, neither, only_a, only_b,
excluded, p_value, note}]}。"""
from app.eval.stats import rate
outcomes: dict[str, dict[str, Mapping[str, Any]]] = {}
summary = []
for name, rows in arms:
cases: dict[str, list[Mapping[str, Any]]] = {}
for r in rows:
cases.setdefault(str(r.get("case_id")), []).append(r)
outcomes[name] = {cid: case_outcome(rs) for cid, rs in cases.items()}
graded = [r for r in rows if r.get("verdict") in ("pass", "fail")]
summary.append({"arm": name, "accuracy": rate(sum(1 for r in graded if r.get("verdict") == "pass"), len(graded))})
pairs = []
names = [n for n, _ in arms]
for i, a in enumerate(names):
for b in names[i + 1:]:
both = neither = only_a = only_b = excluded = 0
for cid in dict.fromkeys([*outcomes[a], *outcomes[b]]):
x, y = outcomes[a].get(cid), outcomes[b].get(cid)
if x is None or y is None or "none" in (x["verdict"], y["verdict"]):
excluded += 1
continue
pa, pb = x["verdict"] == "pass", y["verdict"] == "pass"
both += pa and pb
neither += not pa and not pb
only_a += pa and not pb
only_b += pb and not pa
from app.eval.stats import mcnemar_exact
p = mcnemar_exact(only_a, only_b)
pairs.append({"a": a, "b": b, "both": both, "neither": neither, "only_a": only_a, "only_b": only_b,
"excluded": excluded, "p_value": p, "note": NO_DIFFERENCE if p >= SIGNIFICANT else None})
return {"arms": summary, "pairs": pairs}
__all__ = ["CAUSES", "CAUSE_LABEL", "FINGERPRINT_LABEL", "NO_DIFFERENCE", "TRANSITIONS", "TRANSITION_LABEL",
"case_outcome", "cause", "fingerprint_diff", "pair_table", "pick_baseline", "sql_change", "transitions"]