mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:01:58 +00:00
合入 W2-b 补充：评测变体加「分析工具」开 / 关与「分析工具贡献对照」预设（指纹、对比、A/B 认得出这一项）；评分认正文里「两期不能直接比」的说法（只给评分用的词表）；文档补变体与预设
This commit is contained in:
ThearchyHelios
@@ -18,8 +18,8 @@
|
||||
POST /catalog-snapshots/shadow（后台起草，202）、GET /catalog-snapshots/{id}、POST /catalog-snapshots/{id}/stop。
|
||||
- 评测（app/eval/runner.py，后台任务）：POST /suites/{id}/evaluations/estimate（发起前的估算：几项、按上一次每题平均花费估的
|
||||
金额、生效的预算、同时评几题）；POST /suites/{id}/evaluations（发起，202；variant 一臂，或 arms 几臂的 A/B，或
|
||||
preset=catalog / catalog_reviewed「目录贡献」预设对照）；GET /suites/{id}/evaluations（历次）；GET /evaluations/{id}（进度、
|
||||
总表、和上一次比的计数）；GET /evaluations/{id}/results（逐题）、GET /evaluations/{id}/results/{rid}（一题的明细）；
|
||||
preset=catalog / catalog_reviewed「目录贡献」、preset=analysis_tools「分析工具贡献」预设对照）；
|
||||
GET /suites/{id}/evaluations（历次）；GET /evaluations/{id}（进度、总表、和上一次比的计数）；GET /evaluations/{id}/results（逐题）、GET /evaluations/{id}/results/{rid}（一题的明细）；
|
||||
POST /evaluations/{id}/stop、/retry（只重做没做完的）、/rescore（按新评分规则重评，只读封存、不调模型）、/pin（钉住当基线）；
|
||||
GET /evaluations/{id}/compare?base=（和上一次或点名的那一次比：指纹差异、逐题转移和归因）；GET /pairs/{pair_id}（A/B 配对表和
|
||||
精确 McNemar）；POST /suites/{id}/cleanup（按保留规则清理更早的明细）。「口径命中与重复度」的 E1–E4 直接用口径账本的
|
||||
@@ -855,8 +855,8 @@ EVALUATION_NOT_FOUND = "eval_evaluation_not_found"
|
||||
PAIR_NOT_FOUND = "eval_pair_not_found"
|
||||
RESULT_NOT_FOUND = "eval_result_not_found"
|
||||
CLEANED = "eval_details_cleaned"
|
||||
#: 预设对照
|
||||
PRESETS = ("catalog", "catalog_reviewed")
|
||||
#: 预设对照：目录贡献（当前目录 vs 关，可加只用已核实的项）、分析工具贡献（开 vs 关）
|
||||
PRESETS = ("catalog", "catalog_reviewed", "analysis_tools")
|
||||
|
||||
|
||||
def _refused(e: CaseInvalid) -> CodedHTTPException:
|
||||
@@ -865,7 +865,8 @@ def _refused(e: CaseInvalid) -> CodedHTTPException:
|
||||
|
||||
async def _arms(session: AsyncSession, suite: EvalSuite, body: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""请求体 → 规范化的几臂变体：{variant}（一臂，不写用题集的缺省变体）、{arms: [变体…]}（A/B）或 {preset: catalog |
|
||||
catalog_reviewed}（目录贡献对照：当前目录 vs 关，可加只用已核实的项；模型照 variant 或题集的缺省）。A/B 缺省每题 3 次。"""
|
||||
catalog_reviewed | analysis_tools}（目录贡献对照：当前目录 vs 关，可加只用已核实的项；分析工具贡献对照：开 vs 关；别的项
|
||||
照 variant 或题集的缺省）。A/B 缺省每题 3 次。"""
|
||||
from app.eval import variants as variant_rules
|
||||
|
||||
judge = bool((await _site(session))["judge"])
|
||||
@@ -877,6 +878,8 @@ async def _arms(session: AsyncSession, suite: EvalSuite, body: dict[str, Any]) -
|
||||
raise CaseInvalid(f"预设对照只能是 {'、'.join(PRESETS)}", "preset", variant_rules.VARIANT_INVALID)
|
||||
base = variant_rules.normalize_variant({**dict(suite.default_variant or {}), **dict(body.get("variant") or {})},
|
||||
judge_default=judge, repeats_default=variant_rules.PAIR_REPEATS)
|
||||
if preset == "analysis_tools":
|
||||
return variant_rules.analysis_tools_preset(base)
|
||||
return variant_rules.catalog_preset(base, reviewed=preset == "catalog_reviewed")
|
||||
if raw_arms is not None:
|
||||
if not isinstance(raw_arms, list) or not 2 <= len(raw_arms) <= variant_rules.ARMS_MAX:
|
||||
|
||||
@@ -7,7 +7,8 @@ app/chat/job.py），生成过程是 api/copilot.answer_ops，几个「决定」
|
||||
answer_ops → final.next（chat/turn.next_step）→ launch_run(origin=eval) → 等终态 → review_of_run → 要的话带原图重来一次
|
||||
|
||||
- 生成：问法就是 instruction，intent=answer，限定的数据源同题目；助手模型按变体覆盖（GenerateIn.provider / model，挑表、
|
||||
复核跟着换）；读哪份目录按变体（answer_ops 的 catalog_view，查看的表详情、自查的 SQL 检查器都按它）。
|
||||
复核跟着换）；读哪份目录按变体（answer_ops 的 catalog_view，查看的表详情、自查的 SQL 检查器都按它）；分析工具开不开按
|
||||
变体（answer_ops 的 analysis_tools，经 TurnDriver 透传：关的那一臂工具清单不列分析工具，节点手册不讲分析卡）。
|
||||
- 结局：reply 直接回答、asked 反问、fail 判失败（建图失败）、ready 只建不跑、retry（老服务端那条路：没自查过的图带着原因
|
||||
重来一次）、run 运行。生成过程交出 error（助手本轮未完成）、开流前就拒绝（模型没配好、数据源不在）都是建图失败，另记
|
||||
「助手调用失败」——连续几题都这样就止损（runner.py）；只有生成过程一个操作都没给（只写了思考，error.next=retry）时按
|
||||
@@ -71,6 +72,8 @@ class TurnContext:
|
||||
view: Mapping[str, Any] | None = None
|
||||
#: 这次评测的分析设置快照（有分析期望的题集才有）：作答的运行按它判（Run.options['analysis']），和算期望同一份
|
||||
analysis: Mapping[str, Any] | None = None
|
||||
#: 变体：分析工具开不开（variants.analysis_tools）。关的那一臂建图时助手看不到分析工具（TurnDriver 的 analysis_tools）
|
||||
analysis_tools: bool = True
|
||||
|
||||
|
||||
async def writer_usage(run_ids: list[str]) -> dict[str, Any]:
|
||||
@@ -153,13 +156,14 @@ async def stop_run(run_id: str | None, actor: str | None = None) -> None:
|
||||
|
||||
class Driver(TurnDriver):
|
||||
"""一轮作答（流程在 TurnDriver）。drive() 跑完一轮，结果在 turn 上；on_phase(阶段) 在每一步开始时调（runner 据此写
|
||||
「正在做什么」）。和问数据不一样的：助手模型、目录视图按变体；写作模型按变体写进图；来源 eval、只读和遮罩；停在审批上
|
||||
直接取消（评测不代批）；复核一直等。"""
|
||||
「正在做什么」）。和问数据不一样的：助手模型、目录视图、分析工具开不开按变体；写作模型按变体写进图；来源 eval、只读和
|
||||
遮罩；停在审批上直接取消（评测不代批）；复核一直等。"""
|
||||
|
||||
def __init__(self, question: str, *, datasource_ids: list[str], ctx: TurnContext,
|
||||
on_phase: Callable[[str], Awaitable[None]] | None = None) -> None:
|
||||
super().__init__(question, datasource_ids=datasource_ids, on_phase=on_phase,
|
||||
provider=ctx.assistant.get("provider"), model=ctx.assistant.get("model"))
|
||||
provider=ctx.assistant.get("provider"), model=ctx.assistant.get("model"),
|
||||
analysis_tools=ctx.analysis_tools)
|
||||
self.ctx = ctx
|
||||
|
||||
def catalog_view(self) -> Any:
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
「只参数变」（多半是时间范围理解变了）；作答路径变了是「路径变」；其余「说不清」。SQL 的 same / params_only / changed 和运行
|
||||
对比（engine/compare）是同一条规则：data/sql_semantics.normalize_sql 的规范文本和字面量。
|
||||
|
||||
**指纹差异**：助手模型、写作模型、目录快照、提示词指纹、表结构、题目、评分规则、分析设置（有分析期望的题集开评时拍的
|
||||
那一份判定阈值），哪几项变了。
|
||||
**指纹差异**：助手模型、写作模型、目录快照、分析工具（开、关；加这一项之前的评测按开）、提示词指纹、表结构、题目、
|
||||
评分规则、分析设置（有分析期望的题集开评时拍的那一份判定阈值），哪几项变了。
|
||||
|
||||
**A/B**：同一次发起的几臂（共用 pair_id、共用期望）逐题配对，给两臂各自的准确率、配对表（都对 / 都错 / 只 A 对 / 只 B 对）和
|
||||
精确 McNemar 双侧 p 值；p ≥ 0.05 时照实写「这批题上看不出差别」。不能判的题不进配对表（单列个数）。
|
||||
@@ -31,8 +31,10 @@ CAUSES = ("data", "method", "params", "path", "unknown")
|
||||
CAUSE_LABEL = {"data": "数据变", "method": "做法变", "params": "只参数变", "path": "路径变", "unknown": "说不清"}
|
||||
#: 指纹里比的几项和叫法
|
||||
FINGERPRINT_LABEL = {"assistant_model": "助手模型", "writer_model": "写作模型", "catalog": "目录快照",
|
||||
"prompt": "提示词", "schemas": "表结构", "cases": "题目", "scorer_version": "评分规则",
|
||||
"analysis": "分析设置"}
|
||||
"analysis_tools": "分析工具", "prompt": "提示词", "schemas": "表结构", "cases": "题目",
|
||||
"scorer_version": "评分规则", "analysis": "分析设置"}
|
||||
#: 指纹里照实写前后两边的几项（别的只说变没变）
|
||||
_SHOWN = ("assistant_model", "writer_model", "catalog", "analysis_tools", "scorer_version")
|
||||
NO_DIFFERENCE = "这批题上看不出差别"
|
||||
SIGNIFICANT = 0.05
|
||||
#: 基准能是哪几种状态
|
||||
@@ -128,15 +130,22 @@ def transitions(base: Sequence[Mapping[str, Any]], target: Sequence[Mapping[str,
|
||||
return {"counts": counts, "causes": causes, "items": items}
|
||||
|
||||
|
||||
def _tools(fingerprint: Mapping[str, Any]) -> str:
|
||||
"""分析工具这一项给人看的样子：开 / 关。加这一项之前的评测指纹里没有它，那时一直开着。"""
|
||||
from app.eval.variants import ANALYSIS_TOOLS_LABEL
|
||||
|
||||
return ANALYSIS_TOOLS_LABEL["off" if fingerprint.get("analysis_tools") == "off" else "on"]
|
||||
|
||||
|
||||
def fingerprint_diff(base: Mapping[str, Any], target: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||||
"""两次之间指纹里哪几项变了：[{key, label, before, after}]（表结构、题目只说变没变，不展开）。"""
|
||||
"""两次之间指纹里哪几项变了：[{key, label, before, after}]（表结构、题目只说变没变，不展开；分析工具写开、关）。"""
|
||||
out = []
|
||||
for key, label in FINGERPRINT_LABEL.items():
|
||||
a, b = base.get(key), target.get(key)
|
||||
a, b = (_tools(base), _tools(target)) if key == "analysis_tools" else (base.get(key), target.get(key))
|
||||
if a == b:
|
||||
continue
|
||||
item: dict[str, Any] = {"key": key, "label": label}
|
||||
if key in ("assistant_model", "writer_model", "catalog", "scorer_version"):
|
||||
if key in _SHOWN:
|
||||
item.update(before=a, after=b)
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
@@ -21,7 +21,10 @@ comparable 可比（comparable、conditional）/ incomparable 不可比。
|
||||
- 对且可追溯：答案引用了（[[a:]]、[[v:A…]]、[[table:A…]]、[[see:A…]]）判定对得上的分析结论（同一个算子，变化拆解还要同一种
|
||||
模式；主要来源要点名期望的那几段，方向一致；差异检验比的是同两组）；引用它的那句话里的判断词却和期望对不上的不算。
|
||||
- 对但没引用：没引用对得上的结论，但正文按说法表（engine/analysis_claims，同一套词表、同一套绑定规则）写对了：说法表认出的
|
||||
这一类判断词，拿期望的那份分析结论核对得上（点名的段、方向、连续几期都核）。可比性没有对应的判断词，只能引用。
|
||||
这一类判断词，拿期望的那份分析结论核对得上（点名的段、方向、连续几期都核）。可比性不进说法表（说法核对不该打回「本周数据
|
||||
不完整，不能直接比」这种谨慎的提醒），评分另用一张只给自己用的小词表（_INCOMPARABLE）：期望是不可比的，正文明确说了两期
|
||||
不能直接比（不能直接比、不可比、本期不完整、还没过完、只有几天、天数不同、口径不同……；分句里先有否定的、句子里先有假设
|
||||
或疑问的、问句都不算）也算；期望是可比的只能靠引用。
|
||||
- 错：引用的分析判定对不上、正文写的和期望相反（把常规波动说成异常、把上升说成下降、主要来源说成别的段），或者根本没做也
|
||||
没说。
|
||||
- 不能评分：期望算不了。
|
||||
@@ -80,6 +83,38 @@ _AFFIRMS: dict[str, frozenset[str]] = {
|
||||
#: 判断词所在的句子最多留多少字给人看
|
||||
_SNIPPET_MAX = 40
|
||||
|
||||
#: 数字（阿拉伯、全角、汉字）、期次、分句里的字（不跨句读）、比
|
||||
_N = r"[0-9０-９一二两三四五六七八九十]+"
|
||||
_PERIOD = "周|月|季|年|期|天|日"
|
||||
_IN_CLAUSE = r"[^，,；;。！？!?\n]"
|
||||
_COMPARE = "比较|对比|相比|比"
|
||||
#: 正文里明确说两期不能直接比的说法：直说不能比的，和说出不能直接比的原因的（本期不完整、还没过完、只有几天、天数不同、
|
||||
#: 口径不同）。只给评分用，不进说法表（engine/analysis_claims）：说法核对不该打回「本周数据不完整，不能直接比」这种谨慎的
|
||||
#: 提醒
|
||||
_INCOMPARABLE = re.compile(
|
||||
# 直说不能比：不能直接比、无法直接比较、不宜简单对比、难以比较……「无法比拟」「比不上」「比得上」不是
|
||||
rf"(?:不能|不可以|无法|不宜|不应该?|不便|难以|很难|不好|没办法)(?:直接|简单)?地?(?:拿来|放在一起)?"
|
||||
rf"(?:{_COMPARE})(?![拟肩]|得上|不上)"
|
||||
# 「没法」要带「直接」「简单」，或者说「比较」「对比」：光说「没法比」多半是说差得远
|
||||
rf"|没法(?:直接|简单)地?(?:拿来|放在一起)?(?:{_COMPARE})(?![拟肩]|得上|不上)|没法(?:比较|对比|相比)"
|
||||
# 不能直接拿本周和上周比
|
||||
rf"|(?:不能|无法|不宜|不好|没法)(?:直接|简单)?(?:拿|把|将){_IN_CLAUSE}{{1,12}}?(?:和|跟|与|同){_IN_CLAUSE}{{1,12}}?"
|
||||
rf"(?:{_COMPARE})(?![拟肩]|得上|不上)"
|
||||
r"|不可比(?!拟)|不具备?可比性|没有可比性|缺乏可比性|可比性(?:不足|不够|存疑|较差|很差)"
|
||||
# 原因：本期不完整、还没过完、只有几天、差几天、不满一周、天数不同、口径不同
|
||||
rf"|(?:数据|统计|{_PERIOD}){_IN_CLAUSE}{{0,4}}?(?:不完整|不全(?!是))"
|
||||
rf"|(?:{_PERIOD}){_IN_CLAUSE}{{0,4}}?(?:还|尚)?(?:没|未)(?:过完|结束|走完)"
|
||||
rf"|(?:只有|仅有|才有|只统计了|只算了|只过了|才过了|才过去|才)\s*{_N}\s*(?:天|日)"
|
||||
rf"|(?:还差|少了?|相差|差了?)\s*{_N}\s*(?:天|日)"
|
||||
r"|(?:不满|不足|不到)\s*(?:一|1)\s*整?(?:周|个?月|季度?|年)"
|
||||
r"|(?:天数|期长|时长|长度)也?(?:不同|不一样|不一致|不等|不相等|对不上|不足|不够)"
|
||||
r"|口径也?(?:不同|不一样|不一致|变了|有变|有变化|发生了?变化|调整过?|改过|改了|有调整)|(?:换了|改了|调整了)(?:统计)?口径"
|
||||
)
|
||||
#: 可比性的说法前面（同一分句里）有否定的不算说了：「不是不能比」「并非不可比」
|
||||
_INCOMPARABLE_DENIED = re.compile(r"不是|并非|并不|未必|不一定")
|
||||
#: 句子里在它前面有假设、疑问的不算说了：「如果口径不同，就不能直接比」「是不是不能直接比」
|
||||
_INCOMPARABLE_SUPPOSED = re.compile(r"如果|假如|要是|倘若|假设|若是|是否|是不是|能不能|有没有")
|
||||
|
||||
|
||||
class ParamsInvalid(ValueError):
|
||||
"""期望的分析参数写得不对（保存题目时、算期望时都可能）。message 给人看，key 是出错的那个参数。"""
|
||||
@@ -362,10 +397,17 @@ def mismatch(want: Mapping[str, Any], got: Mapping[str, Any]) -> str | None:
|
||||
_SENTENCE_END = re.compile(r"(?<=[。！？!?\n])")
|
||||
|
||||
|
||||
def _said_something(text: str) -> bool:
|
||||
"""这句话里有没有评分要看的说法：说法表的判断词，或者明确说两期不能直接比（_INCOMPARABLE）。"""
|
||||
from app.engine import analysis_claims as said
|
||||
|
||||
return bool(said.scan(text) or incomparable_spans(text))
|
||||
|
||||
|
||||
def sentences_of_doc(doc: Mapping[str, Any], aliases: Iterable[str]) -> list[dict[str, Any]]:
|
||||
"""报告文档里有判断词的句子：{text: 写作者自己写的字（引用渲染出来的值、判定语、引文涂成 \\x00）, names: 点名用的样子
|
||||
（引用渲染出来的值留着）, analyses: 这句挂的分析结论编号}。标题、代码块、表格不算（同说法核对）。aliases 是文档目录里
|
||||
分析结论的编号。"""
|
||||
"""报告文档里有判断词（或者明确说两期不能直接比）的句子：{text: 写作者自己写的字（引用渲染出来的值、判定语、引文涂成
|
||||
\\x00）, names: 点名用的样子（引用渲染出来的值留着）, analyses: 这句挂的分析结论编号}。标题、代码块、表格不算（同说法
|
||||
核对）。aliases 是文档目录里分析结论的编号。"""
|
||||
from app.engine import analysis_claims as said
|
||||
from app.engine.evidence import iter_units
|
||||
|
||||
@@ -376,19 +418,17 @@ def sentences_of_doc(doc: Mapping[str, Any], aliases: Iterable[str]) -> list[dic
|
||||
or unit.get("loc") is not None:
|
||||
continue
|
||||
sentence = said._sentence(unit, unit.get("cites") or [], known)
|
||||
if sentence is not None and said.scan(sentence.text):
|
||||
if sentence is not None and _said_something(sentence.text):
|
||||
out.append({"text": sentence.text, "names": sentence.names, "analyses": list(sentence.analyses)})
|
||||
return out
|
||||
|
||||
|
||||
def sentences_of_text(text: str) -> list[dict[str, Any]]:
|
||||
"""没有报告文档时（成果里的文字）：按句末标点切开，有判断词的句子。"""
|
||||
from app.engine import analysis_claims as said
|
||||
|
||||
"""没有报告文档时（成果里的文字）：按句末标点切开，有判断词（或者明确说两期不能直接比）的句子。"""
|
||||
out = []
|
||||
for piece in _SENTENCE_END.split(text or ""):
|
||||
piece = piece.strip()
|
||||
if piece and said.scan(piece):
|
||||
if piece and _said_something(piece):
|
||||
out.append({"text": piece, "names": piece, "analyses": []})
|
||||
return out
|
||||
|
||||
@@ -459,6 +499,33 @@ def said_in(want: Mapping[str, Any], sentences: Sequence[Mapping[str, Any]], *,
|
||||
return good, bad
|
||||
|
||||
|
||||
def incomparable_spans(text: str) -> list[tuple[int, int]]:
|
||||
"""一句话（写作者自己写的字）里明确说两期不能直接比的几处 [(起, 止)]（_INCOMPARABLE）。同一分句里前面有否定的、
|
||||
句子里前面有假设或疑问的不算，问句整句不算。"""
|
||||
from app.engine import analysis_claims as said
|
||||
|
||||
if text.rstrip("\x00 ").endswith(("？", "?")):
|
||||
return []
|
||||
out = []
|
||||
for m in _INCOMPARABLE.finditer(text):
|
||||
left, _ = said._clause(text, m.start(), m.end())
|
||||
if _INCOMPARABLE_DENIED.search(text[left:m.start()]) or _INCOMPARABLE_SUPPOSED.search(text[:m.start()]):
|
||||
continue
|
||||
out.append((m.start(), m.end()))
|
||||
return out
|
||||
|
||||
|
||||
def said_incomparable(sentences: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""正文里明确说两期不能直接比的几处（可比性的期望是不可比时，算说出了它）：{text: 所在分句, judgement: incomparable}。"""
|
||||
out = []
|
||||
for raw in sentences:
|
||||
text = str(raw.get("text") or "")
|
||||
names = str(raw.get("names") or text)
|
||||
for start, end in incomparable_spans(text):
|
||||
out.append({"text": _snippet(names, start, end), "judgement": "incomparable", "subjects": []})
|
||||
return out
|
||||
|
||||
|
||||
def _covered(want: Mapping[str, Any], good: Sequence[Mapping[str, Any]]) -> list[str]:
|
||||
"""主要来源期望点名的段里，正文说对的那几处一处都没点到的（top_n 要点名的前几段都要说到）。"""
|
||||
if want.get("judgement") != "source":
|
||||
@@ -492,6 +559,8 @@ def score(exp: Mapping[str, Any], value: Mapping[str, Any], answer: Any) -> dict
|
||||
"reason": f"引用了 {found['alias']}，同一句却写成「{bad[0]['text']}」：{bad[0]['reason']}"}
|
||||
return {"outcome": "traced", "want": want, "found": view(found)}
|
||||
good, bad = said_in(value, sentences)
|
||||
if value.get("judgement") == "incomparable":
|
||||
good += said_incomparable(sentences)
|
||||
if good:
|
||||
missing = _covered(value, good)
|
||||
if not missing:
|
||||
@@ -517,5 +586,6 @@ def score(exp: Mapping[str, Any], value: Mapping[str, Any], answer: Any) -> dict
|
||||
|
||||
|
||||
__all__ = ["ANALYSIS_KINDS", "GOLD_REF", "JUDGEMENTS", "ParamsInvalid", "TOP_N_MAX", "VERDICT_OPS", "check_params",
|
||||
"expected", "judgement_code", "mismatch", "norm", "op_text", "param_names", "record_of", "request_of",
|
||||
"said_in", "score", "sentences_of_doc", "sentences_of_text", "steady", "template_texts", "view"]
|
||||
"expected", "incomparable_spans", "judgement_code", "mismatch", "norm", "op_text", "param_names",
|
||||
"record_of", "request_of", "said_in", "said_incomparable", "score", "sentences_of_doc", "sentences_of_text",
|
||||
"steady", "template_texts", "view"]
|
||||
|
||||
@@ -239,6 +239,7 @@ async def start(session: AsyncSession, suite: EvalSuite, variants: Sequence[Mapp
|
||||
fingerprint={"variant_key": variant_rules.variant_key(variant, models),
|
||||
"assistant_model": models.get("assistant"), "writer_model": models.get("writer"),
|
||||
"catalog_mode": (variant.get("catalog") or {}).get("mode"),
|
||||
"analysis_tools": variant_rules.analysis_tools(variant),
|
||||
"snapshot_artifact": snapshot, "cron": cron},
|
||||
counts={"total": len(cases) * repeats, "done": 0})
|
||||
session.add(run)
|
||||
@@ -531,6 +532,7 @@ class EvalExecution(runtime.JobRun):
|
||||
|
||||
async def _answer(self, arm: _Arm, unit: _Unit, case: Mapping[str, Any], question: str,
|
||||
values: Mapping[str, str], timeout: float) -> Any:
|
||||
from app.eval import variants as variant_rules
|
||||
from app.eval.answer import Driver, TurnContext, stop_run
|
||||
|
||||
result_id = unit.results[arm.run_id]
|
||||
@@ -542,7 +544,8 @@ class EvalExecution(runtime.JobRun):
|
||||
case_id=unit.case_id, case_seq=case.get("seq"), actor=self.actor,
|
||||
arm=arm.name, repeat=unit.repeat,
|
||||
assistant=arm.variant.get("assistant") or {}, writer=arm.variant.get("writer") or {},
|
||||
view=arm.view, analysis=self.analysis)
|
||||
view=arm.view, analysis=self.analysis,
|
||||
analysis_tools=variant_rules.analysis_tools(arm.variant) == "on")
|
||||
driver = Driver(question, datasource_ids=list(case.get("datasource_ids") or []), ctx=ctx, on_phase=on_phase)
|
||||
try:
|
||||
if case.get("kind") == "workflow":
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""评测的变体：这一次用哪几个模型、读哪一份目录、重复几次（细案 01 做法 9、12；评测-D5、D10）。
|
||||
"""评测的变体：这一次用哪几个模型、读哪一份目录、给不给分析工具、重复几次（细案 01 做法 9、12；评测-D5、D10）。
|
||||
|
||||
变体原样存进 EvalRun.variant（发起时规范化过），实际解析出的模型 id 另存 resolved_models。几项：
|
||||
|
||||
@@ -11,13 +11,18 @@
|
||||
- catalog {mode, snapshot}：读哪一份目录（评测-D5）——live 当前目录、reviewed 只用已核实和已确认的项、off 关目录、snapshot
|
||||
指定一份目录快照（EvalCatalogSnapshot 的 id，影子起草的或拍好的）。live、reviewed 在开评时都拍一份当前目录的快照，按
|
||||
快照读：评测一跑几十分钟，期间起草、剖析照常改目录，也不影响这次的可比性（做法 9「不占数据源的队」）。
|
||||
- analysis {tools}：分析工具开不开（on 开、off 关，缺省开）。关的那一臂助手建图时看不到分析工具和分析卡（工具清单不列、
|
||||
节点手册和问数据的包装里只讲分析能力的那几段不给，图里照样绑了分析工具的挡回去改；api/copilot.answer_ops 的
|
||||
analysis_tools，经 chat/drive.TurnDriver 透传）。工作流题不建图，按钉住的那一版跑，不受这一项影响。
|
||||
- repeats：每题重复几次（1–5）。定时跑 1 次，对照缺省 3 次（评测-D10）。
|
||||
- label：这一臂叫什么（A/B 时给人看，「当前目录」「关目录」……）。
|
||||
|
||||
**同一变体**（和上一次比时挑基准用，做法 12）：助手、写作两项实际解析出的模型、目录的 mode（指定快照的连同快照 id）；
|
||||
重复次数、裁判开不开不算——它们不改变答案本身。variant_key 是它的指纹。
|
||||
**同一变体**（和上一次比时挑基准用，做法 12）：助手、写作两项实际解析出的模型、目录的 mode（指定快照的连同快照 id）、
|
||||
分析工具开不开；重复次数、裁判开不开不算——它们不改变答案本身。variant_key 是它的指纹（分析工具开着时和加这一项之前
|
||||
一字不差：以前的评测照样认得出是同一变体）。
|
||||
|
||||
**目录贡献对照**（做法 12）：一组预设的 A/B——当前目录 vs 关，可以再加第三臂「只用已核实和已确认的项」（catalog_preset）。
|
||||
**分析工具贡献对照**：一组预设的 A/B——分析工具开 vs 关（analysis_tools_preset），量出分析工具帮了多少。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -37,6 +42,9 @@ VARIANT_UNAVAILABLE = "eval_variant_unavailable"
|
||||
|
||||
CATALOG_MODES = ("live", "reviewed", "off", "snapshot")
|
||||
CATALOG_LABEL = {"live": "当前目录", "reviewed": "只用已核实和已确认的项", "off": "关目录", "snapshot": "目录快照"}
|
||||
#: 分析工具开、关（变体的 analysis.tools）
|
||||
ANALYSIS_TOOLS = ("on", "off")
|
||||
ANALYSIS_TOOLS_LABEL = {"on": "开", "off": "关"}
|
||||
REPEATS_MAX = 5
|
||||
#: 定时跑 1 次，A/B 对照缺省 3 次（评测-D10）
|
||||
SCHEDULE_REPEATS = 1
|
||||
@@ -44,7 +52,7 @@ PAIR_REPEATS = 3
|
||||
#: 一次 A/B 最多几臂（目录贡献对照最多三臂）
|
||||
ARMS_MAX = 3
|
||||
LABEL_MAX = 40
|
||||
_KEYS = ("assistant", "writer", "judge", "catalog", "repeats", "label")
|
||||
_KEYS = ("assistant", "writer", "judge", "catalog", "analysis", "repeats", "label")
|
||||
|
||||
|
||||
def _invalid(message: str, field_name: str) -> CaseInvalid:
|
||||
@@ -92,6 +100,10 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default:
|
||||
raise _invalid("只有目录选「目录快照」时才指定快照", "variant.catalog.snapshot")
|
||||
else:
|
||||
snapshot = None
|
||||
analysis = {} if raw.get("analysis") is None else raw["analysis"]
|
||||
if not isinstance(analysis, Mapping) or set(analysis) - {"tools"} \
|
||||
or analysis.get("tools", "on") not in ANALYSIS_TOOLS:
|
||||
raise _invalid(f"变体的分析工具需要是 {{tools}}，tools 只能是 {'、'.join(ANALYSIS_TOOLS)}（开、关）", "variant.analysis")
|
||||
repeats = raw.get("repeats", repeats_default)
|
||||
if isinstance(repeats, bool) or not isinstance(repeats, int) or not 1 <= repeats <= REPEATS_MAX:
|
||||
raise _invalid(f"每题重复的次数需要是 1 到 {REPEATS_MAX} 的整数", "variant.repeats")
|
||||
@@ -100,10 +112,16 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default:
|
||||
raise _invalid("这一臂的名字需要是文字", "variant.label")
|
||||
return {"assistant": _model_pair(raw.get("assistant"), "助手模型", "variant.assistant"),
|
||||
"writer": _model_pair(raw.get("writer"), "写作模型", "variant.writer"),
|
||||
"judge": judge, "catalog": {"mode": catalog["mode"], "snapshot": snapshot}, "repeats": repeats,
|
||||
"judge": judge, "catalog": {"mode": catalog["mode"], "snapshot": snapshot},
|
||||
"analysis": {"tools": analysis.get("tools", "on")}, "repeats": repeats,
|
||||
"label": (label or "").strip()[:LABEL_MAX] or None}
|
||||
|
||||
|
||||
def analysis_tools(variant: Mapping[str, Any]) -> str:
|
||||
"""这一臂的分析工具开不开：on / off。加这一项之前的变体（存着的老评测）没有它，按开。"""
|
||||
return "off" if (variant.get("analysis") or {}).get("tools") == "off" else "on"
|
||||
|
||||
|
||||
def catalog_preset(base: Mapping[str, Any], *, reviewed: bool = False) -> list[dict[str, Any]]:
|
||||
"""「目录的贡献」预设对照：当前目录 vs 关（reviewed 时再加「只用已核实和已确认的项」）。模型、裁判、重复次数照 base。"""
|
||||
arms = [("live", CATALOG_LABEL["live"]), ("off", CATALOG_LABEL["off"])]
|
||||
@@ -113,6 +131,12 @@ def catalog_preset(base: Mapping[str, Any], *, reviewed: bool = False) -> list[d
|
||||
for mode, label in arms]
|
||||
|
||||
|
||||
def analysis_tools_preset(base: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||||
"""「分析工具的贡献」预设对照：分析工具开 vs 关。模型、目录、裁判、重复次数照 base。"""
|
||||
return [{**copy.deepcopy(dict(base)), "analysis": {"tools": tools}, "label": f"分析工具{ANALYSIS_TOOLS_LABEL[tools]}"}
|
||||
for tools in ANALYSIS_TOOLS]
|
||||
|
||||
|
||||
def arm_name(variant: Mapping[str, Any], index: int) -> str:
|
||||
"""这一臂的名字：变体里写了就用，否则按目录（只有目录不同时一看就懂），再不然 A、B、C。"""
|
||||
return str(variant.get("label") or "") or "ABC"[index]
|
||||
@@ -188,26 +212,31 @@ async def snapshot_artifact(session: AsyncSession, variant: Mapping[str, Any]) -
|
||||
|
||||
|
||||
def variant_key(variant: Mapping[str, Any], models: Mapping[str, Any]) -> str:
|
||||
"""「同一变体」的指纹：助手、写作实际解析出的模型，目录的 mode（指定快照的连同快照 id）。"""
|
||||
"""「同一变体」的指纹：助手、写作实际解析出的模型，目录的 mode（指定快照的连同快照 id），分析工具开不开。分析工具
|
||||
开着时不写进去：和加这一项之前算出来的一样，以前的评测照样是同一变体（挑基准、趋势线不断）。"""
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
|
||||
catalog = variant.get("catalog") or {}
|
||||
body = {"assistant": models.get("assistant"), "writer": models.get("writer"), "catalog": catalog.get("mode"),
|
||||
"snapshot": catalog.get("snapshot") if catalog.get("mode") == "snapshot" else None}
|
||||
if analysis_tools(variant) == "off":
|
||||
body["analysis_tools"] = "off"
|
||||
return content_hash(canonical_json(body))[:16]
|
||||
|
||||
|
||||
def describe(variant: Mapping[str, Any], models: Mapping[str, Any] | None = None) -> str:
|
||||
"""给人看的一句：「助手 x · 写作 y · 当前目录 · 裁判关 · 每题 1 次」。"""
|
||||
"""给人看的一句：「助手 x · 写作 y · 当前目录 · 分析工具开 · 结论裁判关 · 每题 1 次」。"""
|
||||
models = models or {}
|
||||
catalog = variant.get("catalog") or {}
|
||||
parts = [f"助手 {models.get('assistant') or (variant.get('assistant') or {}).get('model') or '缺省'}",
|
||||
f"写作 {models.get('writer') or (variant.get('writer') or {}).get('model') or '各节点自己的'}",
|
||||
CATALOG_LABEL.get(catalog.get("mode") or "live", "当前目录"),
|
||||
f"分析工具{ANALYSIS_TOOLS_LABEL[analysis_tools(variant)]}",
|
||||
f"结论裁判{'开' if variant.get('judge') else '关'}", f"每题 {variant.get('repeats') or 1} 次"]
|
||||
return " · ".join(parts)
|
||||
|
||||
|
||||
__all__ = ["ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", "PAIR_REPEATS", "REPEATS_MAX", "Resolved", "SCHEDULE_REPEATS",
|
||||
"VARIANT_INVALID", "VARIANT_UNAVAILABLE", "arm_name", "catalog_preset", "describe", "normalize_variant",
|
||||
"resolve_models", "snapshot_artifact", "variant_key", "view_of"]
|
||||
__all__ = ["ANALYSIS_TOOLS", "ANALYSIS_TOOLS_LABEL", "ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", "PAIR_REPEATS",
|
||||
"REPEATS_MAX", "Resolved", "SCHEDULE_REPEATS", "VARIANT_INVALID", "VARIANT_UNAVAILABLE", "analysis_tools",
|
||||
"analysis_tools_preset", "arm_name", "catalog_preset", "describe", "normalize_variant", "resolve_models",
|
||||
"snapshot_artifact", "variant_key", "view_of"]
|
||||
|
||||
@@ -7,6 +7,8 @@ tests/test_eval_analysis_runs.py。地区、门店、数字都是虚构的。
|
||||
- 期望值：判定码归到判定类，主要来源按 top_n 取前几段；比较作答前后两次期望时不看运行当天、现查时刻、输入快照的工件。
|
||||
- 评分五种结论：引用了判定对得上的结论 → 对且可追溯；正文按说法表写对了、没引用 → 对但没引用；引用的判定对不上、正文
|
||||
写反了（常规波动说成异常、方向反了、来源说成别的段）、根本没做也没说 → 错；期望算不了 → 不能评分。
|
||||
- 可比性不进说法表：期望不可比、正文明确说了两期不能直接比的（评分自己的小词表）算对但没引用；假设、疑问、否定的不算；
|
||||
期望可比的只能靠引用。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -259,13 +261,38 @@ def test_doing_nothing_and_saying_nothing_is_wrong():
|
||||
uncited = _score(BAND, fake_result("baseline", None, "within_band"),
|
||||
_answer(_record(fake_result("baseline", None, "within_band"), cited=()), text="上周订单数是 1,203 单。"))
|
||||
assert uncited["outcome"] == "wrong" and "但没有引用" in uncited["reason"], uncited
|
||||
# 可比性没有对应的判断词：只能引用
|
||||
|
||||
|
||||
def test_saying_the_two_periods_cannot_be_compared_directly_is_untraced():
|
||||
"""可比性不进说法表（说法核对不打回谨慎的提醒），评分另认：期望不可比、正文明确说了两期不能直接比（不完整、还没过完、
|
||||
只有几天、天数不同、口径不同、不能直接比……）的算对但没引用；假设、疑问、否定、「没法比」（差得远）不算；期望可比的
|
||||
正文怎么写都不算，只能靠引用。"""
|
||||
from app.engine import analysis_claims
|
||||
|
||||
exp = {**BAND, "op": "comparable"}
|
||||
words = _score(exp, fake_result("comparable", None, "incomparable"), _answer(text="本周还没过完，两周不能直接比。"))
|
||||
assert words["outcome"] == "wrong"
|
||||
cited = _score(exp, fake_result("comparable", None, "incomparable"),
|
||||
_answer(_record(fake_result("comparable", None, "incomparable"))))
|
||||
assert cited["outcome"] == "traced"
|
||||
incomparable = fake_result("comparable", None, "incomparable")
|
||||
for text, said in (("本周还没过完，两周不能直接比。", "本周还没过完"), ("本周数据不完整，不能直接比。", "本周数据不完整"),
|
||||
("两周的天数不同（7 天 vs 3 天），不宜直接比较。", "两周的天数不同（7 天 vs 3 天）"),
|
||||
("统计口径有调整，前后两期不可比。", "统计口径有调整"),
|
||||
("这周才过了三天，放在一起看会失真。", "这周才过了三天"),
|
||||
("不能直接拿本周和上周比较。", "不能直接拿本周和上周比较")):
|
||||
item = _score(exp, incomparable, _answer(text=text))
|
||||
assert item["outcome"] == "untraced", (text, item)
|
||||
assert item["found"] == {"judgement": "incomparable", "text": said}, item
|
||||
assert analysis_claims.scan(text) == [], "说法表照旧不认这类说法：说法核对不会打回"
|
||||
for text in ("如果口径不同，就不能直接比。", "是不是不能直接比？", "不是不能比，只是要按日均看。", "这周跟上周没法比，差远了。",
|
||||
"两周都过完了，可以直接比。", "订单信息不完整的有 3 单。"):
|
||||
item = _score(exp, incomparable, _answer(text=text))
|
||||
assert item["outcome"] == "wrong" and "没有做可比性检查" in item["reason"], (text, item)
|
||||
# 只有说了这类话的句子留下来（和说法表认出判断词的句子一起）
|
||||
assert [x["text"] for x in judgements.sentences_of_text("上周订单数是 1,203 单。本周还没过完，两周不能直接比。")] == \
|
||||
["本周还没过完，两周不能直接比。"]
|
||||
# 期望可比：正文怎么写都不算，只能靠引用
|
||||
comparable = fake_result("comparable", None, "comparable")
|
||||
for text in ("两周都是完整的 7 天，可以直接比。", "本周还没过完，两周不能直接比。"):
|
||||
assert _score(exp, comparable, _answer(text=text))["outcome"] == "wrong", text
|
||||
assert _score(exp, comparable, _answer(_record(comparable)))["outcome"] == "traced"
|
||||
assert _score(exp, incomparable, _answer(_record(incomparable)))["outcome"] == "traced"
|
||||
|
||||
|
||||
def test_an_analysis_that_cannot_be_computed_cannot_be_graded():
|
||||
|
||||
@@ -8,6 +8,8 @@ retail_trends（tests/fixtures/acceptance/retail_trends.py，按今天生成）
|
||||
- 重评老结果：明细里没有分析结论那一项的照样重评（从封存重新读）。
|
||||
- 从头到尾：助手剧本搭「查库 → 分析卡 → 报告」，报告引用判定语；评测给这一题「对且可追溯」，逐题 E1 等价已证明，作答运行和
|
||||
算期望用的是同一份分析设置快照（记进指纹）。
|
||||
- 分析工具开、关（变体 analysis.tools，「分析工具贡献」预设对照）：一次发起两臂；关的那一臂助手建图时工具清单里没有分析工具、
|
||||
节点手册里没有分析卡，作答运行里没有分析结论；开的那一臂照常；两臂的指纹、同一变体的指纹都不同。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -17,7 +19,7 @@ from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core import artifact_store
|
||||
@@ -237,6 +239,16 @@ def _line(op: dict) -> AIMessageChunk:
|
||||
return AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def _graph_lines(graph: dict, summary: str):
|
||||
yield _line({"op": "plan", "summary": summary})
|
||||
for n in graph["nodes"]:
|
||||
yield _line({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["data"]["label"],
|
||||
"config": n["data"]["config"]}})
|
||||
for e in graph["edges"]:
|
||||
yield _line({"op": "add_edge", "edge": e})
|
||||
yield _line({"op": "done", "explanation": "拆一下", "run": True})
|
||||
|
||||
|
||||
class AnalysisCopilot:
|
||||
"""助手剧本：直接搭「入口 → 查库 → 分析卡 → 报告 → 出口」并运行；上线前自查直接过。"""
|
||||
|
||||
@@ -244,14 +256,8 @@ class AnalysisCopilot:
|
||||
if "上线前自查" in messages[1][1]:
|
||||
yield _line({"op": "done"})
|
||||
return
|
||||
yield _line({"op": "plan", "summary": "按地区拆上周销售额的变化"})
|
||||
graph = answer_graph()
|
||||
for n in graph["nodes"]:
|
||||
yield _line({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["data"]["label"],
|
||||
"config": n["data"]["config"]}})
|
||||
for e in graph["edges"]:
|
||||
yield _line({"op": "add_edge", "edge": e})
|
||||
yield _line({"op": "done", "explanation": "拆一下", "run": True})
|
||||
for line in _graph_lines(answer_graph(), "按地区拆上周销售额的变化"):
|
||||
yield line
|
||||
|
||||
|
||||
async def test_an_analysis_case_end_to_end(monkeypatch, source):
|
||||
@@ -313,3 +319,112 @@ async def test_suites_without_analysis_expectations_take_no_settings_snapshot(mo
|
||||
"别的题集的指纹、作答运行的选项和以前一样"
|
||||
finally:
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 分析工具开、关（变体 analysis.tools；「分析工具贡献」预设对照）
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: 关的那一臂搭的图里，报告撰写的要求带这个记号：报告撰写的剧本据此不引用分析结论（那一臂没有）
|
||||
PLAIN_MARK = "只按查询结果写"
|
||||
PLAIN = "## 上周销售额\n\n各地区上周的销售额都在查询结果里。"
|
||||
|
||||
|
||||
def plain_graph() -> dict:
|
||||
"""没有分析卡的图：入口 → 查库 → 报告 → 出口。"""
|
||||
nodes = [_node("start", "input"),
|
||||
_node("q", "tool", tool=f"db_query__{retail_trends.SOURCE_NAME}",
|
||||
args={"sql": rendered(REGION["gold"]["sql"])}),
|
||||
_node("write", "report", instructions=f"{PLAIN_MARK}上周的周报"),
|
||||
_node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}])]
|
||||
ids = [n["id"] for n in nodes]
|
||||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||||
|
||||
|
||||
class SwitchCopilot:
|
||||
"""助手剧本：给的工具清单里有分析工具（开着的那一臂）就搭「查库 → 分析卡 → 报告」，没有（关掉的那一臂）就搭「查库 →
|
||||
报告」——照给它看的用。systems 记下每次建图收到的 system。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.systems: list[str] = []
|
||||
|
||||
async def astream(self, messages):
|
||||
if "上线前自查" in messages[1][1]:
|
||||
yield _line({"op": "done"})
|
||||
return
|
||||
role, system = messages[0]
|
||||
assert role == "system"
|
||||
self.systems.append(system)
|
||||
graph = answer_graph() if "analysis_decompose" in system else plain_graph()
|
||||
for line in _graph_lines(graph, "看上周销售额"):
|
||||
yield line
|
||||
|
||||
|
||||
def use_writers(monkeypatch) -> None:
|
||||
"""报告撰写的剧本：要求里带 PLAIN_MARK 的（关掉分析工具那一臂）写 PLAIN，别的写引用了 A1 的 REPORT。"""
|
||||
from app.providers import mock_model
|
||||
|
||||
def decide(self, messages):
|
||||
said = "\n".join(str(getattr(m, "content", "")) for m in messages)
|
||||
return AIMessage(content=PLAIN if PLAIN_MARK in said else REPORT)
|
||||
|
||||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||||
|
||||
|
||||
async def test_the_analysis_tools_preset_runs_with_and_without_them(monkeypatch, source):
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.eval import compare
|
||||
from app.main import app
|
||||
from app.tools.analysis import ANALYSIS_TOOLS
|
||||
|
||||
copilot = SwitchCopilot()
|
||||
es.use_copilot(monkeypatch, copilot)
|
||||
use_writers(monkeypatch)
|
||||
case = case_of(REGION, source["id"])
|
||||
suite_id = await es.make_suite(source["id"], cases=[{k: case[k] for k in ("question", "datasource_ids", "gold",
|
||||
"expectations")}])
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
started = await client.post(f"/api/evals/suites/{suite_id}/evaluations",
|
||||
json={"preset": "analysis_tools", "variant": {"repeats": 1}})
|
||||
assert started.status_code == 202, started.text
|
||||
body = started.json()
|
||||
on, off = body["evaluations"]
|
||||
assert (on["arm"], off["arm"]) == ("分析工具开", "分析工具关") and body["pair_id"] == on["id"] == off["pair_id"]
|
||||
assert on["variant"]["analysis"] == {"tools": "on"} and off["variant"]["analysis"] == {"tools": "off"}
|
||||
assert on["variant"]["catalog"] == off["variant"]["catalog"] == {"mode": "live", "snapshot": None}
|
||||
assert "· 分析工具开 ·" in on["variant_text"] and "· 分析工具关 ·" in off["variant_text"], off["variant_text"]
|
||||
await es.wait_eval(on["id"])
|
||||
pair = (await client.get(f"/api/evals/pairs/{body['pair_id']}")).json()
|
||||
assert [a["arm"] for a in pair["table"]["arms"]] == ["分析工具开", "分析工具关"]
|
||||
|
||||
async with SessionLocal() as session:
|
||||
runs = {r.arm: r for r in (await session.execute(select(EvalRun).where(EvalRun.pair_id == on["id"]))).scalars()}
|
||||
items = {arm: (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == r.id))).scalar_one()
|
||||
for arm, r in runs.items()}
|
||||
answer_runs = {arm: await session.get(Run, item.run_id) for arm, item in items.items()}
|
||||
assert all(r.status == "succeeded" for r in runs.values()), [(r.arm, r.error) for r in runs.values()]
|
||||
# 建图：两臂各一次，关的那一臂工具清单里没有分析工具、节点手册里没有分析卡；开的那一臂都有
|
||||
assert len(copilot.systems) == 2
|
||||
with_tools = [s for s in copilot.systems if all(name in s for name in ANALYSIS_TOOLS)]
|
||||
without = [s for s in copilot.systems if not any(name in s for name in ANALYSIS_TOOLS)]
|
||||
assert len(with_tools) == len(without) == 1
|
||||
assert "- analysis：分析卡" in with_tools[0] and "- analysis：分析卡" not in without[0]
|
||||
|
||||
# 开的那一臂照常：图里有分析卡，答案引用了分析结论，对且可追溯
|
||||
lit, dark = items["分析工具开"], items["分析工具关"]
|
||||
assert {n["type"] for n in answer_runs["分析工具开"].graph["nodes"]} >= {"analysis", "report"}
|
||||
assert lit.verdict == "pass" and lit.scores[0]["outcome"] == "traced", lit.scores
|
||||
assert [a["alias"] for a in (await score.read_answer(lit.run_id)).analyses] == ["A1"]
|
||||
# 关的那一臂：图里没有分析卡，作答运行里没有分析结论；正文没写这个判断，算错
|
||||
assert "analysis" not in {n["type"] for n in answer_runs["分析工具关"].graph["nodes"]}
|
||||
assert (await score.read_answer(dark.run_id)).analyses == []
|
||||
assert dark.verdict == "fail" and dark.scores[0]["outcome"] == "wrong", dark.scores
|
||||
assert "没有做变化拆解" in dark.scores[0]["reason"], dark.scores[0]
|
||||
# 两臂共用期望；指纹、同一变体的指纹都不同，对比照实写「分析工具：开 → 关」
|
||||
assert lit.gold_run_ids == dark.gold_run_ids
|
||||
fp_on, fp_off = runs["分析工具开"].fingerprint, runs["分析工具关"].fingerprint
|
||||
assert (fp_on["analysis_tools"], fp_off["analysis_tools"]) == ("on", "off")
|
||||
assert fp_on["variant_key"] != fp_off["variant_key"]
|
||||
assert [d for d in compare.fingerprint_diff(fp_on, fp_off) if d["key"] == "analysis_tools"] == \
|
||||
[{"key": "analysis_tools", "label": "分析工具", "before": "开", "after": "关"}]
|
||||
|
||||
@@ -51,6 +51,16 @@ def test_fingerprint_diff_lists_what_changed():
|
||||
assert "before" not in diff[1], "表结构只说变没变"
|
||||
|
||||
|
||||
def test_the_analysis_tools_show_as_on_and_off():
|
||||
"""分析工具这一项写开、关；加这一项之前的评测指纹里没有它，那时一直开着：和现在开着的比不算变了。"""
|
||||
old = {"assistant_model": "m1", "catalog": "live:aaa", "scorer_version": 3}
|
||||
on, off = {**old, "analysis_tools": "on"}, {**old, "analysis_tools": "off"}
|
||||
assert compare.fingerprint_diff(old, on) == [] and compare.fingerprint_diff(on, on) == []
|
||||
item = {"key": "analysis_tools", "label": "分析工具"}
|
||||
assert compare.fingerprint_diff(on, off) == [{**item, "before": "开", "after": "关"}]
|
||||
assert compare.fingerprint_diff(off, old) == [{**item, "before": "关", "after": "开"}]
|
||||
|
||||
|
||||
def test_the_pair_table():
|
||||
a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")]
|
||||
b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")]
|
||||
|
||||
@@ -444,6 +444,67 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch
|
||||
assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid"
|
||||
|
||||
|
||||
async def test_the_variant_switches_the_analysis_tools(client, monkeypatch):
|
||||
"""变体的分析工具（analysis.tools）：缺省开，只认 on、off；同一变体的指纹在开着时和加这一项之前一样（以前的评测照样
|
||||
当基准），关着时不同；变体那一句写「分析工具开 / 关」。「分析工具贡献」预设一次发起开、关两臂，关的那一臂作答的 Driver
|
||||
不给助手分析工具（TurnDriver 的 analysis_tools 透传给 answer_ops，见 test_copilot_analysis）。"""
|
||||
from app.eval import variants
|
||||
from app.eval.cases import CaseInvalid
|
||||
|
||||
on = variants.normalize_variant({})
|
||||
off = variants.normalize_variant({"analysis": {"tools": "off"}})
|
||||
assert on["analysis"] == {"tools": "on"} and off["analysis"] == {"tools": "off"}
|
||||
for bad in ("off", {"tools": "maybe"}, {"tools": None}, {"tools": "off", "mode": "x"}):
|
||||
with pytest.raises(CaseInvalid) as caught:
|
||||
variants.normalize_variant({"analysis": bad})
|
||||
assert (caught.value.field, caught.value.code) == ("variant.analysis", variants.VARIANT_INVALID), bad
|
||||
models = {"assistant": "mock-fast", "writer": None}
|
||||
before = {k: v for k, v in on.items() if k != "analysis"}
|
||||
assert variants.variant_key(on, models) == variants.variant_key(before, models), "开着时和加这一项之前一样"
|
||||
assert variants.variant_key(off, models) != variants.variant_key(on, models)
|
||||
assert variants.describe(before, models) == variants.describe(on, models) \
|
||||
== "助手 mock-fast · 写作 各节点自己的 · 当前目录 · 分析工具开 · 结论裁判关 · 每题 1 次"
|
||||
assert "· 分析工具关 ·" in variants.describe(off, models)
|
||||
arms = variants.analysis_tools_preset({**off, "catalog": {"mode": "off", "snapshot": None}, "repeats": 3,
|
||||
"label": "x"})
|
||||
assert [(a["label"], a["analysis"], a["catalog"]["mode"], a["repeats"]) for a in arms] == [
|
||||
("分析工具开", {"tools": "on"}, "off", 3), ("分析工具关", {"tools": "off"}, "off", 3)]
|
||||
|
||||
seen: list[tuple[str | None, bool]] = []
|
||||
|
||||
async def drive(self):
|
||||
seen.append((self.ctx.arm, self.analysis_tools))
|
||||
self.turn.path = "asked"
|
||||
return self.turn
|
||||
|
||||
monkeypatch.setattr(answer_mod.Driver, "drive", drive)
|
||||
suite_id = await es.make_suite("none", cases=ask_cases(1))
|
||||
estimate = (await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate",
|
||||
json={"preset": "analysis_tools"})).json()
|
||||
assert [(a["label"], a["variant"]["analysis"]["tools"]) for a in estimate["arms"]] == [
|
||||
("分析工具开", "on"), ("分析工具关", "off")]
|
||||
assert estimate["repeats"] == variants.PAIR_REPEATS and "分析工具关" in estimate["arms"][1]["text"]
|
||||
started = await client.post(f"/api/evals/suites/{suite_id}/evaluations",
|
||||
json={"preset": "analysis_tools", "variant": {"repeats": 1}})
|
||||
assert started.status_code == 202, started.text
|
||||
lit, dark = started.json()["evaluations"]
|
||||
await es.wait_eval(lit["id"])
|
||||
assert sorted(seen) == [("分析工具关", False), ("分析工具开", True)]
|
||||
assert (lit["fingerprint"]["analysis_tools"], dark["fingerprint"]["analysis_tools"]) == ("on", "off")
|
||||
assert lit["fingerprint"]["variant_key"] != dark["fingerprint"]["variant_key"]
|
||||
# 一臂、A/B 里写的照样认；不合规的照实拒、落到那一格
|
||||
one = await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate",
|
||||
json={"variant": {"analysis": {"tools": "off"}}})
|
||||
assert one.json()["arms"][0]["variant"]["analysis"] == {"tools": "off"}
|
||||
bad = await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate",
|
||||
json={"arms": [{"label": "开"}, {"label": "关", "analysis": {"tools": "nope"}}]})
|
||||
assert bad.status_code == 422
|
||||
assert (bad.json()["code"], bad.json()["field"]) == ("eval_variant_invalid", "variant.analysis")
|
||||
unknown = await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate", json={"preset": "tools"})
|
||||
assert unknown.status_code == 422 and unknown.json()["field"] == "preset"
|
||||
assert "analysis_tools" in unknown.json()["detail"]
|
||||
|
||||
|
||||
async def test_the_estimate_says_it_cannot_price_instead_of_zero():
|
||||
"""上一次评测的模型都不在价格目录里（价格目录空）：估算以前写「约 $0」，像不花钱。现在一分都没估出来的写估不出；估出了
|
||||
一部分的写「至少」、另有几次估不出（老总表里每题平均记成 0、却有估不出的调用的，也按估不出算）。"""
|
||||
|
||||
@@ -92,6 +92,17 @@ def test_scores_compare_and_catalog_modes():
|
||||
assert _labels("start.catalogMode") == list(CATALOG_MODES)
|
||||
|
||||
|
||||
def test_analysis_tools_and_presets_follow_the_server():
|
||||
"""变体的分析工具开、关和预设对照（api/evals.PRESETS）：前端的取值和服务端一一对上。叫法写「开」「关」，和服务端
|
||||
describe、对比里写的同一种说法。"""
|
||||
from app.api.evals import PRESETS
|
||||
from app.eval.variants import ANALYSIS_TOOLS, ANALYSIS_TOOLS_LABEL
|
||||
|
||||
assert _union("EvalAnalysisTools") == list(ANALYSIS_TOOLS)
|
||||
assert _pairs("start.analysisToolsMode") == list(ANALYSIS_TOOLS_LABEL.items())
|
||||
assert _union("EvalPreset") == list(PRESETS)
|
||||
|
||||
|
||||
def test_signal_bands_follow_the_server():
|
||||
from app.api.evals import SIGNAL_CRITERIA
|
||||
|
||||
|
||||
@@ -31,11 +31,12 @@ Rules:
|
||||
|
||||
### Run an evaluation
|
||||
|
||||
1. Choose the variants: Copilot model, writing model, judge on or off, catalog (current / snapshot / verified only / off), and how many repetitions.
|
||||
2. Check the estimated cost, then start. The Catalog contribution preset runs "current vs. off" in one go.
|
||||
1. Choose the variants: Copilot model, writing model, judge on or off, catalog (current / snapshot / verified only / off), analysis tools (on / off, on by default), and how many repetitions.
|
||||
2. Check the estimated cost, then start. The Catalog contribution preset runs "current vs. off" in one go, and the Analysis tools contribution preset runs "analysis tools on vs. off".
|
||||
|
||||
- Evaluations run in the background, and each question goes exactly the way a user's question would in Ask Data (graph building, run, review). Questions that stop at an approval aren't approved on anyone's behalf and are recorded as they are.
|
||||
- When Copilot returns only thinking and no operations, the question is retried once automatically.
|
||||
- In an arm with analysis tools off, Copilot doesn't see the analysis tools or the analysis card when building the graph, and a graph that still binds analysis tools is sent back to be fixed; workflow questions (which test a pinned version of a published workflow) build no graph and aren't affected. Analysis tools on and off count as different variants, and a comparison with the previous evaluation shows "Analysis tools: on → off".
|
||||
- Concurrency, per-question time limits, and per-evaluation and daily spending all have caps, set under Settings → Preferences → Evaluation. Evaluation spending is tallied separately.
|
||||
- A question's time limit doesn't include time spent queuing. Under full load, evaluations queue behind Ask Data (see [Run concurrency](postgresql-and-scaling.md#run-concurrency)).
|
||||
- An evaluation can be stopped, and you can retry only the unfinished questions. A service restart marks it interrupted, and it doesn't resume by itself.
|
||||
@@ -50,7 +51,7 @@ Rules:
|
||||
Contribution and verdict expectations are scored like this:
|
||||
|
||||
- **Correct and traceable**: the answer cites an analysis conclusion whose verdict matches (the same analysis; for a main source, it names the expected segments with the same direction). If the sentence citing it says the opposite, it is wrong.
|
||||
- **Correct but uncited**: no matching conclusion is cited, but the body states it correctly by the lexicon (the same judgment words and binding rules; see [Analysis claim check](analysis.md#analysis-claim-check)). Comparability has no judgment words, so only a citation counts.
|
||||
- **Correct but uncited**: no matching conclusion is cited, but the body states it correctly by the lexicon (the same judgment words and binding rules; see [Analysis claim check](analysis.md#analysis-claim-check)). Comparability isn't in the lexicon: when the expectation is not comparable, a body that plainly says the two periods can't be compared directly (the current period is incomplete or not over yet, the numbers of days differ, the calibers differ, and so on; statements that are hypothetical, questions, or negated don't count) also falls here; when the expectation is comparable, only a citation counts.
|
||||
- **Wrong**: the cited conclusion's verdict doesn't match, the body says the opposite (calling a usual fluctuation unusual, rising falling, or naming another segment as the main source), the analysis was done but not stated, or it wasn't done at all.
|
||||
- **Ungradable**: the expectation can't be computed.
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ This page lists, newest first, the changes in each update that affect users and
|
||||
- The report export and the verifiable report include analysis conclusions and their input snapshots, recomputed from the sealed inputs for verification.
|
||||
- Settings gain a Preferences → Analysis section (see [Analysis settings](configuration.md#analysis-settings)). It is snapshotted into a run only when the graph has an analysis card or an agent with analysis tools, so the start records of other runs don't change at all.
|
||||
- Ask Data: Copilot gains a graph shape for analytical questions, and its self-check reminds it to give the querying agent the analysis tools (one round only). The review merges several analyses that couldn't be done into one "n analyses couldn't be done" note, without an automatic redo; analysis calls that were wrong but later fixed no longer count as a gap, and ones never fixed no longer trigger a redo of the whole turn.
|
||||
- Evaluation gains contribution and verdict expectations (computed from the expected SQL with the same analyses, never typed in by hand), and the scoring rules move to version 3, so old evaluations can be re-scored under the new rules; Copilot's prompt fingerprint now includes the analysis method, so comparisons with evaluations from before the upgrade flag "Prompt changed" (see [Regression evaluation](evaluation-and-ledger.md#regression-evaluation)).
|
||||
- Evaluation gains contribution and verdict expectations (computed from the expected SQL with the same analyses, never typed in by hand), and the scoring rules move to version 3, so old evaluations can be re-scored under the new rules; Copilot's prompt fingerprint now includes the analysis method, so comparisons with evaluations from before the upgrade flag "Prompt changed"; variants gain an analysis tools switch (on by default) and an Analysis tools contribution preset, and a variant with analysis tools on counts as the same variant as evaluations from before the upgrade (see [Regression evaluation](evaluation-and-ledger.md#regression-evaluation)).
|
||||
|
||||
## 2026-10, orchestration improvements wave 2: PostgreSQL and multiple processes
|
||||
|
||||
|
||||
@@ -31,11 +31,12 @@
|
||||
|
||||
### 发起评测
|
||||
|
||||
1. 选变体：助手模型、写作模型、裁判开关、目录（当前 / 快照 / 只用已核实的 / 关）、重复几次。
|
||||
2. 看预估花费，确认发起。「目录贡献」预设一次跑「当前 vs 关」。
|
||||
1. 选变体：助手模型、写作模型、裁判开关、目录（当前 / 快照 / 只用已核实的 / 关）、分析工具（开 / 关，缺省开）、重复几次。
|
||||
2. 看预估花费，确认发起。「目录贡献」预设一次跑「当前 vs 关」，「分析工具贡献」预设一次跑「分析工具开 vs 关」。
|
||||
|
||||
- 评测在后台跑，每题怎么走和用户在问数据里拿到的一样（建图、运行、复核）。停在审批上的不代批，照实记。
|
||||
- 助手只给了思考、一个操作都没给时，自动重来一次。
|
||||
- 分析工具关掉的那一臂，助手建图时看不到分析工具和分析卡，图里还绑着分析工具的会被挡回去改；工作流题（考一个已发布工作流的固定版本）不建图，不受这一项影响。分析工具开、关不同算不同的变体，和上一次比时写「分析工具：开 → 关」。
|
||||
- 并发、每题时限、单次和每日金额都有上限，在「设置 → 偏好设置 → 评测」里改。评测的花费单独记账。
|
||||
- 每题的时限不算排队的时间。满载时评测会排在问数据后面（见[运行并发](postgresql-and-scaling.md#运行并发)）。
|
||||
- 能停，能只重试没做完的题。服务重启时标成中断，不会自己续跑。
|
||||
@@ -50,7 +51,7 @@
|
||||
贡献、判定两种期望这样评：
|
||||
|
||||
- **对且可追溯**：答案引用了判定对得上的分析结论（同一种分析，主要来源点名的是期望的那几段、方向一致）；引用它的那句话却写反了的算错。
|
||||
- **对但没引用**：没引用对得上的结论，但正文按说法表写对了（同一套判断词和绑定规则，见[分析说法核对](analysis.md#分析说法核对)）。可比性没有对应的判断词，只能靠引用。
|
||||
- **对但没引用**：没引用对得上的结论，但正文按说法表写对了（同一套判断词和绑定规则，见[分析说法核对](analysis.md#分析说法核对)）。可比性不进说法表：期望是不可比的，正文明确说了两期不能直接比（本期不完整、还没过完、天数不同、口径不同……，带假设、疑问或否定的不算）也算这一种；期望是可比的只能靠引用。
|
||||
- **错**：引用的结论判定对不上、正文写反了（把常规波动说成异常、把上升说成下降、主要来源说成别的段）、做了分析没说，或者根本没做。
|
||||
- **不能评分**：期望算不了。
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
- 报告导出、可验证报告带上分析结论和它的输入快照，按封存的输入复算核对。
|
||||
- 设置多了「偏好设置 → 分析」一节（见[分析设置](configuration.md#分析设置)）。只在图里有分析卡、或者 agent 绑了分析工具时，发起运行才把这一组快照进运行，别的运行的发起记录一字不变。
|
||||
- 问数据：助手建图多了分析类问题的形状，自查时提醒查库的 agent 绑分析工具（只提醒一轮）。复核把几次没做成的分析并成一条「有 n 次分析没做成」，只写说明、不自动重来；分析调用写错了、后来改对的不再记成缺口，一直没改对的也不再触发整轮重来。
|
||||
- 评测多了贡献、判定两种期望（由期望 SQL 和同一套分析算出，不许人填），评分规则升到第 3 版，老评测可以按新规则重评；助手提示词的指纹包含了分析方法，和升级前的评测比会标「提示词变了」（见[回归评测](evaluation-and-ledger.md#回归评测)）。
|
||||
- 评测多了贡献、判定两种期望（由期望 SQL 和同一套分析算出，不许人填），评分规则升到第 3 版，老评测可以按新规则重评；助手提示词的指纹包含了分析方法，和升级前的评测比会标「提示词变了」；变体多了「分析工具」开关（缺省开）和「分析工具贡献」预设对照，开着的和升级前的评测算同一变体（见[回归评测](evaluation-and-ledger.md#回归评测)）。
|
||||
|
||||
## 2026-10 编排完善第 2 波：PostgreSQL 与多进程
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
* 统计、结论、能不能做都由服务端给，这里只有叫法和句式：数字一律是服务端算好交来的，这里只负责写成话。
|
||||
*/
|
||||
import type {
|
||||
EvalAnalysisMode, EvalAnalysisOp, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind,
|
||||
EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger,
|
||||
EvalUpgradePolicy, EvalVerdict,
|
||||
EvalAnalysisMode, EvalAnalysisOp, EvalAnalysisTools, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind,
|
||||
EvalGoldKind, EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus,
|
||||
EvalTrigger, EvalUpgradePolicy, EvalVerdict,
|
||||
} from '../typesEval'
|
||||
import { formatCost, formatNumber } from './format'
|
||||
|
||||
@@ -380,12 +380,15 @@ export const EVAL_TEXT = {
|
||||
start: {
|
||||
title: '发起评测',
|
||||
mode: '评测方式',
|
||||
modes: { single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照' } as Record<'single' | 'ab' | 'catalog', string>,
|
||||
modes: {
|
||||
single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照', analysis_tools: '分析工具贡献对照',
|
||||
} as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
|
||||
modeHint: {
|
||||
single: '按一组变体评一遍题集',
|
||||
ab: '两到三臂各自一组变体，共用同一份期望答案，同一题几臂同时作答',
|
||||
catalog: '预设的 A/B：当前目录 vs 关目录，可以再加一臂「只用已核实和已确认的项」，量出数据目录帮了多少',
|
||||
} as Record<'single' | 'ab' | 'catalog', string>,
|
||||
analysis_tools: '预设的 A/B：分析工具开 vs 关，量出分析工具（变化拆解、波动基线……）帮了多少',
|
||||
} as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
|
||||
arm: (name: string) => `${name} 臂`,
|
||||
addArm: '再加一臂',
|
||||
removeArm: '去掉这一臂',
|
||||
@@ -407,6 +410,9 @@ export const EVAL_TEXT = {
|
||||
catalogHint: '当前目录、只用已核实的项都在开评时拍一份快照按它读：评测期间目录被改也不影响可比性',
|
||||
snapshot: '选一份目录快照',
|
||||
snapshotNone: '还没有可用的目录快照',
|
||||
analysisTools: '分析工具',
|
||||
analysisToolsMode: { on: '开', off: '关' } as Record<EvalAnalysisTools, string>,
|
||||
analysisToolsHint: '关掉时助手建图看不到分析工具和分析卡；工作流题按固定的版本运行，不受影响',
|
||||
repeats: '每题重复几次',
|
||||
repeatsHint: '模型有随机性：对照建议每题 3 次',
|
||||
reviewedArm: '再加一臂「只用已核实和已确认的项」',
|
||||
@@ -586,7 +592,7 @@ export const EVAL_TEXT = {
|
||||
baseText: (short: string, when: string) => `${short} · ${when}`,
|
||||
none: '没有可比的上一次：同一题集、同一变体里还没有比这次先发起、已经评完的评测',
|
||||
fingerprint: '两次之间变了什么',
|
||||
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则、分析设置都没变',
|
||||
fingerprintNone: '模型、目录、分析工具、提示词、表结构、题目、评分规则、分析设置都没变',
|
||||
changed: '变了',
|
||||
transitions: '逐题转移',
|
||||
transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record<EvalTransition, string>,
|
||||
|
||||
@@ -6,13 +6,16 @@ import { Field, Modal, Notice, Spinner, toast, useRadioGroup } from '../../compo
|
||||
import { errorMessage } from '../../lib/errors'
|
||||
import { formatCost, formatDateTime, shortId } from '../../lib/format'
|
||||
import { EVAL_TEXT } from '../../lib/termsEval'
|
||||
import type { EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant } from '../../typesEval'
|
||||
import type {
|
||||
EvalAnalysisTools, EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant,
|
||||
} from '../../typesEval'
|
||||
import { ModelPick } from './parts'
|
||||
|
||||
const T = EVAL_TEXT.start
|
||||
const MODES = ['single', 'ab', 'catalog'] as const
|
||||
const MODES = ['single', 'ab', 'catalog', 'analysis_tools'] as const
|
||||
type Mode = (typeof MODES)[number]
|
||||
const CATALOG_MODES: EvalCatalogMode[] = ['live', 'reviewed', 'off', 'snapshot']
|
||||
const ANALYSIS_TOOLS: EvalAnalysisTools[] = ['on', 'off']
|
||||
|
||||
/** 一臂的变体（表单里的样子）：全部键都在，交上去之前不改形状 */
|
||||
export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
|
||||
@@ -21,6 +24,7 @@ export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
|
||||
writer: { provider: base?.writer?.provider ?? null, model: base?.writer?.model ?? null },
|
||||
judge: base?.judge ?? false,
|
||||
catalog: { mode: base?.catalog?.mode ?? 'live', snapshot: base?.catalog?.snapshot ?? null },
|
||||
analysis: { tools: base?.analysis?.tools ?? 'on' },
|
||||
repeats: base?.repeats ?? 1,
|
||||
label: base?.label ?? null,
|
||||
}
|
||||
@@ -37,12 +41,13 @@ export function snapshotLabel(s: EvalSnapshot): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* 一臂变体的表单：助手、写作模型（缺省按设置），结论裁判，数据目录（当前 / 只用已核实的 / 关 / 快照），重复次数，名字。
|
||||
* 题集设置里的「默认变体」也用它（定时评测按默认变体评）
|
||||
* 一臂变体的表单：助手、写作模型（缺省按设置），结论裁判，数据目录（当前 / 只用已核实的 / 关 / 快照），分析工具（开 / 关），
|
||||
* 重复次数，名字。题集设置里的「默认变体」也用它（定时评测按默认变体评）。预设对照里由预设定的那一项（目录、分析工具）不摆
|
||||
*/
|
||||
export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false }: {
|
||||
export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false,
|
||||
hideAnalysis = false }: {
|
||||
idPrefix: string; value: EvalVariant; onChange: (v: EvalVariant) => void; snapshots: EvalSnapshot[]
|
||||
withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean
|
||||
withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean; hideAnalysis?: boolean
|
||||
}) {
|
||||
const set = (patch: Partial<EvalVariant>) => onChange({ ...value, ...patch })
|
||||
const usable = usableSnapshots(snapshots)
|
||||
@@ -82,6 +87,14 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
{!hideAnalysis && (
|
||||
<Field label={T.analysisTools} htmlFor={`${idPrefix}-analysis`} hint={T.analysisToolsHint}>
|
||||
<select id={`${idPrefix}-analysis`} className="field w-24" value={value.analysis.tools} data-eval-analysis-tools=""
|
||||
onChange={(e) => set({ analysis: { tools: e.target.value as EvalAnalysisTools } })}>
|
||||
{ANALYSIS_TOOLS.map((m) => <option key={m} value={m}>{T.analysisToolsMode[m]}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
<Field label={T.repeats} htmlFor={`${idPrefix}-repeats`} hint={T.repeatsHint}>
|
||||
<select id={`${idPrefix}-repeats`} className="field w-24" value={value.repeats} data-eval-repeats=""
|
||||
onChange={(e) => set({ repeats: Number(e.target.value) })}>
|
||||
@@ -98,8 +111,8 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
|
||||
}
|
||||
|
||||
/**
|
||||
* 发起评测：评一次 / A/B 对照（两到三臂） / 目录贡献对照（预设）。填好就向服务端要估算（几项、按上一次每题平均花费估的金额、
|
||||
* 生效的预算、同时评几题、今天评测已经花了多少）——估算和发起是同一份请求体，估出来的就是要发起的那一次。
|
||||
* 发起评测：评一次 / A/B 对照（两到三臂） / 目录贡献对照、分析工具贡献对照（预设）。填好就向服务端要估算（几项、按上一次每题
|
||||
* 平均花费估的金额、生效的预算、同时评几题、今天评测已经花了多少）——估算和发起是同一份请求体，估出来的就是要发起的那一次。
|
||||
*/
|
||||
export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
open: boolean; onClose: () => void; suite: EvalSuite; evaluations: EvalRun[]; onStarted: (run: EvalRun) => void
|
||||
@@ -132,7 +145,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
const repeats = Math.max(first.repeats, 3)
|
||||
return cur.length >= 2 ? cur : [{ ...first, repeats, label: first.label || 'A' }, { ...first, repeats, label: 'B' }]
|
||||
}
|
||||
return [m === 'catalog' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
|
||||
return [m === 'catalog' || m === 'analysis_tools' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
|
||||
})
|
||||
}
|
||||
|
||||
@@ -144,6 +157,10 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
const { catalog: _c, label: _l, ...rest } = v
|
||||
return { preset: reviewedArm ? 'catalog_reviewed' : 'catalog', variant: rest, ...base }
|
||||
}
|
||||
if (mode === 'analysis_tools') {
|
||||
const { analysis: _a, label: _l, ...rest } = v
|
||||
return { preset: 'analysis_tools', variant: rest, ...base }
|
||||
}
|
||||
return { variant: v, ...base }
|
||||
}, [mode, arms, reviewedArm, baseline])
|
||||
const key = JSON.stringify(body)
|
||||
@@ -207,7 +224,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
|
||||
{mode !== 'ab' && (
|
||||
<ArmFields idPrefix="eval-arm-0" value={arms[0] ?? blankVariant()} snapshots={snapshots} hideCatalog={mode === 'catalog'}
|
||||
onChange={(v) => setArms([v])} />
|
||||
hideAnalysis={mode === 'analysis_tools'} onChange={(v) => setArms([v])} />
|
||||
)}
|
||||
{mode === 'catalog' && (
|
||||
<label className="flex items-center gap-2 text-xs">
|
||||
|
||||
@@ -370,26 +370,33 @@ export interface EvalImported {
|
||||
|
||||
export type EvalCatalogMode = 'live' | 'reviewed' | 'off' | 'snapshot'
|
||||
|
||||
/** 分析工具开、关（关的那一臂助手建图时看不到分析工具和分析卡） */
|
||||
export type EvalAnalysisTools = 'on' | 'off'
|
||||
|
||||
/** 预设对照：目录贡献（当前目录 vs 关，可加「只用已核实和已确认的项」）、分析工具贡献（开 vs 关） */
|
||||
export type EvalPreset = 'catalog' | 'catalog_reviewed' | 'analysis_tools'
|
||||
|
||||
export interface EvalModelPair {
|
||||
provider: string | null
|
||||
model: string | null
|
||||
}
|
||||
|
||||
/** 一臂变体：助手、写作模型，结论裁判，目录，重复次数，名字 */
|
||||
/** 一臂变体：助手、写作模型，结论裁判，目录，分析工具，重复次数，名字。加分析工具之前存下的评测记录里没有 analysis（当开） */
|
||||
export interface EvalVariant {
|
||||
assistant: EvalModelPair
|
||||
writer: EvalModelPair
|
||||
judge: boolean
|
||||
catalog: { mode: EvalCatalogMode; snapshot: string | null }
|
||||
analysis: { tools: EvalAnalysisTools }
|
||||
repeats: number
|
||||
label: string | null
|
||||
}
|
||||
|
||||
/** 发起时交上去的：一臂（variant）、几臂的 A/B（arms），或预设的「目录贡献对照」（preset） */
|
||||
/** 发起时交上去的：一臂（variant）、几臂的 A/B（arms），或预设的对照（preset：目录贡献、分析工具贡献） */
|
||||
export interface EvalStartBody {
|
||||
variant?: Partial<EvalVariant>
|
||||
arms?: Partial<EvalVariant>[]
|
||||
preset?: 'catalog' | 'catalog_reviewed'
|
||||
preset?: EvalPreset
|
||||
baseline_id?: string | null
|
||||
}
|
||||
|
||||
|
||||
+49
-7
@@ -135,7 +135,7 @@ const QUERIES = [
|
||||
row_count: 5, truncated: false, masked: true },
|
||||
]
|
||||
const VARIANT = { assistant: { provider: null, model: null }, writer: { provider: null, model: null }, judge: false,
|
||||
catalog: { mode: 'live', snapshot: null }, repeats: 1, label: null }
|
||||
catalog: { mode: 'live', snapshot: null }, analysis: { tools: 'on' }, repeats: 1, label: null }
|
||||
const SUMMARY = {
|
||||
scorer_version: 1, items: 3, done: 3, cases: 3, verdicts: { pass: 2, fail: 1, ungradable: 0, skipped: 0 }, not_run: 0,
|
||||
accuracy: RATE(2, 3), traced_accuracy: RATE(1, 3), outcomes: { traced: 1, untraced: 1, wrong: 1 }, text: {},
|
||||
@@ -152,7 +152,7 @@ const SUMMARY = {
|
||||
}
|
||||
const EVAL_DONE = {
|
||||
id: 'ev-1', suite_id: 'suite-a', status: 'succeeded', trigger: 'manual', schedule_id: null, variant: VARIANT,
|
||||
resolved_models: { assistant: 'model-a', writer: null, judge: false }, variant_text: '助手 model-a · 写作 各节点自己的 · 当前目录 · 结论裁判关 · 每题 1 次',
|
||||
resolved_models: { assistant: 'model-a', writer: null, judge: false }, variant_text: '助手 model-a · 写作 各节点自己的 · 当前目录 · 分析工具开 · 结论裁判关 · 每题 1 次',
|
||||
catalog_snapshot: 'snap-live', as_of: iso(NOW - 2 * H), repeats: 1, fingerprint: {}, pair_id: null, arm: null, baseline_id: 'ev-old',
|
||||
pinned: false, budget: BUDGET, counts: { total: 3, done: 3, passed: 2, failed: 1 }, usage: {}, summary: SUMMARY, stop_reason: null,
|
||||
error: null, created_by: '检查员甲', created_at: iso(NOW - 2 * H), started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H),
|
||||
@@ -219,7 +219,8 @@ const EVAL_AN = { ...EVAL_DONE, id: 'ev-an', counts: { total: 4, done: 4, passed
|
||||
outcomes_by_kind: { contribution: { traced: 1 }, verdict: { untraced: 1, wrong: 1, ungradable: 1 } } } }
|
||||
const COMPARE = {
|
||||
base: EVAL_OLD, cleaned: false,
|
||||
fingerprint_diff: [{ key: 'writer_model', label: '写作模型', before: 'model-a', after: 'model-b' }, { key: 'schemas', label: '表结构' }],
|
||||
fingerprint_diff: [{ key: 'writer_model', label: '写作模型', before: 'model-a', after: 'model-b' },
|
||||
{ key: 'analysis_tools', label: '分析工具', before: '开', after: '关' }, { key: 'schemas', label: '表结构' }],
|
||||
transitions: { counts: { regressed: 1, improved: 1, same: 1, incomparable: 0 }, causes: { data: 0, method: 1, params: 1, path: 0, unknown: 0 },
|
||||
items: [{ case_id: 'case-2', case_seq: 2, before: 'pass', after: 'fail', transition: 'regressed', cause: 'method' },
|
||||
{ case_id: 'case-1', case_seq: 1, before: 'fail', after: 'pass', transition: 'improved', cause: 'params' },
|
||||
@@ -227,12 +228,21 @@ const COMPARE = {
|
||||
}
|
||||
const PAIR_A = { ...EVAL_DONE, id: 'ev-pa', pair_id: 'ev-pa', arm: '当前目录', trigger: 'pair' }
|
||||
const PAIR_B = { ...EVAL_DONE, id: 'ev-pb', pair_id: 'ev-pa', arm: '关目录', trigger: 'pair' }
|
||||
// 分析工具贡献对照的两臂：名字、变体那一句都是服务端给的（variants.analysis_tools_preset、describe）
|
||||
const TOOLS_TEXT = (on) => `助手 model-a · 写作 各节点自己的 · 当前目录 · 分析工具${on ? '开' : '关'} · 结论裁判关 · 每题 3 次`
|
||||
const PAIR_TA = { ...EVAL_DONE, id: 'ev-ta', pair_id: 'ev-ta', arm: '分析工具开', trigger: 'pair', variant_text: TOOLS_TEXT(true),
|
||||
variant: { ...VARIANT, repeats: 3, label: '分析工具开' } }
|
||||
const PAIR_TB = { ...EVAL_DONE, id: 'ev-tb', pair_id: 'ev-ta', arm: '分析工具关', trigger: 'pair', variant_text: TOOLS_TEXT(false),
|
||||
variant: { ...VARIANT, analysis: { tools: 'off' }, repeats: 3, label: '分析工具关' } }
|
||||
const PAIR_TOOLS = { pair_id: 'ev-ta', evaluations: [PAIR_TA, PAIR_TB], cleaned: false,
|
||||
table: { arms: [{ arm: '分析工具开', accuracy: RATE(3, 4) }, { arm: '分析工具关', accuracy: RATE(2, 4) }],
|
||||
pairs: [{ a: '分析工具开', b: '分析工具关', both: 2, neither: 1, only_a: 1, only_b: 0, excluded: 0, p_value: 1, note: '这批题上看不出差别' }] } }
|
||||
const PAIR = { pair_id: 'ev-pa', evaluations: [PAIR_A, PAIR_B], cleaned: false,
|
||||
table: { arms: [{ arm: '当前目录', accuracy: RATE(3, 4) }, { arm: '关目录', accuracy: RATE(1, 4) }],
|
||||
pairs: [{ a: '当前目录', b: '关目录', both: 1, neither: 1, only_a: 2, only_b: 0, excluded: 0, p_value: 0.5, note: '这批题上看不出差别' }] } }
|
||||
const ESTIMATE = (arms) => ({
|
||||
arms: arms.map((a, i) => ({ label: arms.length > 1 ? a : null, variant: VARIANT, models: { assistant: 'model-a', writer: null, judge: false },
|
||||
text: `助手 model-a · 写作 各节点自己的 · ${a} · 结论裁判关 · 每题 ${arms.length > 1 ? 3 : 1} 次` })),
|
||||
text: `助手 model-a · 写作 各节点自己的 · ${a.startsWith('分析工具') ? `当前目录 · ${a}` : `${a} · 分析工具开`} · 结论裁判关 · 每题 ${arms.length > 1 ? 3 : 1} 次` })),
|
||||
cases: 3, repeats: arms.length > 1 ? 3 : 1, estimate: { items: 3 * (arms.length > 1 ? 3 : 1) * arms.length, per_item_usd: 0.014,
|
||||
total_usd: 0.042 * (arms.length > 1 ? 3 : 1) * arms.length, basis: 'ev-1', note: '按这个题集上一次评测的每题平均花费估算' },
|
||||
budget: BUDGET, parallel: 2, spend: { usd: 0.37, calls: 12, unpriced_calls: 0, daily_max_usd: 5 },
|
||||
@@ -282,7 +292,8 @@ const USER = { mode: 'local', username: 'li', display_name: '使用者乙', role
|
||||
function freshState(patch = {}) {
|
||||
return {
|
||||
suites: [clone(SUITE), clone(SUITE_B)], cases: { 'suite-a': [clone(CASE_SQL), clone(CASE_CAL), clone(CASE_ASK)], 'suite-b': [] },
|
||||
evaluations: { 'ev-1': clone(EVAL_DONE), 'ev-old': clone(EVAL_OLD), 'ev-pa': clone(PAIR_A), 'ev-pb': clone(PAIR_B) },
|
||||
evaluations: { 'ev-1': clone(EVAL_DONE), 'ev-old': clone(EVAL_OLD), 'ev-pa': clone(PAIR_A), 'ev-pb': clone(PAIR_B),
|
||||
'ev-ta': clone(PAIR_TA), 'ev-tb': clone(PAIR_TB) },
|
||||
writes: [], me: null, eval: { max_parallel: 2, per_case_timeout_s: 900, per_run_max_usd: 2, daily_max_usd: 5, keep_runs: 20, judge: false },
|
||||
...patch,
|
||||
}
|
||||
@@ -376,8 +387,8 @@ function reply(state, method, path, url, body) {
|
||||
}
|
||||
if (method === 'POST' && rest === '/evaluations/estimate') {
|
||||
write()
|
||||
const arms = body.preset ? (body.preset === 'catalog_reviewed' ? ['当前目录', '只用已核实和已确认的项', '关目录'] : ['当前目录', '关目录'])
|
||||
: body.arms ? body.arms.map((a, i) => a.label || 'ABC'[i]) : ['当前目录']
|
||||
const arms = body.preset ? ({ catalog_reviewed: ['当前目录', '只用已核实和已确认的项', '关目录'], analysis_tools: ['分析工具开', '分析工具关'] }[body.preset]
|
||||
?? ['当前目录', '关目录']) : body.arms ? body.arms.map((a, i) => a.label || 'ABC'[i]) : ['当前目录']
|
||||
return { json: ESTIMATE(arms) }
|
||||
}
|
||||
if (method === 'POST' && rest === '/evaluations') {
|
||||
@@ -433,6 +444,7 @@ function reply(state, method, path, url, body) {
|
||||
if (method === 'POST' && path === '/evals/suites/suite-a/cleanup') { write(); return { json: { runs: 2, results: 6, deleted_runs: 8, files: 3 } } }
|
||||
if (method === 'GET' && path === '/workflows') return { json: [WF_WEEK] }
|
||||
if (method === 'GET' && path === '/evals/pairs/ev-pa') return { json: PAIR }
|
||||
if (method === 'GET' && path === '/evals/pairs/ev-ta') return { json: PAIR_TOOLS }
|
||||
if (method === 'GET' && path === '/evals/calibers') return { json: { calibers: CALIBERS } }
|
||||
if (method === 'GET' && path === '/workflows/wf-week/versions/3') return { json: VERSION_WEEK }
|
||||
if (method === 'GET' && path === '/evals/runs/run-chat-1/queries') return { json: { queries: QUERIES } }
|
||||
@@ -899,7 +911,25 @@ await section('发起评测：估算和发起同一份请求体；评一次、A/
|
||||
await until(async () => (await page.locator('[data-eval-estimate-arm]').count()) === 3)
|
||||
await shot(page, 'start-catalog')
|
||||
check('估算写三臂', await page.locator('[data-eval-estimate-arm]').count() === 3)
|
||||
check('目录贡献对照：目录由预设定、不摆，分析工具照样能选', await page.locator('[data-eval-catalog-mode]').count() === 0
|
||||
&& await page.locator('[data-eval-analysis-tools]').count() === 1)
|
||||
await page.locator('[data-eval-start-mode="analysis_tools"]').click()
|
||||
await until(() => writes(state, /estimate$/).some((w) => w.body?.preset === 'analysis_tools'))
|
||||
const tools = lastWrite(state, /estimate$/)?.body
|
||||
check('分析工具贡献对照：预设 analysis_tools，变体里不带分析工具和名字（开、关两臂由服务端定），每题 3 次',
|
||||
tools?.preset === 'analysis_tools' && !('analysis' in (tools.variant ?? {})) && !('label' in (tools.variant ?? {}))
|
||||
&& tools.variant?.repeats === 3 && tools.variant?.catalog?.mode === 'live', JSON.stringify(tools))
|
||||
check('……分析工具由预设定、不摆，目录照样能选', await page.locator('[data-eval-analysis-tools]').count() === 0
|
||||
&& await page.locator('[data-eval-catalog-mode]').count() === 1)
|
||||
await until(async () => (await page.locator('[data-eval-estimate-arm]').count()) === 2)
|
||||
check('……估算写开、关两臂，各自的变体那一句照服务端写', (await text(page, '[data-eval-estimate-arm="分析工具关"]')).includes('分析工具关')
|
||||
&& (await text(page, '[data-eval-estimate-arm="分析工具开"]')).includes('分析工具开'), await text(page, '[data-eval-estimate]'))
|
||||
await shot(page, 'start-analysis-tools')
|
||||
await page.locator('[data-eval-start-mode="single"]').click()
|
||||
await page.locator('#eval-arm-0-analysis').selectOption('off')
|
||||
await until(() => writes(state, /estimate$/).some((w) => !w.body?.preset && w.body?.variant?.analysis?.tools === 'off'))
|
||||
check('评一次：分析工具选「关」，变体里带 analysis.tools=off', lastWrite(state, /estimate$/)?.body?.variant?.analysis?.tools === 'off',
|
||||
JSON.stringify(lastWrite(state, /estimate$/)?.body))
|
||||
await sleep(500)
|
||||
await page.locator('[data-eval-start-submit]').click()
|
||||
await until(() => lastWrite(state, /^POST \/evals\/suites\/suite-a\/evaluations$/))
|
||||
@@ -982,6 +1012,8 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => {
|
||||
await page.locator('[data-eval-transitions]').waitFor()
|
||||
check('对比：指纹差异写哪几项变了', (await text(page, '[data-eval-fingerprint]')).includes('写作模型：model-a → model-b')
|
||||
&& (await text(page, '[data-eval-fingerprint]')).includes('表结构：变了'))
|
||||
check('对比：分析工具开、关照服务端写前后', (await text(page, '[data-eval-fingerprint]')).includes('分析工具：开 → 关'),
|
||||
await text(page, '[data-eval-fingerprint]'))
|
||||
check('转移和归因', (await text(page, '[data-eval-transitions]')).includes('退步 1') && (await text(page, '[data-eval-transitions]')).includes('做法变 1')
|
||||
&& await page.locator('[data-eval-transition="regressed"]').count() === 1)
|
||||
await shot(page, 'run-compare')
|
||||
@@ -995,6 +1027,16 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => {
|
||||
&& await text(page, '[data-eval-pair-only-b]') === '0' && await text(page, '[data-eval-pair-neither]') === '1')
|
||||
check('p 值和「这批题上看不出差别」照服务端写', (await text(page, '[data-eval-pair-p]')).includes('p = 0.5') && (await text(page, '[data-eval-pair-p]')).includes('看不出差别'))
|
||||
check('两臂各自链到自己的详情', await page.locator('[data-eval-pair-arm="关目录"]').getAttribute('href') === '/evals/runs/ev-pb?tab=pair')
|
||||
|
||||
// 分析工具贡献对照：两臂的名字、变体那一句（分析工具开 / 关）照服务端写
|
||||
await page.goto(`${WEB}/evals/runs/ev-tb?tab=pair`, { waitUntil: 'networkidle' })
|
||||
await page.locator('[data-eval-pair-row]').waitFor()
|
||||
check('分析工具贡献对照：两臂各写自己的分析工具开、关', (await text(page, '[data-eval-pair-arm="分析工具开"]')).includes('分析工具开 · 结论裁判关')
|
||||
&& (await text(page, '[data-eval-pair-arm="分析工具关"]')).includes('分析工具关 · 结论裁判关')
|
||||
&& (await text(page, '[data-eval-pair-row]')).includes('分析工具开 vs 分析工具关'))
|
||||
check('……这一臂的详情页头写它的变体（分析工具关）', (await text(page, '[data-eval-run-head]')).includes(TOOLS_TEXT(false)),
|
||||
await text(page, '[data-eval-run-head]'))
|
||||
await shot(page, 'run-pair-tools')
|
||||
check('没有页面报错', !errors.length, errors.join(' ｜ '))
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user