diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index 325f6ae5..aacfb88a 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -18,8 +18,8 @@ POST /catalog-snapshots/shadow（后台起草，202）、GET /catalog-snapshots/{id}、POST /catalog-snapshots/{id}/stop。 - 评测（app/eval/runner.py，后台任务）：POST /suites/{id}/evaluations/estimate（发起前的估算：几项、按上一次每题平均花费估的 金额、生效的预算、同时评几题）；POST /suites/{id}/evaluations（发起，202；variant 一臂，或 arms 几臂的 A/B，或 - preset=catalog / catalog_reviewed「目录贡献」预设对照）；GET /suites/{id}/evaluations（历次）；GET /evaluations/{id}（进度、 - 总表、和上一次比的计数）；GET /evaluations/{id}/results（逐题）、GET /evaluations/{id}/results/{rid}（一题的明细）； + preset=catalog / catalog_reviewed「目录贡献」、preset=analysis_tools「分析工具贡献」预设对照）； + GET /suites/{id}/evaluations（历次）；GET /evaluations/{id}（进度、总表、和上一次比的计数）；GET /evaluations/{id}/results（逐题）、GET /evaluations/{id}/results/{rid}（一题的明细）； POST /evaluations/{id}/stop、/retry（只重做没做完的）、/rescore（按新评分规则重评，只读封存、不调模型）、/pin（钉住当基线）； GET /evaluations/{id}/compare?base=（和上一次或点名的那一次比：指纹差异、逐题转移和归因）；GET /pairs/{pair_id}（A/B 配对表和 精确 McNemar）；POST /suites/{id}/cleanup（按保留规则清理更早的明细）。「口径命中与重复度」的 E1–E4 直接用口径账本的 @@ -855,8 +855,8 @@ EVALUATION_NOT_FOUND = "eval_evaluation_not_found" PAIR_NOT_FOUND = "eval_pair_not_found" RESULT_NOT_FOUND = "eval_result_not_found" CLEANED = "eval_details_cleaned" -#: 预设对照 -PRESETS = ("catalog", "catalog_reviewed") +#: 预设对照：目录贡献（当前目录 vs 关，可加只用已核实的项）、分析工具贡献（开 vs 关） +PRESETS = ("catalog", "catalog_reviewed", "analysis_tools") def _refused(e: CaseInvalid) -> CodedHTTPException: @@ -865,7 +865,8 @@ def _refused(e: CaseInvalid) -> CodedHTTPException: async def _arms(session: AsyncSession, suite: EvalSuite, body: dict[str, Any]) -> list[dict[str, Any]]: """请求体 → 规范化的几臂变体：{variant}（一臂，不写用题集的缺省变体）、{arms: [变体…]}（A/B）或 {preset: catalog | - catalog_reviewed}（目录贡献对照：当前目录 vs 关，可加只用已核实的项；模型照 variant 或题集的缺省）。A/B 缺省每题 3 次。""" + catalog_reviewed | analysis_tools}（目录贡献对照：当前目录 vs 关，可加只用已核实的项；分析工具贡献对照：开 vs 关；别的项 + 照 variant 或题集的缺省）。A/B 缺省每题 3 次。""" from app.eval import variants as variant_rules judge = bool((await _site(session))["judge"]) @@ -877,6 +878,8 @@ async def _arms(session: AsyncSession, suite: EvalSuite, body: dict[str, Any]) - raise CaseInvalid(f"预设对照只能是 {'、'.join(PRESETS)}", "preset", variant_rules.VARIANT_INVALID) base = variant_rules.normalize_variant({**dict(suite.default_variant or {}), **dict(body.get("variant") or {})}, judge_default=judge, repeats_default=variant_rules.PAIR_REPEATS) + if preset == "analysis_tools": + return variant_rules.analysis_tools_preset(base) return variant_rules.catalog_preset(base, reviewed=preset == "catalog_reviewed") if raw_arms is not None: if not isinstance(raw_arms, list) or not 2 <= len(raw_arms) <= variant_rules.ARMS_MAX: diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py index 326e320d..c2816b7c 100644 --- a/backend/app/eval/answer.py +++ b/backend/app/eval/answer.py @@ -7,7 +7,8 @@ app/chat/job.py），生成过程是 api/copilot.answer_ops，几个「决定」 answer_ops → final.next（chat/turn.next_step）→ launch_run(origin=eval) → 等终态 → review_of_run → 要的话带原图重来一次 - 生成：问法就是 instruction，intent=answer，限定的数据源同题目；助手模型按变体覆盖（GenerateIn.provider / model，挑表、 - 复核跟着换）；读哪份目录按变体（answer_ops 的 catalog_view，查看的表详情、自查的 SQL 检查器都按它）。 + 复核跟着换）；读哪份目录按变体（answer_ops 的 catalog_view，查看的表详情、自查的 SQL 检查器都按它）；分析工具开不开按 + 变体（answer_ops 的 analysis_tools，经 TurnDriver 透传：关的那一臂工具清单不列分析工具，节点手册不讲分析卡）。 - 结局：reply 直接回答、asked 反问、fail 判失败（建图失败）、ready 只建不跑、retry（老服务端那条路：没自查过的图带着原因 重来一次）、run 运行。生成过程交出 error（助手本轮未完成）、开流前就拒绝（模型没配好、数据源不在）都是建图失败，另记 「助手调用失败」——连续几题都这样就止损（runner.py）；只有生成过程一个操作都没给（只写了思考，error.next=retry）时按 @@ -71,6 +72,8 @@ class TurnContext: view: Mapping[str, Any] | None = None #: 这次评测的分析设置快照（有分析期望的题集才有）：作答的运行按它判（Run.options['analysis']），和算期望同一份 analysis: Mapping[str, Any] | None = None + #: 变体：分析工具开不开（variants.analysis_tools）。关的那一臂建图时助手看不到分析工具（TurnDriver 的 analysis_tools） + analysis_tools: bool = True async def writer_usage(run_ids: list[str]) -> dict[str, Any]: @@ -153,13 +156,14 @@ async def stop_run(run_id: str | None, actor: str | None = None) -> None: class Driver(TurnDriver): """一轮作答（流程在 TurnDriver）。drive() 跑完一轮，结果在 turn 上；on_phase(阶段) 在每一步开始时调（runner 据此写 - 「正在做什么」）。和问数据不一样的：助手模型、目录视图按变体；写作模型按变体写进图；来源 eval、只读和遮罩；停在审批上 - 直接取消（评测不代批）；复核一直等。""" + 「正在做什么」）。和问数据不一样的：助手模型、目录视图、分析工具开不开按变体；写作模型按变体写进图；来源 eval、只读和 + 遮罩；停在审批上直接取消（评测不代批）；复核一直等。""" def __init__(self, question: str, *, datasource_ids: list[str], ctx: TurnContext, on_phase: Callable[[str], Awaitable[None]] | None = None) -> None: super().__init__(question, datasource_ids=datasource_ids, on_phase=on_phase, - provider=ctx.assistant.get("provider"), model=ctx.assistant.get("model")) + provider=ctx.assistant.get("provider"), model=ctx.assistant.get("model"), + analysis_tools=ctx.analysis_tools) self.ctx = ctx def catalog_view(self) -> Any: diff --git a/backend/app/eval/compare.py b/backend/app/eval/compare.py index af6c934d..4347b3b4 100644 --- a/backend/app/eval/compare.py +++ b/backend/app/eval/compare.py @@ -11,8 +11,8 @@ 「只参数变」（多半是时间范围理解变了）；作答路径变了是「路径变」；其余「说不清」。SQL 的 same / params_only / changed 和运行 对比（engine/compare）是同一条规则：data/sql_semantics.normalize_sql 的规范文本和字面量。 -**指纹差异**：助手模型、写作模型、目录快照、提示词指纹、表结构、题目、评分规则、分析设置（有分析期望的题集开评时拍的 -那一份判定阈值），哪几项变了。 +**指纹差异**：助手模型、写作模型、目录快照、分析工具（开、关；加这一项之前的评测按开）、提示词指纹、表结构、题目、 +评分规则、分析设置（有分析期望的题集开评时拍的那一份判定阈值），哪几项变了。 **A/B**：同一次发起的几臂（共用 pair_id、共用期望）逐题配对，给两臂各自的准确率、配对表（都对 / 都错 / 只 A 对 / 只 B 对）和 精确 McNemar 双侧 p 值；p ≥ 0.05 时照实写「这批题上看不出差别」。不能判的题不进配对表（单列个数）。 @@ -31,8 +31,10 @@ CAUSES = ("data", "method", "params", "path", "unknown") CAUSE_LABEL = {"data": "数据变", "method": "做法变", "params": "只参数变", "path": "路径变", "unknown": "说不清"} #: 指纹里比的几项和叫法 FINGERPRINT_LABEL = {"assistant_model": "助手模型", "writer_model": "写作模型", "catalog": "目录快照", - "prompt": "提示词", "schemas": "表结构", "cases": "题目", "scorer_version": "评分规则", - "analysis": "分析设置"} + "analysis_tools": "分析工具", "prompt": "提示词", "schemas": "表结构", "cases": "题目", + "scorer_version": "评分规则", "analysis": "分析设置"} +#: 指纹里照实写前后两边的几项（别的只说变没变） +_SHOWN = ("assistant_model", "writer_model", "catalog", "analysis_tools", "scorer_version") NO_DIFFERENCE = "这批题上看不出差别" SIGNIFICANT = 0.05 #: 基准能是哪几种状态 @@ -128,15 +130,22 @@ def transitions(base: Sequence[Mapping[str, Any]], target: Sequence[Mapping[str, return {"counts": counts, "causes": causes, "items": items} +def _tools(fingerprint: Mapping[str, Any]) -> str: + """分析工具这一项给人看的样子：开 / 关。加这一项之前的评测指纹里没有它，那时一直开着。""" + from app.eval.variants import ANALYSIS_TOOLS_LABEL + + return ANALYSIS_TOOLS_LABEL["off" if fingerprint.get("analysis_tools") == "off" else "on"] + + def fingerprint_diff(base: Mapping[str, Any], target: Mapping[str, Any]) -> list[dict[str, Any]]: - """两次之间指纹里哪几项变了：[{key, label, before, after}]（表结构、题目只说变没变，不展开）。""" + """两次之间指纹里哪几项变了：[{key, label, before, after}]（表结构、题目只说变没变，不展开；分析工具写开、关）。""" out = [] for key, label in FINGERPRINT_LABEL.items(): - a, b = base.get(key), target.get(key) + a, b = (_tools(base), _tools(target)) if key == "analysis_tools" else (base.get(key), target.get(key)) if a == b: continue item: dict[str, Any] = {"key": key, "label": label} - if key in ("assistant_model", "writer_model", "catalog", "scorer_version"): + if key in _SHOWN: item.update(before=a, after=b) out.append(item) return out diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index 7fea009c..6d688e14 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -239,6 +239,7 @@ async def start(session: AsyncSession, suite: EvalSuite, variants: Sequence[Mapp fingerprint={"variant_key": variant_rules.variant_key(variant, models), "assistant_model": models.get("assistant"), "writer_model": models.get("writer"), "catalog_mode": (variant.get("catalog") or {}).get("mode"), + "analysis_tools": variant_rules.analysis_tools(variant), "snapshot_artifact": snapshot, "cron": cron}, counts={"total": len(cases) * repeats, "done": 0}) session.add(run) @@ -531,6 +532,7 @@ class EvalExecution(runtime.JobRun): async def _answer(self, arm: _Arm, unit: _Unit, case: Mapping[str, Any], question: str, values: Mapping[str, str], timeout: float) -> Any: + from app.eval import variants as variant_rules from app.eval.answer import Driver, TurnContext, stop_run result_id = unit.results[arm.run_id] @@ -542,7 +544,8 @@ class EvalExecution(runtime.JobRun): case_id=unit.case_id, case_seq=case.get("seq"), actor=self.actor, arm=arm.name, repeat=unit.repeat, assistant=arm.variant.get("assistant") or {}, writer=arm.variant.get("writer") or {}, - view=arm.view, analysis=self.analysis) + view=arm.view, analysis=self.analysis, + analysis_tools=variant_rules.analysis_tools(arm.variant) == "on") driver = Driver(question, datasource_ids=list(case.get("datasource_ids") or []), ctx=ctx, on_phase=on_phase) try: if case.get("kind") == "workflow": diff --git a/backend/app/eval/variants.py b/backend/app/eval/variants.py index f4c1ad0a..3ca58f93 100644 --- a/backend/app/eval/variants.py +++ b/backend/app/eval/variants.py @@ -1,4 +1,4 @@ -"""评测的变体：这一次用哪几个模型、读哪一份目录、重复几次（细案 01 做法 9、12；评测-D5、D10）。 +"""评测的变体：这一次用哪几个模型、读哪一份目录、给不给分析工具、重复几次（细案 01 做法 9、12；评测-D5、D10）。 变体原样存进 EvalRun.variant（发起时规范化过），实际解析出的模型 id 另存 resolved_models。几项： @@ -11,13 +11,18 @@ - catalog {mode, snapshot}：读哪一份目录（评测-D5）——live 当前目录、reviewed 只用已核实和已确认的项、off 关目录、snapshot 指定一份目录快照（EvalCatalogSnapshot 的 id，影子起草的或拍好的）。live、reviewed 在开评时都拍一份当前目录的快照，按 快照读：评测一跑几十分钟，期间起草、剖析照常改目录，也不影响这次的可比性（做法 9「不占数据源的队」）。 +- analysis {tools}：分析工具开不开（on 开、off 关，缺省开）。关的那一臂助手建图时看不到分析工具和分析卡（工具清单不列、 + 节点手册和问数据的包装里只讲分析能力的那几段不给，图里照样绑了分析工具的挡回去改；api/copilot.answer_ops 的 + analysis_tools，经 chat/drive.TurnDriver 透传）。工作流题不建图，按钉住的那一版跑，不受这一项影响。 - repeats：每题重复几次（1–5）。定时跑 1 次，对照缺省 3 次（评测-D10）。 - label：这一臂叫什么（A/B 时给人看，「当前目录」「关目录」……）。 -**同一变体**（和上一次比时挑基准用，做法 12）：助手、写作两项实际解析出的模型、目录的 mode（指定快照的连同快照 id）； -重复次数、裁判开不开不算——它们不改变答案本身。variant_key 是它的指纹。 +**同一变体**（和上一次比时挑基准用，做法 12）：助手、写作两项实际解析出的模型、目录的 mode（指定快照的连同快照 id）、 +分析工具开不开；重复次数、裁判开不开不算——它们不改变答案本身。variant_key 是它的指纹（分析工具开着时和加这一项之前 +一字不差：以前的评测照样认得出是同一变体）。 **目录贡献对照**（做法 12）：一组预设的 A/B——当前目录 vs 关，可以再加第三臂「只用已核实和已确认的项」（catalog_preset）。 +**分析工具贡献对照**：一组预设的 A/B——分析工具开 vs 关（analysis_tools_preset），量出分析工具帮了多少。 """ from __future__ import annotations @@ -37,6 +42,9 @@ VARIANT_UNAVAILABLE = "eval_variant_unavailable" CATALOG_MODES = ("live", "reviewed", "off", "snapshot") CATALOG_LABEL = {"live": "当前目录", "reviewed": "只用已核实和已确认的项", "off": "关目录", "snapshot": "目录快照"} +#: 分析工具开、关（变体的 analysis.tools） +ANALYSIS_TOOLS = ("on", "off") +ANALYSIS_TOOLS_LABEL = {"on": "开", "off": "关"} REPEATS_MAX = 5 #: 定时跑 1 次，A/B 对照缺省 3 次（评测-D10） SCHEDULE_REPEATS = 1 @@ -44,7 +52,7 @@ PAIR_REPEATS = 3 #: 一次 A/B 最多几臂（目录贡献对照最多三臂） ARMS_MAX = 3 LABEL_MAX = 40 -_KEYS = ("assistant", "writer", "judge", "catalog", "repeats", "label") +_KEYS = ("assistant", "writer", "judge", "catalog", "analysis", "repeats", "label") def _invalid(message: str, field_name: str) -> CaseInvalid: @@ -92,6 +100,10 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default: raise _invalid("只有目录选「目录快照」时才指定快照", "variant.catalog.snapshot") else: snapshot = None + analysis = {} if raw.get("analysis") is None else raw["analysis"] + if not isinstance(analysis, Mapping) or set(analysis) - {"tools"} \ + or analysis.get("tools", "on") not in ANALYSIS_TOOLS: + raise _invalid(f"变体的分析工具需要是 {{tools}}，tools 只能是 {'、'.join(ANALYSIS_TOOLS)}（开、关）", "variant.analysis") repeats = raw.get("repeats", repeats_default) if isinstance(repeats, bool) or not isinstance(repeats, int) or not 1 <= repeats <= REPEATS_MAX: raise _invalid(f"每题重复的次数需要是 1 到 {REPEATS_MAX} 的整数", "variant.repeats") @@ -100,10 +112,16 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default: raise _invalid("这一臂的名字需要是文字", "variant.label") return {"assistant": _model_pair(raw.get("assistant"), "助手模型", "variant.assistant"), "writer": _model_pair(raw.get("writer"), "写作模型", "variant.writer"), - "judge": judge, "catalog": {"mode": catalog["mode"], "snapshot": snapshot}, "repeats": repeats, + "judge": judge, "catalog": {"mode": catalog["mode"], "snapshot": snapshot}, + "analysis": {"tools": analysis.get("tools", "on")}, "repeats": repeats, "label": (label or "").strip()[:LABEL_MAX] or None} +def analysis_tools(variant: Mapping[str, Any]) -> str: + """这一臂的分析工具开不开：on / off。加这一项之前的变体（存着的老评测）没有它，按开。""" + return "off" if (variant.get("analysis") or {}).get("tools") == "off" else "on" + + def catalog_preset(base: Mapping[str, Any], *, reviewed: bool = False) -> list[dict[str, Any]]: """「目录的贡献」预设对照：当前目录 vs 关（reviewed 时再加「只用已核实和已确认的项」）。模型、裁判、重复次数照 base。""" arms = [("live", CATALOG_LABEL["live"]), ("off", CATALOG_LABEL["off"])] @@ -113,6 +131,12 @@ def catalog_preset(base: Mapping[str, Any], *, reviewed: bool = False) -> list[d for mode, label in arms] +def analysis_tools_preset(base: Mapping[str, Any]) -> list[dict[str, Any]]: + """「分析工具的贡献」预设对照：分析工具开 vs 关。模型、目录、裁判、重复次数照 base。""" + return [{**copy.deepcopy(dict(base)), "analysis": {"tools": tools}, "label": f"分析工具{ANALYSIS_TOOLS_LABEL[tools]}"} + for tools in ANALYSIS_TOOLS] + + def arm_name(variant: Mapping[str, Any], index: int) -> str: """这一臂的名字：变体里写了就用，否则按目录（只有目录不同时一看就懂），再不然 A、B、C。""" return str(variant.get("label") or "") or "ABC"[index] @@ -188,26 +212,31 @@ async def snapshot_artifact(session: AsyncSession, variant: Mapping[str, Any]) - def variant_key(variant: Mapping[str, Any], models: Mapping[str, Any]) -> str: - """「同一变体」的指纹：助手、写作实际解析出的模型，目录的 mode（指定快照的连同快照 id）。""" + """「同一变体」的指纹：助手、写作实际解析出的模型，目录的 mode（指定快照的连同快照 id），分析工具开不开。分析工具 + 开着时不写进去：和加这一项之前算出来的一样，以前的评测照样是同一变体（挑基准、趋势线不断）。""" from app.core.artifact_store import canonical_json, content_hash catalog = variant.get("catalog") or {} body = {"assistant": models.get("assistant"), "writer": models.get("writer"), "catalog": catalog.get("mode"), "snapshot": catalog.get("snapshot") if catalog.get("mode") == "snapshot" else None} + if analysis_tools(variant) == "off": + body["analysis_tools"] = "off" return content_hash(canonical_json(body))[:16] def describe(variant: Mapping[str, Any], models: Mapping[str, Any] | None = None) -> str: - """给人看的一句：「助手 x · 写作 y · 当前目录 · 裁判关 · 每题 1 次」。""" + """给人看的一句：「助手 x · 写作 y · 当前目录 · 分析工具开 · 结论裁判关 · 每题 1 次」。""" models = models or {} catalog = variant.get("catalog") or {} parts = [f"助手 {models.get('assistant') or (variant.get('assistant') or {}).get('model') or '缺省'}", f"写作 {models.get('writer') or (variant.get('writer') or {}).get('model') or '各节点自己的'}", CATALOG_LABEL.get(catalog.get("mode") or "live", "当前目录"), + f"分析工具{ANALYSIS_TOOLS_LABEL[analysis_tools(variant)]}", f"结论裁判{'开' if variant.get('judge') else '关'}", f"每题 {variant.get('repeats') or 1} 次"] return " · ".join(parts) -__all__ = ["ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", "PAIR_REPEATS", "REPEATS_MAX", "Resolved", "SCHEDULE_REPEATS", - "VARIANT_INVALID", "VARIANT_UNAVAILABLE", "arm_name", "catalog_preset", "describe", "normalize_variant", - "resolve_models", "snapshot_artifact", "variant_key", "view_of"] +__all__ = ["ANALYSIS_TOOLS", "ANALYSIS_TOOLS_LABEL", "ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", "PAIR_REPEATS", + "REPEATS_MAX", "Resolved", "SCHEDULE_REPEATS", "VARIANT_INVALID", "VARIANT_UNAVAILABLE", "analysis_tools", + "analysis_tools_preset", "arm_name", "catalog_preset", "describe", "normalize_variant", "resolve_models", + "snapshot_artifact", "variant_key", "view_of"] diff --git a/backend/tests/test_eval_analysis_runs.py b/backend/tests/test_eval_analysis_runs.py index 61bd3c39..e34e3e04 100644 --- a/backend/tests/test_eval_analysis_runs.py +++ b/backend/tests/test_eval_analysis_runs.py @@ -8,6 +8,8 @@ retail_trends（tests/fixtures/acceptance/retail_trends.py，按今天生成） - 重评老结果：明细里没有分析结论那一项的照样重评（从封存重新读）。 - 从头到尾：助手剧本搭「查库 → 分析卡 → 报告」，报告引用判定语；评测给这一题「对且可追溯」，逐题 E1 等价已证明，作答运行和 算期望用的是同一份分析设置快照（记进指纹）。 +- 分析工具开、关（变体 analysis.tools，「分析工具贡献」预设对照）：一次发起两臂；关的那一臂助手建图时工具清单里没有分析工具、 + 节点手册里没有分析卡，作答运行里没有分析结论；开的那一臂照常；两臂的指纹、同一变体的指纹都不同。 """ from __future__ import annotations @@ -17,7 +19,7 @@ from pathlib import Path from zoneinfo import ZoneInfo import pytest -from langchain_core.messages import AIMessageChunk +from langchain_core.messages import AIMessage, AIMessageChunk from sqlalchemy import select from app.core import artifact_store @@ -237,6 +239,16 @@ def _line(op: dict) -> AIMessageChunk: return AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n") +def _graph_lines(graph: dict, summary: str): + yield _line({"op": "plan", "summary": summary}) + for n in graph["nodes"]: + yield _line({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["data"]["label"], + "config": n["data"]["config"]}}) + for e in graph["edges"]: + yield _line({"op": "add_edge", "edge": e}) + yield _line({"op": "done", "explanation": "拆一下", "run": True}) + + class AnalysisCopilot: """助手剧本：直接搭「入口 → 查库 → 分析卡 → 报告 → 出口」并运行；上线前自查直接过。""" @@ -244,14 +256,8 @@ class AnalysisCopilot: if "上线前自查" in messages[1][1]: yield _line({"op": "done"}) return - yield _line({"op": "plan", "summary": "按地区拆上周销售额的变化"}) - graph = answer_graph() - for n in graph["nodes"]: - yield _line({"op": "add_node", "node": {"id": n["id"], "type": n["type"], "label": n["data"]["label"], - "config": n["data"]["config"]}}) - for e in graph["edges"]: - yield _line({"op": "add_edge", "edge": e}) - yield _line({"op": "done", "explanation": "拆一下", "run": True}) + for line in _graph_lines(answer_graph(), "按地区拆上周销售额的变化"): + yield line async def test_an_analysis_case_end_to_end(monkeypatch, source): @@ -313,3 +319,112 @@ async def test_suites_without_analysis_expectations_take_no_settings_snapshot(mo "别的题集的指纹、作答运行的选项和以前一样" finally: await drop_source(sid) + + +# -------------------------------------------------------------------------- +# 分析工具开、关（变体 analysis.tools；「分析工具贡献」预设对照） +# -------------------------------------------------------------------------- + +#: 关的那一臂搭的图里，报告撰写的要求带这个记号：报告撰写的剧本据此不引用分析结论（那一臂没有） +PLAIN_MARK = "只按查询结果写" +PLAIN = "## 上周销售额\n\n各地区上周的销售额都在查询结果里。" + + +def plain_graph() -> dict: + """没有分析卡的图：入口 → 查库 → 报告 → 出口。""" + nodes = [_node("start", "input"), + _node("q", "tool", tool=f"db_query__{retail_trends.SOURCE_NAME}", + args={"sql": rendered(REGION["gold"]["sql"])}), + _node("write", "report", instructions=f"{PLAIN_MARK}上周的周报"), + _node("out", "output", fields=[{"name": "答案", "value": "{{ nodes.write.text }}"}])] + ids = [n["id"] for n in nodes] + return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} + + +class SwitchCopilot: + """助手剧本：给的工具清单里有分析工具（开着的那一臂）就搭「查库 → 分析卡 → 报告」，没有（关掉的那一臂）就搭「查库 → + 报告」——照给它看的用。systems 记下每次建图收到的 system。""" + + def __init__(self) -> None: + self.systems: list[str] = [] + + async def astream(self, messages): + if "上线前自查" in messages[1][1]: + yield _line({"op": "done"}) + return + role, system = messages[0] + assert role == "system" + self.systems.append(system) + graph = answer_graph() if "analysis_decompose" in system else plain_graph() + for line in _graph_lines(graph, "看上周销售额"): + yield line + + +def use_writers(monkeypatch) -> None: + """报告撰写的剧本：要求里带 PLAIN_MARK 的（关掉分析工具那一臂）写 PLAIN，别的写引用了 A1 的 REPORT。""" + from app.providers import mock_model + + def decide(self, messages): + said = "\n".join(str(getattr(m, "content", "")) for m in messages) + return AIMessage(content=PLAIN if PLAIN_MARK in said else REPORT) + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + + +async def test_the_analysis_tools_preset_runs_with_and_without_them(monkeypatch, source): + from httpx import ASGITransport, AsyncClient + + from app.eval import compare + from app.main import app + from app.tools.analysis import ANALYSIS_TOOLS + + copilot = SwitchCopilot() + es.use_copilot(monkeypatch, copilot) + use_writers(monkeypatch) + case = case_of(REGION, source["id"]) + suite_id = await es.make_suite(source["id"], cases=[{k: case[k] for k in ("question", "datasource_ids", "gold", + "expectations")}]) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + started = await client.post(f"/api/evals/suites/{suite_id}/evaluations", + json={"preset": "analysis_tools", "variant": {"repeats": 1}}) + assert started.status_code == 202, started.text + body = started.json() + on, off = body["evaluations"] + assert (on["arm"], off["arm"]) == ("分析工具开", "分析工具关") and body["pair_id"] == on["id"] == off["pair_id"] + assert on["variant"]["analysis"] == {"tools": "on"} and off["variant"]["analysis"] == {"tools": "off"} + assert on["variant"]["catalog"] == off["variant"]["catalog"] == {"mode": "live", "snapshot": None} + assert "· 分析工具开 ·" in on["variant_text"] and "· 分析工具关 ·" in off["variant_text"], off["variant_text"] + await es.wait_eval(on["id"]) + pair = (await client.get(f"/api/evals/pairs/{body['pair_id']}")).json() + assert [a["arm"] for a in pair["table"]["arms"]] == ["分析工具开", "分析工具关"] + + async with SessionLocal() as session: + runs = {r.arm: r for r in (await session.execute(select(EvalRun).where(EvalRun.pair_id == on["id"]))).scalars()} + items = {arm: (await session.execute(select(EvalResult).where(EvalResult.eval_run_id == r.id))).scalar_one() + for arm, r in runs.items()} + answer_runs = {arm: await session.get(Run, item.run_id) for arm, item in items.items()} + assert all(r.status == "succeeded" for r in runs.values()), [(r.arm, r.error) for r in runs.values()] + # 建图：两臂各一次，关的那一臂工具清单里没有分析工具、节点手册里没有分析卡；开的那一臂都有 + assert len(copilot.systems) == 2 + with_tools = [s for s in copilot.systems if all(name in s for name in ANALYSIS_TOOLS)] + without = [s for s in copilot.systems if not any(name in s for name in ANALYSIS_TOOLS)] + assert len(with_tools) == len(without) == 1 + assert "- analysis：分析卡" in with_tools[0] and "- analysis：分析卡" not in without[0] + + # 开的那一臂照常：图里有分析卡，答案引用了分析结论，对且可追溯 + lit, dark = items["分析工具开"], items["分析工具关"] + assert {n["type"] for n in answer_runs["分析工具开"].graph["nodes"]} >= {"analysis", "report"} + assert lit.verdict == "pass" and lit.scores[0]["outcome"] == "traced", lit.scores + assert [a["alias"] for a in (await score.read_answer(lit.run_id)).analyses] == ["A1"] + # 关的那一臂：图里没有分析卡，作答运行里没有分析结论；正文没写这个判断，算错 + assert "analysis" not in {n["type"] for n in answer_runs["分析工具关"].graph["nodes"]} + assert (await score.read_answer(dark.run_id)).analyses == [] + assert dark.verdict == "fail" and dark.scores[0]["outcome"] == "wrong", dark.scores + assert "没有做变化拆解" in dark.scores[0]["reason"], dark.scores[0] + # 两臂共用期望；指纹、同一变体的指纹都不同，对比照实写「分析工具：开 → 关」 + assert lit.gold_run_ids == dark.gold_run_ids + fp_on, fp_off = runs["分析工具开"].fingerprint, runs["分析工具关"].fingerprint + assert (fp_on["analysis_tools"], fp_off["analysis_tools"]) == ("on", "off") + assert fp_on["variant_key"] != fp_off["variant_key"] + assert [d for d in compare.fingerprint_diff(fp_on, fp_off) if d["key"] == "analysis_tools"] == \ + [{"key": "analysis_tools", "label": "分析工具", "before": "开", "after": "关"}] diff --git a/backend/tests/test_eval_compare.py b/backend/tests/test_eval_compare.py index e8cc87de..78e973f6 100644 --- a/backend/tests/test_eval_compare.py +++ b/backend/tests/test_eval_compare.py @@ -51,6 +51,16 @@ def test_fingerprint_diff_lists_what_changed(): assert "before" not in diff[1], "表结构只说变没变" +def test_the_analysis_tools_show_as_on_and_off(): + """分析工具这一项写开、关；加这一项之前的评测指纹里没有它，那时一直开着：和现在开着的比不算变了。""" + old = {"assistant_model": "m1", "catalog": "live:aaa", "scorer_version": 3} + on, off = {**old, "analysis_tools": "on"}, {**old, "analysis_tools": "off"} + assert compare.fingerprint_diff(old, on) == [] and compare.fingerprint_diff(on, on) == [] + item = {"key": "analysis_tools", "label": "分析工具"} + assert compare.fingerprint_diff(on, off) == [{**item, "before": "开", "after": "关"}] + assert compare.fingerprint_diff(off, old) == [{**item, "before": "关", "after": "开"}] + + def test_the_pair_table(): a = [_row("c1", "pass"), _row("c2", "pass"), _row("c3", "fail"), _row("c4", "ungradable")] b = [_row("c1", "pass"), _row("c2", "fail"), _row("c3", "fail"), _row("c4", "pass")] diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index c23e7b3c..2b074787 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -444,6 +444,67 @@ async def test_the_api_starts_estimates_and_reports_progress(client, monkeypatch assert bad.status_code == 422 and bad.json()["code"] == "eval_variant_invalid" +async def test_the_variant_switches_the_analysis_tools(client, monkeypatch): + """变体的分析工具（analysis.tools）：缺省开，只认 on、off；同一变体的指纹在开着时和加这一项之前一样（以前的评测照样 + 当基准），关着时不同；变体那一句写「分析工具开 / 关」。「分析工具贡献」预设一次发起开、关两臂，关的那一臂作答的 Driver + 不给助手分析工具（TurnDriver 的 analysis_tools 透传给 answer_ops，见 test_copilot_analysis）。""" + from app.eval import variants + from app.eval.cases import CaseInvalid + + on = variants.normalize_variant({}) + off = variants.normalize_variant({"analysis": {"tools": "off"}}) + assert on["analysis"] == {"tools": "on"} and off["analysis"] == {"tools": "off"} + for bad in ("off", {"tools": "maybe"}, {"tools": None}, {"tools": "off", "mode": "x"}): + with pytest.raises(CaseInvalid) as caught: + variants.normalize_variant({"analysis": bad}) + assert (caught.value.field, caught.value.code) == ("variant.analysis", variants.VARIANT_INVALID), bad + models = {"assistant": "mock-fast", "writer": None} + before = {k: v for k, v in on.items() if k != "analysis"} + assert variants.variant_key(on, models) == variants.variant_key(before, models), "开着时和加这一项之前一样" + assert variants.variant_key(off, models) != variants.variant_key(on, models) + assert variants.describe(before, models) == variants.describe(on, models) \ + == "助手 mock-fast · 写作 各节点自己的 · 当前目录 · 分析工具开 · 结论裁判关 · 每题 1 次" + assert "· 分析工具关 ·" in variants.describe(off, models) + arms = variants.analysis_tools_preset({**off, "catalog": {"mode": "off", "snapshot": None}, "repeats": 3, + "label": "x"}) + assert [(a["label"], a["analysis"], a["catalog"]["mode"], a["repeats"]) for a in arms] == [ + ("分析工具开", {"tools": "on"}, "off", 3), ("分析工具关", {"tools": "off"}, "off", 3)] + + seen: list[tuple[str | None, bool]] = [] + + async def drive(self): + seen.append((self.ctx.arm, self.analysis_tools)) + self.turn.path = "asked" + return self.turn + + monkeypatch.setattr(answer_mod.Driver, "drive", drive) + suite_id = await es.make_suite("none", cases=ask_cases(1)) + estimate = (await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate", + json={"preset": "analysis_tools"})).json() + assert [(a["label"], a["variant"]["analysis"]["tools"]) for a in estimate["arms"]] == [ + ("分析工具开", "on"), ("分析工具关", "off")] + assert estimate["repeats"] == variants.PAIR_REPEATS and "分析工具关" in estimate["arms"][1]["text"] + started = await client.post(f"/api/evals/suites/{suite_id}/evaluations", + json={"preset": "analysis_tools", "variant": {"repeats": 1}}) + assert started.status_code == 202, started.text + lit, dark = started.json()["evaluations"] + await es.wait_eval(lit["id"]) + assert sorted(seen) == [("分析工具关", False), ("分析工具开", True)] + assert (lit["fingerprint"]["analysis_tools"], dark["fingerprint"]["analysis_tools"]) == ("on", "off") + assert lit["fingerprint"]["variant_key"] != dark["fingerprint"]["variant_key"] + # 一臂、A/B 里写的照样认；不合规的照实拒、落到那一格 + one = await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate", + json={"variant": {"analysis": {"tools": "off"}}}) + assert one.json()["arms"][0]["variant"]["analysis"] == {"tools": "off"} + bad = await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate", + json={"arms": [{"label": "开"}, {"label": "关", "analysis": {"tools": "nope"}}]}) + assert bad.status_code == 422 + assert (bad.json()["code"], bad.json()["field"]) == ("eval_variant_invalid", "variant.analysis") + unknown = await client.post(f"/api/evals/suites/{suite_id}/evaluations/estimate", json={"preset": "tools"}) + assert unknown.status_code == 422 and unknown.json()["field"] == "preset" + assert "analysis_tools" in unknown.json()["detail"] + + async def test_the_estimate_says_it_cannot_price_instead_of_zero(): """上一次评测的模型都不在价格目录里（价格目录空）：估算以前写「约 $0」，像不花钱。现在一分都没估出来的写估不出；估出了 一部分的写「至少」、另有几次估不出（老总表里每题平均记成 0、却有估不出的调用的，也按估不出算）。"""