评测前端：变体里能选「分析工具」开 / 关，发起评测多一种「分析工具贡献对照」。typesEval 加 EvalAnalysisTools、EvalPreset，EvalVariant 多 analysis；termsEval 加分析工具的叫法、提示和这种对照的说明，对比「都没变」那句带上分析工具。StartDialog：ArmFields 多一格分析工具（题集的默认变体同一张表单），预设对照里由预设定的那一项不摆（目录贡献不摆目录、分析工具贡献不摆分析工具）；分析工具贡献对照交 {preset: analysis_tools, variant}，变体里不带分析工具和名字，缺省每题 3 次。结果页、对比页、A/B 照服务端的变体那一句和指纹差异显示。check-evals：发起时评一次选「关」、两种预设的请求体和估算的两臂，对比写「分析工具：开 → 关」，分析工具贡献对照的两臂和详情页头；test_frontend_eval_unions 对上分析工具的取值、叫法和预设对照

2026-11-15 00:01:58 +00:00 · 2026-10-21 19:32:38 +08:00
parent 8a6ad0e7a2
commit bf89269c11
5 changed files with 109 additions and 26 deletions
@@ -92,6 +92,17 @@ def test_scores_compare_and_catalog_modes():
    assert _labels("start.catalogMode") == list(CATALOG_MODES)


+def test_analysis_tools_and_presets_follow_the_server():
+    """变体的分析工具开、关和预设对照（api/evals.PRESETS）：前端的取值和服务端一一对上。叫法写「开」「关」，和服务端
+    describe、对比里写的同一种说法。"""
+    from app.api.evals import PRESETS
+    from app.eval.variants import ANALYSIS_TOOLS, ANALYSIS_TOOLS_LABEL
+
+    assert _union("EvalAnalysisTools") == list(ANALYSIS_TOOLS)
+    assert _pairs("start.analysisToolsMode") == list(ANALYSIS_TOOLS_LABEL.items())
+    assert _union("EvalPreset") == list(PRESETS)
+
+
 def test_signal_bands_follow_the_server():
    from app.api.evals import SIGNAL_CRITERIA

@@ -5,9 +5,9 @@
 * 统计、结论、能不能做都由服务端给，这里只有叫法和句式：数字一律是服务端算好交来的，这里只负责写成话。
 */
 import type {
-  EvalAnalysisMode, EvalAnalysisOp, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind,
-  EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger,
-  EvalUpgradePolicy, EvalVerdict,
+  EvalAnalysisMode, EvalAnalysisOp, EvalAnalysisTools, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind,
+  EvalGoldKind, EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus,
+  EvalTrigger, EvalUpgradePolicy, EvalVerdict,
 } from '../typesEval'
 import { formatCost, formatNumber } from './format'

@@ -380,12 +380,15 @@ export const EVAL_TEXT = {
  start: {
    title: '发起评测',
    mode: '评测方式',
-    modes: { single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照' } as Record<'single' | 'ab' | 'catalog', string>,
+    modes: {
+      single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照', analysis_tools: '分析工具贡献对照',
+    } as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
    modeHint: {
      single: '按一组变体评一遍题集',
      ab: '两到三臂各自一组变体，共用同一份期望答案，同一题几臂同时作答',
      catalog: '预设的 A/B：当前目录 vs 关目录，可以再加一臂「只用已核实和已确认的项」，量出数据目录帮了多少',
-    } as Record<'single' | 'ab' | 'catalog', string>,
+      analysis_tools: '预设的 A/B：分析工具开 vs 关，量出分析工具（变化拆解、波动基线……）帮了多少',
+    } as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
    arm: (name: string) => `${name} 臂`,
    addArm: '再加一臂',
    removeArm: '去掉这一臂',
@@ -407,6 +410,9 @@ export const EVAL_TEXT = {
    catalogHint: '当前目录、只用已核实的项都在开评时拍一份快照按它读：评测期间目录被改也不影响可比性',
    snapshot: '选一份目录快照',
    snapshotNone: '还没有可用的目录快照',
+    analysisTools: '分析工具',
+    analysisToolsMode: { on: '开', off: '关' } as Record<EvalAnalysisTools, string>,
+    analysisToolsHint: '关掉时助手建图看不到分析工具和分析卡；工作流题按固定的版本运行，不受影响',
    repeats: '每题重复几次',
    repeatsHint: '模型有随机性：对照建议每题 3 次',
    reviewedArm: '再加一臂「只用已核实和已确认的项」',
@@ -586,7 +592,7 @@ export const EVAL_TEXT = {
    baseText: (short: string, when: string) => `${short} · ${when}`,
    none: '没有可比的上一次：同一题集、同一变体里还没有比这次先发起、已经评完的评测',
    fingerprint: '两次之间变了什么',
-    fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则、分析设置都没变',
+    fingerprintNone: '模型、目录、分析工具、提示词、表结构、题目、评分规则、分析设置都没变',
    changed: '变了',
    transitions: '逐题转移',
    transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record<EvalTransition, string>,
@@ -6,13 +6,16 @@ import { Field, Modal, Notice, Spinner, toast, useRadioGroup } from '../../compo
 import { errorMessage } from '../../lib/errors'
 import { formatCost, formatDateTime, shortId } from '../../lib/format'
 import { EVAL_TEXT } from '../../lib/termsEval'
-import type { EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant } from '../../typesEval'
+import type {
+  EvalAnalysisTools, EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant,
+} from '../../typesEval'
 import { ModelPick } from './parts'

 const T = EVAL_TEXT.start
-const MODES = ['single', 'ab', 'catalog'] as const
+const MODES = ['single', 'ab', 'catalog', 'analysis_tools'] as const
 type Mode = (typeof MODES)[number]
 const CATALOG_MODES: EvalCatalogMode[] = ['live', 'reviewed', 'off', 'snapshot']
+const ANALYSIS_TOOLS: EvalAnalysisTools[] = ['on', 'off']

 /** 一臂的变体（表单里的样子）：全部键都在，交上去之前不改形状 */
 export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
@@ -21,6 +24,7 @@ export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
    writer: { provider: base?.writer?.provider ?? null, model: base?.writer?.model ?? null },
    judge: base?.judge ?? false,
    catalog: { mode: base?.catalog?.mode ?? 'live', snapshot: base?.catalog?.snapshot ?? null },
+    analysis: { tools: base?.analysis?.tools ?? 'on' },
    repeats: base?.repeats ?? 1,
    label: base?.label ?? null,
  }
@@ -37,12 +41,13 @@ export function snapshotLabel(s: EvalSnapshot): string {
 }

 /**
- * 一臂变体的表单：助手、写作模型（缺省按设置），结论裁判，数据目录（当前 / 只用已核实的 / 关 / 快照），重复次数，名字。
- * 题集设置里的「默认变体」也用它（定时评测按默认变体评）
+ * 一臂变体的表单：助手、写作模型（缺省按设置），结论裁判，数据目录（当前 / 只用已核实的 / 关 / 快照），分析工具（开 / 关），
+ * 重复次数，名字。题集设置里的「默认变体」也用它（定时评测按默认变体评）。预设对照里由预设定的那一项（目录、分析工具）不摆
 */
-export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false }: {
+export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false,
+  hideAnalysis = false }: {
  idPrefix: string; value: EvalVariant; onChange: (v: EvalVariant) => void; snapshots: EvalSnapshot[]
-  withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean
+  withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean; hideAnalysis?: boolean
 }) {
  const set = (patch: Partial<EvalVariant>) => onChange({ ...value, ...patch })
  const usable = usableSnapshots(snapshots)
@@ -82,6 +87,14 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
          </select>
        </Field>
      )}
+      {!hideAnalysis && (
+        <Field label={T.analysisTools} htmlFor={`${idPrefix}-analysis`} hint={T.analysisToolsHint}>
+          <select id={`${idPrefix}-analysis`} className="field w-24" value={value.analysis.tools} data-eval-analysis-tools=""
+                  onChange={(e) => set({ analysis: { tools: e.target.value as EvalAnalysisTools } })}>
+            {ANALYSIS_TOOLS.map((m) => <option key={m} value={m}>{T.analysisToolsMode[m]}</option>)}
+          </select>
+        </Field>
+      )}
      <Field label={T.repeats} htmlFor={`${idPrefix}-repeats`} hint={T.repeatsHint}>
        <select id={`${idPrefix}-repeats`} className="field w-24" value={value.repeats} data-eval-repeats=""
                onChange={(e) => set({ repeats: Number(e.target.value) })}>
@@ -98,8 +111,8 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
 }

 /**
- * 发起评测：评一次 / A/B 对照（两到三臂） / 目录贡献对照（预设）。填好就向服务端要估算（几项、按上一次每题平均花费估的金额、
- * 生效的预算、同时评几题、今天评测已经花了多少）——估算和发起是同一份请求体，估出来的就是要发起的那一次。
+ * 发起评测：评一次 / A/B 对照（两到三臂） / 目录贡献对照、分析工具贡献对照（预设）。填好就向服务端要估算（几项、按上一次每题
+ * 平均花费估的金额、生效的预算、同时评几题、今天评测已经花了多少）——估算和发起是同一份请求体，估出来的就是要发起的那一次。
 */
 export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
  open: boolean; onClose: () => void; suite: EvalSuite; evaluations: EvalRun[]; onStarted: (run: EvalRun) => void
@@ -132,7 +145,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
        const repeats = Math.max(first.repeats, 3)
        return cur.length >= 2 ? cur : [{ ...first, repeats, label: first.label || 'A' }, { ...first, repeats, label: 'B' }]
      }
-      return [m === 'catalog' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
+      return [m === 'catalog' || m === 'analysis_tools' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
    })
  }

@@ -144,6 +157,10 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
      const { catalog: _c, label: _l, ...rest } = v
      return { preset: reviewedArm ? 'catalog_reviewed' : 'catalog', variant: rest, ...base }
    }
+    if (mode === 'analysis_tools') {
+      const { analysis: _a, label: _l, ...rest } = v
+      return { preset: 'analysis_tools', variant: rest, ...base }
+    }
    return { variant: v, ...base }
  }, [mode, arms, reviewedArm, baseline])
  const key = JSON.stringify(body)
@@ -207,7 +224,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {

        {mode !== 'ab' && (
          <ArmFields idPrefix="eval-arm-0" value={arms[0] ?? blankVariant()} snapshots={snapshots} hideCatalog={mode === 'catalog'}
-                     onChange={(v) => setArms([v])} />
+                     hideAnalysis={mode === 'analysis_tools'} onChange={(v) => setArms([v])} />
        )}
        {mode === 'catalog' && (
          <label className="flex items-center gap-2 text-xs">
@@ -370,26 +370,33 @@ export interface EvalImported {

 export type EvalCatalogMode = 'live' | 'reviewed' | 'off' | 'snapshot'

+/** 分析工具开、关（关的那一臂助手建图时看不到分析工具和分析卡） */
+export type EvalAnalysisTools = 'on' | 'off'
+
+/** 预设对照：目录贡献（当前目录 vs 关，可加「只用已核实和已确认的项」）、分析工具贡献（开 vs 关） */
+export type EvalPreset = 'catalog' | 'catalog_reviewed' | 'analysis_tools'
+
 export interface EvalModelPair {
  provider: string | null
  model: string | null
 }

-/** 一臂变体：助手、写作模型，结论裁判，目录，重复次数，名字 */
+/** 一臂变体：助手、写作模型，结论裁判，目录，分析工具，重复次数，名字。加分析工具之前存下的评测记录里没有 analysis（当开） */
 export interface EvalVariant {
  assistant: EvalModelPair
  writer: EvalModelPair
  judge: boolean
  catalog: { mode: EvalCatalogMode; snapshot: string | null }
+  analysis: { tools: EvalAnalysisTools }
  repeats: number
  label: string | null
 }

-/** 发起时交上去的：一臂（variant）、几臂的 A/B（arms），或预设的「目录贡献对照」（preset） */
+/** 发起时交上去的：一臂（variant）、几臂的 A/B（arms），或预设的对照（preset：目录贡献、分析工具贡献） */
 export interface EvalStartBody {
  variant?: Partial<EvalVariant>
  arms?: Partial<EvalVariant>[]
-  preset?: 'catalog' | 'catalog_reviewed'
+  preset?: EvalPreset
  baseline_id?: string | null
 }

@@ -135,7 +135,7 @@ const QUERIES = [
    row_count: 5, truncated: false, masked: true },
 ]
 const VARIANT = { assistant: { provider: null, model: null }, writer: { provider: null, model: null }, judge: false,
-  catalog: { mode: 'live', snapshot: null }, repeats: 1, label: null }
+  catalog: { mode: 'live', snapshot: null }, analysis: { tools: 'on' }, repeats: 1, label: null }
 const SUMMARY = {
  scorer_version: 1, items: 3, done: 3, cases: 3, verdicts: { pass: 2, fail: 1, ungradable: 0, skipped: 0 }, not_run: 0,
  accuracy: RATE(2, 3), traced_accuracy: RATE(1, 3), outcomes: { traced: 1, untraced: 1, wrong: 1 }, text: {},
@@ -152,7 +152,7 @@ const SUMMARY = {
 }
 const EVAL_DONE = {
  id: 'ev-1', suite_id: 'suite-a', status: 'succeeded', trigger: 'manual', schedule_id: null, variant: VARIANT,
-  resolved_models: { assistant: 'model-a', writer: null, judge: false }, variant_text: '助手 model-a · 写作 各节点自己的 · 当前目录 · 结论裁判关 · 每题 1 次',
+  resolved_models: { assistant: 'model-a', writer: null, judge: false }, variant_text: '助手 model-a · 写作 各节点自己的 · 当前目录 · 分析工具开 · 结论裁判关 · 每题 1 次',
  catalog_snapshot: 'snap-live', as_of: iso(NOW - 2 * H), repeats: 1, fingerprint: {}, pair_id: null, arm: null, baseline_id: 'ev-old',
  pinned: false, budget: BUDGET, counts: { total: 3, done: 3, passed: 2, failed: 1 }, usage: {}, summary: SUMMARY, stop_reason: null,
  error: null, created_by: '检查员甲', created_at: iso(NOW - 2 * H), started_at: iso(NOW - 2 * H), finished_at: iso(NOW - H),
@@ -219,7 +219,8 @@ const EVAL_AN = { ...EVAL_DONE, id: 'ev-an', counts: { total: 4, done: 4, passed
    outcomes_by_kind: { contribution: { traced: 1 }, verdict: { untraced: 1, wrong: 1, ungradable: 1 } } } }
 const COMPARE = {
  base: EVAL_OLD, cleaned: false,
-  fingerprint_diff: [{ key: 'writer_model', label: '写作模型', before: 'model-a', after: 'model-b' }, { key: 'schemas', label: '表结构' }],
+  fingerprint_diff: [{ key: 'writer_model', label: '写作模型', before: 'model-a', after: 'model-b' },
+    { key: 'analysis_tools', label: '分析工具', before: '开', after: '关' }, { key: 'schemas', label: '表结构' }],
  transitions: { counts: { regressed: 1, improved: 1, same: 1, incomparable: 0 }, causes: { data: 0, method: 1, params: 1, path: 0, unknown: 0 },
    items: [{ case_id: 'case-2', case_seq: 2, before: 'pass', after: 'fail', transition: 'regressed', cause: 'method' },
      { case_id: 'case-1', case_seq: 1, before: 'fail', after: 'pass', transition: 'improved', cause: 'params' },
@@ -227,12 +228,21 @@ const COMPARE = {
 }
 const PAIR_A = { ...EVAL_DONE, id: 'ev-pa', pair_id: 'ev-pa', arm: '当前目录', trigger: 'pair' }
 const PAIR_B = { ...EVAL_DONE, id: 'ev-pb', pair_id: 'ev-pa', arm: '关目录', trigger: 'pair' }
+// 分析工具贡献对照的两臂：名字、变体那一句都是服务端给的（variants.analysis_tools_preset、describe）
+const TOOLS_TEXT = (on) => `助手 model-a · 写作 各节点自己的 · 当前目录 · 分析工具${on ? '开' : '关'} · 结论裁判关 · 每题 3 次`
+const PAIR_TA = { ...EVAL_DONE, id: 'ev-ta', pair_id: 'ev-ta', arm: '分析工具开', trigger: 'pair', variant_text: TOOLS_TEXT(true),
+  variant: { ...VARIANT, repeats: 3, label: '分析工具开' } }
+const PAIR_TB = { ...EVAL_DONE, id: 'ev-tb', pair_id: 'ev-ta', arm: '分析工具关', trigger: 'pair', variant_text: TOOLS_TEXT(false),
+  variant: { ...VARIANT, analysis: { tools: 'off' }, repeats: 3, label: '分析工具关' } }
+const PAIR_TOOLS = { pair_id: 'ev-ta', evaluations: [PAIR_TA, PAIR_TB], cleaned: false,
+  table: { arms: [{ arm: '分析工具开', accuracy: RATE(3, 4) }, { arm: '分析工具关', accuracy: RATE(2, 4) }],
+    pairs: [{ a: '分析工具开', b: '分析工具关', both: 2, neither: 1, only_a: 1, only_b: 0, excluded: 0, p_value: 1, note: '这批题上看不出差别' }] } }
 const PAIR = { pair_id: 'ev-pa', evaluations: [PAIR_A, PAIR_B], cleaned: false,
  table: { arms: [{ arm: '当前目录', accuracy: RATE(3, 4) }, { arm: '关目录', accuracy: RATE(1, 4) }],
    pairs: [{ a: '当前目录', b: '关目录', both: 1, neither: 1, only_a: 2, only_b: 0, excluded: 0, p_value: 0.5, note: '这批题上看不出差别' }] } }
 const ESTIMATE = (arms) => ({
  arms: arms.map((a, i) => ({ label: arms.length > 1 ? a : null, variant: VARIANT, models: { assistant: 'model-a', writer: null, judge: false },
-    text: `助手 model-a · 写作 各节点自己的 · ${a} · 结论裁判关 · 每题 ${arms.length > 1 ? 3 : 1} 次` })),
+    text: `助手 model-a · 写作 各节点自己的 · ${a.startsWith('分析工具') ? `当前目录 · ${a}` : `${a} · 分析工具开`} · 结论裁判关 · 每题 ${arms.length > 1 ? 3 : 1} 次` })),
  cases: 3, repeats: arms.length > 1 ? 3 : 1, estimate: { items: 3 * (arms.length > 1 ? 3 : 1) * arms.length, per_item_usd: 0.014,
    total_usd: 0.042 * (arms.length > 1 ? 3 : 1) * arms.length, basis: 'ev-1', note: '按这个题集上一次评测的每题平均花费估算' },
  budget: BUDGET, parallel: 2, spend: { usd: 0.37, calls: 12, unpriced_calls: 0, daily_max_usd: 5 },
@@ -282,7 +292,8 @@ const USER = { mode: 'local', username: 'li', display_name: '使用者乙', role
 function freshState(patch = {}) {
  return {
    suites: [clone(SUITE), clone(SUITE_B)], cases: { 'suite-a': [clone(CASE_SQL), clone(CASE_CAL), clone(CASE_ASK)], 'suite-b': [] },
-    evaluations: { 'ev-1': clone(EVAL_DONE), 'ev-old': clone(EVAL_OLD), 'ev-pa': clone(PAIR_A), 'ev-pb': clone(PAIR_B) },
+    evaluations: { 'ev-1': clone(EVAL_DONE), 'ev-old': clone(EVAL_OLD), 'ev-pa': clone(PAIR_A), 'ev-pb': clone(PAIR_B),
+      'ev-ta': clone(PAIR_TA), 'ev-tb': clone(PAIR_TB) },
    writes: [], me: null, eval: { max_parallel: 2, per_case_timeout_s: 900, per_run_max_usd: 2, daily_max_usd: 5, keep_runs: 20, judge: false },
    ...patch,
  }
@@ -376,8 +387,8 @@ function reply(state, method, path, url, body) {
    }
    if (method === 'POST' && rest === '/evaluations/estimate') {
      write()
-      const arms = body.preset ? (body.preset === 'catalog_reviewed' ? ['当前目录', '只用已核实和已确认的项', '关目录'] : ['当前目录', '关目录'])
-        : body.arms ? body.arms.map((a, i) => a.label || 'ABC'[i]) : ['当前目录']
+      const arms = body.preset ? ({ catalog_reviewed: ['当前目录', '只用已核实和已确认的项', '关目录'], analysis_tools: ['分析工具开', '分析工具关'] }[body.preset]
+        ?? ['当前目录', '关目录']) : body.arms ? body.arms.map((a, i) => a.label || 'ABC'[i]) : ['当前目录']
      return { json: ESTIMATE(arms) }
    }
    if (method === 'POST' && rest === '/evaluations') {
@@ -433,6 +444,7 @@ function reply(state, method, path, url, body) {
  if (method === 'POST' && path === '/evals/suites/suite-a/cleanup') { write(); return { json: { runs: 2, results: 6, deleted_runs: 8, files: 3 } } }
  if (method === 'GET' && path === '/workflows') return { json: [WF_WEEK] }
  if (method === 'GET' && path === '/evals/pairs/ev-pa') return { json: PAIR }
+  if (method === 'GET' && path === '/evals/pairs/ev-ta') return { json: PAIR_TOOLS }
  if (method === 'GET' && path === '/evals/calibers') return { json: { calibers: CALIBERS } }
  if (method === 'GET' && path === '/workflows/wf-week/versions/3') return { json: VERSION_WEEK }
  if (method === 'GET' && path === '/evals/runs/run-chat-1/queries') return { json: { queries: QUERIES } }
@@ -899,7 +911,25 @@ await section('发起评测：估算和发起同一份请求体；评一次、A/
  await until(async () => (await page.locator('[data-eval-estimate-arm]').count()) === 3)
  await shot(page, 'start-catalog')
  check('估算写三臂', await page.locator('[data-eval-estimate-arm]').count() === 3)
+  check('目录贡献对照：目录由预设定、不摆，分析工具照样能选', await page.locator('[data-eval-catalog-mode]').count() === 0
+    && await page.locator('[data-eval-analysis-tools]').count() === 1)
+  await page.locator('[data-eval-start-mode="analysis_tools"]').click()
+  await until(() => writes(state, /estimate$/).some((w) => w.body?.preset === 'analysis_tools'))
+  const tools = lastWrite(state, /estimate$/)?.body
+  check('分析工具贡献对照：预设 analysis_tools，变体里不带分析工具和名字（开、关两臂由服务端定），每题 3 次',
+    tools?.preset === 'analysis_tools' && !('analysis' in (tools.variant ?? {})) && !('label' in (tools.variant ?? {}))
+      && tools.variant?.repeats === 3 && tools.variant?.catalog?.mode === 'live', JSON.stringify(tools))
+  check('……分析工具由预设定、不摆，目录照样能选', await page.locator('[data-eval-analysis-tools]').count() === 0
+    && await page.locator('[data-eval-catalog-mode]').count() === 1)
+  await until(async () => (await page.locator('[data-eval-estimate-arm]').count()) === 2)
+  check('……估算写开、关两臂，各自的变体那一句照服务端写', (await text(page, '[data-eval-estimate-arm="分析工具关"]')).includes('分析工具关')
+    && (await text(page, '[data-eval-estimate-arm="分析工具开"]')).includes('分析工具开'), await text(page, '[data-eval-estimate]'))
+  await shot(page, 'start-analysis-tools')
  await page.locator('[data-eval-start-mode="single"]').click()
+  await page.locator('#eval-arm-0-analysis').selectOption('off')
+  await until(() => writes(state, /estimate$/).some((w) => !w.body?.preset && w.body?.variant?.analysis?.tools === 'off'))
+  check('评一次：分析工具选「关」，变体里带 analysis.tools=off', lastWrite(state, /estimate$/)?.body?.variant?.analysis?.tools === 'off',
+    JSON.stringify(lastWrite(state, /estimate$/)?.body))
  await sleep(500)
  await page.locator('[data-eval-start-submit]').click()
  await until(() => lastWrite(state, /^POST \/evals\/suites\/suite-a\/evaluations$/))
@@ -982,6 +1012,8 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => {
  await page.locator('[data-eval-transitions]').waitFor()
  check('对比：指纹差异写哪几项变了', (await text(page, '[data-eval-fingerprint]')).includes('写作模型：model-a → model-b')
    && (await text(page, '[data-eval-fingerprint]')).includes('表结构：变了'))
+  check('对比：分析工具开、关照服务端写前后', (await text(page, '[data-eval-fingerprint]')).includes('分析工具：开 → 关'),
+    await text(page, '[data-eval-fingerprint]'))
  check('转移和归因', (await text(page, '[data-eval-transitions]')).includes('退步 1') && (await text(page, '[data-eval-transitions]')).includes('做法变 1')
    && await page.locator('[data-eval-transition="regressed"]').count() === 1)
  await shot(page, 'run-compare')
@@ -995,6 +1027,16 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => {
    && await text(page, '[data-eval-pair-only-b]') === '0' && await text(page, '[data-eval-pair-neither]') === '1')
  check('p 值和「这批题上看不出差别」照服务端写', (await text(page, '[data-eval-pair-p]')).includes('p = 0.5') && (await text(page, '[data-eval-pair-p]')).includes('看不出差别'))
  check('两臂各自链到自己的详情', await page.locator('[data-eval-pair-arm="关目录"]').getAttribute('href') === '/evals/runs/ev-pb?tab=pair')
+
+  // 分析工具贡献对照：两臂的名字、变体那一句（分析工具开 / 关）照服务端写
+  await page.goto(`${WEB}/evals/runs/ev-tb?tab=pair`, { waitUntil: 'networkidle' })
+  await page.locator('[data-eval-pair-row]').waitFor()
+  check('分析工具贡献对照：两臂各写自己的分析工具开、关', (await text(page, '[data-eval-pair-arm="分析工具开"]')).includes('分析工具开 · 结论裁判关')
+    && (await text(page, '[data-eval-pair-arm="分析工具关"]')).includes('分析工具关 · 结论裁判关')
+    && (await text(page, '[data-eval-pair-row]')).includes('分析工具开 vs 分析工具关'))
+  check('……这一臂的详情页头写它的变体（分析工具关）', (await text(page, '[data-eval-run-head]')).includes(TOOLS_TEXT(false)),
+    await text(page, '[data-eval-run-head]'))
+  await shot(page, 'run-pair-tools')
  check('没有页面报错', !errors.length, errors.join(' ｜ '))
 })