评测前端:变体里能选「分析工具」开 / 关,发起评测多一种「分析工具贡献对照」。typesEval 加 EvalAnalysisTools、EvalPreset,EvalVariant 多 analysis;termsEval 加分析工具的叫法、提示和这种对照的说明,对比「都没变」那句带上分析工具。StartDialog:ArmFields 多一格分析工具(题集的默认变体同一张表单),预设对照里由预设定的那一项不摆(目录贡献不摆目录、分析工具贡献不摆分析工具);分析工具贡献对照交 {preset: analysis_tools, variant},变体里不带分析工具和名字,缺省每题 3 次。结果页、对比页、A/B 照服务端的变体那一句和指纹差异显示。check-evals:发起时评一次选「关」、两种预设的请求体和估算的两臂,对比写「分析工具:开 → 关」,分析工具贡献对照的两臂和详情页头;test_frontend_eval_unions 对上分析工具的取值、叫法和预设对照

This commit is contained in:
ThearchyHelios
2026-10-21 19:32:38 +08:00
parent 8a6ad0e7a2
commit bf89269c11
5 changed files with 109 additions and 26 deletions
+12 -6
View File
@@ -5,9 +5,9 @@
* 统计、结论、能不能做都由服务端给,这里只有叫法和句式:数字一律是服务端算好交来的,这里只负责写成话。
*/
import type {
EvalAnalysisMode, EvalAnalysisOp, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind,
EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger,
EvalUpgradePolicy, EvalVerdict,
EvalAnalysisMode, EvalAnalysisOp, EvalAnalysisTools, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind,
EvalGoldKind, EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus,
EvalTrigger, EvalUpgradePolicy, EvalVerdict,
} from '../typesEval'
import { formatCost, formatNumber } from './format'
@@ -380,12 +380,15 @@ export const EVAL_TEXT = {
start: {
title: '发起评测',
mode: '评测方式',
modes: { single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照' } as Record<'single' | 'ab' | 'catalog', string>,
modes: {
single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照', analysis_tools: '分析工具贡献对照',
} as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
modeHint: {
single: '按一组变体评一遍题集',
ab: '两到三臂各自一组变体,共用同一份期望答案,同一题几臂同时作答',
catalog: '预设的 A/B:当前目录 vs 关目录,可以再加一臂「只用已核实和已确认的项」,量出数据目录帮了多少',
} as Record<'single' | 'ab' | 'catalog', string>,
analysis_tools: '预设的 A/B:分析工具开 vs 关,量出分析工具(变化拆解、波动基线……)帮了多少',
} as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
arm: (name: string) => `${name} 臂`,
addArm: '再加一臂',
removeArm: '去掉这一臂',
@@ -407,6 +410,9 @@ export const EVAL_TEXT = {
catalogHint: '当前目录、只用已核实的项都在开评时拍一份快照按它读:评测期间目录被改也不影响可比性',
snapshot: '选一份目录快照',
snapshotNone: '还没有可用的目录快照',
analysisTools: '分析工具',
analysisToolsMode: { on: '开', off: '关' } as Record<EvalAnalysisTools, string>,
analysisToolsHint: '关掉时助手建图看不到分析工具和分析卡;工作流题按固定的版本运行,不受影响',
repeats: '每题重复几次',
repeatsHint: '模型有随机性:对照建议每题 3 次',
reviewedArm: '再加一臂「只用已核实和已确认的项」',
@@ -586,7 +592,7 @@ export const EVAL_TEXT = {
baseText: (short: string, when: string) => `${short} · ${when}`,
none: '没有可比的上一次:同一题集、同一变体里还没有比这次先发起、已经评完的评测',
fingerprint: '两次之间变了什么',
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则、分析设置都没变',
fingerprintNone: '模型、目录、分析工具、提示词、表结构、题目、评分规则、分析设置都没变',
changed: '变了',
transitions: '逐题转移',
transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record<EvalTransition, string>,
+27 -10
View File
@@ -6,13 +6,16 @@ import { Field, Modal, Notice, Spinner, toast, useRadioGroup } from '../../compo
import { errorMessage } from '../../lib/errors'
import { formatCost, formatDateTime, shortId } from '../../lib/format'
import { EVAL_TEXT } from '../../lib/termsEval'
import type { EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant } from '../../typesEval'
import type {
EvalAnalysisTools, EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant,
} from '../../typesEval'
import { ModelPick } from './parts'
const T = EVAL_TEXT.start
const MODES = ['single', 'ab', 'catalog'] as const
const MODES = ['single', 'ab', 'catalog', 'analysis_tools'] as const
type Mode = (typeof MODES)[number]
const CATALOG_MODES: EvalCatalogMode[] = ['live', 'reviewed', 'off', 'snapshot']
const ANALYSIS_TOOLS: EvalAnalysisTools[] = ['on', 'off']
/** 一臂的变体(表单里的样子):全部键都在,交上去之前不改形状 */
export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
@@ -21,6 +24,7 @@ export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
writer: { provider: base?.writer?.provider ?? null, model: base?.writer?.model ?? null },
judge: base?.judge ?? false,
catalog: { mode: base?.catalog?.mode ?? 'live', snapshot: base?.catalog?.snapshot ?? null },
analysis: { tools: base?.analysis?.tools ?? 'on' },
repeats: base?.repeats ?? 1,
label: base?.label ?? null,
}
@@ -37,12 +41,13 @@ export function snapshotLabel(s: EvalSnapshot): string {
}
/**
* 一臂变体的表单:助手、写作模型(缺省按设置),结论裁判,数据目录(当前 / 只用已核实的 / 关 / 快照),重复次数,名字。
* 题集设置里的「默认变体」也用它(定时评测按默认变体评)
* 一臂变体的表单:助手、写作模型(缺省按设置),结论裁判,数据目录(当前 / 只用已核实的 / 关 / 快照),分析工具(开 / 关),
* 重复次数,名字。题集设置里的「默认变体」也用它(定时评测按默认变体评)。预设对照里由预设定的那一项(目录、分析工具)不摆
*/
export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false }: {
export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false,
hideAnalysis = false }: {
idPrefix: string; value: EvalVariant; onChange: (v: EvalVariant) => void; snapshots: EvalSnapshot[]
withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean
withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean; hideAnalysis?: boolean
}) {
const set = (patch: Partial<EvalVariant>) => onChange({ ...value, ...patch })
const usable = usableSnapshots(snapshots)
@@ -82,6 +87,14 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
</select>
</Field>
)}
{!hideAnalysis && (
<Field label={T.analysisTools} htmlFor={`${idPrefix}-analysis`} hint={T.analysisToolsHint}>
<select id={`${idPrefix}-analysis`} className="field w-24" value={value.analysis.tools} data-eval-analysis-tools=""
onChange={(e) => set({ analysis: { tools: e.target.value as EvalAnalysisTools } })}>
{ANALYSIS_TOOLS.map((m) => <option key={m} value={m}>{T.analysisToolsMode[m]}</option>)}
</select>
</Field>
)}
<Field label={T.repeats} htmlFor={`${idPrefix}-repeats`} hint={T.repeatsHint}>
<select id={`${idPrefix}-repeats`} className="field w-24" value={value.repeats} data-eval-repeats=""
onChange={(e) => set({ repeats: Number(e.target.value) })}>
@@ -98,8 +111,8 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
}
/**
* 发起评测:评一次 / A/B 对照(两到三臂) / 目录贡献对照(预设)。填好就向服务端要估算(几项、按上一次每题平均花费估的金额、
* 生效的预算、同时评几题、今天评测已经花了多少)——估算和发起是同一份请求体,估出来的就是要发起的那一次。
* 发起评测:评一次 / A/B 对照(两到三臂) / 目录贡献对照、分析工具贡献对照(预设)。填好就向服务端要估算(几项、按上一次每题
* 平均花费估的金额、生效的预算、同时评几题、今天评测已经花了多少)——估算和发起是同一份请求体,估出来的就是要发起的那一次。
*/
export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
open: boolean; onClose: () => void; suite: EvalSuite; evaluations: EvalRun[]; onStarted: (run: EvalRun) => void
@@ -132,7 +145,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
const repeats = Math.max(first.repeats, 3)
return cur.length >= 2 ? cur : [{ ...first, repeats, label: first.label || 'A' }, { ...first, repeats, label: 'B' }]
}
return [m === 'catalog' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
return [m === 'catalog' || m === 'analysis_tools' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
})
}
@@ -144,6 +157,10 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
const { catalog: _c, label: _l, ...rest } = v
return { preset: reviewedArm ? 'catalog_reviewed' : 'catalog', variant: rest, ...base }
}
if (mode === 'analysis_tools') {
const { analysis: _a, label: _l, ...rest } = v
return { preset: 'analysis_tools', variant: rest, ...base }
}
return { variant: v, ...base }
}, [mode, arms, reviewedArm, baseline])
const key = JSON.stringify(body)
@@ -207,7 +224,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
{mode !== 'ab' && (
<ArmFields idPrefix="eval-arm-0" value={arms[0] ?? blankVariant()} snapshots={snapshots} hideCatalog={mode === 'catalog'}
onChange={(v) => setArms([v])} />
hideAnalysis={mode === 'analysis_tools'} onChange={(v) => setArms([v])} />
)}
{mode === 'catalog' && (
<label className="flex items-center gap-2 text-xs">
+10 -3
View File
@@ -370,26 +370,33 @@ export interface EvalImported {
export type EvalCatalogMode = 'live' | 'reviewed' | 'off' | 'snapshot'
/** 分析工具开、关(关的那一臂助手建图时看不到分析工具和分析卡) */
export type EvalAnalysisTools = 'on' | 'off'
/** 预设对照:目录贡献(当前目录 vs 关,可加「只用已核实和已确认的项」)、分析工具贡献(开 vs 关) */
export type EvalPreset = 'catalog' | 'catalog_reviewed' | 'analysis_tools'
export interface EvalModelPair {
provider: string | null
model: string | null
}
/** 一臂变体:助手、写作模型,结论裁判,目录,重复次数,名字 */
/** 一臂变体:助手、写作模型,结论裁判,目录,分析工具,重复次数,名字。加分析工具之前存下的评测记录里没有 analysis(当开) */
export interface EvalVariant {
assistant: EvalModelPair
writer: EvalModelPair
judge: boolean
catalog: { mode: EvalCatalogMode; snapshot: string | null }
analysis: { tools: EvalAnalysisTools }
repeats: number
label: string | null
}
/** 发起时交上去的:一臂(variant)、几臂的 A/B(arms),或预设的「目录贡献对照」(preset) */
/** 发起时交上去的:一臂(variant)、几臂的 A/B(arms),或预设的对照(preset:目录贡献、分析工具贡献) */
export interface EvalStartBody {
variant?: Partial<EvalVariant>
arms?: Partial<EvalVariant>[]
preset?: 'catalog' | 'catalog_reviewed'
preset?: EvalPreset
baseline_id?: string | null
}