mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:21:58 +00:00
评测前端:变体里能选「分析工具」开 / 关,发起评测多一种「分析工具贡献对照」。typesEval 加 EvalAnalysisTools、EvalPreset,EvalVariant 多 analysis;termsEval 加分析工具的叫法、提示和这种对照的说明,对比「都没变」那句带上分析工具。StartDialog:ArmFields 多一格分析工具(题集的默认变体同一张表单),预设对照里由预设定的那一项不摆(目录贡献不摆目录、分析工具贡献不摆分析工具);分析工具贡献对照交 {preset: analysis_tools, variant},变体里不带分析工具和名字,缺省每题 3 次。结果页、对比页、A/B 照服务端的变体那一句和指纹差异显示。check-evals:发起时评一次选「关」、两种预设的请求体和估算的两臂,对比写「分析工具:开 → 关」,分析工具贡献对照的两臂和详情页头;test_frontend_eval_unions 对上分析工具的取值、叫法和预设对照
This commit is contained in:
@@ -5,9 +5,9 @@
|
||||
* 统计、结论、能不能做都由服务端给,这里只有叫法和句式:数字一律是服务端算好交来的,这里只负责写成话。
|
||||
*/
|
||||
import type {
|
||||
EvalAnalysisMode, EvalAnalysisOp, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind,
|
||||
EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger,
|
||||
EvalUpgradePolicy, EvalVerdict,
|
||||
EvalAnalysisMode, EvalAnalysisOp, EvalAnalysisTools, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind,
|
||||
EvalGoldKind, EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus,
|
||||
EvalTrigger, EvalUpgradePolicy, EvalVerdict,
|
||||
} from '../typesEval'
|
||||
import { formatCost, formatNumber } from './format'
|
||||
|
||||
@@ -380,12 +380,15 @@ export const EVAL_TEXT = {
|
||||
start: {
|
||||
title: '发起评测',
|
||||
mode: '评测方式',
|
||||
modes: { single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照' } as Record<'single' | 'ab' | 'catalog', string>,
|
||||
modes: {
|
||||
single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照', analysis_tools: '分析工具贡献对照',
|
||||
} as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
|
||||
modeHint: {
|
||||
single: '按一组变体评一遍题集',
|
||||
ab: '两到三臂各自一组变体,共用同一份期望答案,同一题几臂同时作答',
|
||||
catalog: '预设的 A/B:当前目录 vs 关目录,可以再加一臂「只用已核实和已确认的项」,量出数据目录帮了多少',
|
||||
} as Record<'single' | 'ab' | 'catalog', string>,
|
||||
analysis_tools: '预设的 A/B:分析工具开 vs 关,量出分析工具(变化拆解、波动基线……)帮了多少',
|
||||
} as Record<'single' | 'ab' | 'catalog' | 'analysis_tools', string>,
|
||||
arm: (name: string) => `${name} 臂`,
|
||||
addArm: '再加一臂',
|
||||
removeArm: '去掉这一臂',
|
||||
@@ -407,6 +410,9 @@ export const EVAL_TEXT = {
|
||||
catalogHint: '当前目录、只用已核实的项都在开评时拍一份快照按它读:评测期间目录被改也不影响可比性',
|
||||
snapshot: '选一份目录快照',
|
||||
snapshotNone: '还没有可用的目录快照',
|
||||
analysisTools: '分析工具',
|
||||
analysisToolsMode: { on: '开', off: '关' } as Record<EvalAnalysisTools, string>,
|
||||
analysisToolsHint: '关掉时助手建图看不到分析工具和分析卡;工作流题按固定的版本运行,不受影响',
|
||||
repeats: '每题重复几次',
|
||||
repeatsHint: '模型有随机性:对照建议每题 3 次',
|
||||
reviewedArm: '再加一臂「只用已核实和已确认的项」',
|
||||
@@ -586,7 +592,7 @@ export const EVAL_TEXT = {
|
||||
baseText: (short: string, when: string) => `${short} · ${when}`,
|
||||
none: '没有可比的上一次:同一题集、同一变体里还没有比这次先发起、已经评完的评测',
|
||||
fingerprint: '两次之间变了什么',
|
||||
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则、分析设置都没变',
|
||||
fingerprintNone: '模型、目录、分析工具、提示词、表结构、题目、评分规则、分析设置都没变',
|
||||
changed: '变了',
|
||||
transitions: '逐题转移',
|
||||
transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record<EvalTransition, string>,
|
||||
|
||||
@@ -6,13 +6,16 @@ import { Field, Modal, Notice, Spinner, toast, useRadioGroup } from '../../compo
|
||||
import { errorMessage } from '../../lib/errors'
|
||||
import { formatCost, formatDateTime, shortId } from '../../lib/format'
|
||||
import { EVAL_TEXT } from '../../lib/termsEval'
|
||||
import type { EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant } from '../../typesEval'
|
||||
import type {
|
||||
EvalAnalysisTools, EvalCatalogMode, EvalEstimate, EvalRun, EvalSnapshot, EvalStartBody, EvalSuite, EvalVariant,
|
||||
} from '../../typesEval'
|
||||
import { ModelPick } from './parts'
|
||||
|
||||
const T = EVAL_TEXT.start
|
||||
const MODES = ['single', 'ab', 'catalog'] as const
|
||||
const MODES = ['single', 'ab', 'catalog', 'analysis_tools'] as const
|
||||
type Mode = (typeof MODES)[number]
|
||||
const CATALOG_MODES: EvalCatalogMode[] = ['live', 'reviewed', 'off', 'snapshot']
|
||||
const ANALYSIS_TOOLS: EvalAnalysisTools[] = ['on', 'off']
|
||||
|
||||
/** 一臂的变体(表单里的样子):全部键都在,交上去之前不改形状 */
|
||||
export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
|
||||
@@ -21,6 +24,7 @@ export function blankVariant(base?: Partial<EvalVariant>): EvalVariant {
|
||||
writer: { provider: base?.writer?.provider ?? null, model: base?.writer?.model ?? null },
|
||||
judge: base?.judge ?? false,
|
||||
catalog: { mode: base?.catalog?.mode ?? 'live', snapshot: base?.catalog?.snapshot ?? null },
|
||||
analysis: { tools: base?.analysis?.tools ?? 'on' },
|
||||
repeats: base?.repeats ?? 1,
|
||||
label: base?.label ?? null,
|
||||
}
|
||||
@@ -37,12 +41,13 @@ export function snapshotLabel(s: EvalSnapshot): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* 一臂变体的表单:助手、写作模型(缺省按设置),结论裁判,数据目录(当前 / 只用已核实的 / 关 / 快照),重复次数,名字。
|
||||
* 题集设置里的「默认变体」也用它(定时评测按默认变体评)
|
||||
* 一臂变体的表单:助手、写作模型(缺省按设置),结论裁判,数据目录(当前 / 只用已核实的 / 关 / 快照),分析工具(开 / 关),
|
||||
* 重复次数,名字。题集设置里的「默认变体」也用它(定时评测按默认变体评)。预设对照里由预设定的那一项(目录、分析工具)不摆
|
||||
*/
|
||||
export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false }: {
|
||||
export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = false, labelIndex = 0, hideCatalog = false,
|
||||
hideAnalysis = false }: {
|
||||
idPrefix: string; value: EvalVariant; onChange: (v: EvalVariant) => void; snapshots: EvalSnapshot[]
|
||||
withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean
|
||||
withLabel?: boolean; labelIndex?: number; hideCatalog?: boolean; hideAnalysis?: boolean
|
||||
}) {
|
||||
const set = (patch: Partial<EvalVariant>) => onChange({ ...value, ...patch })
|
||||
const usable = usableSnapshots(snapshots)
|
||||
@@ -82,6 +87,14 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
{!hideAnalysis && (
|
||||
<Field label={T.analysisTools} htmlFor={`${idPrefix}-analysis`} hint={T.analysisToolsHint}>
|
||||
<select id={`${idPrefix}-analysis`} className="field w-24" value={value.analysis.tools} data-eval-analysis-tools=""
|
||||
onChange={(e) => set({ analysis: { tools: e.target.value as EvalAnalysisTools } })}>
|
||||
{ANALYSIS_TOOLS.map((m) => <option key={m} value={m}>{T.analysisToolsMode[m]}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
<Field label={T.repeats} htmlFor={`${idPrefix}-repeats`} hint={T.repeatsHint}>
|
||||
<select id={`${idPrefix}-repeats`} className="field w-24" value={value.repeats} data-eval-repeats=""
|
||||
onChange={(e) => set({ repeats: Number(e.target.value) })}>
|
||||
@@ -98,8 +111,8 @@ export function ArmFields({ idPrefix, value, onChange, snapshots, withLabel = fa
|
||||
}
|
||||
|
||||
/**
|
||||
* 发起评测:评一次 / A/B 对照(两到三臂) / 目录贡献对照(预设)。填好就向服务端要估算(几项、按上一次每题平均花费估的金额、
|
||||
* 生效的预算、同时评几题、今天评测已经花了多少)——估算和发起是同一份请求体,估出来的就是要发起的那一次。
|
||||
* 发起评测:评一次 / A/B 对照(两到三臂) / 目录贡献对照、分析工具贡献对照(预设)。填好就向服务端要估算(几项、按上一次每题
|
||||
* 平均花费估的金额、生效的预算、同时评几题、今天评测已经花了多少)——估算和发起是同一份请求体,估出来的就是要发起的那一次。
|
||||
*/
|
||||
export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
open: boolean; onClose: () => void; suite: EvalSuite; evaluations: EvalRun[]; onStarted: (run: EvalRun) => void
|
||||
@@ -132,7 +145,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
const repeats = Math.max(first.repeats, 3)
|
||||
return cur.length >= 2 ? cur : [{ ...first, repeats, label: first.label || 'A' }, { ...first, repeats, label: 'B' }]
|
||||
}
|
||||
return [m === 'catalog' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
|
||||
return [m === 'catalog' || m === 'analysis_tools' ? { ...first, repeats: Math.max(first.repeats, 3) } : first]
|
||||
})
|
||||
}
|
||||
|
||||
@@ -144,6 +157,10 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
const { catalog: _c, label: _l, ...rest } = v
|
||||
return { preset: reviewedArm ? 'catalog_reviewed' : 'catalog', variant: rest, ...base }
|
||||
}
|
||||
if (mode === 'analysis_tools') {
|
||||
const { analysis: _a, label: _l, ...rest } = v
|
||||
return { preset: 'analysis_tools', variant: rest, ...base }
|
||||
}
|
||||
return { variant: v, ...base }
|
||||
}, [mode, arms, reviewedArm, baseline])
|
||||
const key = JSON.stringify(body)
|
||||
@@ -207,7 +224,7 @@ export function StartDialog({ open, onClose, suite, evaluations, onStarted }: {
|
||||
|
||||
{mode !== 'ab' && (
|
||||
<ArmFields idPrefix="eval-arm-0" value={arms[0] ?? blankVariant()} snapshots={snapshots} hideCatalog={mode === 'catalog'}
|
||||
onChange={(v) => setArms([v])} />
|
||||
hideAnalysis={mode === 'analysis_tools'} onChange={(v) => setArms([v])} />
|
||||
)}
|
||||
{mode === 'catalog' && (
|
||||
<label className="flex items-center gap-2 text-xs">
|
||||
|
||||
@@ -370,26 +370,33 @@ export interface EvalImported {
|
||||
|
||||
export type EvalCatalogMode = 'live' | 'reviewed' | 'off' | 'snapshot'
|
||||
|
||||
/** 分析工具开、关(关的那一臂助手建图时看不到分析工具和分析卡) */
|
||||
export type EvalAnalysisTools = 'on' | 'off'
|
||||
|
||||
/** 预设对照:目录贡献(当前目录 vs 关,可加「只用已核实和已确认的项」)、分析工具贡献(开 vs 关) */
|
||||
export type EvalPreset = 'catalog' | 'catalog_reviewed' | 'analysis_tools'
|
||||
|
||||
export interface EvalModelPair {
|
||||
provider: string | null
|
||||
model: string | null
|
||||
}
|
||||
|
||||
/** 一臂变体:助手、写作模型,结论裁判,目录,重复次数,名字 */
|
||||
/** 一臂变体:助手、写作模型,结论裁判,目录,分析工具,重复次数,名字。加分析工具之前存下的评测记录里没有 analysis(当开) */
|
||||
export interface EvalVariant {
|
||||
assistant: EvalModelPair
|
||||
writer: EvalModelPair
|
||||
judge: boolean
|
||||
catalog: { mode: EvalCatalogMode; snapshot: string | null }
|
||||
analysis: { tools: EvalAnalysisTools }
|
||||
repeats: number
|
||||
label: string | null
|
||||
}
|
||||
|
||||
/** 发起时交上去的:一臂(variant)、几臂的 A/B(arms),或预设的「目录贡献对照」(preset) */
|
||||
/** 发起时交上去的:一臂(variant)、几臂的 A/B(arms),或预设的对照(preset:目录贡献、分析工具贡献) */
|
||||
export interface EvalStartBody {
|
||||
variant?: Partial<EvalVariant>
|
||||
arms?: Partial<EvalVariant>[]
|
||||
preset?: 'catalog' | 'catalog_reviewed'
|
||||
preset?: EvalPreset
|
||||
baseline_id?: string | null
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user