评测前端认两种分析期望:题目编辑器在预览之后选贡献或判定(分析方法、模式、按哪几列、期次、点名前几段),试算写出服务端算的判定和判定语;结果页逐题写期望和答案的判定,点开写判定语、引用了哪一份结论或正文那一句;总览按期望的种类分开写结论

- typesEval.ts:期望种类加 contribution、verdict,期望带 op、mode、params;EvalAnalysisOp、EvalAnalysisMode、EvalJudgement(同服务端契约和 judgements.JUDGEMENTS);期望值、评分带判定类、点名的段、方向、判定语;总表加 outcomes_by_kind
- termsEval.ts:贡献、判定的叫法;分析方法、模式的叫法和契约的 OP_LABELS、MODE_LABELS 一字不差(test_frontend_eval_unions 逐个对照,连同判定类);判定类、方向、参数的叫法;指纹差异一句补上分析设置
- parts.tsx:期望写成「变化拆解(可加) · 按地区 · 销售额」;判定写成「主要来源:华东(下降)」「超出常规波动(上升)」「差异站得住(西南高于华东)」,算不了的照实写原因;判定类、段、方向都是服务端给的,界面只套叫法
- CaseEditor.tsx:分析期望的表单按分析方法列参数(参数名同分析工具,列从预览结果里选),换方法、模式只留新方法还认的参数;编辑配了几个期望的题时另外的期望列出来、保存时照旧带上(以前只留第一个,第二个期望会丢)
- EvaluationPage.tsx:逐题表、展开的明细、总览按种类
- scripts/check-evals.mjs:夹具补分析期望(加题选算子、试算、保存的请求体不带来源方向判定,编辑保留另外的期望;结果页四种结论、点开的明细、总览按种类),共 130 项
This commit is contained in:
ThearchyHelios
2026-10-21 18:23:57 +08:00
parent 83cd5da72e
commit 9358d9ec48
7 changed files with 591 additions and 23 deletions
+71 -4
View File
@@ -5,8 +5,9 @@
* 统计、结论、能不能做都由服务端给,这里只有叫法和句式:数字一律是服务端算好交来的,这里只负责写成话。
*/
import type {
EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind, EvalOutcome, EvalPath,
EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger, EvalUpgradePolicy, EvalVerdict,
EvalAnalysisMode, EvalAnalysisOp, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind,
EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger,
EvalUpgradePolicy, EvalVerdict,
} from '../typesEval'
import { formatCost, formatNumber } from './format'
@@ -150,7 +151,8 @@ export const EVAL_TEXT = {
},
expectation: {
kind: {
scalar: '一个数', table: '多行', set: '名单', none: '没有数据', ask: '反问', text: '文字要点',
scalar: '一个数', table: '多行', set: '名单', none: '没有数据', ask: '反问', text: '文字要点', contribution: '贡献',
verdict: '判定',
} as Record<EvalExpectationKind, string>,
scalarCell: (column: string, row: number) => `「${column}」第 ${row + 1} 行`,
scalarWhere: (column: string, where: string) => `「${column}」(${where})`,
@@ -160,6 +162,9 @@ export const EVAL_TEXT = {
none: '期望答「没有数据」',
ask: '期望反问',
text: (n: number) => `${n} 条文字要点(模型判断,不计入准确率)`,
/** 贡献、判定:「变化拆解(可加)· 按地区」「波动基线 · 订单数」 */
analysis: (method: string, about: string) => (about ? `${method} · ${about}` : method),
topN: (n: number) => `点名前 ${n} 段`,
optional: '非必需',
tolerance: (t: number) => `允许误差 ${(t * 100).toFixed(t < 0.01 ? 2 : 1)}%`,
},
@@ -254,6 +259,21 @@ export const EVAL_TEXT = {
specialNoneHint: '写一条查这件事的 SQL,它应当返回 0 行',
specialAsk: '期望助手反问',
specialAskHint: '问法含糊、该先问清楚的题:助手反问了就算对,不需要期望答案',
// 分析期望
analysisLabel: '或者定分析期望',
analysisHint: '在这条 SQL 的结果上按平台的分析方法算一次:期望的主要来源、判定都由算子算出来,不靠人填;阈值按评测时的分析设置,和作答用同一份',
analysisKind: {
contribution: '贡献:变化从哪来', verdict: '判定:算不算异常、有没有趋势、差异站不站得住、能不能比',
} as Record<'contribution' | 'verdict', string>,
analysisOp: '分析方法',
analysisMode: '模式',
analysisTopN: '期望点名主要来源的前几段',
analysisColumn: '选一列',
analysisNone: '不填',
analysisPeriodHint: '不填基期、本期时,期望 SQL 只能返回两期:先的一期是基期、后的一期是本期;可以写 {{ schedule.last_week_start }} 这类时间变量',
analysisPairsHint: '每行一对,两组之间用逗号隔开,例如:西南,华东',
analysisPick: '定这个分析期望',
analysisOthers: '这道题另外的期望(照旧保留)',
// 文字要点
points: '文字要点(可选)',
pointsHint: '一行一条;由结论裁判判断有没有答到,单列「模型判断」,不计入准确率',
@@ -285,6 +305,47 @@ export const EVAL_TEXT = {
table: (n: number) => `${formatNumber(n)} 行`,
none: '0 行(没有数据)',
dims: (d: string) => `(${d})`,
/** 分析期望:判定类和系统渲染的判定语 */
analysis: (judgement: string, phrase: string) => (phrase ? `${judgement} · ${phrase}` : judgement),
refused: (reason: string) => `算不了:${reason}(这一项不能评分)`,
},
/**
* 分析期望(贡献、判定)的叫法。分析方法、模式的叫法和服务端契约(app/analysis/contract 的 OP_LABELS、MODE_LABELS)一字
* 不差(tests/test_frontend_eval_unions.py 逐个对照);判定类是评测自己的(app/eval/judgements.JUDGEMENTS)
*/
analysis: {
op: {
decompose: '变化拆解', baseline: '波动基线', trend: '趋势判断', differ: '差异检验', comparable: '可比性检查',
} as Record<EvalAnalysisOp, string>,
mode: {
additive: '可加', ratio: '比率', factor: '乘积', rate: '比率', mean: '均值', rows: '逐行明细',
} as Record<EvalAnalysisMode, string>,
judgement: {
source: '主要来源', dispersed: '分散,没有主要来源', flat: '基本持平', mix: '主要来自结构变化', rate: '主要来自各段自身的变化',
mixed: '几方面都有影响', normal: '常规波动', anomaly: '超出常规波动', rising: '上升趋势', falling: '下降趋势',
no_trend: '没有明显趋势', differs: '差异站得住', no_evidence: '不足以下结论', comparable: '可比', incomparable: '不可比',
} as Record<EvalJudgement, string>,
direction: { up: '上升', down: '下降' } as Record<'up' | 'down', string>,
/** 「主要来源:华东(下降)」「西南高于华东」 */
withSubjects: (judgement: string, subjects: string) => `${judgement}:${subjects}`,
withDirection: (text: string, direction: string) => `${text}(${direction})`,
higher: (a: string, b: string) => `${a}高于${b}`,
lower: (a: string, b: string) => `${a}低于${b}`,
pair: (a: string, b: string) => `${a}和${b}`,
/** 方法的叫法加模式:「变化拆解(可加)」 */
method: (op: string, mode: string | null) => (mode ? `${op}(${mode})` : op),
by: (columns: string) => `按${columns}`,
ratio: (numerator: string, denominator: string) => `${numerator} ÷ ${denominator}`,
/** 答案里同类的分析没有做成 */
refused: (reason: string) => `没有做成:${reason}`,
param: {
by: '维度列', period: '期次列', base: '基期', target: '本期', measure: '度量列', numerator: '分子列', denominator: '分母列',
product: '乘积列', factors: '因子列', label: '度量的叫法', basis: '比较方式', grain: '粒度', lag: '和往前第几期比',
pairs: '比哪几对', reference: '对照组', mean: '均值列', sd: '标准差列', sum: '合计列', sumsq: '平方和列', n: '样本数列',
value: '逐行的值列',
} as Record<string, string>,
basis: { change: '和比较期比的变化率', level: '值本身' } as Record<string, string>,
grain: { day: '天', week: '周', month: '月', quarter: '季度' } as Record<string, string>,
},
upgrade: {
title: (seq: number) => `第 ${seq} 题:处置口径升版`,
@@ -451,6 +512,7 @@ export const EVAL_TEXT = {
role: { assistant: '助手', writer: '写作', judge: '结论裁判', gold: '算期望' } as Record<string, string>,
passRates: '重复评的题的通过率',
outcomes: '各期望的结论',
outcomesByKind: '按期望的种类',
baseline: '和上一次比',
baselineText: (regressed: number, improved: number) => `退步 ${formatNumber(regressed)} 题 · 进步 ${formatNumber(improved)} 题`,
baselineNone: '没有可比的上一次',
@@ -509,6 +571,11 @@ export const EVAL_TEXT = {
retriesWhy: (reasons: string[]) => `:${reasons.join(';')}`,
turnReason: '作答的经过',
answer: '答案原文',
/** 分析期望展开的明细 */
analysisWant: '期望的判定',
analysisFound: '答案',
analysisCited: (alias: string) => `引用了 ${alias}`,
analysisSaid: (text: string) => `正文写的「${text}」`,
expand: '展开',
collapse: '收起',
},
@@ -519,7 +586,7 @@ export const EVAL_TEXT = {
baseText: (short: string, when: string) => `${short} · ${when}`,
none: '没有可比的上一次:同一题集、同一变体里还没有比这次先发起、已经评完的评测',
fingerprint: '两次之间变了什么',
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则都没变',
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则、分析设置都没变',
changed: '变了',
transitions: '逐题转移',
transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record<EvalTransition, string>,
+191 -1
View File
@@ -10,7 +10,8 @@ import { EVAL_TEXT } from '../../lib/termsEval'
import { useCatalog } from '../../store/catalog'
import type { Run } from '../../types'
import type {
EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview, EvalRunQuery, EvalSuite, EvalTrial,
EvalAnalysisMode, EvalAnalysisOp, EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview,
EvalRunQuery, EvalSuite, EvalTrial,
} from '../../typesEval'
import { graphFields, type InputField } from '../schedules/model'
import { cellText, expectationText, goldValueText } from './parts'
@@ -31,6 +32,11 @@ function modeOf(c: EvalCase | null | undefined): Mode {
/** 题目里那个要答出数(或答「没有数据」、反问)的期望:编辑时据它回填 */
const mainOf = (c: EvalCase | null | undefined) => c?.expectations.find((e) => e.kind !== 'text') ?? null
/** 别的期望(一道题配了不止一个,比如贡献再配一个「一个数」):编辑框不改它们,保存时照旧带上 */
const othersOf = (c: EvalCase | null | undefined) => {
const main = mainOf(c)
return (c?.expectations ?? []).filter((e) => e !== main && e.kind !== 'text')
}
const pointsOf = (c: EvalCase | null | undefined) => (c?.expectations.find((e) => e.kind === 'text')?.points ?? []).join('\n')
/**
@@ -57,6 +63,8 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
const [tolerance, setTolerance] = useState('')
/** 要答出数的那一个期望(点选出来的,或编辑时回填的) */
const [main, setMain] = useState<EvalExpectation | null>(null)
/** 编辑的题另外配的期望:原样保留 */
const [others, setOthers] = useState<EvalExpectation[]>([])
// SQL(写 SQL 定期望、期望没有数据)
const [sourceId, setSourceId] = useState('')
const [sql, setSql] = useState('')
@@ -95,6 +103,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
setPoints(pointsOf(c))
const m = mainOf(c)
setMain(m)
setOthers(othersOf(c))
setRequired(m?.required !== false)
setTolerance(m?.tolerance != null ? String(m.tolerance) : '')
setSourceId(c?.gold?.datasource_id ?? '')
@@ -118,6 +127,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
if (m === mode || editing) return
setMode(m)
setMain(null)
setOthers([])
setTrial(null)
setError(null)
setPreview(null)
@@ -134,6 +144,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
if (t && (e.kind === 'scalar' || e.kind === 'table')) e.tolerance = Number(t)
out.push(e)
}
out.push(...others)
const list = points.split('\n').map((p) => p.trim()).filter(Boolean)
if (list.length) out.push({ kind: 'text', points: list })
return out
@@ -339,6 +350,14 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
</div>
)}
{needsMain && !main && mode === 'sql' && <p className="text-2xs text-faint" data-eval-case-no-pick="">{T.noPick}</p>}
{!!others.length && (
<div className="space-y-0.5 rounded-md border border-dashed p-2.5 text-xs" data-eval-case-others={others.length}>
<div className="label">{T.analysisOthers}</div>
{others.map((e, i) => (
<div key={e.id ?? i} className="text-dim">{EVAL_TEXT.expectation.kind[e.kind]}:{expectationText(e)}</div>
))}
</div>
)}
<div className="grid gap-3 sm:grid-cols-2">
<Field label={T.scope} htmlFor="eval-case-scope" hint={T.scopeHint}>
@@ -479,10 +498,181 @@ function SqlPart({ suiteId: _suiteId, sourceId, setSourceId, sql, setSql, previe
{ok && preview && !!preview.rows.length && (
<PreviewGrid preview={preview} main={main} setMain={setMain} pickable={pickable} />
)}
{ok && preview && !!preview.rows.length && pickable && (
<AnalysisPick columns={preview.columns} main={main} setMain={setMain} />
)}
</div>
)
}
// ---------------------------------------------------------------------------
// 分析期望:贡献、判定
// ---------------------------------------------------------------------------
type ParamKind = 'column' | 'columns' | 'text' | 'int' | 'choice' | 'pairs'
interface ParamField { key: string; kind: ParamKind; choices?: Record<string, string> }
const A = EVAL_TEXT.analysis
/** 判定期望能用的分析(贡献恒为变化拆解)、各自的模式(同服务端 contract.MODES) */
const VERDICT_OPS: EvalAnalysisOp[] = ['baseline', 'trend', 'differ', 'comparable']
const OP_MODES: Partial<Record<EvalAnalysisOp, EvalAnalysisMode[]>> = {
decompose: ['additive', 'ratio', 'factor'], differ: ['rate', 'mean', 'rows'],
}
const col = (key: string): ParamField => ({ key, kind: 'column' })
const PERIODS: ParamField[] = [col('period'), { key: 'base', kind: 'text' }, { key: 'target', kind: 'text' }]
const LABEL: ParamField = { key: 'label', kind: 'text' }
const GRAIN: ParamField = { key: 'grain', kind: 'choice', choices: A.grain }
const GROUPS: ParamField[] = [{ key: 'by', kind: 'columns' }, { key: 'pairs', kind: 'pairs' }, { key: 'reference', kind: 'text' }]
/** 每种分析(模式)在表单里填哪几个参数:参数名同分析工具(服务端按同一份参数定义校验),叫法在 termsEval */
const PARAM_FIELDS: Record<string, ParamField[]> = {
'decompose:additive': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('measure'), LABEL],
'decompose:ratio': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('numerator'), col('denominator'), LABEL],
'decompose:factor': [...PERIODS, col('product'), { key: 'factors', kind: 'columns' }, LABEL],
baseline: [col('period'), col('measure'), { key: 'target', kind: 'text' },
{ key: 'basis', kind: 'choice', choices: A.basis }, GRAIN, { key: 'lag', kind: 'int' }, LABEL],
trend: [col('period'), col('measure'), GRAIN, LABEL],
'differ:rate': [...GROUPS, col('numerator'), col('denominator'), LABEL],
'differ:mean': [...GROUPS, col('mean'), col('sd'), col('sum'), col('sumsq'), col('n'), LABEL],
'differ:rows': [...GROUPS, col('value'), LABEL],
comparable: [...PERIODS, col('measure'), GRAIN],
}
const fieldsOf = (e: EvalExpectation): ParamField[] =>
PARAM_FIELDS[e.op === 'decompose' || e.op === 'differ' ? `${e.op}:${e.mode}` : e.op ?? ''] ?? []
/** 换了分析方法、模式时,只留下新的那种分析还认的参数 */
function keepParams(e: EvalExpectation): EvalExpectation {
const keys = new Set(fieldsOf(e).map((f) => f.key))
const params = Object.fromEntries(Object.entries(e.params ?? {}).filter(([k]) => keys.has(k)))
return { ...e, params }
}
/**
* 在预览的结果上定一个分析期望:贡献(变化拆解:选模式、按哪几列拆、期次、度量……,期望点名前几段)或判定(波动基线、
* 趋势判断、差异检验、可比性检查)。只填做哪种分析、按哪几列;主要来源、判定由服务端在期望 SQL 的结果上按平台的分析方法算,
* 试算时写出来(界面不判)。列从预览结果里选;基期、本期、对照组是期次列、分组列里的值,可以写时间变量
*/
function AnalysisPick({ columns, main, setMain }: {
columns: string[]; main: EvalExpectation | null; setMain: (e: EvalExpectation | null) => void
}) {
const current = main && (main.kind === 'contribution' || main.kind === 'verdict') ? main : null
const start = (kind: 'contribution' | 'verdict') => setMain(kind === 'contribution'
? { kind, op: 'decompose', mode: 'additive', params: {}, top_n: 1 }
: { kind, op: 'baseline', params: {} })
const update = (patch: Partial<EvalExpectation>) => current && setMain(keepParams({ ...current, ...patch }))
const setParam = (key: string, value: unknown) => {
if (!current) return
const params = { ...(current.params ?? {}) }
if (value === '' || value == null || (Array.isArray(value) && !value.length)) delete params[key]
else params[key] = value
setMain({ ...current, params })
}
const modes = current?.op ? OP_MODES[current.op] : undefined
return (
<div className="space-y-2 rounded-md border p-2.5 text-xs" data-eval-analysis={current?.kind ?? ''}>
<div className="label">{T.analysisLabel}</div>
<p className="text-2xs leading-relaxed text-faint">{T.analysisHint}</p>
<div className="flex flex-wrap gap-1.5">
{(['contribution', 'verdict'] as const).map((k) => (
<button key={k} type="button" data-eval-analysis-kind={k} onClick={() => start(k)}
className={clsx('rounded-md border px-2 py-1 text-left', current?.kind === k ? 'border-[var(--accent)] bg-accent-soft text-fg' : 'text-dim hover:bg-hover')}>
{T.analysisKind[k]}
</button>
))}
</div>
{current && (
<div className="grid gap-2 sm:grid-cols-2">
{current.kind === 'verdict' && (
<Field label={T.analysisOp} htmlFor="eval-analysis-op">
<select id="eval-analysis-op" className="field" value={current.op} data-eval-analysis-op=""
onChange={(e) => {
const op = e.target.value as EvalAnalysisOp
update({ op, mode: OP_MODES[op]?.[0] })
}}>
{VERDICT_OPS.map((o) => <option key={o} value={o}>{A.op[o]}</option>)}
</select>
</Field>
)}
{modes && (
<Field label={T.analysisMode} htmlFor="eval-analysis-mode">
<select id="eval-analysis-mode" className="field" value={current.mode ?? ''} data-eval-analysis-mode=""
onChange={(e) => update({ mode: e.target.value as EvalAnalysisMode })}>
{modes.map((m) => <option key={m} value={m}>{A.method(A.op[current.op!], A.mode[m])}</option>)}
</select>
</Field>
)}
{current.kind === 'contribution' && (
<Field label={T.analysisTopN} htmlFor="eval-analysis-topn">
<input id="eval-analysis-topn" className="field tnum w-24" inputMode="numeric" value={current.top_n ?? 1}
data-eval-analysis-topn="" onChange={(e) => update({ top_n: Number(e.target.value) || 1 })} />
</Field>
)}
{fieldsOf(current).map((f) => (
<ParamInput key={`${current.op}:${current.mode}:${f.key}`} field={f} columns={columns}
value={(current.params ?? {})[f.key]} onChange={(v) => setParam(f.key, v)} />
))}
{fieldsOf(current).some((f) => f.key === 'base') && (
<p className="text-2xs leading-relaxed text-faint sm:col-span-2">{T.analysisPeriodHint}</p>
)}
</div>
)}
</div>
)
}
function ParamInput({ field, columns, value, onChange }: {
field: ParamField; columns: string[]; value: unknown; onChange: (v: unknown) => void
}) {
const id = `eval-analysis-param-${field.key}`
const label = A.param[field.key] ?? field.key
if (field.kind === 'columns') {
const picked = Array.isArray(value) ? value.map(String) : []
return (
<div className="sm:col-span-2" data-eval-analysis-param={field.key}>
<div className="label">{label}</div>
<div className="flex flex-wrap gap-2">
{columns.map((c) => (
<label key={c} className="flex items-center gap-1">
<input type="checkbox" checked={picked.includes(c)} data-eval-analysis-param-col={c}
onChange={(e) => onChange(e.target.checked ? [...picked, c] : picked.filter((x) => x !== c))} />
{c}
</label>
))}
</div>
</div>
)
}
if (field.kind === 'pairs') {
const text = Array.isArray(value) ? value.map((p) => (Array.isArray(p) ? p.join(',') : String(p))).join('\n') : ''
return (
<Field label={label} htmlFor={id} hint={T.analysisPairsHint} className="sm:col-span-2">
<textarea id={id} className="field min-h-12 text-xs" defaultValue={text} data-eval-analysis-param={field.key}
onChange={(e) => onChange(e.target.value.split('\n').map((line) => line.split(/[,,、]/).map((x) => x.trim())
.filter(Boolean)).filter((pair) => pair.length))} />
</Field>
)
}
if (field.kind === 'column' || field.kind === 'choice') {
const options = field.kind === 'column' ? Object.fromEntries(columns.map((c) => [c, c])) : field.choices ?? {}
return (
<Field label={label} htmlFor={id}>
<select id={id} className="field" value={value == null ? '' : String(value)} data-eval-analysis-param={field.key}
onChange={(e) => onChange(e.target.value)}>
<option value="">{field.kind === 'column' ? T.analysisColumn : T.analysisNone}</option>
{Object.entries(options).map(([k, v]) => <option key={k} value={k}>{v}</option>)}
</select>
</Field>
)
}
return (
<Field label={label} htmlFor={id}>
<input id={id} className={clsx('field', field.kind === 'int' && 'tnum w-24')} value={value == null ? '' : String(value)}
inputMode={field.kind === 'int' ? 'numeric' : undefined} placeholder={field.kind === 'text' ? T.analysisNone : undefined}
data-eval-analysis-param={field.key}
onChange={(e) => onChange(field.kind === 'int' ? (e.target.value.trim() ? Number(e.target.value) : '') : e.target.value)} />
</Field>
)
}
/** 预览结果:点一格定「一个数」,点列名定「名单」;「多行」选配对的列。格里的值是服务端遮罩过、确认不涉及遮罩列的 */
function PreviewGrid({ preview, main, setMain, pickable }: {
preview: EvalPreview; main: EvalExpectation | null; setMain: (e: EvalExpectation | null) => void; pickable: boolean
+59 -9
View File
@@ -10,7 +10,7 @@ import { formatCost, formatDateTime, formatDuration, formatNumber, formatTokens,
import { EVAL_TEXT, distText, pct, rateText } from '../../lib/termsEval'
import { useOnReconnect } from '../../store/catalog'
import type { EvalCompare, EvalPair, EvalResult, EvalRun, EvalRunDetail, EvalScore, EvalSuiteDetail, EvalSummary } from '../../typesEval'
import { EvalStatusPill, RateCard, StatCard, VerdictChip, cellText, usageCost } from './parts'
import { EvalStatusPill, RateCard, StatCard, VerdictChip, cellText, judgementText, usageCost } from './parts'
const T = EVAL_TEXT
type TabKey = 'overview' | 'results' | 'compare' | 'pair'
@@ -223,6 +223,18 @@ function Overview({ run }: { run: EvalRun }) {
{!!s.not_run && <p className="text-dim" data-eval-not-run={s.not_run}>{O.notRun(s.not_run)}</p>}
{!!s.errors && <p className="text-dim" data-eval-errors={s.errors}>{O.errors(s.errors)}</p>}
<p className="text-dim">{O.outcomes}:{distText(s.outcomes as Record<string, number>, T.outcome)}</p>
{!!s.outcomes_by_kind && Object.keys(s.outcomes_by_kind).length > 0 && (
<div className="text-dim" data-eval-outcomes-by-kind="">
<span>{O.outcomesByKind}:</span>
<ul className="mt-0.5 space-y-0.5 pl-3">
{Object.entries(s.outcomes_by_kind).map(([kind, dist]) => (
<li key={kind} data-eval-outcomes-kind={kind}>
{T.expectation.kind[kind as keyof typeof T.expectation.kind] ?? kind}:{distText(dist as Record<string, number>, T.outcome)}
</li>
))}
</ul>
</div>
)}
{s.text && Object.keys(s.text).length > 0 && <p className="text-dim">{O.text}:{distText(s.text as Record<string, number>, T.textOutcome)}</p>}
</Section>
{base && (
@@ -394,8 +406,17 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
<td className="px-2 py-1.5">
{counted.length ? counted.map((s) => (
<div key={s.id} className="tnum" data-eval-score={s.outcome}>
{s.expected !== undefined && <span>{cellText(s.expected)}</span>}
{s.got !== undefined && <span className="text-dim"> → {cellText(s.got)}</span>}
{s.want ? (
<span data-eval-score-judgement="">
{judgementText(s.want)}
{s.found && <span className="text-dim"> → {judgementText(s.found)}</span>}
</span>
) : (
<>
{s.expected !== undefined && <span>{cellText(s.expected)}</span>}
{s.got !== undefined && <span className="text-dim"> → {cellText(s.got)}</span>}
</>
)}
<span className="ml-1 text-2xs text-faint">{scoreText(s)}</span>
{s.weak && <span className="ml-1 chip text-[var(--warn)]" title={R.weakHint}>{R.weak}</span>}
</div>
@@ -420,12 +441,15 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
<td colSpan={8} className="space-y-1.5 px-3 py-2">
{r.error && <p className="text-dim"><span className="text-faint">{R.reason}:</span>{r.error}</p>}
{r.scores.map((s) => (
<p key={s.id} className="text-dim">
<span className="text-faint">{T.expectation.kind[s.kind]}{s.label ? `「${s.label}」` : ''}:</span>
{scoreText(s)}
{s.key && <span className="text-faint">({Object.entries(s.key).map(([k, v]) => `${k}=${cellText(v)}`).join('、')})</span>}
{s.reason && <span> · {s.reason}</span>}
</p>
<Fragment key={s.id}>
<p className="text-dim">
<span className="text-faint">{T.expectation.kind[s.kind]}{s.label ? `「${s.label}」` : ''}:</span>
{scoreText(s)}
{s.key && <span className="text-faint">({Object.entries(s.key).map(([k, v]) => `${k}=${cellText(v)}`).join('、')})</span>}
{s.reason && <span> · {s.reason}</span>}
</p>
{s.want && <AnalysisDetail score={s} />}
</Fragment>
))}
{r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && (
<p className="text-faint" data-eval-sql-basis={r.sql_match.basis}>{T.sqlMatchBasis[r.sql_match.basis]}</p>
@@ -450,6 +474,32 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
)
}
/**
* 分析期望的明细:期望的判定和系统渲染的判定语(期望 SQL 上算的),答案里对上(或对不上)的那一份——引用的分析结论编号和
* 它的判定语,或者正文里说出这个判断的那一句。都是服务端给的,照原样写
*/
function AnalysisDetail({ score }: { score: EvalScore }) {
const R = T.results
const want = score.want ?? {}
const found = score.found
return (
<div className="space-y-0.5 border-l-2 pl-2 text-dim" data-eval-analysis-detail={score.id}>
<p data-eval-analysis-want="">
<span className="text-faint">{R.analysisWant}:</span>{judgementText(want)}
{want.method && <span className="text-faint"> · {want.method}</span>}
{want.phrase && <span className="block text-faint">{want.phrase}</span>}
</p>
{found && (
<p data-eval-analysis-found={found.alias ?? 'text'}>
<span className="text-faint">{R.analysisFound}:</span>
{found.alias ? R.analysisCited(found.alias) : found.text ? R.analysisSaid(found.text) : judgementText(found)}
{found.alias && found.phrase && <span className="block text-faint">{found.phrase}</span>}
</p>
)}
</div>
)
}
/** 一题的明细(GET …/results/{id} 的明细工件):这一轮作答的经过(重来了几次、原因)和读出来的答案原文。明细清理了的不显示 */
function ResultDetail({ evaluationId, resultId }: { evaluationId: string; resultId: string }) {
const [details, setDetails] = useState<Record<string, any> | null | undefined>(undefined)
+43 -1
View File
@@ -6,7 +6,8 @@ import { formatCost, formatDateTime, formatNumber } from '../../lib/format'
import { EVAL_TEXT as T, pct, rateText } from '../../lib/termsEval'
import { runCostText } from '../../lib/terms'
import type {
EvalExpectation, EvalGoldValue, EvalModelPair, EvalRate, EvalStatus, EvalTrendPoint, EvalUsage, EvalVerdict,
EvalExpectation, EvalGoldValue, EvalJudgementView, EvalModelPair, EvalRate, EvalStatus, EvalTrendPoint, EvalUsage,
EvalVerdict,
} from '../../typesEval'
/*
@@ -144,6 +145,8 @@ export function expectationText(e: EvalExpectation): string {
case 'none': text = E.none; break
case 'ask': text = E.ask; break
case 'text': text = E.text(e.points?.length ?? 0); break
case 'contribution':
case 'verdict': text = analysisText(e); break
default: text = T.expectation.kind[e.kind as keyof typeof T.expectation.kind] ?? String(e.kind)
}
const extra = [
@@ -154,11 +157,50 @@ export function expectationText(e: EvalExpectation): string {
return extra.length ? `${text}(${extra.join(',')})` : text
}
/** 分析期望做哪种分析、分析什么:「变化拆解(可加) · 按地区 · 销售额」,贡献点名不止一段时另写几段 */
function analysisText(e: EvalExpectation): string {
const A = T.analysis
const op = e.op ?? 'decompose'
const method = A.method(A.op[op] ?? op, e.mode ? A.mode[e.mode] ?? e.mode : null)
const p = e.params ?? {}
const words = (v: unknown) => (Array.isArray(v) ? v.map(String).join('、') : v == null ? '' : String(v))
const about = [
words(p.by) && A.by(words(p.by)),
words(p.label) || words(p.measure) || words(p.product) || words(p.value)
|| (p.numerator && p.denominator ? A.ratio(words(p.numerator), words(p.denominator)) : ''),
].filter(Boolean).join(' · ')
const text = T.expectation.analysis(method, about)
return e.kind === 'contribution' && (e.top_n ?? 1) > 1 ? `${text} · ${T.expectation.topN(e.top_n ?? 1)}` : text
}
/**
* 分析期望的判定(期望、答案各一份)写成字:「主要来源:华东(下降)」「超出常规波动(上升)」「差异站得住(西南高于华东)」。
* 判定类、点名的段、方向都是服务端给的,这里只套叫法;没有判定类的(正文里一句对不上的话)写那一句
*/
export function judgementText(v: EvalJudgementView | EvalGoldValue | null | undefined): string {
if (!v) return '—'
const A = T.analysis
if (v.status === 'refused') return A.refused(v.reason ?? '')
if (!v.judgement) return 'text' in v && v.text ? `「${v.text}」` : '—'
const label = A.judgement[v.judgement] ?? v.judgement
const subjects = v.subjects ?? []
if (subjects.length === 2 && (v.judgement === 'differs' || v.judgement === 'no_evidence')) {
const [a, b] = subjects
const which = v.judgement === 'no_evidence' || !v.direction ? A.pair(a, b) : v.direction === 'up' ? A.higher(a, b) : A.lower(a, b)
return A.withDirection(label, which)
}
const text = v.judgement === 'source' && subjects.length ? A.withSubjects(label, subjects.join('、')) : label
return v.direction ? A.withDirection(text, A.direction[v.direction] ?? v.direction) : text
}
/** 试算算出来的一份期望值写成字(值都是服务端算好的) */
export function goldValueText(v: EvalGoldValue | undefined): string {
if (!v) return '—'
const G = T.goldValue
switch (v.kind) {
case 'contribution':
case 'verdict':
return v.status === 'refused' ? G.refused(v.reason ?? '') : G.analysis(judgementText(v), v.phrase ?? '')
case 'scalar': {
const dims = v.dims && Object.keys(v.dims).length
? G.dims(Object.entries(v.dims).map(([k, x]) => `${k}=${cellText(x)}`).join('、')) : ''
+55 -2
View File
@@ -88,10 +88,23 @@ export interface EvalSuiteDetail extends EvalSuite {
// ---------------------------------------------------------------------------
export type EvalCaseKind = 'ask' | 'workflow'
export type EvalExpectationKind = 'scalar' | 'table' | 'set' | 'none' | 'ask' | 'text'
export type EvalExpectationKind = 'scalar' | 'table' | 'set' | 'none' | 'ask' | 'text' | 'contribution' | 'verdict'
/** 分析期望用的分析方法(服务端 app/analysis/contract.OPS):贡献恒为 decompose,判定是另外四种 */
export type EvalAnalysisOp = 'decompose' | 'baseline' | 'trend' | 'differ' | 'comparable'
/** 分析方法的模式(contract.MODES):变化拆解 additive / ratio / factor,差异检验 rate / mean / rows */
export type EvalAnalysisMode = 'additive' | 'ratio' | 'factor' | 'rate' | 'mean' | 'rows'
/**
* 判定类(服务端 app/eval/judgements.JUDGEMENTS):期望和答案里的分析结论都按它比。贡献:source 主要来源、dispersed 分散、
* flat 基本持平、mix 结构变化为主、rate 各段自身的变化为主、mixed 两方面都有影响;判定:normal 常规波动、anomaly 超出常规波动、
* rising / falling / no_trend、differs 差异站得住、no_evidence 不足以下结论、comparable / incomparable
*/
export type EvalJudgement = 'source' | 'dispersed' | 'flat' | 'mix' | 'rate' | 'mixed' | 'normal' | 'anomaly' | 'rising'
| 'falling' | 'no_trend' | 'differs' | 'no_evidence' | 'comparable' | 'incomparable'
/**
* 一个期望。数值期望不许人填(评测-D4):只写从期望答案的结果里取哪一格、哪一列、整张表,值由 SQL / 口径算出来
* 一个期望。数值期望不许人填(评测-D4):只写从期望答案的结果里取哪一格、哪一列、整张表,值由 SQL / 口径算出来;分析期望
* (贡献、判定)只写做哪种分析、参数,来源、方向、判定由平台的分析方法在期望 SQL 的结果上算
*/
export interface EvalExpectation {
id?: string
@@ -112,10 +125,15 @@ export interface EvalExpectation {
keys?: string[]
/** table:比哪几列(缺省全部) */
columns?: string[]
/** table、set:前几行;contribution:期望点名主要来源的前几段(缺省 1) */
top_n?: number
ordered?: boolean
/** text:文字要点(由裁判判,单列,不计入准确率) */
points?: string[]
/** contribution、verdict:分析方法、模式、参数(同分析工具的参数名:by、period、base、target、measure……) */
op?: EvalAnalysisOp
mode?: EvalAnalysisMode
params?: Record<string, unknown>
}
export type EvalGoldKind = 'sql' | 'caliber' | 'from_run'
@@ -233,6 +251,21 @@ export interface EvalGoldSet {
export interface EvalGoldValue {
kind: EvalExpectationKind
/** contribution、verdict:分析做成了(ok)还是算不了(refused,这一项不能评分,原因在 reason) */
status?: 'ok' | 'refused'
op?: EvalAnalysisOp
mode?: EvalAnalysisMode | null
/** 「变化拆解(可加)」:服务端按契约的叫法拼好 */
method?: string
judgement?: EvalJudgement | null
/** 主要来源点名的段(按 top_n 取前几段)、差异检验比的两组 */
subjects?: string[]
direction?: 'up' | 'down' | null
/** 判定语(系统渲染,照原样显示) */
phrase?: string
criterion?: string
reason?: string | null
reason_code?: string | null
value?: unknown
column?: string
row?: number
@@ -444,6 +477,8 @@ export interface EvalSummary {
accuracy?: EvalRate
traced_accuracy?: EvalRate
outcomes?: Partial<Record<EvalOutcome, number>>
/** 各期望的结论按期望的种类分开(必需的、文字要点除外) */
outcomes_by_kind?: Partial<Record<EvalExpectationKind, Partial<Record<EvalOutcome, number>>>>
text?: Partial<Record<EvalTextOutcome, number>>
paths?: Partial<Record<EvalPath, number>>
tiers?: Partial<Record<'formal' | 'degraded' | 'withheld' | 'explore', number>>
@@ -504,6 +539,21 @@ export interface EvalRunDetail extends EvalRun {
actions?: { stop: boolean; retry: boolean; rescore: boolean; pin: boolean; unpin: boolean; open: number; cleaned: boolean }
}
/** 分析期望的评分里期望、答案各一份(judgements.view):叫法在 termsEval,判定语照原样显示 */
export interface EvalJudgementView {
judgement?: EvalJudgement
subjects?: string[]
direction?: 'up' | 'down'
phrase?: string
method?: string
/** 答案引用的分析结论编号(A1) */
alias?: string
status?: 'ok' | 'refused'
/** 正文里说出这个判断的那一句 */
text?: string
reason?: string
}
export type EvalVerdict = 'pass' | 'fail' | 'ungradable' | 'skipped'
export type EvalOutcome = 'traced' | 'untraced' | 'wrong' | 'scale' | 'ungradable'
export type EvalTextOutcome = 'covered' | 'partial' | 'missing' | 'unjudged'
@@ -528,6 +578,9 @@ export interface EvalScore {
items?: number
missing?: unknown[]
points?: number
/** contribution、verdict:期望的判定、答案里对上的(或对不上的)那一份 */
want?: EvalJudgementView
found?: EvalJudgementView
}
/** 逐题的 E1:答案的查询和期望的取数等价与否、差在哪个方面 */