mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:21:58 +00:00
评测前端认两种分析期望:题目编辑器在预览之后选贡献或判定(分析方法、模式、按哪几列、期次、点名前几段),试算写出服务端算的判定和判定语;结果页逐题写期望和答案的判定,点开写判定语、引用了哪一份结论或正文那一句;总览按期望的种类分开写结论
- typesEval.ts:期望种类加 contribution、verdict,期望带 op、mode、params;EvalAnalysisOp、EvalAnalysisMode、EvalJudgement(同服务端契约和 judgements.JUDGEMENTS);期望值、评分带判定类、点名的段、方向、判定语;总表加 outcomes_by_kind - termsEval.ts:贡献、判定的叫法;分析方法、模式的叫法和契约的 OP_LABELS、MODE_LABELS 一字不差(test_frontend_eval_unions 逐个对照,连同判定类);判定类、方向、参数的叫法;指纹差异一句补上分析设置 - parts.tsx:期望写成「变化拆解(可加) · 按地区 · 销售额」;判定写成「主要来源:华东(下降)」「超出常规波动(上升)」「差异站得住(西南高于华东)」,算不了的照实写原因;判定类、段、方向都是服务端给的,界面只套叫法 - CaseEditor.tsx:分析期望的表单按分析方法列参数(参数名同分析工具,列从预览结果里选),换方法、模式只留新方法还认的参数;编辑配了几个期望的题时另外的期望列出来、保存时照旧带上(以前只留第一个,第二个期望会丢) - EvaluationPage.tsx:逐题表、展开的明细、总览按种类 - scripts/check-evals.mjs:夹具补分析期望(加题选算子、试算、保存的请求体不带来源方向判定,编辑保留另外的期望;结果页四种结论、点开的明细、总览按种类),共 130 项
This commit is contained in:
@@ -5,8 +5,9 @@
|
||||
* 统计、结论、能不能做都由服务端给,这里只有叫法和句式:数字一律是服务端算好交来的,这里只负责写成话。
|
||||
*/
|
||||
import type {
|
||||
EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind, EvalOutcome, EvalPath,
|
||||
EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger, EvalUpgradePolicy, EvalVerdict,
|
||||
EvalAnalysisMode, EvalAnalysisOp, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind,
|
||||
EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger,
|
||||
EvalUpgradePolicy, EvalVerdict,
|
||||
} from '../typesEval'
|
||||
import { formatCost, formatNumber } from './format'
|
||||
|
||||
@@ -150,7 +151,8 @@ export const EVAL_TEXT = {
|
||||
},
|
||||
expectation: {
|
||||
kind: {
|
||||
scalar: '一个数', table: '多行', set: '名单', none: '没有数据', ask: '反问', text: '文字要点',
|
||||
scalar: '一个数', table: '多行', set: '名单', none: '没有数据', ask: '反问', text: '文字要点', contribution: '贡献',
|
||||
verdict: '判定',
|
||||
} as Record<EvalExpectationKind, string>,
|
||||
scalarCell: (column: string, row: number) => `「${column}」第 ${row + 1} 行`,
|
||||
scalarWhere: (column: string, where: string) => `「${column}」(${where})`,
|
||||
@@ -160,6 +162,9 @@ export const EVAL_TEXT = {
|
||||
none: '期望答「没有数据」',
|
||||
ask: '期望反问',
|
||||
text: (n: number) => `${n} 条文字要点(模型判断,不计入准确率)`,
|
||||
/** 贡献、判定:「变化拆解(可加)· 按地区」「波动基线 · 订单数」 */
|
||||
analysis: (method: string, about: string) => (about ? `${method} · ${about}` : method),
|
||||
topN: (n: number) => `点名前 ${n} 段`,
|
||||
optional: '非必需',
|
||||
tolerance: (t: number) => `允许误差 ${(t * 100).toFixed(t < 0.01 ? 2 : 1)}%`,
|
||||
},
|
||||
@@ -254,6 +259,21 @@ export const EVAL_TEXT = {
|
||||
specialNoneHint: '写一条查这件事的 SQL,它应当返回 0 行',
|
||||
specialAsk: '期望助手反问',
|
||||
specialAskHint: '问法含糊、该先问清楚的题:助手反问了就算对,不需要期望答案',
|
||||
// 分析期望
|
||||
analysisLabel: '或者定分析期望',
|
||||
analysisHint: '在这条 SQL 的结果上按平台的分析方法算一次:期望的主要来源、判定都由算子算出来,不靠人填;阈值按评测时的分析设置,和作答用同一份',
|
||||
analysisKind: {
|
||||
contribution: '贡献:变化从哪来', verdict: '判定:算不算异常、有没有趋势、差异站不站得住、能不能比',
|
||||
} as Record<'contribution' | 'verdict', string>,
|
||||
analysisOp: '分析方法',
|
||||
analysisMode: '模式',
|
||||
analysisTopN: '期望点名主要来源的前几段',
|
||||
analysisColumn: '选一列',
|
||||
analysisNone: '不填',
|
||||
analysisPeriodHint: '不填基期、本期时,期望 SQL 只能返回两期:先的一期是基期、后的一期是本期;可以写 {{ schedule.last_week_start }} 这类时间变量',
|
||||
analysisPairsHint: '每行一对,两组之间用逗号隔开,例如:西南,华东',
|
||||
analysisPick: '定这个分析期望',
|
||||
analysisOthers: '这道题另外的期望(照旧保留)',
|
||||
// 文字要点
|
||||
points: '文字要点(可选)',
|
||||
pointsHint: '一行一条;由结论裁判判断有没有答到,单列「模型判断」,不计入准确率',
|
||||
@@ -285,6 +305,47 @@ export const EVAL_TEXT = {
|
||||
table: (n: number) => `${formatNumber(n)} 行`,
|
||||
none: '0 行(没有数据)',
|
||||
dims: (d: string) => `(${d})`,
|
||||
/** 分析期望:判定类和系统渲染的判定语 */
|
||||
analysis: (judgement: string, phrase: string) => (phrase ? `${judgement} · ${phrase}` : judgement),
|
||||
refused: (reason: string) => `算不了:${reason}(这一项不能评分)`,
|
||||
},
|
||||
/**
|
||||
* 分析期望(贡献、判定)的叫法。分析方法、模式的叫法和服务端契约(app/analysis/contract 的 OP_LABELS、MODE_LABELS)一字
|
||||
* 不差(tests/test_frontend_eval_unions.py 逐个对照);判定类是评测自己的(app/eval/judgements.JUDGEMENTS)
|
||||
*/
|
||||
analysis: {
|
||||
op: {
|
||||
decompose: '变化拆解', baseline: '波动基线', trend: '趋势判断', differ: '差异检验', comparable: '可比性检查',
|
||||
} as Record<EvalAnalysisOp, string>,
|
||||
mode: {
|
||||
additive: '可加', ratio: '比率', factor: '乘积', rate: '比率', mean: '均值', rows: '逐行明细',
|
||||
} as Record<EvalAnalysisMode, string>,
|
||||
judgement: {
|
||||
source: '主要来源', dispersed: '分散,没有主要来源', flat: '基本持平', mix: '主要来自结构变化', rate: '主要来自各段自身的变化',
|
||||
mixed: '几方面都有影响', normal: '常规波动', anomaly: '超出常规波动', rising: '上升趋势', falling: '下降趋势',
|
||||
no_trend: '没有明显趋势', differs: '差异站得住', no_evidence: '不足以下结论', comparable: '可比', incomparable: '不可比',
|
||||
} as Record<EvalJudgement, string>,
|
||||
direction: { up: '上升', down: '下降' } as Record<'up' | 'down', string>,
|
||||
/** 「主要来源:华东(下降)」「西南高于华东」 */
|
||||
withSubjects: (judgement: string, subjects: string) => `${judgement}:${subjects}`,
|
||||
withDirection: (text: string, direction: string) => `${text}(${direction})`,
|
||||
higher: (a: string, b: string) => `${a}高于${b}`,
|
||||
lower: (a: string, b: string) => `${a}低于${b}`,
|
||||
pair: (a: string, b: string) => `${a}和${b}`,
|
||||
/** 方法的叫法加模式:「变化拆解(可加)」 */
|
||||
method: (op: string, mode: string | null) => (mode ? `${op}(${mode})` : op),
|
||||
by: (columns: string) => `按${columns}`,
|
||||
ratio: (numerator: string, denominator: string) => `${numerator} ÷ ${denominator}`,
|
||||
/** 答案里同类的分析没有做成 */
|
||||
refused: (reason: string) => `没有做成:${reason}`,
|
||||
param: {
|
||||
by: '维度列', period: '期次列', base: '基期', target: '本期', measure: '度量列', numerator: '分子列', denominator: '分母列',
|
||||
product: '乘积列', factors: '因子列', label: '度量的叫法', basis: '比较方式', grain: '粒度', lag: '和往前第几期比',
|
||||
pairs: '比哪几对', reference: '对照组', mean: '均值列', sd: '标准差列', sum: '合计列', sumsq: '平方和列', n: '样本数列',
|
||||
value: '逐行的值列',
|
||||
} as Record<string, string>,
|
||||
basis: { change: '和比较期比的变化率', level: '值本身' } as Record<string, string>,
|
||||
grain: { day: '天', week: '周', month: '月', quarter: '季度' } as Record<string, string>,
|
||||
},
|
||||
upgrade: {
|
||||
title: (seq: number) => `第 ${seq} 题:处置口径升版`,
|
||||
@@ -451,6 +512,7 @@ export const EVAL_TEXT = {
|
||||
role: { assistant: '助手', writer: '写作', judge: '结论裁判', gold: '算期望' } as Record<string, string>,
|
||||
passRates: '重复评的题的通过率',
|
||||
outcomes: '各期望的结论',
|
||||
outcomesByKind: '按期望的种类',
|
||||
baseline: '和上一次比',
|
||||
baselineText: (regressed: number, improved: number) => `退步 ${formatNumber(regressed)} 题 · 进步 ${formatNumber(improved)} 题`,
|
||||
baselineNone: '没有可比的上一次',
|
||||
@@ -509,6 +571,11 @@ export const EVAL_TEXT = {
|
||||
retriesWhy: (reasons: string[]) => `:${reasons.join(';')}`,
|
||||
turnReason: '作答的经过',
|
||||
answer: '答案原文',
|
||||
/** 分析期望展开的明细 */
|
||||
analysisWant: '期望的判定',
|
||||
analysisFound: '答案',
|
||||
analysisCited: (alias: string) => `引用了 ${alias}`,
|
||||
analysisSaid: (text: string) => `正文写的「${text}」`,
|
||||
expand: '展开',
|
||||
collapse: '收起',
|
||||
},
|
||||
@@ -519,7 +586,7 @@ export const EVAL_TEXT = {
|
||||
baseText: (short: string, when: string) => `${short} · ${when}`,
|
||||
none: '没有可比的上一次:同一题集、同一变体里还没有比这次先发起、已经评完的评测',
|
||||
fingerprint: '两次之间变了什么',
|
||||
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则都没变',
|
||||
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则、分析设置都没变',
|
||||
changed: '变了',
|
||||
transitions: '逐题转移',
|
||||
transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record<EvalTransition, string>,
|
||||
|
||||
@@ -10,7 +10,8 @@ import { EVAL_TEXT } from '../../lib/termsEval'
|
||||
import { useCatalog } from '../../store/catalog'
|
||||
import type { Run } from '../../types'
|
||||
import type {
|
||||
EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview, EvalRunQuery, EvalSuite, EvalTrial,
|
||||
EvalAnalysisMode, EvalAnalysisOp, EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview,
|
||||
EvalRunQuery, EvalSuite, EvalTrial,
|
||||
} from '../../typesEval'
|
||||
import { graphFields, type InputField } from '../schedules/model'
|
||||
import { cellText, expectationText, goldValueText } from './parts'
|
||||
@@ -31,6 +32,11 @@ function modeOf(c: EvalCase | null | undefined): Mode {
|
||||
|
||||
/** 题目里那个要答出数(或答「没有数据」、反问)的期望:编辑时据它回填 */
|
||||
const mainOf = (c: EvalCase | null | undefined) => c?.expectations.find((e) => e.kind !== 'text') ?? null
|
||||
/** 别的期望(一道题配了不止一个,比如贡献再配一个「一个数」):编辑框不改它们,保存时照旧带上 */
|
||||
const othersOf = (c: EvalCase | null | undefined) => {
|
||||
const main = mainOf(c)
|
||||
return (c?.expectations ?? []).filter((e) => e !== main && e.kind !== 'text')
|
||||
}
|
||||
const pointsOf = (c: EvalCase | null | undefined) => (c?.expectations.find((e) => e.kind === 'text')?.points ?? []).join('\n')
|
||||
|
||||
/**
|
||||
@@ -57,6 +63,8 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
const [tolerance, setTolerance] = useState('')
|
||||
/** 要答出数的那一个期望(点选出来的,或编辑时回填的) */
|
||||
const [main, setMain] = useState<EvalExpectation | null>(null)
|
||||
/** 编辑的题另外配的期望:原样保留 */
|
||||
const [others, setOthers] = useState<EvalExpectation[]>([])
|
||||
// SQL(写 SQL 定期望、期望没有数据)
|
||||
const [sourceId, setSourceId] = useState('')
|
||||
const [sql, setSql] = useState('')
|
||||
@@ -95,6 +103,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
setPoints(pointsOf(c))
|
||||
const m = mainOf(c)
|
||||
setMain(m)
|
||||
setOthers(othersOf(c))
|
||||
setRequired(m?.required !== false)
|
||||
setTolerance(m?.tolerance != null ? String(m.tolerance) : '')
|
||||
setSourceId(c?.gold?.datasource_id ?? '')
|
||||
@@ -118,6 +127,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
if (m === mode || editing) return
|
||||
setMode(m)
|
||||
setMain(null)
|
||||
setOthers([])
|
||||
setTrial(null)
|
||||
setError(null)
|
||||
setPreview(null)
|
||||
@@ -134,6 +144,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
if (t && (e.kind === 'scalar' || e.kind === 'table')) e.tolerance = Number(t)
|
||||
out.push(e)
|
||||
}
|
||||
out.push(...others)
|
||||
const list = points.split('\n').map((p) => p.trim()).filter(Boolean)
|
||||
if (list.length) out.push({ kind: 'text', points: list })
|
||||
return out
|
||||
@@ -339,6 +350,14 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
</div>
|
||||
)}
|
||||
{needsMain && !main && mode === 'sql' && <p className="text-2xs text-faint" data-eval-case-no-pick="">{T.noPick}</p>}
|
||||
{!!others.length && (
|
||||
<div className="space-y-0.5 rounded-md border border-dashed p-2.5 text-xs" data-eval-case-others={others.length}>
|
||||
<div className="label">{T.analysisOthers}</div>
|
||||
{others.map((e, i) => (
|
||||
<div key={e.id ?? i} className="text-dim">{EVAL_TEXT.expectation.kind[e.kind]}:{expectationText(e)}</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
<Field label={T.scope} htmlFor="eval-case-scope" hint={T.scopeHint}>
|
||||
@@ -479,10 +498,181 @@ function SqlPart({ suiteId: _suiteId, sourceId, setSourceId, sql, setSql, previe
|
||||
{ok && preview && !!preview.rows.length && (
|
||||
<PreviewGrid preview={preview} main={main} setMain={setMain} pickable={pickable} />
|
||||
)}
|
||||
{ok && preview && !!preview.rows.length && pickable && (
|
||||
<AnalysisPick columns={preview.columns} main={main} setMain={setMain} />
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 分析期望:贡献、判定
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type ParamKind = 'column' | 'columns' | 'text' | 'int' | 'choice' | 'pairs'
|
||||
interface ParamField { key: string; kind: ParamKind; choices?: Record<string, string> }
|
||||
|
||||
const A = EVAL_TEXT.analysis
|
||||
/** 判定期望能用的分析(贡献恒为变化拆解)、各自的模式(同服务端 contract.MODES) */
|
||||
const VERDICT_OPS: EvalAnalysisOp[] = ['baseline', 'trend', 'differ', 'comparable']
|
||||
const OP_MODES: Partial<Record<EvalAnalysisOp, EvalAnalysisMode[]>> = {
|
||||
decompose: ['additive', 'ratio', 'factor'], differ: ['rate', 'mean', 'rows'],
|
||||
}
|
||||
const col = (key: string): ParamField => ({ key, kind: 'column' })
|
||||
const PERIODS: ParamField[] = [col('period'), { key: 'base', kind: 'text' }, { key: 'target', kind: 'text' }]
|
||||
const LABEL: ParamField = { key: 'label', kind: 'text' }
|
||||
const GRAIN: ParamField = { key: 'grain', kind: 'choice', choices: A.grain }
|
||||
const GROUPS: ParamField[] = [{ key: 'by', kind: 'columns' }, { key: 'pairs', kind: 'pairs' }, { key: 'reference', kind: 'text' }]
|
||||
/** 每种分析(模式)在表单里填哪几个参数:参数名同分析工具(服务端按同一份参数定义校验),叫法在 termsEval */
|
||||
const PARAM_FIELDS: Record<string, ParamField[]> = {
|
||||
'decompose:additive': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('measure'), LABEL],
|
||||
'decompose:ratio': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('numerator'), col('denominator'), LABEL],
|
||||
'decompose:factor': [...PERIODS, col('product'), { key: 'factors', kind: 'columns' }, LABEL],
|
||||
baseline: [col('period'), col('measure'), { key: 'target', kind: 'text' },
|
||||
{ key: 'basis', kind: 'choice', choices: A.basis }, GRAIN, { key: 'lag', kind: 'int' }, LABEL],
|
||||
trend: [col('period'), col('measure'), GRAIN, LABEL],
|
||||
'differ:rate': [...GROUPS, col('numerator'), col('denominator'), LABEL],
|
||||
'differ:mean': [...GROUPS, col('mean'), col('sd'), col('sum'), col('sumsq'), col('n'), LABEL],
|
||||
'differ:rows': [...GROUPS, col('value'), LABEL],
|
||||
comparable: [...PERIODS, col('measure'), GRAIN],
|
||||
}
|
||||
const fieldsOf = (e: EvalExpectation): ParamField[] =>
|
||||
PARAM_FIELDS[e.op === 'decompose' || e.op === 'differ' ? `${e.op}:${e.mode}` : e.op ?? ''] ?? []
|
||||
|
||||
/** 换了分析方法、模式时,只留下新的那种分析还认的参数 */
|
||||
function keepParams(e: EvalExpectation): EvalExpectation {
|
||||
const keys = new Set(fieldsOf(e).map((f) => f.key))
|
||||
const params = Object.fromEntries(Object.entries(e.params ?? {}).filter(([k]) => keys.has(k)))
|
||||
return { ...e, params }
|
||||
}
|
||||
|
||||
/**
|
||||
* 在预览的结果上定一个分析期望:贡献(变化拆解:选模式、按哪几列拆、期次、度量……,期望点名前几段)或判定(波动基线、
|
||||
* 趋势判断、差异检验、可比性检查)。只填做哪种分析、按哪几列;主要来源、判定由服务端在期望 SQL 的结果上按平台的分析方法算,
|
||||
* 试算时写出来(界面不判)。列从预览结果里选;基期、本期、对照组是期次列、分组列里的值,可以写时间变量
|
||||
*/
|
||||
function AnalysisPick({ columns, main, setMain }: {
|
||||
columns: string[]; main: EvalExpectation | null; setMain: (e: EvalExpectation | null) => void
|
||||
}) {
|
||||
const current = main && (main.kind === 'contribution' || main.kind === 'verdict') ? main : null
|
||||
const start = (kind: 'contribution' | 'verdict') => setMain(kind === 'contribution'
|
||||
? { kind, op: 'decompose', mode: 'additive', params: {}, top_n: 1 }
|
||||
: { kind, op: 'baseline', params: {} })
|
||||
const update = (patch: Partial<EvalExpectation>) => current && setMain(keepParams({ ...current, ...patch }))
|
||||
const setParam = (key: string, value: unknown) => {
|
||||
if (!current) return
|
||||
const params = { ...(current.params ?? {}) }
|
||||
if (value === '' || value == null || (Array.isArray(value) && !value.length)) delete params[key]
|
||||
else params[key] = value
|
||||
setMain({ ...current, params })
|
||||
}
|
||||
const modes = current?.op ? OP_MODES[current.op] : undefined
|
||||
return (
|
||||
<div className="space-y-2 rounded-md border p-2.5 text-xs" data-eval-analysis={current?.kind ?? ''}>
|
||||
<div className="label">{T.analysisLabel}</div>
|
||||
<p className="text-2xs leading-relaxed text-faint">{T.analysisHint}</p>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{(['contribution', 'verdict'] as const).map((k) => (
|
||||
<button key={k} type="button" data-eval-analysis-kind={k} onClick={() => start(k)}
|
||||
className={clsx('rounded-md border px-2 py-1 text-left', current?.kind === k ? 'border-[var(--accent)] bg-accent-soft text-fg' : 'text-dim hover:bg-hover')}>
|
||||
{T.analysisKind[k]}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
{current && (
|
||||
<div className="grid gap-2 sm:grid-cols-2">
|
||||
{current.kind === 'verdict' && (
|
||||
<Field label={T.analysisOp} htmlFor="eval-analysis-op">
|
||||
<select id="eval-analysis-op" className="field" value={current.op} data-eval-analysis-op=""
|
||||
onChange={(e) => {
|
||||
const op = e.target.value as EvalAnalysisOp
|
||||
update({ op, mode: OP_MODES[op]?.[0] })
|
||||
}}>
|
||||
{VERDICT_OPS.map((o) => <option key={o} value={o}>{A.op[o]}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
{modes && (
|
||||
<Field label={T.analysisMode} htmlFor="eval-analysis-mode">
|
||||
<select id="eval-analysis-mode" className="field" value={current.mode ?? ''} data-eval-analysis-mode=""
|
||||
onChange={(e) => update({ mode: e.target.value as EvalAnalysisMode })}>
|
||||
{modes.map((m) => <option key={m} value={m}>{A.method(A.op[current.op!], A.mode[m])}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
{current.kind === 'contribution' && (
|
||||
<Field label={T.analysisTopN} htmlFor="eval-analysis-topn">
|
||||
<input id="eval-analysis-topn" className="field tnum w-24" inputMode="numeric" value={current.top_n ?? 1}
|
||||
data-eval-analysis-topn="" onChange={(e) => update({ top_n: Number(e.target.value) || 1 })} />
|
||||
</Field>
|
||||
)}
|
||||
{fieldsOf(current).map((f) => (
|
||||
<ParamInput key={`${current.op}:${current.mode}:${f.key}`} field={f} columns={columns}
|
||||
value={(current.params ?? {})[f.key]} onChange={(v) => setParam(f.key, v)} />
|
||||
))}
|
||||
{fieldsOf(current).some((f) => f.key === 'base') && (
|
||||
<p className="text-2xs leading-relaxed text-faint sm:col-span-2">{T.analysisPeriodHint}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ParamInput({ field, columns, value, onChange }: {
|
||||
field: ParamField; columns: string[]; value: unknown; onChange: (v: unknown) => void
|
||||
}) {
|
||||
const id = `eval-analysis-param-${field.key}`
|
||||
const label = A.param[field.key] ?? field.key
|
||||
if (field.kind === 'columns') {
|
||||
const picked = Array.isArray(value) ? value.map(String) : []
|
||||
return (
|
||||
<div className="sm:col-span-2" data-eval-analysis-param={field.key}>
|
||||
<div className="label">{label}</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{columns.map((c) => (
|
||||
<label key={c} className="flex items-center gap-1">
|
||||
<input type="checkbox" checked={picked.includes(c)} data-eval-analysis-param-col={c}
|
||||
onChange={(e) => onChange(e.target.checked ? [...picked, c] : picked.filter((x) => x !== c))} />
|
||||
{c}
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
if (field.kind === 'pairs') {
|
||||
const text = Array.isArray(value) ? value.map((p) => (Array.isArray(p) ? p.join(',') : String(p))).join('\n') : ''
|
||||
return (
|
||||
<Field label={label} htmlFor={id} hint={T.analysisPairsHint} className="sm:col-span-2">
|
||||
<textarea id={id} className="field min-h-12 text-xs" defaultValue={text} data-eval-analysis-param={field.key}
|
||||
onChange={(e) => onChange(e.target.value.split('\n').map((line) => line.split(/[,,、]/).map((x) => x.trim())
|
||||
.filter(Boolean)).filter((pair) => pair.length))} />
|
||||
</Field>
|
||||
)
|
||||
}
|
||||
if (field.kind === 'column' || field.kind === 'choice') {
|
||||
const options = field.kind === 'column' ? Object.fromEntries(columns.map((c) => [c, c])) : field.choices ?? {}
|
||||
return (
|
||||
<Field label={label} htmlFor={id}>
|
||||
<select id={id} className="field" value={value == null ? '' : String(value)} data-eval-analysis-param={field.key}
|
||||
onChange={(e) => onChange(e.target.value)}>
|
||||
<option value="">{field.kind === 'column' ? T.analysisColumn : T.analysisNone}</option>
|
||||
{Object.entries(options).map(([k, v]) => <option key={k} value={k}>{v}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<Field label={label} htmlFor={id}>
|
||||
<input id={id} className={clsx('field', field.kind === 'int' && 'tnum w-24')} value={value == null ? '' : String(value)}
|
||||
inputMode={field.kind === 'int' ? 'numeric' : undefined} placeholder={field.kind === 'text' ? T.analysisNone : undefined}
|
||||
data-eval-analysis-param={field.key}
|
||||
onChange={(e) => onChange(field.kind === 'int' ? (e.target.value.trim() ? Number(e.target.value) : '') : e.target.value)} />
|
||||
</Field>
|
||||
)
|
||||
}
|
||||
|
||||
/** 预览结果:点一格定「一个数」,点列名定「名单」;「多行」选配对的列。格里的值是服务端遮罩过、确认不涉及遮罩列的 */
|
||||
function PreviewGrid({ preview, main, setMain, pickable }: {
|
||||
preview: EvalPreview; main: EvalExpectation | null; setMain: (e: EvalExpectation | null) => void; pickable: boolean
|
||||
|
||||
@@ -10,7 +10,7 @@ import { formatCost, formatDateTime, formatDuration, formatNumber, formatTokens,
|
||||
import { EVAL_TEXT, distText, pct, rateText } from '../../lib/termsEval'
|
||||
import { useOnReconnect } from '../../store/catalog'
|
||||
import type { EvalCompare, EvalPair, EvalResult, EvalRun, EvalRunDetail, EvalScore, EvalSuiteDetail, EvalSummary } from '../../typesEval'
|
||||
import { EvalStatusPill, RateCard, StatCard, VerdictChip, cellText, usageCost } from './parts'
|
||||
import { EvalStatusPill, RateCard, StatCard, VerdictChip, cellText, judgementText, usageCost } from './parts'
|
||||
|
||||
const T = EVAL_TEXT
|
||||
type TabKey = 'overview' | 'results' | 'compare' | 'pair'
|
||||
@@ -223,6 +223,18 @@ function Overview({ run }: { run: EvalRun }) {
|
||||
{!!s.not_run && <p className="text-dim" data-eval-not-run={s.not_run}>{O.notRun(s.not_run)}</p>}
|
||||
{!!s.errors && <p className="text-dim" data-eval-errors={s.errors}>{O.errors(s.errors)}</p>}
|
||||
<p className="text-dim">{O.outcomes}:{distText(s.outcomes as Record<string, number>, T.outcome)}</p>
|
||||
{!!s.outcomes_by_kind && Object.keys(s.outcomes_by_kind).length > 0 && (
|
||||
<div className="text-dim" data-eval-outcomes-by-kind="">
|
||||
<span>{O.outcomesByKind}:</span>
|
||||
<ul className="mt-0.5 space-y-0.5 pl-3">
|
||||
{Object.entries(s.outcomes_by_kind).map(([kind, dist]) => (
|
||||
<li key={kind} data-eval-outcomes-kind={kind}>
|
||||
{T.expectation.kind[kind as keyof typeof T.expectation.kind] ?? kind}:{distText(dist as Record<string, number>, T.outcome)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
{s.text && Object.keys(s.text).length > 0 && <p className="text-dim">{O.text}:{distText(s.text as Record<string, number>, T.textOutcome)}</p>}
|
||||
</Section>
|
||||
{base && (
|
||||
@@ -394,8 +406,17 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
|
||||
<td className="px-2 py-1.5">
|
||||
{counted.length ? counted.map((s) => (
|
||||
<div key={s.id} className="tnum" data-eval-score={s.outcome}>
|
||||
{s.expected !== undefined && <span>{cellText(s.expected)}</span>}
|
||||
{s.got !== undefined && <span className="text-dim"> → {cellText(s.got)}</span>}
|
||||
{s.want ? (
|
||||
<span data-eval-score-judgement="">
|
||||
{judgementText(s.want)}
|
||||
{s.found && <span className="text-dim"> → {judgementText(s.found)}</span>}
|
||||
</span>
|
||||
) : (
|
||||
<>
|
||||
{s.expected !== undefined && <span>{cellText(s.expected)}</span>}
|
||||
{s.got !== undefined && <span className="text-dim"> → {cellText(s.got)}</span>}
|
||||
</>
|
||||
)}
|
||||
<span className="ml-1 text-2xs text-faint">{scoreText(s)}</span>
|
||||
{s.weak && <span className="ml-1 chip text-[var(--warn)]" title={R.weakHint}>{R.weak}</span>}
|
||||
</div>
|
||||
@@ -420,12 +441,15 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
|
||||
<td colSpan={8} className="space-y-1.5 px-3 py-2">
|
||||
{r.error && <p className="text-dim"><span className="text-faint">{R.reason}:</span>{r.error}</p>}
|
||||
{r.scores.map((s) => (
|
||||
<p key={s.id} className="text-dim">
|
||||
<span className="text-faint">{T.expectation.kind[s.kind]}{s.label ? `「${s.label}」` : ''}:</span>
|
||||
{scoreText(s)}
|
||||
{s.key && <span className="text-faint">({Object.entries(s.key).map(([k, v]) => `${k}=${cellText(v)}`).join('、')})</span>}
|
||||
{s.reason && <span> · {s.reason}</span>}
|
||||
</p>
|
||||
<Fragment key={s.id}>
|
||||
<p className="text-dim">
|
||||
<span className="text-faint">{T.expectation.kind[s.kind]}{s.label ? `「${s.label}」` : ''}:</span>
|
||||
{scoreText(s)}
|
||||
{s.key && <span className="text-faint">({Object.entries(s.key).map(([k, v]) => `${k}=${cellText(v)}`).join('、')})</span>}
|
||||
{s.reason && <span> · {s.reason}</span>}
|
||||
</p>
|
||||
{s.want && <AnalysisDetail score={s} />}
|
||||
</Fragment>
|
||||
))}
|
||||
{r.sql_match?.basis && T.sqlMatchBasis[r.sql_match.basis] && (
|
||||
<p className="text-faint" data-eval-sql-basis={r.sql_match.basis}>{T.sqlMatchBasis[r.sql_match.basis]}</p>
|
||||
@@ -450,6 +474,32 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 分析期望的明细:期望的判定和系统渲染的判定语(期望 SQL 上算的),答案里对上(或对不上)的那一份——引用的分析结论编号和
|
||||
* 它的判定语,或者正文里说出这个判断的那一句。都是服务端给的,照原样写
|
||||
*/
|
||||
function AnalysisDetail({ score }: { score: EvalScore }) {
|
||||
const R = T.results
|
||||
const want = score.want ?? {}
|
||||
const found = score.found
|
||||
return (
|
||||
<div className="space-y-0.5 border-l-2 pl-2 text-dim" data-eval-analysis-detail={score.id}>
|
||||
<p data-eval-analysis-want="">
|
||||
<span className="text-faint">{R.analysisWant}:</span>{judgementText(want)}
|
||||
{want.method && <span className="text-faint"> · {want.method}</span>}
|
||||
{want.phrase && <span className="block text-faint">{want.phrase}</span>}
|
||||
</p>
|
||||
{found && (
|
||||
<p data-eval-analysis-found={found.alias ?? 'text'}>
|
||||
<span className="text-faint">{R.analysisFound}:</span>
|
||||
{found.alias ? R.analysisCited(found.alias) : found.text ? R.analysisSaid(found.text) : judgementText(found)}
|
||||
{found.alias && found.phrase && <span className="block text-faint">{found.phrase}</span>}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** 一题的明细(GET …/results/{id} 的明细工件):这一轮作答的经过(重来了几次、原因)和读出来的答案原文。明细清理了的不显示 */
|
||||
function ResultDetail({ evaluationId, resultId }: { evaluationId: string; resultId: string }) {
|
||||
const [details, setDetails] = useState<Record<string, any> | null | undefined>(undefined)
|
||||
|
||||
@@ -6,7 +6,8 @@ import { formatCost, formatDateTime, formatNumber } from '../../lib/format'
|
||||
import { EVAL_TEXT as T, pct, rateText } from '../../lib/termsEval'
|
||||
import { runCostText } from '../../lib/terms'
|
||||
import type {
|
||||
EvalExpectation, EvalGoldValue, EvalModelPair, EvalRate, EvalStatus, EvalTrendPoint, EvalUsage, EvalVerdict,
|
||||
EvalExpectation, EvalGoldValue, EvalJudgementView, EvalModelPair, EvalRate, EvalStatus, EvalTrendPoint, EvalUsage,
|
||||
EvalVerdict,
|
||||
} from '../../typesEval'
|
||||
|
||||
/*
|
||||
@@ -144,6 +145,8 @@ export function expectationText(e: EvalExpectation): string {
|
||||
case 'none': text = E.none; break
|
||||
case 'ask': text = E.ask; break
|
||||
case 'text': text = E.text(e.points?.length ?? 0); break
|
||||
case 'contribution':
|
||||
case 'verdict': text = analysisText(e); break
|
||||
default: text = T.expectation.kind[e.kind as keyof typeof T.expectation.kind] ?? String(e.kind)
|
||||
}
|
||||
const extra = [
|
||||
@@ -154,11 +157,50 @@ export function expectationText(e: EvalExpectation): string {
|
||||
return extra.length ? `${text}(${extra.join(',')})` : text
|
||||
}
|
||||
|
||||
/** 分析期望做哪种分析、分析什么:「变化拆解(可加) · 按地区 · 销售额」,贡献点名不止一段时另写几段 */
|
||||
function analysisText(e: EvalExpectation): string {
|
||||
const A = T.analysis
|
||||
const op = e.op ?? 'decompose'
|
||||
const method = A.method(A.op[op] ?? op, e.mode ? A.mode[e.mode] ?? e.mode : null)
|
||||
const p = e.params ?? {}
|
||||
const words = (v: unknown) => (Array.isArray(v) ? v.map(String).join('、') : v == null ? '' : String(v))
|
||||
const about = [
|
||||
words(p.by) && A.by(words(p.by)),
|
||||
words(p.label) || words(p.measure) || words(p.product) || words(p.value)
|
||||
|| (p.numerator && p.denominator ? A.ratio(words(p.numerator), words(p.denominator)) : ''),
|
||||
].filter(Boolean).join(' · ')
|
||||
const text = T.expectation.analysis(method, about)
|
||||
return e.kind === 'contribution' && (e.top_n ?? 1) > 1 ? `${text} · ${T.expectation.topN(e.top_n ?? 1)}` : text
|
||||
}
|
||||
|
||||
/**
|
||||
* 分析期望的判定(期望、答案各一份)写成字:「主要来源:华东(下降)」「超出常规波动(上升)」「差异站得住(西南高于华东)」。
|
||||
* 判定类、点名的段、方向都是服务端给的,这里只套叫法;没有判定类的(正文里一句对不上的话)写那一句
|
||||
*/
|
||||
export function judgementText(v: EvalJudgementView | EvalGoldValue | null | undefined): string {
|
||||
if (!v) return '—'
|
||||
const A = T.analysis
|
||||
if (v.status === 'refused') return A.refused(v.reason ?? '')
|
||||
if (!v.judgement) return 'text' in v && v.text ? `「${v.text}」` : '—'
|
||||
const label = A.judgement[v.judgement] ?? v.judgement
|
||||
const subjects = v.subjects ?? []
|
||||
if (subjects.length === 2 && (v.judgement === 'differs' || v.judgement === 'no_evidence')) {
|
||||
const [a, b] = subjects
|
||||
const which = v.judgement === 'no_evidence' || !v.direction ? A.pair(a, b) : v.direction === 'up' ? A.higher(a, b) : A.lower(a, b)
|
||||
return A.withDirection(label, which)
|
||||
}
|
||||
const text = v.judgement === 'source' && subjects.length ? A.withSubjects(label, subjects.join('、')) : label
|
||||
return v.direction ? A.withDirection(text, A.direction[v.direction] ?? v.direction) : text
|
||||
}
|
||||
|
||||
/** 试算算出来的一份期望值写成字(值都是服务端算好的) */
|
||||
export function goldValueText(v: EvalGoldValue | undefined): string {
|
||||
if (!v) return '—'
|
||||
const G = T.goldValue
|
||||
switch (v.kind) {
|
||||
case 'contribution':
|
||||
case 'verdict':
|
||||
return v.status === 'refused' ? G.refused(v.reason ?? '') : G.analysis(judgementText(v), v.phrase ?? '')
|
||||
case 'scalar': {
|
||||
const dims = v.dims && Object.keys(v.dims).length
|
||||
? G.dims(Object.entries(v.dims).map(([k, x]) => `${k}=${cellText(x)}`).join('、')) : ''
|
||||
|
||||
@@ -88,10 +88,23 @@ export interface EvalSuiteDetail extends EvalSuite {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type EvalCaseKind = 'ask' | 'workflow'
|
||||
export type EvalExpectationKind = 'scalar' | 'table' | 'set' | 'none' | 'ask' | 'text'
|
||||
export type EvalExpectationKind = 'scalar' | 'table' | 'set' | 'none' | 'ask' | 'text' | 'contribution' | 'verdict'
|
||||
|
||||
/** 分析期望用的分析方法(服务端 app/analysis/contract.OPS):贡献恒为 decompose,判定是另外四种 */
|
||||
export type EvalAnalysisOp = 'decompose' | 'baseline' | 'trend' | 'differ' | 'comparable'
|
||||
/** 分析方法的模式(contract.MODES):变化拆解 additive / ratio / factor,差异检验 rate / mean / rows */
|
||||
export type EvalAnalysisMode = 'additive' | 'ratio' | 'factor' | 'rate' | 'mean' | 'rows'
|
||||
/**
|
||||
* 判定类(服务端 app/eval/judgements.JUDGEMENTS):期望和答案里的分析结论都按它比。贡献:source 主要来源、dispersed 分散、
|
||||
* flat 基本持平、mix 结构变化为主、rate 各段自身的变化为主、mixed 两方面都有影响;判定:normal 常规波动、anomaly 超出常规波动、
|
||||
* rising / falling / no_trend、differs 差异站得住、no_evidence 不足以下结论、comparable / incomparable
|
||||
*/
|
||||
export type EvalJudgement = 'source' | 'dispersed' | 'flat' | 'mix' | 'rate' | 'mixed' | 'normal' | 'anomaly' | 'rising'
|
||||
| 'falling' | 'no_trend' | 'differs' | 'no_evidence' | 'comparable' | 'incomparable'
|
||||
|
||||
/**
|
||||
* 一个期望。数值期望不许人填(评测-D4):只写从期望答案的结果里取哪一格、哪一列、整张表,值由 SQL / 口径算出来
|
||||
* 一个期望。数值期望不许人填(评测-D4):只写从期望答案的结果里取哪一格、哪一列、整张表,值由 SQL / 口径算出来;分析期望
|
||||
* (贡献、判定)只写做哪种分析、参数,来源、方向、判定由平台的分析方法在期望 SQL 的结果上算
|
||||
*/
|
||||
export interface EvalExpectation {
|
||||
id?: string
|
||||
@@ -112,10 +125,15 @@ export interface EvalExpectation {
|
||||
keys?: string[]
|
||||
/** table:比哪几列(缺省全部) */
|
||||
columns?: string[]
|
||||
/** table、set:前几行;contribution:期望点名主要来源的前几段(缺省 1) */
|
||||
top_n?: number
|
||||
ordered?: boolean
|
||||
/** text:文字要点(由裁判判,单列,不计入准确率) */
|
||||
points?: string[]
|
||||
/** contribution、verdict:分析方法、模式、参数(同分析工具的参数名:by、period、base、target、measure……) */
|
||||
op?: EvalAnalysisOp
|
||||
mode?: EvalAnalysisMode
|
||||
params?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export type EvalGoldKind = 'sql' | 'caliber' | 'from_run'
|
||||
@@ -233,6 +251,21 @@ export interface EvalGoldSet {
|
||||
|
||||
export interface EvalGoldValue {
|
||||
kind: EvalExpectationKind
|
||||
/** contribution、verdict:分析做成了(ok)还是算不了(refused,这一项不能评分,原因在 reason) */
|
||||
status?: 'ok' | 'refused'
|
||||
op?: EvalAnalysisOp
|
||||
mode?: EvalAnalysisMode | null
|
||||
/** 「变化拆解(可加)」:服务端按契约的叫法拼好 */
|
||||
method?: string
|
||||
judgement?: EvalJudgement | null
|
||||
/** 主要来源点名的段(按 top_n 取前几段)、差异检验比的两组 */
|
||||
subjects?: string[]
|
||||
direction?: 'up' | 'down' | null
|
||||
/** 判定语(系统渲染,照原样显示) */
|
||||
phrase?: string
|
||||
criterion?: string
|
||||
reason?: string | null
|
||||
reason_code?: string | null
|
||||
value?: unknown
|
||||
column?: string
|
||||
row?: number
|
||||
@@ -444,6 +477,8 @@ export interface EvalSummary {
|
||||
accuracy?: EvalRate
|
||||
traced_accuracy?: EvalRate
|
||||
outcomes?: Partial<Record<EvalOutcome, number>>
|
||||
/** 各期望的结论按期望的种类分开(必需的、文字要点除外) */
|
||||
outcomes_by_kind?: Partial<Record<EvalExpectationKind, Partial<Record<EvalOutcome, number>>>>
|
||||
text?: Partial<Record<EvalTextOutcome, number>>
|
||||
paths?: Partial<Record<EvalPath, number>>
|
||||
tiers?: Partial<Record<'formal' | 'degraded' | 'withheld' | 'explore', number>>
|
||||
@@ -504,6 +539,21 @@ export interface EvalRunDetail extends EvalRun {
|
||||
actions?: { stop: boolean; retry: boolean; rescore: boolean; pin: boolean; unpin: boolean; open: number; cleaned: boolean }
|
||||
}
|
||||
|
||||
/** 分析期望的评分里期望、答案各一份(judgements.view):叫法在 termsEval,判定语照原样显示 */
|
||||
export interface EvalJudgementView {
|
||||
judgement?: EvalJudgement
|
||||
subjects?: string[]
|
||||
direction?: 'up' | 'down'
|
||||
phrase?: string
|
||||
method?: string
|
||||
/** 答案引用的分析结论编号(A1) */
|
||||
alias?: string
|
||||
status?: 'ok' | 'refused'
|
||||
/** 正文里说出这个判断的那一句 */
|
||||
text?: string
|
||||
reason?: string
|
||||
}
|
||||
|
||||
export type EvalVerdict = 'pass' | 'fail' | 'ungradable' | 'skipped'
|
||||
export type EvalOutcome = 'traced' | 'untraced' | 'wrong' | 'scale' | 'ungradable'
|
||||
export type EvalTextOutcome = 'covered' | 'partial' | 'missing' | 'unjudged'
|
||||
@@ -528,6 +578,9 @@ export interface EvalScore {
|
||||
items?: number
|
||||
missing?: unknown[]
|
||||
points?: number
|
||||
/** contribution、verdict:期望的判定、答案里对上的(或对不上的)那一份 */
|
||||
want?: EvalJudgementView
|
||||
found?: EvalJudgementView
|
||||
}
|
||||
|
||||
/** 逐题的 E1:答案的查询和期望的取数等价与否、差在哪个方面 */
|
||||
|
||||
Reference in New Issue
Block a user