diff --git a/backend/tests/test_frontend_eval_unions.py b/backend/tests/test_frontend_eval_unions.py index 39967637..4c79c18c 100644 --- a/backend/tests/test_frontend_eval_unions.py +++ b/backend/tests/test_frontend_eval_unions.py @@ -27,8 +27,8 @@ def _union(name: str) -> list[str]: return re.findall(r"'([^']+)'", found.group(1)) -def _labels(path: str) -> list[str]: - """EVAL_TEXT 里某一张叫法表（a.b 这样的路径）的键，按写的顺序。只认一行写完、或几行写完的对象字面量。""" +def _pairs(path: str) -> list[tuple[str, str]]: + """EVAL_TEXT 里某一张叫法表（a.b 这样的路径）的 (键, 叫法)，按写的顺序。只认一行写完、或几行写完的对象字面量。""" text = _source(TERMS_EVAL) *parents, leaf = path.split(".") start = 0 @@ -38,7 +38,12 @@ def _labels(path: str) -> list[str]: start += hit.end() hit = re.search(rf"\n\s*{leaf}: \{{(.*?)\}}( as Record<[^>]+>)?,", text[start:], re.S) assert hit is not None, f"EVAL_TEXT 里没有 {path}" - return re.findall(r"(\w+): '", hit.group(1)) + return re.findall(r"(\w+): '([^']*)'", hit.group(1)) + + +def _labels(path: str) -> list[str]: + """EVAL_TEXT 里某一张叫法表的键，按写的顺序。""" + return [key for key, _ in _pairs(path)] def test_statuses_triggers_and_kinds(): @@ -93,3 +98,18 @@ def test_signal_bands_follow_the_server(): assert _union("EvalE1Band") == [c["band"] for c in SIGNAL_CRITERIA["e1"]] assert _union("EvalE3Band") == [c["band"] for c in SIGNAL_CRITERIA["e3"]] assert set(_labels("signals.band")) == {c["band"] for c in [*SIGNAL_CRITERIA["e1"], *SIGNAL_CRITERIA["e3"]]} + + +def test_analysis_expectations_follow_the_server(): + """分析期望：判定类和服务端 app/eval/judgements 一致；分析方法、模式的叫法和契约（app/analysis/contract 的 OP_LABELS、 + MODE_LABELS）一字不差——界面上的「变化拆解（可加）」和服务端拼的、证据面板写的是同一种说法。""" + from app.analysis.contract import MODE_LABELS, MODES, OP_LABELS, OPS + from app.eval.judgements import JUDGEMENTS + + assert _union("EvalJudgement") == list(JUDGEMENTS) + assert _labels("analysis.judgement") == list(JUDGEMENTS) + assert _union("EvalAnalysisOp") == list(OPS) + assert _pairs("analysis.op") == list(OP_LABELS.items()) + assert _union("EvalAnalysisMode") == list(MODE_LABELS) + assert _pairs("analysis.mode") == list(MODE_LABELS.items()) + assert {m for modes in MODES.values() for m in modes} == set(MODE_LABELS) diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts index 85bd5f31..a3a5d615 100644 --- a/frontend/src/lib/termsEval.ts +++ b/frontend/src/lib/termsEval.ts @@ -5,8 +5,9 @@ * 统计、结论、能不能做都由服务端给，这里只有叫法和句式：数字一律是服务端算好交来的，这里只负责写成话。 */ import type { - EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind, EvalOutcome, EvalPath, - EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger, EvalUpgradePolicy, EvalVerdict, + EvalAnalysisMode, EvalAnalysisOp, EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind, + EvalJudgement, EvalOutcome, EvalPath, EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger, + EvalUpgradePolicy, EvalVerdict, } from '../typesEval' import { formatCost, formatNumber } from './format' @@ -150,7 +151,8 @@ export const EVAL_TEXT = { }, expectation: { kind: { - scalar: '一个数', table: '多行', set: '名单', none: '没有数据', ask: '反问', text: '文字要点', + scalar: '一个数', table: '多行', set: '名单', none: '没有数据', ask: '反问', text: '文字要点', contribution: '贡献', + verdict: '判定', } as Record, scalarCell: (column: string, row: number) => `「${column}」第 ${row + 1} 行`, scalarWhere: (column: string, where: string) => `「${column}」（${where}）`, @@ -160,6 +162,9 @@ export const EVAL_TEXT = { none: '期望答「没有数据」', ask: '期望反问', text: (n: number) => `${n} 条文字要点（模型判断，不计入准确率）`, + /** 贡献、判定：「变化拆解（可加）· 按地区」「波动基线 · 订单数」 */ + analysis: (method: string, about: string) => (about ? `${method} · ${about}` : method), + topN: (n: number) => `点名前 ${n} 段`, optional: '非必需', tolerance: (t: number) => `允许误差 ${(t * 100).toFixed(t < 0.01 ? 2 : 1)}%`, }, @@ -254,6 +259,21 @@ export const EVAL_TEXT = { specialNoneHint: '写一条查这件事的 SQL，它应当返回 0 行', specialAsk: '期望助手反问', specialAskHint: '问法含糊、该先问清楚的题：助手反问了就算对，不需要期望答案', + // 分析期望 + analysisLabel: '或者定分析期望', + analysisHint: '在这条 SQL 的结果上按平台的分析方法算一次：期望的主要来源、判定都由算子算出来，不靠人填；阈值按评测时的分析设置，和作答用同一份', + analysisKind: { + contribution: '贡献：变化从哪来', verdict: '判定：算不算异常、有没有趋势、差异站不站得住、能不能比', + } as Record<'contribution' | 'verdict', string>, + analysisOp: '分析方法', + analysisMode: '模式', + analysisTopN: '期望点名主要来源的前几段', + analysisColumn: '选一列', + analysisNone: '不填', + analysisPeriodHint: '不填基期、本期时，期望 SQL 只能返回两期：先的一期是基期、后的一期是本期；可以写 {{ schedule.last_week_start }} 这类时间变量', + analysisPairsHint: '每行一对，两组之间用逗号隔开，例如：西南，华东', + analysisPick: '定这个分析期望', + analysisOthers: '这道题另外的期望（照旧保留）', // 文字要点 points: '文字要点（可选）', pointsHint: '一行一条；由结论裁判判断有没有答到，单列「模型判断」，不计入准确率', @@ -285,6 +305,47 @@ export const EVAL_TEXT = { table: (n: number) => `${formatNumber(n)} 行`, none: '0 行（没有数据）', dims: (d: string) => `（${d}）`, + /** 分析期望：判定类和系统渲染的判定语 */ + analysis: (judgement: string, phrase: string) => (phrase ? `${judgement} · ${phrase}` : judgement), + refused: (reason: string) => `算不了：${reason}（这一项不能评分）`, + }, + /** + * 分析期望（贡献、判定）的叫法。分析方法、模式的叫法和服务端契约（app/analysis/contract 的 OP_LABELS、MODE_LABELS）一字 + * 不差（tests/test_frontend_eval_unions.py 逐个对照）；判定类是评测自己的（app/eval/judgements.JUDGEMENTS） + */ + analysis: { + op: { + decompose: '变化拆解', baseline: '波动基线', trend: '趋势判断', differ: '差异检验', comparable: '可比性检查', + } as Record, + mode: { + additive: '可加', ratio: '比率', factor: '乘积', rate: '比率', mean: '均值', rows: '逐行明细', + } as Record, + judgement: { + source: '主要来源', dispersed: '分散，没有主要来源', flat: '基本持平', mix: '主要来自结构变化', rate: '主要来自各段自身的变化', + mixed: '几方面都有影响', normal: '常规波动', anomaly: '超出常规波动', rising: '上升趋势', falling: '下降趋势', + no_trend: '没有明显趋势', differs: '差异站得住', no_evidence: '不足以下结论', comparable: '可比', incomparable: '不可比', + } as Record, + direction: { up: '上升', down: '下降' } as Record<'up' | 'down', string>, + /** 「主要来源：华东（下降）」「西南高于华东」 */ + withSubjects: (judgement: string, subjects: string) => `${judgement}：${subjects}`, + withDirection: (text: string, direction: string) => `${text}（${direction}）`, + higher: (a: string, b: string) => `${a}高于${b}`, + lower: (a: string, b: string) => `${a}低于${b}`, + pair: (a: string, b: string) => `${a}和${b}`, + /** 方法的叫法加模式：「变化拆解（可加）」 */ + method: (op: string, mode: string | null) => (mode ? `${op}（${mode}）` : op), + by: (columns: string) => `按${columns}`, + ratio: (numerator: string, denominator: string) => `${numerator} ÷ ${denominator}`, + /** 答案里同类的分析没有做成 */ + refused: (reason: string) => `没有做成：${reason}`, + param: { + by: '维度列', period: '期次列', base: '基期', target: '本期', measure: '度量列', numerator: '分子列', denominator: '分母列', + product: '乘积列', factors: '因子列', label: '度量的叫法', basis: '比较方式', grain: '粒度', lag: '和往前第几期比', + pairs: '比哪几对', reference: '对照组', mean: '均值列', sd: '标准差列', sum: '合计列', sumsq: '平方和列', n: '样本数列', + value: '逐行的值列', + } as Record, + basis: { change: '和比较期比的变化率', level: '值本身' } as Record, + grain: { day: '天', week: '周', month: '月', quarter: '季度' } as Record, }, upgrade: { title: (seq: number) => `第 ${seq} 题：处置口径升版`, @@ -451,6 +512,7 @@ export const EVAL_TEXT = { role: { assistant: '助手', writer: '写作', judge: '结论裁判', gold: '算期望' } as Record, passRates: '重复评的题的通过率', outcomes: '各期望的结论', + outcomesByKind: '按期望的种类', baseline: '和上一次比', baselineText: (regressed: number, improved: number) => `退步 ${formatNumber(regressed)} 题 · 进步 ${formatNumber(improved)} 题`, baselineNone: '没有可比的上一次', @@ -509,6 +571,11 @@ export const EVAL_TEXT = { retriesWhy: (reasons: string[]) => `：${reasons.join('；')}`, turnReason: '作答的经过', answer: '答案原文', + /** 分析期望展开的明细 */ + analysisWant: '期望的判定', + analysisFound: '答案', + analysisCited: (alias: string) => `引用了 ${alias}`, + analysisSaid: (text: string) => `正文写的「${text}」`, expand: '展开', collapse: '收起', }, @@ -519,7 +586,7 @@ export const EVAL_TEXT = { baseText: (short: string, when: string) => `${short} · ${when}`, none: '没有可比的上一次：同一题集、同一变体里还没有比这次先发起、已经评完的评测', fingerprint: '两次之间变了什么', - fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则都没变', + fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则、分析设置都没变', changed: '变了', transitions: '逐题转移', transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record, diff --git a/frontend/src/pages/evals/CaseEditor.tsx b/frontend/src/pages/evals/CaseEditor.tsx index 12a27a9c..b2f7bb4c 100644 --- a/frontend/src/pages/evals/CaseEditor.tsx +++ b/frontend/src/pages/evals/CaseEditor.tsx @@ -10,7 +10,8 @@ import { EVAL_TEXT } from '../../lib/termsEval' import { useCatalog } from '../../store/catalog' import type { Run } from '../../types' import type { - EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview, EvalRunQuery, EvalSuite, EvalTrial, + EvalAnalysisMode, EvalAnalysisOp, EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview, + EvalRunQuery, EvalSuite, EvalTrial, } from '../../typesEval' import { graphFields, type InputField } from '../schedules/model' import { cellText, expectationText, goldValueText } from './parts' @@ -31,6 +32,11 @@ function modeOf(c: EvalCase | null | undefined): Mode { /** 题目里那个要答出数（或答「没有数据」、反问）的期望：编辑时据它回填 */ const mainOf = (c: EvalCase | null | undefined) => c?.expectations.find((e) => e.kind !== 'text') ?? null +/** 别的期望（一道题配了不止一个，比如贡献再配一个「一个数」）：编辑框不改它们，保存时照旧带上 */ +const othersOf = (c: EvalCase | null | undefined) => { + const main = mainOf(c) + return (c?.expectations ?? []).filter((e) => e !== main && e.kind !== 'text') +} const pointsOf = (c: EvalCase | null | undefined) => (c?.expectations.find((e) => e.kind === 'text')?.points ?? []).join('\n') /** @@ -57,6 +63,8 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS const [tolerance, setTolerance] = useState('') /** 要答出数的那一个期望（点选出来的，或编辑时回填的） */ const [main, setMain] = useState(null) + /** 编辑的题另外配的期望：原样保留 */ + const [others, setOthers] = useState([]) // SQL（写 SQL 定期望、期望没有数据） const [sourceId, setSourceId] = useState('') const [sql, setSql] = useState('') @@ -95,6 +103,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS setPoints(pointsOf(c)) const m = mainOf(c) setMain(m) + setOthers(othersOf(c)) setRequired(m?.required !== false) setTolerance(m?.tolerance != null ? String(m.tolerance) : '') setSourceId(c?.gold?.datasource_id ?? '') @@ -118,6 +127,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS if (m === mode || editing) return setMode(m) setMain(null) + setOthers([]) setTrial(null) setError(null) setPreview(null) @@ -134,6 +144,7 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS if (t && (e.kind === 'scalar' || e.kind === 'table')) e.tolerance = Number(t) out.push(e) } + out.push(...others) const list = points.split('\n').map((p) => p.trim()).filter(Boolean) if (list.length) out.push({ kind: 'text', points: list }) return out @@ -339,6 +350,14 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS )} {needsMain && !main && mode === 'sql' &&

{T.noPick}

} + {!!others.length && ( +
+
{T.analysisOthers}
+ {others.map((e, i) => ( +
{EVAL_TEXT.expectation.kind[e.kind]}：{expectationText(e)}
+ ))} +
+ )}
@@ -479,10 +498,181 @@ function SqlPart({ suiteId: _suiteId, sourceId, setSourceId, sql, setSql, previe {ok && preview && !!preview.rows.length && ( )} + {ok && preview && !!preview.rows.length && pickable && ( + + )}
) } +// --------------------------------------------------------------------------- +// 分析期望：贡献、判定 +// --------------------------------------------------------------------------- + +type ParamKind = 'column' | 'columns' | 'text' | 'int' | 'choice' | 'pairs' +interface ParamField { key: string; kind: ParamKind; choices?: Record } + +const A = EVAL_TEXT.analysis +/** 判定期望能用的分析（贡献恒为变化拆解）、各自的模式（同服务端 contract.MODES） */ +const VERDICT_OPS: EvalAnalysisOp[] = ['baseline', 'trend', 'differ', 'comparable'] +const OP_MODES: Partial> = { + decompose: ['additive', 'ratio', 'factor'], differ: ['rate', 'mean', 'rows'], +} +const col = (key: string): ParamField => ({ key, kind: 'column' }) +const PERIODS: ParamField[] = [col('period'), { key: 'base', kind: 'text' }, { key: 'target', kind: 'text' }] +const LABEL: ParamField = { key: 'label', kind: 'text' } +const GRAIN: ParamField = { key: 'grain', kind: 'choice', choices: A.grain } +const GROUPS: ParamField[] = [{ key: 'by', kind: 'columns' }, { key: 'pairs', kind: 'pairs' }, { key: 'reference', kind: 'text' }] +/** 每种分析（模式）在表单里填哪几个参数：参数名同分析工具（服务端按同一份参数定义校验），叫法在 termsEval */ +const PARAM_FIELDS: Record = { + 'decompose:additive': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('measure'), LABEL], + 'decompose:ratio': [{ key: 'by', kind: 'columns' }, ...PERIODS, col('numerator'), col('denominator'), LABEL], + 'decompose:factor': [...PERIODS, col('product'), { key: 'factors', kind: 'columns' }, LABEL], + baseline: [col('period'), col('measure'), { key: 'target', kind: 'text' }, + { key: 'basis', kind: 'choice', choices: A.basis }, GRAIN, { key: 'lag', kind: 'int' }, LABEL], + trend: [col('period'), col('measure'), GRAIN, LABEL], + 'differ:rate': [...GROUPS, col('numerator'), col('denominator'), LABEL], + 'differ:mean': [...GROUPS, col('mean'), col('sd'), col('sum'), col('sumsq'), col('n'), LABEL], + 'differ:rows': [...GROUPS, col('value'), LABEL], + comparable: [...PERIODS, col('measure'), GRAIN], +} +const fieldsOf = (e: EvalExpectation): ParamField[] => + PARAM_FIELDS[e.op === 'decompose' || e.op === 'differ' ? `${e.op}:${e.mode}` : e.op ?? ''] ?? [] + +/** 换了分析方法、模式时，只留下新的那种分析还认的参数 */ +function keepParams(e: EvalExpectation): EvalExpectation { + const keys = new Set(fieldsOf(e).map((f) => f.key)) + const params = Object.fromEntries(Object.entries(e.params ?? {}).filter(([k]) => keys.has(k))) + return { ...e, params } +} + +/** + * 在预览的结果上定一个分析期望：贡献（变化拆解：选模式、按哪几列拆、期次、度量……，期望点名前几段）或判定（波动基线、 + * 趋势判断、差异检验、可比性检查）。只填做哪种分析、按哪几列；主要来源、判定由服务端在期望 SQL 的结果上按平台的分析方法算， + * 试算时写出来（界面不判）。列从预览结果里选；基期、本期、对照组是期次列、分组列里的值，可以写时间变量 + */ +function AnalysisPick({ columns, main, setMain }: { + columns: string[]; main: EvalExpectation | null; setMain: (e: EvalExpectation | null) => void +}) { + const current = main && (main.kind === 'contribution' || main.kind === 'verdict') ? main : null + const start = (kind: 'contribution' | 'verdict') => setMain(kind === 'contribution' + ? { kind, op: 'decompose', mode: 'additive', params: {}, top_n: 1 } + : { kind, op: 'baseline', params: {} }) + const update = (patch: Partial) => current && setMain(keepParams({ ...current, ...patch })) + const setParam = (key: string, value: unknown) => { + if (!current) return + const params = { ...(current.params ?? {}) } + if (value === '' || value == null || (Array.isArray(value) && !value.length)) delete params[key] + else params[key] = value + setMain({ ...current, params }) + } + const modes = current?.op ? OP_MODES[current.op] : undefined + return ( +
+
{T.analysisLabel}
+

{T.analysisHint}

+
+ {(['contribution', 'verdict'] as const).map((k) => ( + + ))} +
+ {current && ( +
+ {current.kind === 'verdict' && ( + + + + )} + {modes && ( + + + + )} + {current.kind === 'contribution' && ( + + update({ top_n: Number(e.target.value) || 1 })} /> + + )} + {fieldsOf(current).map((f) => ( + setParam(f.key, v)} /> + ))} + {fieldsOf(current).some((f) => f.key === 'base') && ( +

{T.analysisPeriodHint}

+ )} +
+ )} +
+ ) +} + +function ParamInput({ field, columns, value, onChange }: { + field: ParamField; columns: string[]; value: unknown; onChange: (v: unknown) => void +}) { + const id = `eval-analysis-param-${field.key}` + const label = A.param[field.key] ?? field.key + if (field.kind === 'columns') { + const picked = Array.isArray(value) ? value.map(String) : [] + return ( +
+
{label}
+
+ {columns.map((c) => ( + + ))} +
+
+ ) + } + if (field.kind === 'pairs') { + const text = Array.isArray(value) ? value.map((p) => (Array.isArray(p) ? p.join('，') : String(p))).join('\n') : '' + return ( + +