mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 14:31:59 +00:00
ccb2b2baa8
验收:评测停止或中断后,没跑的题计入 errors、done = total、stop_reason 为 null,读起来像全评完了、出了很多错。 原因:runner._counts 把 cancelled、skipped 都算进 done(TASK_DONE),cancelled 又算进 errors;停止、中断走 _close → _conclude(status, None, error),原因写进了 error、stop_reason 是 None;服务重启后的整理(startup、reconcile_stale) 和排队中就停掉(cancel_queued)只改了逐项状态,计数还是旧的、stop_reason 也没写。 改法: - _counts:done 只算真的跑过的(ok、partial、failed);停止、中断时没评完的(cancelled)和到了上限、止损之后没开评 的(skipped)单独计 not_run;errors 只算 failed;skipped 只算评了、但期望算不出、不评分的(verdict),不再混进没开评 的。done + not_run + running + pending = total。 - 停止、中断是「停下」不是「出错」:_close 把原因记进 stop_reason(STOPPED / INTERRUPTED),error 留空;整体出错照旧 记 error。到了上限、止损本来就写 stop_reason,不变。还没备好几臂就停下的那一支(bare)同样写 stop_reason 并重算计数。 - core/job_runtime 的 close_job、cancel_queued 加一个可选的 finish:状态改好、提交之前在同一个事务里补一步。评测的 startup、reconcile_stale、stop 用它记 stop_reason、按改好的逐项状态重算计数(_closing)。起草、剖析、探查不传, 行为不变(它们的测试全过)。 - stats.summarize 的 not_run 也只算没跑的(cancelled、skipped),出错的另列 errors(以前 not_run 里混着出错的)。 前端:评测页总览「N 项没评(停止、中断,或者到了上限、止损之后没开评)」和「M 项出错」分两行;类型补上 not_run、 errors 的说明。停下的原因本来就照 stop_reason 显示。 测试(test_eval_runner):停止之后 stop_reason 是「评测已停止」、error 为空,errors 为 0、done 是评完的题数、not_run 是其余,总表的 not_run、errors 同样分开;重启整理成 interrupted 时 stop_reason 是中断的说法、计数重算(not_run 1、 running 0);新增一条排队中就停掉的:not_run 等于题数、pending 归零;到了金额上限没开评的那题记 not_run、不记跳过。 去掉修复回跑挂 4 条。check-evals 的停止夹具照服务端改,核对停下的原因、不当成出错、没评的单列。评测、起草、剖析、 探查、定时维护相关的测试 548 条全过;tsc、check-copy 通过,check-evals 103 项全过。
682 lines
37 KiB
TypeScript
682 lines
37 KiB
TypeScript
/**
|
||
* 评测页的文案(docs/design/next-round/01-效果度量与回归评测.md 第 18 项;总方案 X10 的叫法:评测、题集、口径对账……)。
|
||
* 规矩同 terms.ts 开头的术语表:「固定版本」不写钉住;「运行」不写跑;「服务端」不写后端;不在句子里裸写键名;全角标点。
|
||
*
|
||
* 统计、结论、能不能做都由服务端给,这里只有叫法和句式:数字一律是服务端算好交来的,这里只负责写成话。
|
||
*/
|
||
import type {
|
||
EvalCatalogMode, EvalCause, EvalE1Band, EvalE3Band, EvalExpectationKind, EvalGoldKind, EvalOutcome, EvalPath,
|
||
EvalRate, EvalStatus, EvalTextOutcome, EvalTransition, EvalTrialStatus, EvalTrigger, EvalUpgradePolicy, EvalVerdict,
|
||
} from '../typesEval'
|
||
import { formatCost, formatNumber } from './format'
|
||
|
||
/** 比率写成百分数:0.756 → 「75.6%」;没有分母(null)写「—」 */
|
||
export function pct(rate: number | null | undefined, digits = 1): string {
|
||
if (rate == null || !Number.isFinite(rate)) return '—'
|
||
const v = rate * 100
|
||
return `${Number.isInteger(v) ? v : v.toFixed(digits)}%`
|
||
}
|
||
|
||
/** 「3 / 4 题通过(75%,95% 区间 30%–95%)」:数都是服务端的 stats.rate */
|
||
export function rateText(r: EvalRate | null | undefined, unit = '题'): string {
|
||
if (!r || !r.n) return EVAL_TEXT.rate.none
|
||
const ci = r.ci ? `,95% 区间 ${pct(r.ci[0], 0)}–${pct(r.ci[1], 0)}` : ''
|
||
return `${formatNumber(r.passed)} / ${formatNumber(r.n)} ${unit}(${pct(r.rate)}${ci})`
|
||
}
|
||
|
||
export const EVAL_TEXT = {
|
||
nav: '评测',
|
||
title: '评测',
|
||
hint: '用题集和由 SQL、口径算出的期望答案,量问数据的效果:换模型、改提示词、改目录之后自动回归',
|
||
tabs: { suites: '题集', signals: '口径命中与重复度', snapshots: '目录快照' } as Record<'suites' | 'signals' | 'snapshots', string>,
|
||
tabsLabel: '评测的几部分',
|
||
loadFailed: '无法加载评测',
|
||
notFound: '没有这个评测页面',
|
||
back: '返回题集',
|
||
backToSuite: '返回题集',
|
||
|
||
rate: { none: '还没有能判的题' },
|
||
status: {
|
||
queued: '排队中', running: '评测中', succeeded: '已评完', partial: '部分评完', failed: '失败', cancelled: '已停止',
|
||
interrupted: '已中断',
|
||
} as Record<EvalStatus, string>,
|
||
trigger: { manual: '手动发起', schedule: '定时发起', pair: 'A/B 对照' } as Record<EvalTrigger, string>,
|
||
|
||
// ---- 题集列表
|
||
list: {
|
||
create: '新建题集',
|
||
createTitle: '新建题集',
|
||
createName: '题集名称',
|
||
createNamePlaceholder: '例如:零售问数据回归',
|
||
created: (name: string) => `已建题集「${name}」`,
|
||
import: '导入题集',
|
||
showArchived: '显示已归档的题集',
|
||
emptyTitle: '还没有题集',
|
||
emptyBody: '题集是一组固定的问题,每题的期望答案由 SQL、已治理的口径或认定过的运行算出,不靠人填数。建好题集后可以一键发起评测,换模型、改目录前后各评一次就能比出差别。',
|
||
label: '题集列表',
|
||
cases: (n: number, enabled: number) => (n === enabled ? `${formatNumber(n)} 道题` : `${formatNumber(n)} 道题,${formatNumber(enabled)} 道启用`),
|
||
noRun: '还没有评测过',
|
||
latest: '最近一次',
|
||
trend: '同一变体最近几次的准确率',
|
||
trendTitle: (points: string) => `同一变体最近几次的准确率(旧的在前):${points}`,
|
||
trendPoint: (when: string, rate: string) => `${when} ${rate}`,
|
||
schedules: '定时',
|
||
noSchedule: '没有定时',
|
||
addSchedule: '设定时',
|
||
addScheduleTitle: '新建一个定时评测这个题集的定时任务',
|
||
paused: '已暂停',
|
||
archived: '已归档',
|
||
},
|
||
|
||
// ---- 题集详情
|
||
suite: {
|
||
start: '发起评测',
|
||
export: '导出',
|
||
exported: (name: string) => `已导出「${name}」`,
|
||
settings: '题集设置',
|
||
archive: '归档',
|
||
unarchive: '取消归档',
|
||
archivedNote: '这个题集已归档:不能发起评测,题目照旧保留',
|
||
delete: '删除题集',
|
||
deleteTitle: (name: string) => `删除题集「${name}」?`,
|
||
deleteConsequences: ['题目一起删除,不可恢复', '已经评测过的题集不能删除,只能归档'],
|
||
deleted: (name: string) => `已删除题集「${name}」`,
|
||
archivedOk: (name: string) => `已归档「${name}」`,
|
||
unarchivedOk: (name: string) => `已取消归档「${name}」`,
|
||
tabs: { cases: '题目', runs: '评测记录' } as Record<'cases' | 'runs', string>,
|
||
addCase: '加题',
|
||
casesEmpty: '还没有题目',
|
||
casesEmptyBody: '加题有四种方式:写 SQL 定期望、选一个已发布的口径、从问数据的运行里认定一条查询、期望没有数据或期望反问。每道题保存前都会试算一次。',
|
||
upgradeBanner: (n: number) => `有 ${formatNumber(n)} 道口径题固定的口径已升版,待处置:处置之前这几道题评测时跳过`,
|
||
upgradeHint: '每道题三选一,没有缺省:改为固定新版本(历史在这里断开)、新旧两版各算一份期望、或旧题归档另建新题',
|
||
runsEmpty: '还没有评测记录',
|
||
runsLabel: '历次评测',
|
||
cleanup: '清理更早的明细',
|
||
cleanupHint: '评测收尾时会按保留规则自动清理,这里可以手动再清一次',
|
||
cleanupTitle: (name: string) => `清理「${name}」更早的评测明细?`,
|
||
cleanupConsequences: ['按设置里的保留次数清理更早的评测:逐题结果、作答和算期望的运行一起删除,不可恢复',
|
||
'设为基线的那次、A/B 的另几臂留着;每次评测的总表永久保留'],
|
||
cleaned: (runs: number, files: number) => (runs ? `已清理 ${runs} 次评测的明细,回收了 ${files} 个工件文件` : '没有要清理的评测'),
|
||
schedules: '定时评测',
|
||
noSchedules: '这个题集还没有定时任务',
|
||
budget: '预算',
|
||
budgetText: (parallel: number, timeout: number, run: number, daily: number) =>
|
||
`同时 ${parallel} 题 · 每题 ${timeout} 秒 · 单次 ${formatCost(run)} · 每日 ${formatCost(daily)}`,
|
||
timezone: (tz: string) => `题目里的时间变量按 ${tz} 计算`,
|
||
},
|
||
|
||
suiteSettings: {
|
||
description: '说明',
|
||
timezone: '时区',
|
||
timezoneHint: '题目、期望 SQL、口径输入里的时间变量按这个时区计算',
|
||
variant: '默认变体',
|
||
variantHint: '发起评测时缺省按它,定时评测就按它评',
|
||
budgetHint: '只填要改的项,留空用设置里的缺省;每日上限只能往下收',
|
||
},
|
||
|
||
// ---- 题目
|
||
caseRow: {
|
||
seq: (n: number) => `第 ${n} 题`,
|
||
kind: { ask: '问数据题', workflow: '工作流题' } as Record<'ask' | 'workflow', string>,
|
||
workflow: (name: string, v: number | null) => `工作流 ${name} v${v ?? '?'}`,
|
||
disabled: '未启用',
|
||
archived: '已归档',
|
||
supersededBy: (n: string) => `已被新题 ${n} 取代`,
|
||
enable: (seq: number) => `启用第 ${seq} 题`,
|
||
trial: '试算',
|
||
trialTitle: '按此刻重新试算这道题的期望答案',
|
||
edit: '编辑',
|
||
delete: '删除',
|
||
deleteTitle: (seq: number) => `删除第 ${seq} 题?`,
|
||
deleteConsequences: ['题目删除后不可恢复', '已有的评测结果照旧保留'],
|
||
deleted: (seq: number) => `已删除第 ${seq} 题`,
|
||
lastTrial: '最近一次试算',
|
||
noTrial: '还没有试算过',
|
||
trialAt: (when: string, who: string | null) => `${when}${who ? ` · ${who}` : ''}`,
|
||
dispose: '处置升版',
|
||
tags: '标签',
|
||
datasources: '限定的数据源',
|
||
question: '问法',
|
||
expectations: '期望',
|
||
goldSource: '期望答案的来源',
|
||
},
|
||
gold: {
|
||
kind: { sql: 'SQL', caliber: '口径', from_run: '从运行认定' } as Record<EvalGoldKind, string>,
|
||
sql: (source: string | null) => `SQL · ${source ?? '数据源已不在'}`,
|
||
caliber: (workflow: string, v: number) => `口径 · ${workflow} v${v}`,
|
||
fromRun: (source: string | null) => `从运行认定 · ${source ?? '数据源已不在'}`,
|
||
confirmed: (who: string | null, when: string) => `认定人:${who || '未署名'} · ${when}`,
|
||
none: '不需要期望答案',
|
||
},
|
||
expectation: {
|
||
kind: {
|
||
scalar: '一个数', table: '多行', set: '名单', none: '没有数据', ask: '反问', text: '文字要点',
|
||
} as Record<EvalExpectationKind, string>,
|
||
scalarCell: (column: string, row: number) => `「${column}」第 ${row + 1} 行`,
|
||
scalarWhere: (column: string, where: string) => `「${column}」(${where})`,
|
||
scalarMetric: (metric: string) => `指标「${metric}」`,
|
||
set: (column: string) => `「${column}」这一列`,
|
||
table: (keys: string) => `按「${keys}」配对的多行`,
|
||
none: '期望答「没有数据」',
|
||
ask: '期望反问',
|
||
text: (n: number) => `${n} 条文字要点(模型判断,不计入准确率)`,
|
||
optional: '非必需',
|
||
tolerance: (t: number) => `允许误差 ${(t * 100).toFixed(t < 0.01 ? 2 : 1)}%`,
|
||
},
|
||
|
||
// ---- 加题、改题
|
||
editor: {
|
||
addTitle: '加题',
|
||
editTitle: (seq: number) => `编辑第 ${seq} 题`,
|
||
modeLabel: '期望答案怎么定',
|
||
mode: {
|
||
sql: '写 SQL 定期望', caliber: '选口径', from_run: '从问数据运行认定', special: '期望没有数据或期望反问',
|
||
} as Record<'sql' | 'caliber' | 'from_run' | 'special', string>,
|
||
modeHint: {
|
||
sql: '选数据源、写 SQL、点选结果里的格或列:期望值每次评测都由这条 SQL 重算',
|
||
caliber: '选一个已发布的口径卡上的指标:期望值由那一版口径卡算,取整、单位都由口径卡定',
|
||
from_run: '从一次问数据运行执行过的查询里认定一条:拷出它的 SQL,之后每次评测按这条 SQL 重算',
|
||
special: '问法问的是没有的东西(期望答「没有数据」),或者问法含糊(期望助手反问)',
|
||
} as Record<'sql' | 'caliber' | 'from_run' | 'special', string>,
|
||
question: '问法',
|
||
workflowQuestion: '这道题考什么(可选)',
|
||
workflowQuestionHint: '工作流题不需要问法:写一句说明,列表里认得出是哪道题',
|
||
caseKind: '题目种类',
|
||
caseKindHint: {
|
||
ask: '问法交给助手:建图、运行、复核,和问数据页同一条路',
|
||
workflow: '固定一个工作流的已发布版本,按输入运行一次(不建图、不复核),考的是这一版本身',
|
||
} as Record<'ask' | 'workflow', string>,
|
||
workflow: '工作流',
|
||
workflowPlaceholder: '选择已发布的工作流',
|
||
workflowNone: '还没有已发布的工作流',
|
||
workflowVersion: '固定的版本',
|
||
questionPlaceholder: '例如:上周按支付日统计、不含退款的销售额是多少?',
|
||
questionHint: '可以写 {{ schedule.last_week_start }} 这类时间变量,按评测的截至时间计算',
|
||
scope: '限定的数据源',
|
||
scopeHint: '不选就是不限(同问数据页的数据源范围)',
|
||
scopeAll: '不限',
|
||
tags: '标签',
|
||
tagsPlaceholder: '用逗号分隔,例如:口径陷阱,相对日期',
|
||
enabled: '启用',
|
||
required: '必需',
|
||
tolerance: '允许的相对误差',
|
||
tolerancePlaceholder: '不填按显示精度推',
|
||
toleranceHint: '0.01 表示 1%',
|
||
// SQL
|
||
source: '数据源',
|
||
sourcePlaceholder: '选择数据源',
|
||
sql: '期望 SQL',
|
||
sqlPlaceholder: 'SELECT SUM(amount) AS gmv FROM orders WHERE …',
|
||
sqlHint: '只读查询;可以写 {{ schedule.* }} 时间变量。用到遮罩列的结果不能当期望',
|
||
preview: '预览结果',
|
||
previewing: '正在查询…',
|
||
previewRows: (shown: number, total: number | null, truncated: boolean) =>
|
||
`${total != null ? `共 ${formatNumber(total)} 行` : ''}${truncated ? `,只显示前 ${formatNumber(shown)} 行` : ''}`,
|
||
previewEmpty: '这条 SQL 没有返回行:期望没有数据的题请选「期望没有数据或期望反问」',
|
||
previewPick: '点结果里的一格定「一个数」,点列名定「名单」,或者选配对的列定「多行」',
|
||
previewRendered: '按此刻渲染的 SQL',
|
||
pickKind: '期望的种类',
|
||
pickCell: (column: string, row: number) => `「${column}」第 ${row + 1} 行`,
|
||
pickColumn: (column: string) => `「${column}」这一列`,
|
||
whereLabel: '按这一行的哪几列找这一行(不选就按第几行)',
|
||
keysLabel: '或者定「多行」:按哪几列配对',
|
||
columnsLabel: '比哪几列(不选就比全部)',
|
||
topN: '只看前几行',
|
||
ordered: '要按顺序',
|
||
noPick: '还没有选期望:先预览结果,再点选格或列',
|
||
// 口径
|
||
caliberLoading: '正在读取口径清单…',
|
||
caliberEmpty: '还没有能选的口径:口径来自已发布、未退役的工作流里的口径卡',
|
||
caliberSearch: '搜索工作流、口径卡或指标',
|
||
caliberNotReusable: '不能当口径题',
|
||
caliberInputs: '口径的输入',
|
||
caliberInputsHint: '口径卡读的取数节点要的入口字段,可以写时间变量',
|
||
caliberInputsNone: '这个口径不需要输入',
|
||
caliberInputsLoading: '正在读取入口字段…',
|
||
caliberPicked: (workflow: string, v: number, metric: string) => `${workflow} v${v} · ${metric}`,
|
||
// 从运行认定
|
||
runPick: '选一次问数据运行',
|
||
runRecent: '最近的问数据运行',
|
||
runRecentEmpty: '最近没有问数据运行',
|
||
runId: '或者填运行编号',
|
||
runIdPlaceholder: '运行编号',
|
||
runLoad: '读取查询',
|
||
runQueries: '这次运行执行过的查询',
|
||
runQueriesEmpty: '这次运行没有执行过查询',
|
||
runQueryMasked: '用到了遮罩列,不能认定',
|
||
runQueryRows: (n: number | null) => (n != null ? `${formatNumber(n)} 行` : ''),
|
||
runQueryPick: '认定这一条',
|
||
runColumnPick: '从这条查询的结果里取',
|
||
fromRunNote: '认定人记的是你;之后每次评测按这条 SQL 重算,不冻结数值',
|
||
// 期望没有数据或反问
|
||
specialKind: '期望的是',
|
||
specialNone: '期望答「没有数据」',
|
||
specialNoneHint: '写一条查这件事的 SQL,它应当返回 0 行',
|
||
specialAsk: '期望助手反问',
|
||
specialAskHint: '问法含糊、该先问清楚的题:助手反问了就算对,不需要期望答案',
|
||
// 文字要点
|
||
points: '文字要点(可选)',
|
||
pointsHint: '一行一条;由结论裁判判断有没有答到,单列「模型判断」,不计入准确率',
|
||
// 试算、保存
|
||
trial: '试算',
|
||
trialing: '正在试算…',
|
||
trialOk: '试算通过',
|
||
trialExpected: '算出的期望',
|
||
trialAsOf: (when: string) => `按 ${when} 计算`,
|
||
save: '保存',
|
||
saving: '正在保存…',
|
||
saved: (seq: number) => `已保存第 ${seq} 题`,
|
||
saveFailed: '没有保存',
|
||
saveHint: '保存前服务端会再试算一次,没过不保存',
|
||
cancel: '取消',
|
||
leaveTitle: '这道题还没有保存',
|
||
leaveConsequences: ['关掉之后填写的内容会丢失'],
|
||
conflict: '这道题已在别处被修改,请关闭后重新打开再改',
|
||
fromRunGold: '这道题的期望答案是从运行认定的:要改 SQL,请另建一道写 SQL 定期望的题',
|
||
},
|
||
trialStatus: {
|
||
ok: '通过', refused: '拒收', no_value: '没有可用的值', failed: '失败', timeout: '超时', skipped: '跳过',
|
||
drifted: '数据变了', invalid: '有误',
|
||
} as Record<EvalTrialStatus | 'invalid', string>,
|
||
/** 试算通过时一个期望算出了什么(值都是服务端算好的) */
|
||
goldValue: {
|
||
scalar: (v: string, unit?: string) => `${v}${unit ? ` ${unit}` : ''}`,
|
||
set: (n: number) => `${formatNumber(n)} 项`,
|
||
table: (n: number) => `${formatNumber(n)} 行`,
|
||
none: '0 行(没有数据)',
|
||
dims: (d: string) => `(${d})`,
|
||
},
|
||
upgrade: {
|
||
title: (seq: number) => `第 ${seq} 题:处置口径升版`,
|
||
current: (pinned: number | null, latest: number | null | undefined) =>
|
||
`固定的是 v${pinned ?? '?'}${latest ? `,最新发布的是 v${latest}` : ''}`,
|
||
policyLabel: '怎么处置',
|
||
noDefault: '没有缺省,请选一种',
|
||
confirm: '处置',
|
||
done: (seq: number) => `已处置第 ${seq} 题`,
|
||
newCase: (seq: number) => `旧题已归档,按新版本另建了第 ${seq} 题`,
|
||
},
|
||
|
||
// ---- 导入
|
||
import: {
|
||
title: '导入题集',
|
||
file: '题集文件',
|
||
fileHint: '导出的题集 JSON:按数据源名、工作流名对上,不含任何期望值、答案或运行',
|
||
reading: '正在读取…',
|
||
preview: (ok: number, skipped: number) => `能导入 ${formatNumber(ok)} 道,跳过 ${formatNumber(skipped)} 道`,
|
||
previewHint: '导入时每道题都会试算一次,没过的跳过并写明原因',
|
||
name: '新题集的名称',
|
||
sources: '数据源对照',
|
||
sourceMissing: '这里没有同名的数据源',
|
||
skipped: '跳过的题',
|
||
confirm: '导入',
|
||
importing: '正在导入并试算…',
|
||
done: (name: string, n: number, skipped: number) => `已导入「${name}」:${n} 道题${skipped ? `,跳过 ${skipped} 道` : ''}`,
|
||
notJson: '这个文件不是 JSON',
|
||
},
|
||
|
||
// ---- 发起评测
|
||
start: {
|
||
title: '发起评测',
|
||
mode: '评测方式',
|
||
modes: { single: '评一次', ab: 'A/B 对照', catalog: '目录贡献对照' } as Record<'single' | 'ab' | 'catalog', string>,
|
||
modeHint: {
|
||
single: '按一组变体评一遍题集',
|
||
ab: '两到三臂各自一组变体,共用同一份期望答案,同一题几臂同时作答',
|
||
catalog: '预设的 A/B:当前目录 vs 关目录,可以再加一臂「只用已核实和已确认的项」,量出数据目录帮了多少',
|
||
} as Record<'single' | 'ab' | 'catalog', string>,
|
||
arm: (name: string) => `${name} 臂`,
|
||
addArm: '再加一臂',
|
||
removeArm: '去掉这一臂',
|
||
armLabel: '这一臂的名字',
|
||
armLabelPlaceholder: (i: number) => `例如:${['新模型', '旧模型', '第三臂'][i] ?? '对照'}`,
|
||
assistant: '助手模型',
|
||
assistantHint: '建图、挑表、复核用的模型;不选按「编排」页助手栏的设置',
|
||
writer: '写作模型',
|
||
writerHint: '答案运行里模型节点的缺省;节点自己选了模型的不覆盖',
|
||
modelDefault: '缺省',
|
||
modelProvider: '模型接入',
|
||
modelName: '模型',
|
||
judge: '结论裁判',
|
||
judgeHint: '打开后逐句判断结论有没有证据支撑、判文字要点,花费记在评测的账上',
|
||
catalog: '数据目录',
|
||
catalogMode: {
|
||
live: '当前目录', reviewed: '只用已核实和已确认的项', off: '关目录', snapshot: '目录快照',
|
||
} as Record<EvalCatalogMode, string>,
|
||
catalogHint: '当前目录、只用已核实的项都在开评时拍一份快照按它读:评测期间目录被改也不影响可比性',
|
||
snapshot: '选一份目录快照',
|
||
snapshotNone: '还没有可用的目录快照',
|
||
repeats: '每题重复几次',
|
||
repeatsHint: '模型有随机性:对照建议每题 3 次',
|
||
reviewedArm: '再加一臂「只用已核实和已确认的项」',
|
||
baseline: '和哪一次比',
|
||
baselineAuto: '自动:同一变体里设为基线的那次,没有就是上一次',
|
||
estimate: '预估',
|
||
estimating: '正在估算…',
|
||
estimateText: (items: number, cases: number, repeats: number, arms: number) =>
|
||
`${formatNumber(cases)} 道题 × 每题 ${repeats} 次${arms > 1 ? ` × ${arms} 臂` : ''},共 ${formatNumber(items)} 项`,
|
||
cost: (total: string, per: string) => `约 ${total}(每项约 ${per})`,
|
||
costNone: '估不出花费',
|
||
budget: '生效的预算',
|
||
parallel: (n: number) => `同时评 ${n} 题`,
|
||
spendToday: (usd: string, daily: string | null) => `评测今天已花 ${usd}${daily ? ` / 每日上限 ${daily}` : ''}`,
|
||
spendUnpriced: (n: number) => `(另有 ${formatNumber(n)} 次调用金额无法估算)`,
|
||
models: (assistant: string | null | undefined, writer: string | null | undefined) =>
|
||
`助手 ${assistant || '缺省'} · 写作 ${writer || '各节点自己的'}`,
|
||
submit: '开始评测',
|
||
starting: '正在发起…',
|
||
started: '已发起评测',
|
||
estimateFailed: '无法估算',
|
||
},
|
||
|
||
// ---- 评测详情
|
||
run: {
|
||
/** short 是 shortId 给的「#a1b2c3」 */
|
||
title: (short: string) => `评测 ${short}`,
|
||
arm: (name: string) => `${name} 臂`,
|
||
of: (suite: string) => `题集「${suite}」`,
|
||
variant: '变体',
|
||
asOf: (when: string) => `截至 ${when}`,
|
||
startedBy: (who: string | null) => `${who || '未署名'} 发起`,
|
||
stop: '停止',
|
||
stopping: '正在停止…',
|
||
stopped: '已停止评测',
|
||
retry: '重试没做完的',
|
||
retryHint: (n: number) => `只把没做完的 ${n} 项接着做(停止、中断、出错、到了上限的)`,
|
||
retried: '已重新排队,只做没做完的题',
|
||
rescore: '按新规则重评',
|
||
rescoreHint: '只读封存的作答运行和存下的期望,不调模型、不查业务库',
|
||
rescored: (n: number) => (n ? `已重新评分,${n} 项结论有变化` : '已重新评分,结论都没变'),
|
||
pin: '设为基线',
|
||
pinHint: '和上一次比时优先和它比,清理时也留着它',
|
||
unpin: '取消基线',
|
||
pinned: '基线',
|
||
pinnedOk: '已设为基线',
|
||
unpinnedOk: '已取消基线',
|
||
cleaned: '明细已清理:总表还在,逐题结果、对比不再可看',
|
||
progress: (done: number, total: number) => `已评 ${formatNumber(done)} / ${formatNumber(total)} 项`,
|
||
current: '正在评',
|
||
currentItem: (seq: number | null, repeat: number, arm: string | null) =>
|
||
`第 ${seq ?? '?'} 题${repeat ? ` · 第 ${repeat + 1} 次` : ''}${arm ? ` · ${arm} 臂` : ''}`,
|
||
stopReason: '停下的原因',
|
||
error: '出错',
|
||
tabs: { overview: '总览', results: '逐题', compare: '和上一次比', pair: 'A/B' } as Record<'overview' | 'results' | 'compare' | 'pair', string>,
|
||
noSummary: '评完之后才有总表',
|
||
openRun: '查看运行',
|
||
goldRuns: '算期望的运行',
|
||
},
|
||
overview: {
|
||
accuracy: '准确率',
|
||
accuracyHint: '分母是能判的题(通过 + 不通过);不能评分、跳过的单列,既不算对也不算错',
|
||
traced: '可追溯准确率',
|
||
tracedHint: '必需的期望全部经引用对上(答案里的数点得开、来自查询的那一格或口径卡的指标)',
|
||
verdicts: '结论',
|
||
notRun: (n: number) => `${formatNumber(n)} 项没评(停止、中断,或者到了上限、止损之后没开评)`,
|
||
errors: (n: number) => `${formatNumber(n)} 项出错`,
|
||
cost: '花费',
|
||
costHint: '助手(建图、挑表、复核)+ 写作(答案运行里的模型调用)+ 结论裁判;算期望不调模型',
|
||
perCase: (v: string) => `每题约 ${v}`,
|
||
e1: '逐题的口径命中(E1)',
|
||
e1Hint: '答案执行的 SQL 和期望的取数被证明等价的比例(口径账本同一个比对函数)',
|
||
citations: '报告引用率',
|
||
citationsText: (cited: number, all: number) => `${formatNumber(cited)} / ${formatNumber(all)} 个数有引用`,
|
||
uncited: (n: number, unresolved: number) => `正文裸写 ${formatNumber(n)} 个,引用没解析 ${formatNumber(unresolved)} 个`,
|
||
paths: '作答路径',
|
||
tiers: '出具档位',
|
||
tier: { formal: '完整出具', degraded: '降档出具', withheld: '不予出具', explore: '探索问答' } as Record<string, string>,
|
||
review: '复核',
|
||
reviewVerdict: { ok: '没发现问题', annotated: '加了说明', rewritten: '改写了答案', reverted: '退回原答案', failed: '复核出错' } as Record<string, string>,
|
||
reviewRetries: (n: number) => `自动重来 ${formatNumber(n)} 次`,
|
||
reviewSlow: (n: number) => `${formatNumber(n)} 题复核超过 25 秒(问数据页不会等这么久)`,
|
||
judge: '结论裁判',
|
||
judgeOff: '这次没有打开结论裁判',
|
||
claims: { supported: '有证据支撑', partial: '部分支撑', contradicted: '和证据矛盾', insufficient: '证据不足', unjudged: '未判' } as Record<string, string>,
|
||
text: '文字要点(模型判断,不计入准确率)',
|
||
looks: '查看',
|
||
looksText: (model: number, system: number) => `模型要求 ${formatNumber(model)} 次 · 系统预先 ${formatNumber(system)} 次`,
|
||
looksPer: (v: number | null) => (v != null ? `每题 ${v} 次` : ''),
|
||
looksLimit: (v: string) => `用满上限的占 ${v}`,
|
||
calls: (n: number, per: number | null) => `助手调用 ${formatNumber(n)} 次${per != null ? `(每题 ${per} 次)` : ''}`,
|
||
timings: '耗时(P50 / P90)',
|
||
timing: { build_ms: '建图', run_wall_ms: '运行(墙钟)', run_active_ms: '运行(执行)', review_ms: '复核', total_ms: '整题' } as Record<string, string>,
|
||
usage: '按角色的用量',
|
||
tokens: (input: string, output: string) => `输入 ${input} · 输出 ${output}`,
|
||
callsN: (n: number) => `${formatNumber(n)} 次调用`,
|
||
role: { assistant: '助手', writer: '写作', judge: '结论裁判', gold: '算期望' } as Record<string, string>,
|
||
passRates: '重复评的题的通过率',
|
||
outcomes: '各期望的结论',
|
||
baseline: '和上一次比',
|
||
baselineText: (regressed: number, improved: number) => `退步 ${formatNumber(regressed)} 题 · 进步 ${formatNumber(improved)} 题`,
|
||
baselineNone: '没有可比的上一次',
|
||
},
|
||
verdict: { pass: '通过', fail: '不通过', ungradable: '不能评分', skipped: '跳过' } as Record<EvalVerdict, string>,
|
||
outcome: {
|
||
traced: '对且可追溯', untraced: '对但没引用', wrong: '错', scale: '量纲不同', ungradable: '不能评分',
|
||
} as Record<EvalOutcome, string>,
|
||
textOutcome: { covered: '都答到了', partial: '答到一部分', missing: '没答到', unjudged: '没有判' } as Record<EvalTextOutcome, string>,
|
||
path: {
|
||
reply: '直接回答', asked: '反问', ready: '只建图未运行', build_failed: '建图失败', run_failed: '运行失败', ran: '运行完成',
|
||
rejected: '被驳回',
|
||
} as Record<EvalPath, string>,
|
||
itemStatus: {
|
||
pending: '待评', running: '评测中', ok: '评完', partial: '部分评完', failed: '出错', skipped: '跳过', cancelled: '没做完',
|
||
} as Record<string, string>,
|
||
sqlMatch: {
|
||
proven: '等价已证明', related: '不等价', none: '答案没有查询', skipped: '没比对',
|
||
} as Record<string, string>,
|
||
aspect: {
|
||
time: '时间口径', time_range: '时间范围', filter: '筛选条件', aggregate: '聚合', expression: '计算式', join: '关联',
|
||
tables: '用到的表', group_by: '分组', limit: '行数限制', other: '其他', parse: '解析不了',
|
||
} as Record<string, string>,
|
||
results: {
|
||
label: '逐题结果',
|
||
empty: '还没有结果',
|
||
cleaned: '明细已清理',
|
||
seq: '题',
|
||
question: '问法',
|
||
/** 逐题表里实际问的问题下面一行:题目原文和实际问的不同(带变量的模板,或者题目之后改过)时写原文 */
|
||
questionTemplate: (text: string) => `题目原文:${text}`,
|
||
verdict: '结论',
|
||
expected: '期望',
|
||
got: '答案里的数',
|
||
traced: '引用',
|
||
tracedYes: '有引用',
|
||
tracedNo: '没引用',
|
||
sql: 'SQL 差在哪',
|
||
looks: '查看',
|
||
looksCell: (model: number, system: number) => `${model} · ${system}`,
|
||
time: '耗时',
|
||
cost: '花费',
|
||
path: '路径',
|
||
weak: '弱匹配',
|
||
weakHint: '期望是 0–10 的小整数又没有维度可配:可能是巧合',
|
||
reason: '原因',
|
||
filterAll: '全部',
|
||
filter: '只看',
|
||
repeat: (n: number) => `第 ${n + 1} 次`,
|
||
attempts: (n: number) => `这一题自动重来了 ${n} 次`,
|
||
turnReason: '作答的经过',
|
||
answer: '答案原文',
|
||
expand: '展开',
|
||
collapse: '收起',
|
||
},
|
||
compare: {
|
||
base: '基准',
|
||
baseAuto: '按规则挑(同一变体里设为基线的那次,没有就是上一次)',
|
||
/** short 是 shortId 给的「#a1b2c3」 */
|
||
baseText: (short: string, when: string) => `${short} · ${when}`,
|
||
none: '没有可比的上一次:同一题集、同一变体里还没有比这次先发起、已经评完的评测',
|
||
fingerprint: '两次之间变了什么',
|
||
fingerprintNone: '模型、目录、提示词、表结构、题目、评分规则都没变',
|
||
changed: '变了',
|
||
transitions: '逐题转移',
|
||
transition: { regressed: '退步', improved: '进步', same: '不变', incomparable: '不可比' } as Record<EvalTransition, string>,
|
||
cause: { data: '数据变', method: '做法变', params: '只参数变', path: '路径变', unknown: '说不清' } as Record<EvalCause, string>,
|
||
causes: '退步和进步的归因',
|
||
causeHint: {
|
||
data: '期望值变了',
|
||
method: '答案执行的 SQL 写法不同',
|
||
params: '只有字面量不同(常见于时间范围理解变了)',
|
||
path: '作答路径变了',
|
||
unknown: '期望、SQL、路径都没变',
|
||
} as Record<EvalCause, string>,
|
||
cleaned: '有一边的明细已清理,逐题部分不可看',
|
||
items: '转移了的题',
|
||
noItems: '没有转移的题',
|
||
case: (seq: number | null) => `第 ${seq ?? '?'} 题`,
|
||
},
|
||
pair: {
|
||
title: 'A/B 配对表',
|
||
arms: '各臂的准确率',
|
||
pairs: '两两配对',
|
||
both: '都对',
|
||
neither: '都错',
|
||
onlyA: (a: string) => `只 ${a} 对`,
|
||
onlyB: (b: string) => `只 ${b} 对`,
|
||
excluded: (n: number) => `${formatNumber(n)} 题有一边不能判,不进配对表`,
|
||
pValue: (p: string) => `精确 McNemar 双侧 p = ${p}`,
|
||
significant: '差别显著(p < 0.05)',
|
||
cleaned: '有一臂的明细已清理,配对表不可看',
|
||
open: (arm: string) => `打开 ${arm} 臂`,
|
||
},
|
||
|
||
// ---- 口径命中与重复度
|
||
signals: {
|
||
title: '口径命中与重复度',
|
||
hint: '问数据的 SQL 有多少能证明和已治理的口径等价(E1),问题有多少在反复问(E3)。评测运行不计入',
|
||
window: '统计窗口',
|
||
days: (n: number) => `最近 ${n} 天`,
|
||
e1: '口径命中(E1)',
|
||
e1Hint: '有候选口径的查询里,被证明和口径等价的比例',
|
||
e3: '问题重复度(E3)',
|
||
e3Hint: (n: number) => `问数据的问题里,落在成员不少于 ${n} 个的类里的比例`,
|
||
lowerBound: (v: string) => `按 SQL 形状重复的运行占 ${v}(下限)`,
|
||
n: (n: number) => `样本 ${formatNumber(n)}`,
|
||
noData: '这个窗口里没有数据',
|
||
band: {
|
||
route: '值得按已治理的口径走', normalize: '先加强规范化', reconcile: '先只做口径对账',
|
||
worth: '沉淀有价值', not_yet: '反复问的问题还不多',
|
||
} as Record<EvalE1Band | EvalE3Band, string>,
|
||
criteria: '参考线',
|
||
trend: '趋势',
|
||
trendHint: (days: number) => `每 ${days} 天一个窗口,旧的在前`,
|
||
trendWindow: (since: string, until: string) => `${since} – ${until}`,
|
||
detail: '明细',
|
||
status: { proven: '等价已证明', composed: '助手选用', related: '相关但不等价', none: '没有相关口径', skipped: '没比对' } as Record<string, string>,
|
||
diffAspects: '不等价时差在哪',
|
||
unproven: '没证明的原因',
|
||
shapes: (n: number, repeated: number) => `${formatNumber(n)} 种 SQL 形状,${formatNumber(repeated)} 种在重复`,
|
||
embedder: (name: string) => `聚类用的向量模型:${name}(本地哈希向量的语义能力弱,结果是下限)`,
|
||
clusters: '反复问的问题',
|
||
clustersHint: (n: number) => `成员不少于 ${n} 个的类:可以加入题集(期望答案仍要按 SQL 或口径给),或从最近一次的运行提取模板`,
|
||
clustersEmpty: '这个窗口里还没有反复问的问题',
|
||
clusterCount: (n: number) => `问了 ${formatNumber(n)} 次`,
|
||
members: '问法',
|
||
addToSuite: '加入题集',
|
||
addToSuiteTitle: '把这个问题加入题集',
|
||
pickSuite: '加到哪个题集',
|
||
noSuites: '还没有题集,请先新建一个',
|
||
go: '去加题',
|
||
extract: '提取模板',
|
||
extractHint: '把最近一次的运行提取成工作流草稿,在画布上审改',
|
||
extractNone: '这一类的问题都没有挂着运行,不能提取',
|
||
extracted: (name: string) => `已提取为草稿「${name}」`,
|
||
ledgerFailed: '无法读取口径账本的统计',
|
||
},
|
||
|
||
// ---- 目录快照
|
||
snapshots: {
|
||
title: '目录快照',
|
||
hint: '评测按快照读数据目录:做「当前目录 vs 关」、起草模型 A 的目录 vs B 的这类对照,线上目录不受影响',
|
||
take: '拍当前目录',
|
||
takeTitle: '拍一份当前目录',
|
||
takeHint: '选定数据源此刻的目录,拍好就不再变,可以被多次评测引用',
|
||
taken: '已拍好目录快照',
|
||
shadow: '影子起草',
|
||
shadowTitle: '影子起草一份目录',
|
||
shadowHint: '用指定的起草模型把题集用得到的表重新起草进一份快照:不碰线上目录、不查业务库,按表计费',
|
||
sources: '数据源',
|
||
sourcesHint: '一次最多 20 个',
|
||
tables: (name: string) => `「${name}」要起草的表`,
|
||
tablesHint: '一行一张表;留空就按题集的期望 SQL 推',
|
||
fromSuite: '按题集推要起草的表',
|
||
fromSuiteNone: '不按题集推',
|
||
provider: '起草用的模型接入',
|
||
model: '起草模型',
|
||
modelDefault: '按设置里起草模型的取值链',
|
||
guidance: '起草要求(可选)',
|
||
note: '这份快照的说明(可选)',
|
||
strip: '先去掉模型起草、还没人审过的项',
|
||
estimate: '估算',
|
||
estimating: '正在估算…',
|
||
estimateText: (tables: number, calls: number, input: string, output: string, cost: string) =>
|
||
`${formatNumber(tables)} 张表 · 调用模型约 ${formatNumber(calls)} 次 · 输入约 ${input}、输出约 ${output} · ${cost}`,
|
||
estimateUnpriced: '这个模型不在价格目录中,金额无法估算',
|
||
unknownTables: (name: string, list: string) => `「${name}」的表结构里没有:${list}`,
|
||
startShadow: '开始起草',
|
||
started: '已开始影子起草',
|
||
stop: '停止',
|
||
stopped: '已停止影子起草',
|
||
empty: '还没有目录快照',
|
||
emptyBody: '开评时选「当前目录」「只用已核实的项」会自动拍一份;要做起草模型对照,先在这里影子起草一份。',
|
||
origin: { live: '当前目录', shadow: '影子起草' } as Record<'live' | 'shadow', string>,
|
||
status: {
|
||
queued: '排队中', running: '起草中', succeeded: '已完成', partial: '部分完成', failed: '失败', cancelled: '已停止',
|
||
interrupted: '已中断',
|
||
} as Record<string, string>,
|
||
progress: (done: number, total: number) => `已起草 ${formatNumber(done)} / ${formatNumber(total)} 张表`,
|
||
by: (who: string | null, when: string) => `${who || '未署名'} · ${when}`,
|
||
usage: (calls: number, cost: string) => `调用模型 ${formatNumber(calls)} 次 · ${cost}`,
|
||
failures: (n: number) => `${formatNumber(n)} 张表没起草成`,
|
||
model_: (m: string) => `起草模型:${m}`,
|
||
stripped: '起草前去掉了模型起草、未审过的项',
|
||
use: '可在发起评测时选它',
|
||
},
|
||
|
||
// ---- 记录页
|
||
runSource: {
|
||
tag: '评测',
|
||
tagTitle: '这次运行由评测发起:点一下去对应的那道题',
|
||
tagSr: '由评测发起',
|
||
link: (seq: string) => `来自评测 · ${seq}`,
|
||
linkTitle: '去评测里对应的那道题',
|
||
role: { answer: '作答', gold: '算期望', gold_after: '作答后复算期望' } as Record<string, string>,
|
||
trial: '试算',
|
||
caseUnknown: '题目',
|
||
},
|
||
|
||
// ---- 设置
|
||
settings: {
|
||
title: '评测',
|
||
hint: '保存在服务端、全站共用:题集上改过的预算项盖过这里的缺省,评测每日上限题集只能往下收',
|
||
max_parallel: '同时评几道题',
|
||
per_case_timeout_s: '每题时限(秒)',
|
||
per_run_max_usd: '单次评测的金额上限(美元)',
|
||
daily_max_usd: '评测每日金额上限(美元)',
|
||
keep_runs: '每个题集保留最近几次的明细',
|
||
keepHint: '更早的明细由清理任务删掉;设为基线的那次、A/B 的另几臂另外留着;每次评测的总表永久保留',
|
||
judge: '发起时缺省打开结论裁判',
|
||
judgeHint: '结论裁判按句数花钱;发起评测时可以按变体再开关',
|
||
spend: (usd: string, daily: string, unpriced: number) =>
|
||
`今天评测已花 ${usd} / 每日上限 ${daily}${unpriced ? `(另有 ${unpriced} 次调用金额无法估算)` : ''}`,
|
||
save: '保存评测设置',
|
||
saving: '正在保存…',
|
||
saved: '已保存评测设置',
|
||
discard: '放弃修改',
|
||
readOnly: '只有管理员能修改评测设置',
|
||
invalid: {
|
||
int: (low: number, high: number) => `请填写 ${low} 到 ${high} 之间的整数`,
|
||
usd: (low: number, high: number) => `请填写 ${low} 到 ${high} 之间的金额`,
|
||
runOverDaily: '单次评测的金额上限不能高于每日上限',
|
||
},
|
||
leaveTitle: (n: number) => `评测设置有 ${n} 项还没保存`,
|
||
leaveConsequences: ['离开后这些修改会丢失'],
|
||
},
|
||
}
|
||
|
||
/** 一种分布({键: 个数})写成「通过 3 · 不通过 1」:叫法表里没有的键原样写 */
|
||
export function distText(dist: Record<string, number> | null | undefined, labels: Record<string, string>): string {
|
||
const parts = Object.entries(dist ?? {}).filter(([, n]) => n > 0).map(([k, n]) => `${labels[k] ?? k} ${formatNumber(n)}`)
|
||
return parts.length ? parts.join(' · ') : '—'
|
||
}
|