mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:21:58 +00:00
评测页补齐:工作流题、手动清理更早的明细、逐题明细里的作答经过和答案原文
前三波的接口这一波都要有界面(收口):
- 加题时「写 SQL 定期望」「选口径」两种可以选题目种类——问数据题(问法交给助手)或工作流题(固定一个已发布版本、按入口字段填
输入模板,考的是这一版本身);题目表里工作流题写工作流和版本。反问、从运行认定只对问数据题有意义,不给选。
- 评测记录页签加「清理更早的明细」(POST /suites/{id}/cleanup,先确认、写清后果;收尾时本来就按保留规则自动清,这里是手动再清一次)。
- 逐题表展开时读这一题的明细工件(GET …/results/{id}):自动重来了几次、作答的经过、答案原文——答错的题能看到它当时答了什么。
check-evals 跟着加三处(工作流题的请求体、清理先确认再发、明细),101 项。
This commit is contained in:
@@ -91,6 +91,12 @@ export const EVAL_TEXT = {
|
||||
upgradeHint: '每道题三选一,没有缺省:改为固定新版本(历史在这里断开)、新旧两版各算一份期望、或旧题归档另建新题',
|
||||
runsEmpty: '还没有评测记录',
|
||||
runsLabel: '历次评测',
|
||||
cleanup: '清理更早的明细',
|
||||
cleanupHint: '评测收尾时会按保留规则自动清理,这里可以手动再清一次',
|
||||
cleanupTitle: (name: string) => `清理「${name}」更早的评测明细?`,
|
||||
cleanupConsequences: ['按设置里的保留次数清理更早的评测:逐题结果、作答和算期望的运行一起删除,不可恢复',
|
||||
'设为基线的那次、A/B 的另几臂留着;每次评测的总表永久保留'],
|
||||
cleaned: (runs: number, files: number) => (runs ? `已清理 ${runs} 次评测的明细,回收了 ${files} 个工件文件` : '没有要清理的评测'),
|
||||
schedules: '定时评测',
|
||||
noSchedules: '这个题集还没有定时任务',
|
||||
budget: '预算',
|
||||
@@ -112,6 +118,7 @@ export const EVAL_TEXT = {
|
||||
caseRow: {
|
||||
seq: (n: number) => `第 ${n} 题`,
|
||||
kind: { ask: '问数据题', workflow: '工作流题' } as Record<'ask' | 'workflow', string>,
|
||||
workflow: (name: string, v: number | null) => `工作流 ${name} v${v ?? '?'}`,
|
||||
disabled: '未启用',
|
||||
archived: '已归档',
|
||||
supersededBy: (n: string) => `已被新题 ${n} 取代`,
|
||||
@@ -172,6 +179,17 @@ export const EVAL_TEXT = {
|
||||
special: '问法问的是没有的东西(期望答「没有数据」),或者问法含糊(期望助手反问)',
|
||||
} as Record<'sql' | 'caliber' | 'from_run' | 'special', string>,
|
||||
question: '问法',
|
||||
workflowQuestion: '这道题考什么(可选)',
|
||||
workflowQuestionHint: '工作流题不需要问法:写一句说明,列表里认得出是哪道题',
|
||||
caseKind: '题目种类',
|
||||
caseKindHint: {
|
||||
ask: '问法交给助手:建图、运行、复核,和问数据页同一条路',
|
||||
workflow: '固定一个工作流的已发布版本,按输入运行一次(不建图、不复核),考的是这一版本身',
|
||||
} as Record<'ask' | 'workflow', string>,
|
||||
workflow: '工作流',
|
||||
workflowPlaceholder: '选择已发布的工作流',
|
||||
workflowNone: '还没有已发布的工作流',
|
||||
workflowVersion: '固定的版本',
|
||||
questionPlaceholder: '例如:上周按支付日统计、不含退款的销售额是多少?',
|
||||
questionHint: '可以写 {{ schedule.last_week_start }} 这类时间变量,按评测的截至时间计算',
|
||||
scope: '限定的数据源',
|
||||
@@ -473,6 +491,9 @@ export const EVAL_TEXT = {
|
||||
filterAll: '全部',
|
||||
filter: '只看',
|
||||
repeat: (n: number) => `第 ${n + 1} 次`,
|
||||
attempts: (n: number) => `这一题自动重来了 ${n} 次`,
|
||||
turnReason: '作答的经过',
|
||||
answer: '答案原文',
|
||||
expand: '展开',
|
||||
collapse: '收起',
|
||||
},
|
||||
|
||||
@@ -10,7 +10,7 @@ import { EVAL_TEXT } from '../../lib/termsEval'
|
||||
import { useCatalog } from '../../store/catalog'
|
||||
import type { Run } from '../../types'
|
||||
import type {
|
||||
EvalCaliber, EvalCase, EvalCaseBody, EvalExpectation, EvalGold, EvalPreview, EvalRunQuery, EvalSuite, EvalTrial,
|
||||
EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview, EvalRunQuery, EvalSuite, EvalTrial,
|
||||
} from '../../typesEval'
|
||||
import { graphFields, type InputField } from '../schedules/model'
|
||||
import { cellText, expectationText, goldValueText } from './parts'
|
||||
@@ -69,6 +69,11 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
// 从运行认定
|
||||
const [runId, setRunId] = useState('')
|
||||
const [queryId, setQueryId] = useState('')
|
||||
// 题目种类:问数据题(问法交给助手作答)或工作流题(固定一个已发布版本,按输入跑一次,考的是这一版本身)
|
||||
const [caseKind, setCaseKind] = useState<EvalCaseKind>('ask')
|
||||
const [workflowId, setWorkflowId] = useState('')
|
||||
const [workflowVersion, setWorkflowVersion] = useState<number | null>(null)
|
||||
const [workflowInputs, setWorkflowInputs] = useState<Record<string, string>>({})
|
||||
// 试算、保存
|
||||
const [trial, setTrial] = useState<EvalTrial | null>(null)
|
||||
const [trialing, setTrialing] = useState(false)
|
||||
@@ -101,6 +106,10 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
setInputs(c?.gold?.kind === 'caliber' ? { ...(c.gold.inputs ?? {}) } : {})
|
||||
setRunId(c?.gold?.kind === 'from_run' ? c.gold.run_id ?? '' : '')
|
||||
setQueryId(c?.gold?.kind === 'from_run' ? c.gold.query ?? '' : '')
|
||||
setCaseKind(c?.kind ?? 'ask')
|
||||
setWorkflowId(c?.workflow_id ?? '')
|
||||
setWorkflowVersion(c?.workflow_version ?? null)
|
||||
setWorkflowInputs({ ...(c?.input_template ?? {}) })
|
||||
setTrial(null)
|
||||
setError(null)
|
||||
}, [open, editing?.id, editing?.version, initialQuestion])
|
||||
@@ -144,7 +153,13 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
tags: tags.split(/[,,、\n]/).map((t) => t.trim()).filter(Boolean),
|
||||
}
|
||||
if (!fixedFromRun) out.gold = gold()
|
||||
if (!editing) out.kind = 'ask'
|
||||
const workflow = caseKind === 'workflow' && (mode === 'sql' || mode === 'caliber')
|
||||
if (!editing) out.kind = workflow ? 'workflow' : 'ask'
|
||||
if (workflow) {
|
||||
out.workflow_id = workflowId
|
||||
out.workflow_version = workflowVersion
|
||||
out.input_template = Object.fromEntries(Object.entries(workflowInputs).filter(([, v]) => v.trim() !== ''))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -215,12 +230,16 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
|
||||
const fieldError = (...fields: string[]) => (error && error.field && fields.some((f) => error.field === f || error.field!.startsWith(`${f}.`))
|
||||
? error.message : undefined)
|
||||
const generalError = error && !(error.field && ['question', 'gold.sql', 'gold.datasource_id', 'gold.inputs', 'query', 'tags']
|
||||
const generalError = error && !(error.field && ['question', 'gold.sql', 'gold.datasource_id', 'gold.inputs', 'query', 'tags',
|
||||
'workflow_id', 'workflow_version', 'input_template']
|
||||
.some((f) => error.field === f || error.field!.startsWith(`${f}.`))) ? error.message : null
|
||||
const needsMain = mode !== 'special'
|
||||
/** 工作流题只配「写 SQL 定期望」「选口径」两种:反问是助手的事,工作流不会反问;从运行认定只认问数据运行里的查询 */
|
||||
const workflowCase = caseKind === 'workflow' && (mode === 'sql' || mode === 'caliber')
|
||||
// 关掉之前要不要先问:新建时填了东西、编辑时改了要试算的那几样
|
||||
const dirty = editing ? changed() : !!(question.trim() || sql.trim() || main || points.trim() || runId)
|
||||
const ready = (!needsMain || !!main) && (mode !== 'from_run' || fixedFromRun || (!!runId && !!queryId))
|
||||
&& (!workflowCase || (!!workflowId && !!workflowVersion))
|
||||
&& (mode !== 'caliber' || !!caliber) && (!(mode === 'sql' || (mode === 'special' && special === 'none')) || (!!sourceId && !!sql.trim()))
|
||||
|
||||
return (
|
||||
@@ -256,8 +275,16 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
<p className="mt-1 text-2xs leading-relaxed text-faint">{T.modeHint[mode]}</p>
|
||||
</div>
|
||||
|
||||
<Field label={T.question} htmlFor="eval-case-question" hint={T.questionHint} error={fieldError('question')}
|
||||
required={mode !== 'from_run'}>
|
||||
{(mode === 'sql' || mode === 'caliber') && (
|
||||
<CaseKindPart kind={caseKind} setKind={setCaseKind} disabled={!!editing} workflowId={workflowId}
|
||||
setWorkflowId={(v) => { setWorkflowId(v); setWorkflowInputs({}) }} version={workflowVersion}
|
||||
setVersion={setWorkflowVersion} inputs={workflowInputs} setInputs={setWorkflowInputs}
|
||||
error={fieldError('workflow_id', 'workflow_version', 'input_template')} />
|
||||
)}
|
||||
|
||||
<Field label={workflowCase ? T.workflowQuestion : T.question} htmlFor="eval-case-question"
|
||||
hint={workflowCase ? T.workflowQuestionHint : T.questionHint} error={fieldError('question')}
|
||||
required={mode !== 'from_run' && !workflowCase}>
|
||||
<textarea id="eval-case-question" className="field min-h-16" value={question} maxLength={2000}
|
||||
placeholder={T.questionPlaceholder} onChange={(e) => setQuestion(e.target.value)} data-eval-case-question="" />
|
||||
</Field>
|
||||
@@ -342,6 +369,72 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 题目种类:问数据题 / 工作流题
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 工作流题:固定一个工作流的已发布版本(服务端核对已发布、未退役),按输入模板跑一次探索运行,考的是这一版本身(不建图、
|
||||
* 不复核)。输入模板按那一版入口节点的字段填,可以写时间变量
|
||||
*/
|
||||
function CaseKindPart({ kind, setKind, disabled, workflowId, setWorkflowId, version, setVersion, inputs, setInputs, error }: {
|
||||
kind: EvalCaseKind; setKind: (k: EvalCaseKind) => void; disabled: boolean; workflowId: string; setWorkflowId: (v: string) => void
|
||||
version: number | null; setVersion: (v: number | null) => void; inputs: Record<string, string>
|
||||
setInputs: (v: Record<string, string>) => void; error?: string
|
||||
}) {
|
||||
const workflows = useCatalog((s) => s.workflows)
|
||||
const published = workflows.filter((w) => w.published_version)
|
||||
const [fields, setFields] = useState<InputField[] | null>(null)
|
||||
useEffect(() => {
|
||||
if (kind !== 'workflow' || !workflowId || !version) { setFields(null); return }
|
||||
setFields(null)
|
||||
api.workflows.version(workflowId, version).then((v) => setFields(graphFields(v.graph)), () => setFields([]))
|
||||
}, [kind, workflowId, version])
|
||||
return (
|
||||
<div className="space-y-2" data-eval-case-kind={kind}>
|
||||
<div className="label">{T.caseKind}</div>
|
||||
<div className="flex flex-wrap gap-3 text-xs">
|
||||
{(['ask', 'workflow'] as const).map((k) => (
|
||||
<label key={k} className="flex items-center gap-1.5">
|
||||
<input type="radio" name="eval-case-kind" checked={kind === k} disabled={disabled} onChange={() => setKind(k)}
|
||||
data-eval-case-kind-pick={k} />
|
||||
{EVAL_TEXT.caseRow.kind[k]}
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-2xs leading-relaxed text-faint">{T.caseKindHint[kind]}</p>
|
||||
{kind === 'workflow' && (
|
||||
<div className="grid gap-3 rounded-md border p-2.5 sm:grid-cols-[minmax(0,1fr)_8rem]">
|
||||
<Field label={T.workflow} htmlFor="eval-case-workflow" required>
|
||||
<select id="eval-case-workflow" className="field" value={workflowId} disabled={disabled} data-eval-case-workflow=""
|
||||
onChange={(e) => {
|
||||
const w = published.find((x) => x.id === e.target.value)
|
||||
setWorkflowId(e.target.value)
|
||||
setVersion(w?.published_version ?? null)
|
||||
}}>
|
||||
<option value="">{published.length ? T.workflowPlaceholder : T.workflowNone}</option>
|
||||
{published.map((w) => <option key={w.id} value={w.id}>{w.name}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label={T.workflowVersion} htmlFor="eval-case-workflow-version" required>
|
||||
<input id="eval-case-workflow-version" className="field tnum" inputMode="numeric" value={version ?? ''} disabled={disabled}
|
||||
onChange={(e) => setVersion(e.target.value.trim() ? Number(e.target.value) : null)} />
|
||||
</Field>
|
||||
{fields === null && workflowId && version && <p className="text-2xs text-dim sm:col-span-2"><Spinner size={10} /> {T.caliberInputsLoading}</p>}
|
||||
{fields?.map((f) => (
|
||||
<Field key={f.name} label={f.name} htmlFor={`eval-case-input-${f.name}`} required={!!f.required} hint={f.description}
|
||||
className="sm:col-span-2">
|
||||
<input id={`eval-case-input-${f.name}`} className="field mono text-xs" value={inputs[f.name] ?? ''}
|
||||
onChange={(e) => setInputs({ ...inputs, [f.name]: e.target.value })} data-eval-case-input={f.name} />
|
||||
</Field>
|
||||
))}
|
||||
{error && <p className="text-2xs text-[var(--err)] sm:col-span-2" role="alert">{error}</p>}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 写 SQL 定期望
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -414,6 +414,7 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
|
||||
</p>
|
||||
))}
|
||||
{r.sql_match?.reason && <p className="text-faint">{r.sql_match.reason}</p>}
|
||||
<ResultDetail evaluationId={id} resultId={r.id} />
|
||||
<div className="flex flex-wrap gap-2">
|
||||
{r.run_id && <Link className="btn btn-sm" to={`/runs/${encodeURIComponent(r.run_id)}`} data-eval-result-run="">{T.run.openRun}</Link>}
|
||||
{r.gold_run_ids.slice(0, 2).map((g) => (
|
||||
@@ -432,6 +433,31 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
|
||||
)
|
||||
}
|
||||
|
||||
/** 一题的明细(GET …/results/{id} 的明细工件):这一轮作答的经过(重来了几次、原因)和读出来的答案原文。明细清理了的不显示 */
|
||||
function ResultDetail({ evaluationId, resultId }: { evaluationId: string; resultId: string }) {
|
||||
const [details, setDetails] = useState<Record<string, any> | null | undefined>(undefined)
|
||||
useEffect(() => {
|
||||
api.evals.result(evaluationId, resultId).then((r) => setDetails(r.details ?? null), () => setDetails(null))
|
||||
}, [evaluationId, resultId])
|
||||
if (details === undefined) return <Spinner size={11} />
|
||||
if (!details) return null
|
||||
const turn = details.turn ?? {}
|
||||
const answer = String(details.answer?.text ?? turn.reply ?? '').trim()
|
||||
const R = T.results
|
||||
return (
|
||||
<div className="space-y-1" data-eval-result-details="">
|
||||
{(turn.attempts ?? 0) > 0 && <p className="text-faint">{R.attempts(turn.attempts)}</p>}
|
||||
{turn.reason && <p className="text-dim"><span className="text-faint">{R.turnReason}:</span>{turn.reason}</p>}
|
||||
{answer && (
|
||||
<details>
|
||||
<summary className="cursor-pointer text-2xs text-dim">{R.answer}</summary>
|
||||
<p className="mt-1 max-h-48 overflow-auto whitespace-pre-wrap break-words text-dim">{answer}</p>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 和上一次比
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -11,7 +11,7 @@ import { formatDateTime, formatTime } from '../../lib/format'
|
||||
import { timezoneOptions } from '../../lib/freq'
|
||||
import { leavePass } from '../../lib/leave'
|
||||
import { EVAL_TEXT, pct, rateText } from '../../lib/termsEval'
|
||||
import { useOnReconnect } from '../../store/catalog'
|
||||
import { useCatalog, useOnReconnect } from '../../store/catalog'
|
||||
import type {
|
||||
EvalCase, EvalRun, EvalSnapshot, EvalSuiteDetail, EvalTrial, EvalUpgradePolicy, EvalUpgradeState, EvalVariant,
|
||||
} from '../../typesEval'
|
||||
@@ -111,6 +111,14 @@ export function SuiteDetail({ id }: { id: string }) {
|
||||
toast.ok(T.suite.deleted(suite.name))
|
||||
navigate('/evals', leavePass({ replace: true }))
|
||||
})
|
||||
const cleanup = () => act('cleanup', async () => {
|
||||
if (!suite) return
|
||||
if (!(await confirmDialog({ title: T.suite.cleanupTitle(suite.name), consequences: T.suite.cleanupConsequences,
|
||||
confirmLabel: T.suite.cleanup, danger: true }))) return
|
||||
const r = await api.evals.cleanup(suite.id) as { runs?: number; files?: number }
|
||||
toast.ok(T.suite.cleaned(Number(r.runs ?? 0), Number(r.files ?? 0)))
|
||||
await load()
|
||||
})
|
||||
const toggleCase = (c: EvalCase, enabled: boolean) => act(`toggle:${c.id}`, async () => {
|
||||
if (!suite) return
|
||||
await api.evals.updateCase(suite.id, c.id, { enabled, version: c.version })
|
||||
@@ -210,7 +218,14 @@ export function SuiteDetail({ id }: { id: string }) {
|
||||
)}
|
||||
</div>
|
||||
<div role="tabpanel" id="eval-suite-panel-runs" aria-labelledby="eval-suite-tab-runs" hidden={tab !== 'runs'}
|
||||
className="mx-auto max-w-6xl p-4">
|
||||
className="mx-auto max-w-6xl space-y-3 p-4">
|
||||
<div className="flex flex-wrap items-center justify-end gap-2">
|
||||
<span className="text-2xs text-faint">{T.suite.cleanupHint}</span>
|
||||
<button type="button" className="btn btn-sm" onClick={() => void cleanup()} disabled={!!busy || !runs?.length}
|
||||
data-eval-suite-cleanup="">
|
||||
{busy === 'cleanup' && <Spinner size={11} />} {T.suite.cleanup}
|
||||
</button>
|
||||
</div>
|
||||
<RunList runs={runs ?? []} />
|
||||
</div>
|
||||
</div>
|
||||
@@ -242,6 +257,8 @@ function CaseRow({ c, busy, trial, archivedSuite, focused, onEdit, onToggle, onT
|
||||
}) {
|
||||
const last = c.trial
|
||||
const pending = !!c.upgrade?.pending
|
||||
const workflows = useCatalog((s) => s.workflows)
|
||||
const workflowName = (id: string | null) => workflows.find((w) => w.id === id)?.name ?? id ?? ''
|
||||
return (
|
||||
<li className={clsx('rounded-lg border bg-panel px-3 py-2.5', (!c.enabled || c.archived) && 'opacity-75',
|
||||
focused && 'border-[var(--accent)]')} data-eval-case={c.id} data-eval-case-focused={focused ? '' : undefined}
|
||||
@@ -257,6 +274,9 @@ function CaseRow({ c, busy, trial, archivedSuite, focused, onEdit, onToggle, onT
|
||||
</div>
|
||||
<div className="mt-1 break-words text-sm" data-eval-case-text="">{c.question || '—'}</div>
|
||||
<div className="mt-1 text-2xs text-dim">
|
||||
{c.kind === 'workflow' && (
|
||||
<span data-eval-case-workflow={c.workflow_id ?? ''}>{T.caseRow.workflow(workflowName(c.workflow_id), c.workflow_version)} · </span>
|
||||
)}
|
||||
<span title={T.caseRow.goldSource}>{goldText(c)}</span>
|
||||
{c.datasources.length > 0 && <span className="text-faint"> · {T.caseRow.datasources}:{c.datasources.map((d) => d.name ?? d.id).join('、')}</span>}
|
||||
</div>
|
||||
|
||||
@@ -231,6 +231,9 @@ const EVAL_RUN = { id: 'run-eval-ans', workflow_id: null, workflow_name: '评测
|
||||
origin_kind: 'eval', origin: { suite_id: 'suite-a', eval_run_id: 'ev-1', case_id: 'case-2', role: 'answer', repeat: 0 }, schedule_id: null,
|
||||
created_at: iso(NOW - 90 * 60e3), started_at: iso(NOW - 90 * 60e3), finished_at: iso(NOW - 89 * 60e3) }
|
||||
const RUN_GRAPH = { nodes: [{ id: 'out', type: 'output', position: { x: 0, y: 0 }, data: { label: '成果', config: {} } }], edges: [] }
|
||||
/** 已发布的工作流(工作流题要固定一个已发布版本) */
|
||||
const WF_WEEK = { id: 'wf-week', name: '周销售', description: '', graph: VERSION_WEEK.graph, tags: [], version: 4, is_template: false,
|
||||
status: 'published', published_version: 3, published_level: 'governed', created_at: iso(NOW - 30 * 24 * H), updated_at: iso(NOW - 24 * H) }
|
||||
const BUILDER = { mode: 'local', username: 'jiang', display_name: '检查员甲', role: 'builder', groups: [], verified: true }
|
||||
const USER = { mode: 'local', username: 'li', display_name: '使用者乙', role: 'user', groups: [], verified: true }
|
||||
|
||||
@@ -370,6 +373,14 @@ function reply(state, method, path, url, body) {
|
||||
return { json: ev }
|
||||
}
|
||||
}
|
||||
m = path.match(/^\/evals\/evaluations\/([^/]+)\/results\/([^/]+)$/)
|
||||
if (method === 'GET' && m) {
|
||||
const r = RESULTS.find((x) => x.id === m[2])
|
||||
return r ? { json: { ...r, details: { turn: { attempts: 1, reason: null, reply: null }, answer: { text: `答案原文:上周销售额 ${r.scores[0]?.got ?? ''} 元` } } } }
|
||||
: { status: 404, json: { detail: '这一题的结果不存在' } }
|
||||
}
|
||||
if (method === 'POST' && path === '/evals/suites/suite-a/cleanup') { write(); return { json: { runs: 2, results: 6, deleted_runs: 8, files: 3 } } }
|
||||
if (method === 'GET' && path === '/workflows') return { json: [WF_WEEK] }
|
||||
if (method === 'GET' && path === '/evals/pairs/ev-pa') return { json: PAIR }
|
||||
if (method === 'GET' && path === '/evals/calibers') return { json: { calibers: CALIBERS } }
|
||||
if (method === 'GET' && path === '/workflows/wf-week/versions/3') return { json: VERSION_WEEK }
|
||||
@@ -565,6 +576,10 @@ await section('题集详情:题目表、口径升版处置、启用开关、
|
||||
await page.locator('[data-eval-runs]').count()
|
||||
await page.getByRole('tab', { name: '评测记录' }).click()
|
||||
check('评测记录:历次评测链到详情', await page.locator('[data-eval-run-row="ev-1"]').getAttribute('href') === '/evals/runs/ev-1')
|
||||
await page.locator('[data-eval-suite-cleanup]').click()
|
||||
await page.locator('[role="dialog"]').getByRole('button', { name: '清理更早的明细' }).click()
|
||||
check('清理更早的明细:先确认、再发一次,提示清了几次', await until(() => writes(state, /cleanup$/).length === 1)
|
||||
&& await until(async () => (await page.locator('[data-toast]').allInnerTexts()).some((x) => x.includes('已清理 2 次评测的明细'))))
|
||||
check('没有页面报错', !errors.length, errors.join(' | '))
|
||||
})
|
||||
|
||||
@@ -652,6 +667,20 @@ await section('加题:选口径、从运行认定、期望反问', async () =>
|
||||
const ask = lastWrite(state, /^POST \/evals\/suites\/suite-b\/cases$/)?.body
|
||||
check('期望反问:期望只有「反问」,不带期望答案的来源', JSON.stringify(ask?.expectations) === JSON.stringify([{ kind: 'ask' }]) && ask?.gold === null,
|
||||
JSON.stringify(ask))
|
||||
await page.locator('[data-eval-add-case]').click()
|
||||
await page.locator('[data-eval-case-kind-pick="workflow"]').check()
|
||||
await page.locator('[data-eval-case-workflow]').selectOption('wf-week')
|
||||
await page.locator('[data-eval-case-input="week"]').fill('{{ schedule.last_week_start }}')
|
||||
await page.locator('[data-eval-case-source]').selectOption('ds-shop')
|
||||
await page.locator('[data-eval-case-sql]').fill('SELECT region AS 地区, SUM(amount) AS gmv FROM orders GROUP BY region')
|
||||
await page.locator('[data-eval-case-preview]').click()
|
||||
await page.locator('[data-eval-preview-cell="0:gmv"]').click()
|
||||
await page.locator('[data-eval-case-save]').click()
|
||||
await until(() => writes(state, /^POST \/evals\/suites\/suite-b\/cases$/).length === 3)
|
||||
const wf = lastWrite(state, /^POST \/evals\/suites\/suite-b\/cases$/)?.body
|
||||
check('工作流题:固定已发布的 v3、按入口字段填输入模板,期望照样由 SQL 算', wf?.kind === 'workflow' && wf.workflow_id === 'wf-week'
|
||||
&& wf.workflow_version === 3 && wf.input_template?.week === '{{ schedule.last_week_start }}' && wf.gold?.kind === 'sql',
|
||||
JSON.stringify(wf))
|
||||
check('没有页面报错', !errors.length, errors.join(' | '))
|
||||
})
|
||||
|
||||
@@ -744,6 +773,9 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => {
|
||||
check('逐题:结论、期望和答案里的数、SQL 差在哪', t2.includes('不通过') && t2.includes('330') && t2.includes('310') && t2.includes('时间口径'), t2)
|
||||
check('一题的花费照服务端加好的写', (await page.locator('[data-eval-result="r-1"]').innerText()).includes('$0.014'))
|
||||
check('点开去运行详情', await page.locator('[data-eval-result-detail="r-2"] [data-eval-result-run]').getAttribute('href') === '/runs/run-answer-2')
|
||||
await page.locator('[data-eval-result-details]').waitFor()
|
||||
check('展开的明细写作答的经过和答案原文(明细工件)', (await text(page, '[data-eval-result-details]')).includes('自动重来了 1 次')
|
||||
&& (await page.locator('[data-eval-result-details] details').count()) === 1)
|
||||
|
||||
await page.getByRole('tab', { name: '和上一次比' }).click()
|
||||
await page.locator('[data-eval-transitions]').waitFor()
|
||||
|
||||
Reference in New Issue
Block a user