diff --git a/frontend/src/lib/termsEval.ts b/frontend/src/lib/termsEval.ts
index cfc9f652..04e268cd 100644
--- a/frontend/src/lib/termsEval.ts
+++ b/frontend/src/lib/termsEval.ts
@@ -91,6 +91,12 @@ export const EVAL_TEXT = {
upgradeHint: '每道题三选一,没有缺省:改为固定新版本(历史在这里断开)、新旧两版各算一份期望、或旧题归档另建新题',
runsEmpty: '还没有评测记录',
runsLabel: '历次评测',
+ cleanup: '清理更早的明细',
+ cleanupHint: '评测收尾时会按保留规则自动清理,这里可以手动再清一次',
+ cleanupTitle: (name: string) => `清理「${name}」更早的评测明细?`,
+ cleanupConsequences: ['按设置里的保留次数清理更早的评测:逐题结果、作答和算期望的运行一起删除,不可恢复',
+ '设为基线的那次、A/B 的另几臂留着;每次评测的总表永久保留'],
+ cleaned: (runs: number, files: number) => (runs ? `已清理 ${runs} 次评测的明细,回收了 ${files} 个工件文件` : '没有要清理的评测'),
schedules: '定时评测',
noSchedules: '这个题集还没有定时任务',
budget: '预算',
@@ -112,6 +118,7 @@ export const EVAL_TEXT = {
caseRow: {
seq: (n: number) => `第 ${n} 题`,
kind: { ask: '问数据题', workflow: '工作流题' } as Record<'ask' | 'workflow', string>,
+ workflow: (name: string, v: number | null) => `工作流 ${name} v${v ?? '?'}`,
disabled: '未启用',
archived: '已归档',
supersededBy: (n: string) => `已被新题 ${n} 取代`,
@@ -172,6 +179,17 @@ export const EVAL_TEXT = {
special: '问法问的是没有的东西(期望答「没有数据」),或者问法含糊(期望助手反问)',
} as Record<'sql' | 'caliber' | 'from_run' | 'special', string>,
question: '问法',
+ workflowQuestion: '这道题考什么(可选)',
+ workflowQuestionHint: '工作流题不需要问法:写一句说明,列表里认得出是哪道题',
+ caseKind: '题目种类',
+ caseKindHint: {
+ ask: '问法交给助手:建图、运行、复核,和问数据页同一条路',
+ workflow: '固定一个工作流的已发布版本,按输入运行一次(不建图、不复核),考的是这一版本身',
+ } as Record<'ask' | 'workflow', string>,
+ workflow: '工作流',
+ workflowPlaceholder: '选择已发布的工作流',
+ workflowNone: '还没有已发布的工作流',
+ workflowVersion: '固定的版本',
questionPlaceholder: '例如:上周按支付日统计、不含退款的销售额是多少?',
questionHint: '可以写 {{ schedule.last_week_start }} 这类时间变量,按评测的截至时间计算',
scope: '限定的数据源',
@@ -473,6 +491,9 @@ export const EVAL_TEXT = {
filterAll: '全部',
filter: '只看',
repeat: (n: number) => `第 ${n + 1} 次`,
+ attempts: (n: number) => `这一题自动重来了 ${n} 次`,
+ turnReason: '作答的经过',
+ answer: '答案原文',
expand: '展开',
collapse: '收起',
},
diff --git a/frontend/src/pages/evals/CaseEditor.tsx b/frontend/src/pages/evals/CaseEditor.tsx
index e89fac44..12a27a9c 100644
--- a/frontend/src/pages/evals/CaseEditor.tsx
+++ b/frontend/src/pages/evals/CaseEditor.tsx
@@ -10,7 +10,7 @@ import { EVAL_TEXT } from '../../lib/termsEval'
import { useCatalog } from '../../store/catalog'
import type { Run } from '../../types'
import type {
- EvalCaliber, EvalCase, EvalCaseBody, EvalExpectation, EvalGold, EvalPreview, EvalRunQuery, EvalSuite, EvalTrial,
+ EvalCaliber, EvalCase, EvalCaseBody, EvalCaseKind, EvalExpectation, EvalGold, EvalPreview, EvalRunQuery, EvalSuite, EvalTrial,
} from '../../typesEval'
import { graphFields, type InputField } from '../schedules/model'
import { cellText, expectationText, goldValueText } from './parts'
@@ -69,6 +69,11 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
// 从运行认定
const [runId, setRunId] = useState('')
const [queryId, setQueryId] = useState('')
+ // 题目种类:问数据题(问法交给助手作答)或工作流题(固定一个已发布版本,按输入跑一次,考的是这一版本身)
+ const [caseKind, setCaseKind] = useState('ask')
+ const [workflowId, setWorkflowId] = useState('')
+ const [workflowVersion, setWorkflowVersion] = useState(null)
+ const [workflowInputs, setWorkflowInputs] = useState>({})
// 试算、保存
const [trial, setTrial] = useState(null)
const [trialing, setTrialing] = useState(false)
@@ -101,6 +106,10 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
setInputs(c?.gold?.kind === 'caliber' ? { ...(c.gold.inputs ?? {}) } : {})
setRunId(c?.gold?.kind === 'from_run' ? c.gold.run_id ?? '' : '')
setQueryId(c?.gold?.kind === 'from_run' ? c.gold.query ?? '' : '')
+ setCaseKind(c?.kind ?? 'ask')
+ setWorkflowId(c?.workflow_id ?? '')
+ setWorkflowVersion(c?.workflow_version ?? null)
+ setWorkflowInputs({ ...(c?.input_template ?? {}) })
setTrial(null)
setError(null)
}, [open, editing?.id, editing?.version, initialQuestion])
@@ -144,7 +153,13 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
tags: tags.split(/[,,、\n]/).map((t) => t.trim()).filter(Boolean),
}
if (!fixedFromRun) out.gold = gold()
- if (!editing) out.kind = 'ask'
+ const workflow = caseKind === 'workflow' && (mode === 'sql' || mode === 'caliber')
+ if (!editing) out.kind = workflow ? 'workflow' : 'ask'
+ if (workflow) {
+ out.workflow_id = workflowId
+ out.workflow_version = workflowVersion
+ out.input_template = Object.fromEntries(Object.entries(workflowInputs).filter(([, v]) => v.trim() !== ''))
+ }
return out
}
@@ -215,12 +230,16 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
const fieldError = (...fields: string[]) => (error && error.field && fields.some((f) => error.field === f || error.field!.startsWith(`${f}.`))
? error.message : undefined)
- const generalError = error && !(error.field && ['question', 'gold.sql', 'gold.datasource_id', 'gold.inputs', 'query', 'tags']
+ const generalError = error && !(error.field && ['question', 'gold.sql', 'gold.datasource_id', 'gold.inputs', 'query', 'tags',
+ 'workflow_id', 'workflow_version', 'input_template']
.some((f) => error.field === f || error.field!.startsWith(`${f}.`))) ? error.message : null
const needsMain = mode !== 'special'
+ /** 工作流题只配「写 SQL 定期望」「选口径」两种:反问是助手的事,工作流不会反问;从运行认定只认问数据运行里的查询 */
+ const workflowCase = caseKind === 'workflow' && (mode === 'sql' || mode === 'caliber')
// 关掉之前要不要先问:新建时填了东西、编辑时改了要试算的那几样
const dirty = editing ? changed() : !!(question.trim() || sql.trim() || main || points.trim() || runId)
const ready = (!needsMain || !!main) && (mode !== 'from_run' || fixedFromRun || (!!runId && !!queryId))
+ && (!workflowCase || (!!workflowId && !!workflowVersion))
&& (mode !== 'caliber' || !!caliber) && (!(mode === 'sql' || (mode === 'special' && special === 'none')) || (!!sourceId && !!sql.trim()))
return (
@@ -256,8 +275,16 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
{T.modeHint[mode]}
-
+ {(mode === 'sql' || mode === 'caliber') && (
+ { setWorkflowId(v); setWorkflowInputs({}) }} version={workflowVersion}
+ setVersion={setWorkflowVersion} inputs={workflowInputs} setInputs={setWorkflowInputs}
+ error={fieldError('workflow_id', 'workflow_version', 'input_template')} />
+ )}
+
+
@@ -342,6 +369,72 @@ export function CaseEditor({ open, onClose, suite, editing, initialQuestion, onS
)
}
+// ---------------------------------------------------------------------------
+// 题目种类:问数据题 / 工作流题
+// ---------------------------------------------------------------------------
+
+/**
+ * 工作流题:固定一个工作流的已发布版本(服务端核对已发布、未退役),按输入模板跑一次探索运行,考的是这一版本身(不建图、
+ * 不复核)。输入模板按那一版入口节点的字段填,可以写时间变量
+ */
+function CaseKindPart({ kind, setKind, disabled, workflowId, setWorkflowId, version, setVersion, inputs, setInputs, error }: {
+ kind: EvalCaseKind; setKind: (k: EvalCaseKind) => void; disabled: boolean; workflowId: string; setWorkflowId: (v: string) => void
+ version: number | null; setVersion: (v: number | null) => void; inputs: Record
+ setInputs: (v: Record) => void; error?: string
+}) {
+ const workflows = useCatalog((s) => s.workflows)
+ const published = workflows.filter((w) => w.published_version)
+ const [fields, setFields] = useState(null)
+ useEffect(() => {
+ if (kind !== 'workflow' || !workflowId || !version) { setFields(null); return }
+ setFields(null)
+ api.workflows.version(workflowId, version).then((v) => setFields(graphFields(v.graph)), () => setFields([]))
+ }, [kind, workflowId, version])
+ return (
+
+
{T.caseKind}
+
+ {(['ask', 'workflow'] as const).map((k) => (
+
+ ))}
+
+
{T.caseKindHint[kind]}
+ {kind === 'workflow' && (
+
+
+
+
+
+ setVersion(e.target.value.trim() ? Number(e.target.value) : null)} />
+
+ {fields === null && workflowId && version &&
{T.caliberInputsLoading}
}
+ {fields?.map((f) => (
+
+ setInputs({ ...inputs, [f.name]: e.target.value })} data-eval-case-input={f.name} />
+
+ ))}
+ {error &&
{error}
}
+
+ )}
+
+ )
+}
+
// ---------------------------------------------------------------------------
// 写 SQL 定期望
// ---------------------------------------------------------------------------
diff --git a/frontend/src/pages/evals/EvaluationPage.tsx b/frontend/src/pages/evals/EvaluationPage.tsx
index c1784b63..11d473df 100644
--- a/frontend/src/pages/evals/EvaluationPage.tsx
+++ b/frontend/src/pages/evals/EvaluationPage.tsx
@@ -414,6 +414,7 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
))}
{r.sql_match?.reason && {r.sql_match.reason}
}
+
{r.run_id &&
{T.run.openRun}}
{r.gold_run_ids.slice(0, 2).map((g) => (
@@ -432,6 +433,31 @@ function Results({ id, live, focusCase }: { id: string; live: boolean; focusCase
)
}
+/** 一题的明细(GET …/results/{id} 的明细工件):这一轮作答的经过(重来了几次、原因)和读出来的答案原文。明细清理了的不显示 */
+function ResultDetail({ evaluationId, resultId }: { evaluationId: string; resultId: string }) {
+ const [details, setDetails] = useState
| null | undefined>(undefined)
+ useEffect(() => {
+ api.evals.result(evaluationId, resultId).then((r) => setDetails(r.details ?? null), () => setDetails(null))
+ }, [evaluationId, resultId])
+ if (details === undefined) return
+ if (!details) return null
+ const turn = details.turn ?? {}
+ const answer = String(details.answer?.text ?? turn.reply ?? '').trim()
+ const R = T.results
+ return (
+
+ {(turn.attempts ?? 0) > 0 &&
{R.attempts(turn.attempts)}
}
+ {turn.reason &&
{R.turnReason}:{turn.reason}
}
+ {answer && (
+
+ {R.answer}
+ {answer}
+
+ )}
+
+ )
+}
+
// ---------------------------------------------------------------------------
// 和上一次比
// ---------------------------------------------------------------------------
diff --git a/frontend/src/pages/evals/SuiteDetail.tsx b/frontend/src/pages/evals/SuiteDetail.tsx
index 7faeda59..36d5e250 100644
--- a/frontend/src/pages/evals/SuiteDetail.tsx
+++ b/frontend/src/pages/evals/SuiteDetail.tsx
@@ -11,7 +11,7 @@ import { formatDateTime, formatTime } from '../../lib/format'
import { timezoneOptions } from '../../lib/freq'
import { leavePass } from '../../lib/leave'
import { EVAL_TEXT, pct, rateText } from '../../lib/termsEval'
-import { useOnReconnect } from '../../store/catalog'
+import { useCatalog, useOnReconnect } from '../../store/catalog'
import type {
EvalCase, EvalRun, EvalSnapshot, EvalSuiteDetail, EvalTrial, EvalUpgradePolicy, EvalUpgradeState, EvalVariant,
} from '../../typesEval'
@@ -111,6 +111,14 @@ export function SuiteDetail({ id }: { id: string }) {
toast.ok(T.suite.deleted(suite.name))
navigate('/evals', leavePass({ replace: true }))
})
+ const cleanup = () => act('cleanup', async () => {
+ if (!suite) return
+ if (!(await confirmDialog({ title: T.suite.cleanupTitle(suite.name), consequences: T.suite.cleanupConsequences,
+ confirmLabel: T.suite.cleanup, danger: true }))) return
+ const r = await api.evals.cleanup(suite.id) as { runs?: number; files?: number }
+ toast.ok(T.suite.cleaned(Number(r.runs ?? 0), Number(r.files ?? 0)))
+ await load()
+ })
const toggleCase = (c: EvalCase, enabled: boolean) => act(`toggle:${c.id}`, async () => {
if (!suite) return
await api.evals.updateCase(suite.id, c.id, { enabled, version: c.version })
@@ -210,7 +218,14 @@ export function SuiteDetail({ id }: { id: string }) {
)}
+ className="mx-auto max-w-6xl space-y-3 p-4">
+
+ {T.suite.cleanupHint}
+
+
@@ -242,6 +257,8 @@ function CaseRow({ c, busy, trial, archivedSuite, focused, onEdit, onToggle, onT
}) {
const last = c.trial
const pending = !!c.upgrade?.pending
+ const workflows = useCatalog((s) => s.workflows)
+ const workflowName = (id: string | null) => workflows.find((w) => w.id === id)?.name ?? id ?? ''
return (
{c.question || '—'}
+ {c.kind === 'workflow' && (
+ {T.caseRow.workflow(workflowName(c.workflow_id), c.workflow_version)} ·
+ )}
{goldText(c)}
{c.datasources.length > 0 && · {T.caseRow.datasources}:{c.datasources.map((d) => d.name ?? d.id).join('、')}}
diff --git a/scripts/check-evals.mjs b/scripts/check-evals.mjs
index d65f5594..50e238e7 100644
--- a/scripts/check-evals.mjs
+++ b/scripts/check-evals.mjs
@@ -231,6 +231,9 @@ const EVAL_RUN = { id: 'run-eval-ans', workflow_id: null, workflow_name: '评测
origin_kind: 'eval', origin: { suite_id: 'suite-a', eval_run_id: 'ev-1', case_id: 'case-2', role: 'answer', repeat: 0 }, schedule_id: null,
created_at: iso(NOW - 90 * 60e3), started_at: iso(NOW - 90 * 60e3), finished_at: iso(NOW - 89 * 60e3) }
const RUN_GRAPH = { nodes: [{ id: 'out', type: 'output', position: { x: 0, y: 0 }, data: { label: '成果', config: {} } }], edges: [] }
+/** 已发布的工作流(工作流题要固定一个已发布版本) */
+const WF_WEEK = { id: 'wf-week', name: '周销售', description: '', graph: VERSION_WEEK.graph, tags: [], version: 4, is_template: false,
+ status: 'published', published_version: 3, published_level: 'governed', created_at: iso(NOW - 30 * 24 * H), updated_at: iso(NOW - 24 * H) }
const BUILDER = { mode: 'local', username: 'jiang', display_name: '检查员甲', role: 'builder', groups: [], verified: true }
const USER = { mode: 'local', username: 'li', display_name: '使用者乙', role: 'user', groups: [], verified: true }
@@ -370,6 +373,14 @@ function reply(state, method, path, url, body) {
return { json: ev }
}
}
+ m = path.match(/^\/evals\/evaluations\/([^/]+)\/results\/([^/]+)$/)
+ if (method === 'GET' && m) {
+ const r = RESULTS.find((x) => x.id === m[2])
+ return r ? { json: { ...r, details: { turn: { attempts: 1, reason: null, reply: null }, answer: { text: `答案原文:上周销售额 ${r.scores[0]?.got ?? ''} 元` } } } }
+ : { status: 404, json: { detail: '这一题的结果不存在' } }
+ }
+ if (method === 'POST' && path === '/evals/suites/suite-a/cleanup') { write(); return { json: { runs: 2, results: 6, deleted_runs: 8, files: 3 } } }
+ if (method === 'GET' && path === '/workflows') return { json: [WF_WEEK] }
if (method === 'GET' && path === '/evals/pairs/ev-pa') return { json: PAIR }
if (method === 'GET' && path === '/evals/calibers') return { json: { calibers: CALIBERS } }
if (method === 'GET' && path === '/workflows/wf-week/versions/3') return { json: VERSION_WEEK }
@@ -565,6 +576,10 @@ await section('题集详情:题目表、口径升版处置、启用开关、
await page.locator('[data-eval-runs]').count()
await page.getByRole('tab', { name: '评测记录' }).click()
check('评测记录:历次评测链到详情', await page.locator('[data-eval-run-row="ev-1"]').getAttribute('href') === '/evals/runs/ev-1')
+ await page.locator('[data-eval-suite-cleanup]').click()
+ await page.locator('[role="dialog"]').getByRole('button', { name: '清理更早的明细' }).click()
+ check('清理更早的明细:先确认、再发一次,提示清了几次', await until(() => writes(state, /cleanup$/).length === 1)
+ && await until(async () => (await page.locator('[data-toast]').allInnerTexts()).some((x) => x.includes('已清理 2 次评测的明细'))))
check('没有页面报错', !errors.length, errors.join(' | '))
})
@@ -652,6 +667,20 @@ await section('加题:选口径、从运行认定、期望反问', async () =>
const ask = lastWrite(state, /^POST \/evals\/suites\/suite-b\/cases$/)?.body
check('期望反问:期望只有「反问」,不带期望答案的来源', JSON.stringify(ask?.expectations) === JSON.stringify([{ kind: 'ask' }]) && ask?.gold === null,
JSON.stringify(ask))
+ await page.locator('[data-eval-add-case]').click()
+ await page.locator('[data-eval-case-kind-pick="workflow"]').check()
+ await page.locator('[data-eval-case-workflow]').selectOption('wf-week')
+ await page.locator('[data-eval-case-input="week"]').fill('{{ schedule.last_week_start }}')
+ await page.locator('[data-eval-case-source]').selectOption('ds-shop')
+ await page.locator('[data-eval-case-sql]').fill('SELECT region AS 地区, SUM(amount) AS gmv FROM orders GROUP BY region')
+ await page.locator('[data-eval-case-preview]').click()
+ await page.locator('[data-eval-preview-cell="0:gmv"]').click()
+ await page.locator('[data-eval-case-save]').click()
+ await until(() => writes(state, /^POST \/evals\/suites\/suite-b\/cases$/).length === 3)
+ const wf = lastWrite(state, /^POST \/evals\/suites\/suite-b\/cases$/)?.body
+ check('工作流题:固定已发布的 v3、按入口字段填输入模板,期望照样由 SQL 算', wf?.kind === 'workflow' && wf.workflow_id === 'wf-week'
+ && wf.workflow_version === 3 && wf.input_template?.week === '{{ schedule.last_week_start }}' && wf.gold?.kind === 'sql',
+ JSON.stringify(wf))
check('没有页面报错', !errors.length, errors.join(' | '))
})
@@ -744,6 +773,9 @@ await section('逐题结果、和上一次比、A/B 配对表', async () => {
check('逐题:结论、期望和答案里的数、SQL 差在哪', t2.includes('不通过') && t2.includes('330') && t2.includes('310') && t2.includes('时间口径'), t2)
check('一题的花费照服务端加好的写', (await page.locator('[data-eval-result="r-1"]').innerText()).includes('$0.014'))
check('点开去运行详情', await page.locator('[data-eval-result-detail="r-2"] [data-eval-result-run]').getAttribute('href') === '/runs/run-answer-2')
+ await page.locator('[data-eval-result-details]').waitFor()
+ check('展开的明细写作答的经过和答案原文(明细工件)', (await text(page, '[data-eval-result-details]')).includes('自动重来了 1 次')
+ && (await page.locator('[data-eval-result-details] details').count()) === 1)
await page.getByRole('tab', { name: '和上一次比' }).click()
await page.locator('[data-eval-transitions]').waitFor()