mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 15:31:59 +00:00
工具审批三档、智能体护栏取代固定 12 步、检查脚本分道并行
MCP / 自定义工具的审批三档(tools/trust.py) - 平台看不出这两类工具做什么,以前审批关卡一律当它们安全、照跑不误。现在默认每次调用都等审批; 工具页可以逐个改成「始终允许 · 门控把关」或「始终允许」,审批卡上的「始终允许」等于前者 - 门控把关:每次调用先由门控模型审一遍工具说明、参数和节点任务;判可疑、出错、超时、答不成格式, 一律交还给人,不会变成放行。判定包进 checkpoint,审批恢复重放时不再问一遍 - agent 节点、工具节点、协作团队三处关卡都接上;团队成员停不下来等人,门控没放行的调用不执行 - 每次运行在发起时快照档位(runs.tool_trust),恢复和接着跑沿用同一份;正式运行不看档位,一律等审批; 升级前发起的运行照旧不问——它们可能正停在某个审批上,多问一次答复就会对错号 - 接口:GET /api/tools 带 trust / trust_key,PUT /api/tools/trust;设置新增门控模型;新事件 tool.gated - 界面:工具页三选一、审批卡「始终允许」、设置页门控模型、运行记录解码 tool.gated; 顺手修了 MCP 工具详情 404、工具审批被驳回却显示成放行 智能体护栏(engine/guards.py),取代固定的「最多 12 步」 - 步数只留兜底:默认 100,设置里可改;以前画布自动写进节点的 12 按「没填」处理 - 同一个工具、同样的参数第二次不再执行,把上次的结果交还给它;连续 3 步拿不到新信息就收尾 - 每个节点的令牌预算(默认 200 万)和金额预算(默认不限),都可以在设置里改成不限 - 上下文用到窗口的 60% 先压缩早期的工具结果,到 85% 就收尾;临近任何一个上限先提醒模型一次 - 下游结构化校验失败时写明是上游哪个 agent 提前收了尾;复核信号带上收尾原因, 问数据页只在真是步数用满时才给「放宽步数重跑」 - 上限在发起时快照(runs.agent_limits);升级前发起的运行一字不改走旧逻辑 检查脚本 - check-all 默认 4 道并行:量时间最多的两项互不重叠;并行时没过的最后单独重跑一遍,重跑才过的 单列成「偶发」。16 项全量从约 21 分钟降到约 7 分钟(加上一次重跑约 10 分钟);--lanes 1 回到串行 顺带修的 - 设置接口里的 limits 改为只读:库里存着旧版设置页写进去的一份(max_agent_steps: 25),从不生效, 却盖住了真实上限,问数据页照着错的数许诺 - test_kb 里嵌入模型那条测试不再真连 api.openai.com(代理慢时一条要等近 4 分钟) - check-stream 读滚动容器前先等它挂上,并行跑时不再偶发 验证(沙箱):tsc 干净,生产构建通过,pytest 1655 passed(两遍),check-all 16 项检查、3562 项全过; 新增后端测试 53 条,关键逻辑做了 22 处变异验证、全部抓到。
This commit is contained in:
+100
-18
@@ -1,15 +1,21 @@
|
||||
// 按顺序跑全部检查,汇总成一张表,给一个总的退出码。
|
||||
// 分道并行跑全部检查,汇总成一张表,给一个总的退出码。
|
||||
//
|
||||
// node scripts/check-all.mjs 全部跑一遍
|
||||
// node scripts/check-all.mjs ui runs 只跑 check-ui、check-runs(名字去掉 check- 前缀)
|
||||
// node scripts/check-all.mjs --repeat 2 整套连跑两遍,列出只在某一遍失败的项(偶发)
|
||||
// node scripts/check-all.mjs --lanes 1 一个接一个地跑(以前的跑法);也可以写 CHECK_LANES=1
|
||||
//
|
||||
// 默认连 5273 / 8000。对别的实例(比如一份沙箱拷贝)跑时带上地址,会原样传给每个检查:
|
||||
// AGENTLAB_WEB=http://localhost:<前端端口> AGENTLAB_API=http://localhost:<后端端口>/api node scripts/check-all.mjs
|
||||
// 每个检查的完整输出写进 CHECK_LOGS(默认 /tmp/agentlab-checks/<时间>/),终端只给汇总。
|
||||
//
|
||||
// 一个接一个地跑,不并行:它们共用同一个 vite 开发服务器,有几项还量时间(审批卡多少毫秒
|
||||
// 出现、计时器走没走),挤在一起跑只会多出偶发失败。
|
||||
// 默认 4 道并行。各检查对后端只读(几处 POST 都是纯计算的排版接口),截图各有各的目录,
|
||||
// 时间大多花在 waitForTimeout 的固定等待上,所以并行几乎是线性提速。代价是共用同一个
|
||||
// vite 开发服务器,有几项还量时间(审批卡多少毫秒出现、计时器走没走)。两道防线:
|
||||
// - 量时间最多的几项(EXCLUSIVE)互相不重叠,同一时刻最多跑其中一项;
|
||||
// - 并行时没过的,全部跑完后再单独、一个接一个地重跑一遍。重跑通过的算通过,但单列成
|
||||
// 「偶发」:它在挤的时候会挂,迟早也会在别处挂。连跑几遍(--repeat)时不重跑,要的就是
|
||||
// 原始的偶发率。
|
||||
//
|
||||
// 各检查自己的段过滤(CHECK_ONLY、ONLY、THEMES……)不往下传:整套检查就是整套,只跑某几段
|
||||
// 请直接跑那个脚本。
|
||||
@@ -43,6 +49,14 @@ const ORDER = [
|
||||
'manage', // 数据、工具、知识、设置
|
||||
'ui', // 各页面端到端,连真数据
|
||||
]
|
||||
// 最近一次全量里各项的耗时(秒)。只决定排队顺序:长的先开跑,短的填空,结果不受影响。
|
||||
// 新加的检查不在表里就按 60 秒算
|
||||
const ESTIMATE = {
|
||||
'run-states': 220, 'canvas-fx': 181, stream: 162, studio: 125, chat: 124, manage: 115, runs: 91,
|
||||
shell: 84, 'ui-kit': 69, ui: 61, evidence: 21, guards: 10, 'canvas-layout': 4, tokens: 1, trace: 1, decode: 1,
|
||||
}
|
||||
// 量时间最多、以前并行最容易挂的:同一时刻最多跑其中一项
|
||||
const EXCLUSIVE = new Set(['run-states', 'canvas-fx'])
|
||||
const TIMEOUT_MS = Number(process.env.CHECK_TIMEOUT_MS ?? 15 * 60_000)
|
||||
// 各检查只跑其中几段、只跑一套主题的开关。壳里留着一个 ONLY=… 就会让整套检查悄悄只跑一小截,
|
||||
// 却照样报「全部通过」,所以一律不传给子进程
|
||||
@@ -55,10 +69,13 @@ const unlisted = readdirSync(HERE)
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
let repeat = 1
|
||||
let lanes = Math.max(1, Number(process.env.CHECK_LANES) || 4)
|
||||
const picked = []
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--repeat') repeat = Math.max(1, Number(args[++i]) || 1)
|
||||
else if (args[i].startsWith('--repeat=')) repeat = Math.max(1, Number(args[i].slice(9)) || 1)
|
||||
else if (args[i] === '--lanes') lanes = Math.max(1, Number(args[++i]) || 1)
|
||||
else if (args[i].startsWith('--lanes=')) lanes = Math.max(1, Number(args[i].slice(8)) || 1)
|
||||
else picked.push(args[i].replace(/^check-/, '').replace(/\.mjs$/, ''))
|
||||
}
|
||||
const unknown = picked.filter((n) => !ORDER.includes(n))
|
||||
@@ -96,7 +113,8 @@ mkdirSync(LOGS, { recursive: true })
|
||||
console.log(`前端 ${WEB} · 后端 ${API}`)
|
||||
const dropped = FILTERS.filter((k) => process.env[k])
|
||||
if (dropped.length) console.log(`没传给各检查的段过滤:${dropped.map((k) => `${k}=${process.env[k]}`).join(' ')}(只跑某几段请直接跑那个脚本)`)
|
||||
console.log(`${names.length} 项检查${repeat > 1 ? ` × ${repeat} 遍` : ''},完整输出在 ${LOGS}/\n`)
|
||||
lanes = Math.min(lanes, names.length)
|
||||
console.log(`${names.length} 项检查${repeat > 1 ? ` × ${repeat} 遍` : ''},${lanes > 1 ? `${lanes} 道并行` : '一个接一个'},完整输出在 ${LOGS}/\n`)
|
||||
|
||||
const childEnv = { ...process.env, AGENTLAB_WEB: WEB, AGENTLAB_API: API }
|
||||
for (const k of FILTERS) delete childEnv[k]
|
||||
@@ -105,8 +123,8 @@ for (const k of FILTERS) delete childEnv[k]
|
||||
* 跑一个检查,输出落盘,数 ✓ / ✗。退出码非 0 却一个 ✗ 都没有的,是中途崩了;
|
||||
* 退出码 0 却一个 ✓ 都没有的,是什么都没查——都不算通过
|
||||
*/
|
||||
function runOne(name, pass) {
|
||||
const file = join(LOGS, repeat > 1 ? `${name}.${pass}.log` : `${name}.log`)
|
||||
function runOne(name, pass, suffix = '') {
|
||||
const file = join(LOGS, `${name}${repeat > 1 ? `.${pass}` : ''}${suffix}.log`)
|
||||
const out = createWriteStream(file)
|
||||
const started = Date.now()
|
||||
return new Promise((resolve) => {
|
||||
@@ -150,21 +168,83 @@ const verdict = (r) => r.timedOut ? `超时(${limit})`
|
||||
: r.empty ? '一项都没跑'
|
||||
: r.passed ? '通过'
|
||||
: `${r.bad.length} 项没过`
|
||||
let spent = 0 // 各项累计的秒数,重跑的那次也算
|
||||
const report = (r) => {
|
||||
spent += r.secs
|
||||
const mark = r.passed ? '✓' : '✗'
|
||||
console.log(`${mark} ${`check-${r.name}`.padEnd(20)} ${verdict(r).padEnd(10)} ${String(r.ok).padStart(4)} 项通过 · ${r.secs}s`)
|
||||
for (const l of r.bad.slice(0, 5)) console.log(` ${l}`)
|
||||
if (r.bad.length > 5) console.log(` ……另有 ${r.bad.length - 5} 项,见 ${r.file}`)
|
||||
if (r.tail) console.log(` ${r.tail}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* 一遍:lanes 道同时从队里取。长的排前面;EXCLUSIVE 里的一项在跑时,另一项先让给队里
|
||||
* 后面的。结果按跑完的先后打印,返回时按 ORDER 排好
|
||||
*/
|
||||
async function runPass(pass) {
|
||||
const queue = [...names].sort((a, b) => (ESTIMATE[b] ?? 60) - (ESTIMATE[a] ?? 60))
|
||||
const done = []
|
||||
let exclusiveBusy = false
|
||||
let waiting = [] // 等 EXCLUSIVE 那一项跑完的几道
|
||||
const take = () => {
|
||||
const i = queue.findIndex((n) => !(EXCLUSIVE.has(n) && exclusiveBusy))
|
||||
return i < 0 ? null : queue.splice(i, 1)[0]
|
||||
}
|
||||
const lane = async () => {
|
||||
while (queue.length) {
|
||||
const name = take()
|
||||
if (!name) {
|
||||
// 队里只剩 EXCLUSIVE 的,而另一项正在跑:等它跑完再取
|
||||
await new Promise((resolve) => waiting.push(resolve))
|
||||
continue
|
||||
}
|
||||
const exclusive = EXCLUSIVE.has(name)
|
||||
if (exclusive) exclusiveBusy = true
|
||||
const r = await runOne(name, pass)
|
||||
if (exclusive) {
|
||||
exclusiveBusy = false
|
||||
const woken = waiting
|
||||
waiting = []
|
||||
for (const resolve of woken) resolve()
|
||||
}
|
||||
done.push(r)
|
||||
report(r)
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: lanes }, lane))
|
||||
return names.map((n) => done.find((r) => r.name === n))
|
||||
}
|
||||
|
||||
const results = []
|
||||
const flakyRerun = [] // 并行时没过、单独重跑通过的
|
||||
const started = Date.now()
|
||||
for (let pass = 1; pass <= repeat; pass++) {
|
||||
if (repeat > 1) console.log(`—— 第 ${pass} 遍 ——`)
|
||||
for (const name of names) {
|
||||
const r = await runOne(name, pass)
|
||||
results.push(r)
|
||||
const mark = r.passed ? '✓' : '✗'
|
||||
console.log(`${mark} ${`check-${name}`.padEnd(20)} ${verdict(r).padEnd(10)} ${String(r.ok).padStart(4)} 项通过 · ${r.secs}s`)
|
||||
for (const l of r.bad.slice(0, 5)) console.log(` ${l}`)
|
||||
if (r.bad.length > 5) console.log(` ……另有 ${r.bad.length - 5} 项,见 ${r.file}`)
|
||||
if (r.tail) console.log(` ${r.tail}`)
|
||||
const got = await runPass(pass)
|
||||
if (lanes > 1 && repeat === 1 && got.some((r) => !r.passed)) {
|
||||
const failed = got.filter((r) => !r.passed)
|
||||
console.log(`\n并行时没过的 ${failed.length} 项,单独重跑一遍:`)
|
||||
for (const first of failed) {
|
||||
const again = await runOne(first.name, pass, '.solo')
|
||||
report(again)
|
||||
if (again.passed) flakyRerun.push({ first, again })
|
||||
got[got.indexOf(first)] = again
|
||||
}
|
||||
}
|
||||
results.push(...got)
|
||||
if (repeat > 1) console.log('')
|
||||
}
|
||||
|
||||
if (flakyRerun.length) {
|
||||
console.log('\n偶发(并行时没过、单独重跑通过):')
|
||||
for (const { first } of flakyRerun) {
|
||||
const why = first.bad.length ? first.bad.slice(0, 3).join(';') : verdict(first)
|
||||
console.log(` check-${first.name}:${why}(见 ${first.file})`)
|
||||
}
|
||||
}
|
||||
console.log('')
|
||||
|
||||
// 连跑几遍时:同一项只在部分遍数里失败,就是偶发——比稳定失败更该查。按项名认,不带
|
||||
// 「 — 」后面的读数:每遍都没过、只是毫秒数或像素差不一样的,是稳定失败,不是偶发
|
||||
if (repeat > 1) {
|
||||
@@ -193,11 +273,13 @@ if (unlisted.length && !picked.length) {
|
||||
|
||||
const failedRuns = results.filter((r) => !r.passed)
|
||||
const total = results.reduce((n, r) => n + r.ok, 0)
|
||||
const secs = results.reduce((n, r) => n + r.secs, 0)
|
||||
const clock = `${Math.floor(secs / 60)} 分 ${String(secs % 60).padStart(2, '0')} 秒`
|
||||
const minutes = (secs) => `${Math.floor(secs / 60)} 分 ${String(secs % 60).padStart(2, '0')} 秒`
|
||||
const wall = Math.round((Date.now() - started) / 1000)
|
||||
const clock = lanes > 1 ? `用时 ${minutes(wall)}(各项累计 ${minutes(spent)})` : `共 ${minutes(wall)}`
|
||||
const flakyNote = flakyRerun.length ? `,其中 ${flakyRerun.length} 项是重跑才过的,见上面「偶发」` : ''
|
||||
if (failedRuns.length) {
|
||||
console.log(`✗ ${failedRuns.length}/${results.length} 次检查没通过(${[...new Set(failedRuns.map((r) => `check-${r.name}`))].join('、')})· 共 ${clock}`)
|
||||
console.log(`✗ ${failedRuns.length}/${results.length} 次检查没通过(${[...new Set(failedRuns.map((r) => `check-${r.name}`))].join('、')})· ${clock}`)
|
||||
} else {
|
||||
console.log(`✓ 全部通过:${results.length} 次检查、${total} 项 · 共 ${clock}`)
|
||||
console.log(`✓ 全部通过:${results.length} 次检查、${total} 项${flakyNote} · ${clock}`)
|
||||
}
|
||||
process.exit(failedRuns.length ? 1 : 0)
|
||||
|
||||
+52
-3
@@ -149,7 +149,26 @@ const TRASH_MORE = {
|
||||
trash4: conv('c0trash4', '删掉的对话丁:还在跑', 1, 60 * 24 * 5,
|
||||
{ archived: true, last_status: 'running', last_run_id: 'run-trash-live' }),
|
||||
}
|
||||
/** 只在「护栏」那段出现:放宽步数只在步数用满、而且还有得放宽时给(后端 engine/guards.py) */
|
||||
const GRAPH8 = { ...GRAPH, nodes: GRAPH.nodes.map((n) => (n.type === 'agent'
|
||||
? { ...n, data: { ...n.data, config: { max_steps: 8 } } } : n)) }
|
||||
const guardTurn = (id, graph, reason, detail) => turn(id, '查一下各门店的销量', {
|
||||
answer: '只查了一部分的结论', graph, run_id: `run-${id}`,
|
||||
review: {
|
||||
verdict: 'annotated', note: detail, answer: null, retry: true, severity: 'broken',
|
||||
signals: [{ kind: 'step_limit', detail, severity: 'broken', ...(reason ? { reason } : {}) }],
|
||||
},
|
||||
meta: { v: 1, runId: `run-${id}`, runClass: 'exploratory', runStatus: 'succeeded' },
|
||||
})
|
||||
const GUARD_CONVS = {
|
||||
stall: conv('c0gstall', '护栏:连续几步没进展收的尾', 1, 60 * 24 * 30),
|
||||
steps: conv('c0gsteps', '护栏:写死 8 步、用满了', 1, 60 * 24 * 31),
|
||||
dflt: conv('c0gdflt', '护栏:跟随默认步数', 1, 60 * 24 * 32),
|
||||
}
|
||||
const DETAIL = {
|
||||
c0gstall: [guardTurn('tg1', GRAPH8, 'stall', 'agent 连续 3 步没有拿到新信息,收尾轮也没有给出结论。')],
|
||||
c0gsteps: [guardTurn('tg2', GRAPH8, 'steps', 'agent 用满了 8 步。收尾轮也没有给出结论。')],
|
||||
c0gdflt: [guardTurn('tg3', GRAPH, null, 'agent 用满了 12 步还没给出结论。')],
|
||||
c0busya: [],
|
||||
c0idleb: [turn('tb1', 'B 的问题', { answer: 'B 的答案', graph: GRAPH, run_id: 'run-b1' })],
|
||||
c0longc: Array.from({ length: 6 }, (_, i) =>
|
||||
@@ -316,6 +335,8 @@ const ctl = {
|
||||
few: false, run500: true, reviewDelay: {}, launchFail: true, scope400: false,
|
||||
/** 回收站里多摆两个(TRASH_MORE) */
|
||||
trashMore: false,
|
||||
/** 列表里多摆「护栏」那段的三个会话(GUARD_CONVS) */
|
||||
guardConvs: false,
|
||||
/** 数据源列表取不回来 */
|
||||
sourcesFail: false,
|
||||
/** 数据源列表里多一个别处刚加的库 */
|
||||
@@ -328,7 +349,8 @@ const resetDb = () => {
|
||||
db.purged = new Set()
|
||||
}
|
||||
resetDb()
|
||||
const allConvs = () => [...Object.values(CONVS), ...Object.values(TRASHED), ...(ctl.trashMore ? Object.values(TRASH_MORE) : [])]
|
||||
const allConvs = () => [...Object.values(CONVS), ...Object.values(TRASHED), ...(ctl.trashMore ? Object.values(TRASH_MORE) : []),
|
||||
...(ctl.guardConvs ? Object.values(GUARD_CONVS) : [])]
|
||||
.filter((c) => !db.purged.has(c.id))
|
||||
.map((c) => ({ ...c, archived: db.archived.has(c.id) }))
|
||||
let runSeq = 0
|
||||
@@ -621,7 +643,7 @@ async function fakeStream(ws) {
|
||||
|
||||
const browser = await chromium.launch({ executablePath: CHROME })
|
||||
|
||||
async function open(theme = 'light', { width = 1440, height = 900, reducedMotion = 'no-preference' } = {}) {
|
||||
async function open(theme = 'light', { width = 1440, height = 900, reducedMotion = 'no-preference', settings = null } = {}) {
|
||||
const ctx = await browser.newContext({ viewport: { width, height }, colorScheme: theme, reducedMotion })
|
||||
opened.add(ctx)
|
||||
await ctx.addInitScript((t) => localStorage.setItem('agentlab.theme', t), theme)
|
||||
@@ -632,7 +654,8 @@ async function open(theme = 'light', { width = 1440, height = 900, reducedMotion
|
||||
await page.route('**/api/settings', async (route) => {
|
||||
if (route.request().method() !== 'GET') return route.abort()
|
||||
// 主题以设置为准(沙箱里存的是 light),这里按要看的那套改掉
|
||||
return route.fulfill({ json: { ui: { theme }, run: {}, limits: { max_agent_steps: 25 }, copilot: {}, embedding: {} } })
|
||||
return route.fulfill({ json: { ui: { theme }, run: {}, limits: { max_agent_steps: 25 }, copilot: {}, embedding: {},
|
||||
...(settings ?? {}) } })
|
||||
})
|
||||
await page.route(/\/api\/(conversations|copilot\/(generate-stream|review)|runs|approvals|datasources)(\/|\?|$)/, fakeApi)
|
||||
await page.routeWebSocket(/\/api\/runs\/[^/]+\/stream/, fakeStream)
|
||||
@@ -1573,6 +1596,32 @@ for (const theme of THEMES) {
|
||||
await ctx.close()
|
||||
})
|
||||
|
||||
await section('guard', '放宽步数只在对症时给:停滞、预算收的尾不给,跟随默认 100 步的也不给', async () => {
|
||||
resetDb()
|
||||
ctl.guardConvs = true
|
||||
try {
|
||||
const stepBtn = (page) => page.getByRole('button', { name: /放宽步数重跑/ })
|
||||
// 新后端:默认 100 步、硬上限 100
|
||||
const fresh = { run: { agent_max_steps: 100 }, limits: { max_agent_steps: 100 } }
|
||||
const { page, ctx, errors } = await open(theme, { settings: fresh })
|
||||
await goto(page, 'c0gstall')
|
||||
await shows(page, '只查了一部分的结论')
|
||||
check('连续几步没进展收的尾:不给「放宽步数」(加步数只会原样再撞一次)', await stepBtn(page).count() === 0)
|
||||
await goto(page, 'c0gsteps')
|
||||
await shows(page, '只查了一部分的结论')
|
||||
const eight = stepBtn(page)
|
||||
check('节点写死 8 步、步数用满:给「放宽步数」,从 8 放到 16', await eight.count() === 1
|
||||
&& /8 → 16 步/.test(await eight.innerText()), await eight.innerText().catch(() => '(没有按钮)'))
|
||||
await goto(page, 'c0gdflt')
|
||||
await shows(page, '只查了一部分的结论')
|
||||
check('节点上的 12 跟随默认 100 步、已经到硬上限:不许一个放不宽的数', await stepBtn(page).count() === 0)
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await ctx.close()
|
||||
} finally {
|
||||
ctl.guardConvs = false
|
||||
}
|
||||
})
|
||||
|
||||
await section('rerun', '重跑服务重启挂起的那一轮:旧运行顺手取消', async () => {
|
||||
const { page, ctx, errors } = await open(theme)
|
||||
await goto(page, 'c0suspo')
|
||||
|
||||
@@ -1125,6 +1125,113 @@ await section('报告核对 report.checked、口径卡的台账(可点击证
|
||||
check('载荷缺字段也不崩,只说核对了报告', bare?.title === '核对报告' && !bare.artifact, bare?.title)
|
||||
})
|
||||
|
||||
await section('工具门控 tool.gated(工具信任三档)', async () => {
|
||||
// 契约:data = {tool, verdict: allow|escalate, reason, model, duration_ms, call_id?, agent?},node_id 是所在节点
|
||||
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
|
||||
const gated = (steps) => flatten(steps).filter((s) => s.code === 'tool_gated')
|
||||
const nodeOf = (steps, id) => steps.find((s) => s.nodeId === id && s.kind === 'node')
|
||||
|
||||
// 放行:info 级的一行,挂在 agent 节点下,接着是正常的工具调用
|
||||
const allowRun = mod.decodeRun([
|
||||
E(1, 'node.started', 'agent', { node_type: 'agent', label: '查档案' }),
|
||||
E(2, 'tool.gated', 'agent', { tool: 'crm_lookup', verdict: 'allow', reason: '只读查询,参数正常', model: 'tiny-1', duration_ms: 820, call_id: 'c1' }),
|
||||
E(3, 'tool.start', 'agent', { tool: 'crm_lookup', args: { id: 'C-1' }, call_id: 'c1' }),
|
||||
E(4, 'tool.end', 'agent', { tool: 'crm_lookup', call_id: 'c1', preview: '{"name":"示例客户"}', duration_ms: 40 }),
|
||||
])
|
||||
const allow = gated(allowRun)[0]
|
||||
check('放行:说「门控放行 <工具>:<理由>」', allow?.title === '门控放行 crm_lookup:只读查询,参数正常', allow?.title)
|
||||
check('……info 级,不是提醒', allow?.level === 'info', allow?.level)
|
||||
check('……挂在所在节点下面,排在工具调用前', nodeOf(allowRun, 'agent')?.children?.[0] === allow
|
||||
&& nodeOf(allowRun, 'agent')?.children?.[1]?.kind === 'tool', nodeOf(allowRun, 'agent')?.children?.map((c) => c.kind).join(','))
|
||||
check('……行尾写门控用的模型和耗时', allow?.meta === '门控 tiny-1 · 820 ms', allow?.meta)
|
||||
// 问门控时那次模型调用已经答完了:不能在门控那几秒里还转着「思考并作答」
|
||||
const thinking = flatten(mod.decodeRun([E(1, 'node.started', 'agent', { node_type: 'agent' }), E(2, 'llm.start', 'agent', { model: 'm' }),
|
||||
E(3, 'tool.gated', 'agent', { tool: 'crm_lookup', verdict: 'allow', reason: '只读' })])).find((s) => s.kind === 'llm')
|
||||
check('……门控一来,前面那次模型调用收成完成', thinking?.status === 'done', thinking?.status)
|
||||
|
||||
// 拦下(agent 节点):warn 级,接着是 human.requested,交给人批
|
||||
const escEvents = [
|
||||
E(1, 'node.started', 'agent', { node_type: 'agent', label: '改档案' }),
|
||||
E(2, 'tool.gated', 'agent', { tool: 'mcp:demo/search', verdict: 'escalate', reason: '参数里带了删除条件', model: 'tiny-1', duration_ms: 910 }),
|
||||
E(3, 'human.requested', 'agent', { mode: 'approve', tool: 'mcp:demo/search', args: {}, title: 'Agent 想调用工具 mcp:demo/search', trust_key: 'mcp:demo/search' }),
|
||||
E(4, 'run.interrupted', 'agent', { payload: { kind: 'tool_approval', node_id: 'agent', tool: 'mcp:demo/search', trust_key: 'mcp:demo/search' } }),
|
||||
]
|
||||
const escRun = mod.decodeRun(escEvents)
|
||||
const esc = gated(escRun)[0]
|
||||
check('拦下:说「门控拦下 <工具>,交给人工审批:<理由>」', esc?.title === '门控拦下 mcp:demo/search,交给人工审批:参数里带了删除条件', esc?.title)
|
||||
check('……warn 级', esc?.level === 'warn', esc?.level)
|
||||
check('……挂在所在节点下面,后面跟着等审批的那一行', nodeOf(escRun, 'agent')?.children?.includes(esc)
|
||||
&& flatten(escRun).some((s) => s.kind === 'human' && s.status === 'waiting'))
|
||||
|
||||
// 协作团队里拦下:成员停不下来,紧接着一条 tool_needs_approval 的 log,调用不执行。
|
||||
// 两个成员并行,两条门控都拦下之后 log 才到:按工具名各认各的(按先后取最近的一条会配错)
|
||||
const teamRun = mod.decodeRun([
|
||||
E(1, 'node.started', 'team', { node_type: 'supervisor', label: '调研团队' }),
|
||||
E(2, 'tool.start', 'team', { tool: 'crm_lookup', args: {}, agent: '研究员', call_id: 't1' }),
|
||||
E(3, 'tool.start', 'team', { tool: 'weather', args: {}, agent: '分析员', call_id: 't2' }),
|
||||
E(4, 'tool.gated', 'team', { tool: 'crm_lookup', verdict: 'escalate', reason: '要批量导出', model: 'tiny-1', duration_ms: 700, agent: '研究员', call_id: 't1' }),
|
||||
E(5, 'tool.gated', 'team', { tool: 'weather', verdict: 'escalate', reason: '门控模型没答上来:超时', model: 'tiny-1', duration_ms: 20000, agent: '分析员', call_id: 't2' }),
|
||||
E(6, 'log', 'team', { level: 'warn', code: 'tool_needs_approval', message: '研究员 想调用 crm_lookup,需要人工确认,协作节点里不执行' }),
|
||||
E(7, 'log', 'team', { level: 'warn', code: 'tool_needs_approval', message: '分析员 想调用 weather,需要人工确认,协作节点里不执行' }),
|
||||
E(8, 'tool.end', 'team', { tool: 'crm_lookup', agent: '研究员', call_id: 't1', preview: '没有执行:crm_lookup 这次调用需要人工确认', duration_ms: 710 }),
|
||||
E(9, 'tool.end', 'team', { tool: 'weather', agent: '分析员', call_id: 't2', preview: '没有执行:weather 这次调用需要人工确认', duration_ms: 20010 }),
|
||||
])
|
||||
const teamRows = gated(teamRun)
|
||||
const crm = teamRows.find((s) => s.title.includes('crm_lookup'))
|
||||
const wx = teamRows.find((s) => s.title.includes('weather'))
|
||||
check('团队里拦下:改说「门控拦下 <工具>,协作团队里不执行:<理由>」', crm?.title === '门控拦下 crm_lookup,协作团队里不执行:要批量导出', crm?.title)
|
||||
check('……并行的两条按工具名各认各的:标题和折进来的原话都对得上', wx?.title === '门控拦下 weather,协作团队里不执行:门控模型没答上来:超时'
|
||||
&& !!crm?.detail?.includes('研究员 想调用 crm_lookup') && !!wx.detail?.includes('分析员 想调用 weather'), `${wx?.title} / ${crm?.detail} / ${wx?.detail}`)
|
||||
check('……副标题写是哪个成员调的', crm?.sub === '研究员 调用' && wx?.sub === '分析员 调用', `${crm?.sub} / ${wx?.sub}`)
|
||||
check('……那条 log 折进门控那一行,不另起一行', !flatten(teamRun).some((s) => s.code === 'tool_needs_approval')
|
||||
&& !!crm?.detail?.includes('协作节点里不执行'), flatten(teamRun).filter((s) => s.kind === 'note').map((s) => s.title).join(' | '))
|
||||
check('……挂在团队节点下面', !!nodeOf(teamRun, 'team')?.children?.includes(crm))
|
||||
// 没有门控的团队审批(ask 档)照旧:log 自己成一行
|
||||
const plain = flatten(mod.decodeRun([E(1, 'node.started', 'team', { node_type: 'supervisor' }),
|
||||
E(2, 'log', 'team', { level: 'warn', code: 'tool_needs_approval', message: '研究员 想调用 crm_lookup,需要人工确认,协作节点里不执行' })]))
|
||||
check('……没过门控的 tool_needs_approval 照旧自己成一行', plain.some((s) => s.code === 'tool_needs_approval'))
|
||||
// 拦下后交给了人(agent 节点),之后同一节点再来的 log 不改写它
|
||||
const handed = mod.decodeRun([E(1, 'node.started', 'a', { node_type: 'agent' }),
|
||||
E(2, 'tool.gated', 'a', { tool: 'crm_lookup', verdict: 'escalate', reason: '可疑' }),
|
||||
E(3, 'human.requested', 'a', { mode: 'approve', tool: 'crm_lookup', title: '要调用 crm_lookup' }),
|
||||
E(4, 'log', 'a', { level: 'warn', code: 'tool_needs_approval', message: 'x 想调用 crm_lookup' })])
|
||||
check('……已经交给人工的,不会被后来的 log 改口', gated(handed)[0]?.title === '门控拦下 crm_lookup,交给人工审批:可疑', gated(handed)[0]?.title)
|
||||
|
||||
// 审批卡上点了「始终允许」:后端的 human.resolved 带 always(工具审批是平铺的 {tool, approved, note, always})
|
||||
const resolved = (data) => flatten(mod.decodeRun([...escEvents, E(5, 'run.resumed', null, { response: data, actor: '张工' }),
|
||||
E(6, 'human.resolved', 'agent', data)]))
|
||||
.find((s) => s.kind === 'human')
|
||||
const always = resolved({ tool: 'mcp:demo/search', approved: true, note: '', actor: '张工', always: true })
|
||||
check('始终允许:审批那一行写「放行了,并设为「始终允许 · 门控把关」」', always?.title === 'Agent 想调用工具 mcp:demo/search → 张工 放行了,并设为「始终允许 · 门控把关」',
|
||||
always?.title)
|
||||
const once = resolved({ tool: 'mcp:demo/search', approved: true, note: '', actor: '张工' })
|
||||
check('……只是通过的不带这句', once?.title === 'Agent 想调用工具 mcp:demo/search → 张工 放行了', once?.title)
|
||||
const denied = resolved({ tool: 'mcp:demo/search', approved: false, note: '不许导出', actor: '张工' })
|
||||
check('……工具审批被驳回写「驳回了」(以前只读 response,平铺的 approved 读不到,一律写放行)',
|
||||
denied?.title === 'Agent 想调用工具 mcp:demo/search → 张工 驳回了' && !!denied.detail?.includes('备注:不许导出'), `${denied?.title} / ${denied?.detail}`)
|
||||
|
||||
// 缺字段:不崩,不出 undefined,不当成没翻译的记录
|
||||
let bare = []
|
||||
let thrown = null
|
||||
try {
|
||||
bare = flatten(mod.decodeRun([
|
||||
E(1, 'tool.gated', null, {}),
|
||||
E(2, 'node.started', 'x', { node_type: 'agent' }),
|
||||
E(3, 'tool.gated', 'x', { verdict: 'allow' }),
|
||||
E(4, 'tool.gated', 'x', { tool: 'crm_lookup', verdict: 'escalate', reason: null, duration_ms: 'slow' }),
|
||||
E(5, 'log', 'x', { level: 'warn', code: 'tool_needs_approval' }),
|
||||
]))
|
||||
} catch (e) { thrown = e }
|
||||
const rows = bare.filter((s) => s.code === 'tool_gated')
|
||||
check('缺字段:不抛异常,每条都成一行', !thrown && rows.length === 3, thrown ? String(thrown.message) : `${rows.length} 行`)
|
||||
check('……没有工具名说「一次工具调用」,没有理由就不带冒号', rows[0]?.title === '门控拦下一次工具调用,交给人工审批'
|
||||
&& rows[1]?.title === '门控放行一次工具调用', rows.map((r) => r.title).join(' | '))
|
||||
check('……理由、耗时不是字符串 / 数字时不拼出 undefined / null', !rows.some((r) => /undefined|null|NaN/.test(`${r.title}${r.meta ?? ''}${r.detail ?? ''}`)),
|
||||
rows.map((r) => `${r.title} ${r.meta ?? ''}`).join(' | '))
|
||||
check('……没有消息的 tool_needs_approval 也折进去', rows[2]?.title === '门控拦下 crm_lookup,协作团队里不执行', rows[2]?.title)
|
||||
check('……不是「一条还没翻译的记录」', !bare.some((s) => s.title === '一条还没翻译的记录'))
|
||||
})
|
||||
|
||||
await section('术语', async () => {
|
||||
// 没起名的节点退到类型名,类型名跟全站同一张表
|
||||
const h = mod.decodeRun([{ seq: 1, type: 'node.started', node_id: 'h', ts: 1, data: { node_type: 'human', label: 'h' } }])
|
||||
|
||||
@@ -493,6 +493,122 @@ await section('设置 · 偏好', async () => {
|
||||
await off.close()
|
||||
})
|
||||
|
||||
await section('设置 · 门控模型', async () => {
|
||||
// 设置读写都走假的一份;接入列表也换成假的两家(日志里只出现这些通用名)
|
||||
let stored = await get('/settings')
|
||||
stored = { ...stored, run: { ...(stored.run ?? {}), tool_gate_provider: null, tool_gate_model: null } }
|
||||
const scope0 = stored.run.default_memory_scope ?? 'default'
|
||||
const PROVIDERS = [
|
||||
{ id: 'check-gate-p1', name: 'demo-main', kind: 'openai', base_url: 'https://example.com/v1', default_model: 'big-model',
|
||||
models: [{ id: 'big-model' }], enabled: true, extra: {}, api_key_masked: '', has_key: true },
|
||||
{ id: 'check-gate-p2', name: 'demo-small', kind: 'openai', base_url: 'https://example.com/v1', default_model: 'tiny-1',
|
||||
models: [{ id: 'tiny-1' }, { id: 'tiny-2' }], enabled: true, extra: {}, api_key_masked: '', has_key: true },
|
||||
]
|
||||
const { page, sent, errors, close } = await open('/settings/prefs', {
|
||||
handlers: [
|
||||
[/^GET \/settings$/, (route) => json(stored)(route)],
|
||||
[/^PUT \/settings$/, (route, { body }) => { stored = { ...stored, ...body.values }; return json(stored)(route) }],
|
||||
[/^GET \/providers$/, json(PROVIDERS)],
|
||||
],
|
||||
})
|
||||
const box = page.locator('[data-gate-model]')
|
||||
const hint = await box.innerText().catch(() => '')
|
||||
check('运行默认值里有「门控模型」,提示留空用默认、建议选小模型', hint.includes('门控模型') && hint.includes('留空') && hint.includes('小模型'),
|
||||
hint.replace(/\s+/g, ' ').slice(0, 100))
|
||||
const provider = page.getByLabel('门控模型 · 接入')
|
||||
const model = page.getByLabel('门控模型 · 模型')
|
||||
check('……留空时写明默认是哪家(第一个启用的接入)', (await provider.locator('option').first().innerText()).includes('demo-main'))
|
||||
await provider.selectOption('demo-small')
|
||||
await model.fill('tiny-2')
|
||||
const bar = await page.locator('[data-prefs-bar]').innerText().catch(() => '')
|
||||
check('……改了两项:保存条写「门控模型的接入、门控模型」', bar.includes('有 2 项未保存') && bar.includes('门控模型的接入') && bar.includes('门控模型'),
|
||||
bar.replace(/\s+/g, ' '))
|
||||
await page.getByRole('button', { name: /保存设置/ }).click()
|
||||
await page.waitForTimeout(500)
|
||||
const put = sent.filter((s) => s.key === 'PUT /settings').at(-1)
|
||||
const run = put?.body?.values?.run ?? {}
|
||||
// run 组里别的键是沙箱的真实配置:只报门控这两项
|
||||
check('保存只 PUT run 一组,写 tool_gate_provider / tool_gate_model', Object.keys(put?.body?.values ?? {}).join() === 'run'
|
||||
&& run.tool_gate_provider === 'demo-small' && run.tool_gate_model === 'tiny-2',
|
||||
`provider=${run.tool_gate_provider ?? '—'} · model=${run.tool_gate_model ?? '—'}`)
|
||||
check('……其余运行默认值原样带着', run.default_memory_scope === scope0 && 'confirm_dangerous_tools' in run)
|
||||
await shot(page, 'prefs-gate-model')
|
||||
|
||||
// 换接入:原来填的模型不在新接入里就清掉;都清空保存成 null(= 用默认接入的默认模型)
|
||||
await provider.selectOption('demo-main')
|
||||
check('换了接入,旧接入的模型清掉', await model.inputValue() === '', await model.inputValue())
|
||||
await provider.selectOption('')
|
||||
await page.getByRole('button', { name: /保存设置/ }).click()
|
||||
await page.waitForTimeout(500)
|
||||
const run2 = sent.filter((s) => s.key === 'PUT /settings').at(-1)?.body?.values?.run ?? {}
|
||||
check('……两项都留空:存成 null', run2.tool_gate_provider === null && run2.tool_gate_model === null,
|
||||
`provider=${JSON.stringify(run2.tool_gate_provider)} · model=${JSON.stringify(run2.tool_gate_model)}`)
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await close()
|
||||
|
||||
// 老后端:run 组里没有这两项,读进来就是空,不算改动
|
||||
let legacy = await get('/settings')
|
||||
legacy = { ...legacy, run: Object.fromEntries(Object.entries(legacy.run ?? {}).filter(([k]) => !k.startsWith('tool_gate_'))) }
|
||||
const old = await open('/settings/prefs', { handlers: [[/^GET \/settings$/, (route) => json(legacy)(route)], [/^GET \/providers$/, json(PROVIDERS)]] })
|
||||
check('老后端没有这两项:显示空、没有保存条', await old.page.getByLabel('门控模型 · 模型').inputValue() === ''
|
||||
&& await old.page.locator('[data-prefs-bar]').count() === 0)
|
||||
await old.close()
|
||||
})
|
||||
|
||||
await section('设置 · 智能体护栏', async () => {
|
||||
// 默认步数、令牌预算、金额预算(后端 engine/guards.py)。预算留空 = 不限,存 null
|
||||
let stored = await get('/settings')
|
||||
stored = { ...stored, run: { ...(stored.run ?? {}), agent_max_steps: 100, agent_budget_tokens: 2000000, agent_budget_usd: null },
|
||||
limits: { ...(stored.limits ?? {}), max_agent_steps: 100 } }
|
||||
const { page, sent, errors, close } = await open('/settings/prefs', {
|
||||
handlers: [
|
||||
[/^GET \/settings$/, (route) => json(stored)(route)],
|
||||
[/^PUT \/settings$/, (route, { body }) => { stored = { ...stored, ...body.values }; return json(stored)(route) }],
|
||||
],
|
||||
})
|
||||
const box = page.locator('[data-agent-guard]')
|
||||
const text = (await box.innerText().catch(() => '')).replace(/\s+/g, ' ')
|
||||
check('运行默认值里有「智能体护栏」,说清不再被固定步数掐断', text.includes('智能体护栏') && text.includes('按查到的部分收尾'),
|
||||
text.slice(0, 100))
|
||||
const steps = page.getByLabel('默认最大步数')
|
||||
const tokens = page.getByLabel('令牌预算(每个节点)')
|
||||
const usd = page.getByLabel('金额预算(美元)')
|
||||
check('……读回设置里的值:100 步、200 万令牌、金额不限', await steps.inputValue() === '100'
|
||||
&& await tokens.inputValue() === '2000000' && await usd.inputValue() === '')
|
||||
check('……金额留空写明「不限」', text.includes('不限:费用只受步数兜底和上下文约束'))
|
||||
|
||||
await steps.fill('500')
|
||||
const save = page.getByRole('button', { name: /保存设置/ })
|
||||
check('步数超过硬上限:标红、保存按钮不可用', await save.isDisabled()
|
||||
&& (await box.innerText()).includes('填 1 到 100 的整数'))
|
||||
await steps.fill('60')
|
||||
await tokens.fill('')
|
||||
await usd.fill('1.5')
|
||||
const bar = (await page.locator('[data-prefs-bar]').innerText().catch(() => '')).replace(/\s+/g, ' ')
|
||||
check('……改了三项:保存条写明是哪三项', bar.includes('有 3 项未保存') && bar.includes('默认最大步数')
|
||||
&& bar.includes('令牌预算') && bar.includes('金额预算'), bar)
|
||||
await save.click()
|
||||
await page.waitForTimeout(500)
|
||||
const run = sent.filter((s) => s.key === 'PUT /settings').at(-1)?.body?.values?.run ?? {}
|
||||
check('保存:步数写数字,令牌留空存 null(不限),金额写数字', run.agent_max_steps === 60
|
||||
&& run.agent_budget_tokens === null && run.agent_budget_usd === 1.5,
|
||||
`steps=${JSON.stringify(run.agent_max_steps)} tokens=${JSON.stringify(run.agent_budget_tokens)} usd=${JSON.stringify(run.agent_budget_usd)}`)
|
||||
await tokens.fill('500')
|
||||
check('令牌预算少于 1000:标红、不能保存', await save.isDisabled())
|
||||
await shot(page, 'prefs-agent-guard')
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
await close()
|
||||
|
||||
// 存着的步数比硬上限大(硬上限后来调小了):没动它就不挡别的设置保存,后端发起运行时自己会截
|
||||
const tight = { ...stored, run: { ...stored.run, agent_max_steps: 100 }, limits: { ...stored.limits, max_agent_steps: 25 } }
|
||||
const t = await open('/settings/prefs', { handlers: [[/^GET \/settings$/, (route) => json(tight)(route)],
|
||||
[/^PUT \/settings$/, (route) => json(tight)(route)]] })
|
||||
await t.page.getByLabel('金额预算(美元)').fill('2')
|
||||
check('存着的步数超过硬上限、这次没改它:只改金额照样能保存',
|
||||
!(await t.page.getByRole('button', { name: /保存设置/ }).isDisabled()))
|
||||
await t.close()
|
||||
})
|
||||
|
||||
await section('数据 · 数据库', async () => {
|
||||
const dbs = sources.filter((s) => !/[\\/]uploads[\\/]tables[\\/]/.test(s.database ?? ''))
|
||||
const first = dbs.find((s) => s.table_count > 0) ?? dbs[0]
|
||||
@@ -1369,6 +1485,143 @@ await section('自定义工具 · 已有工具', async () => {
|
||||
await close()
|
||||
})
|
||||
|
||||
await section('工具 · 信任三档', async () => {
|
||||
// 后端的 trust / trust_key 可能还没上线:GET /tools 在真列表后面补三件假的(一个自定义、两个 MCP),
|
||||
// PUT /tools/trust 一律拦下来。fail 打开时回 422,看界面退回原样、说出后端给的原因
|
||||
const custom = { id: 'crm_lookup', name: 'crm_lookup', description: '按客户编号查客户档案', category: '自定义 · http',
|
||||
source: 'custom', dangerous: true, runtime_approval: true, schema: {}, problem: null, trust: 'ask', trust_key: 'crm_lookup' }
|
||||
const search = { id: 'mcp:demo/search', name: 'search', server: 'demo', description: '全文检索', category: 'MCP · demo',
|
||||
source: 'mcp', dangerous: true, runtime_approval: true, schema: {}, trust: 'gated', trust_key: 'mcp:demo/search' }
|
||||
const weather = { id: 'mcp:demo/weather', name: 'weather', server: 'demo', description: '查天气', category: 'MCP · demo',
|
||||
source: 'mcp', dangerous: true, runtime_approval: false, schema: {}, trust: 'always', trust_key: 'mcp:demo/weather' }
|
||||
const customRow = { id: 'check-trust-crm', name: 'crm_lookup', kind: 'http', description: '按客户编号查客户档案', parameters: {},
|
||||
config: { method: 'GET', url: 'https://example.com/crm' }, enabled: true, problem: null }
|
||||
const server = { id: 'check-trust-demo', name: 'demo', transport: 'http', command: null, args: [], env: {}, url: 'https://example.com/mcp',
|
||||
enabled: true, status: 'ok', last_error: null, tools_cache: ['search', 'weather'], last_checked_at: null, last_check_ok: null, last_latency_ms: null }
|
||||
let fail = false
|
||||
const builtin = tools.find((t) => t.source === 'builtin')
|
||||
const { page, sent, natives, errors, close } = await open('/tools/library/crm_lookup', {
|
||||
handlers: [
|
||||
[/^GET \/tools$/, async (route) => json([...await (await route.fetch()).json(), custom, search, weather])(route)],
|
||||
[/^GET \/custom-tools$/, json([customRow])],
|
||||
[/^GET \/mcp\/servers$/, json([server])],
|
||||
[/^PUT \/tools\/trust$/, (route, { body }) => (fail
|
||||
? delayed(500, { detail: '信任档只能是 ask、gated、always 之一' }, 422)(route)
|
||||
: delayed(150, { key: body?.key, trust: body?.trust })(route))],
|
||||
],
|
||||
})
|
||||
const puts = () => sent.filter((s) => s.key === 'PUT /tools/trust')
|
||||
const group = (name) => page.getByRole('radiogroup', { name: `${name} 的运行时审批` })
|
||||
const checkedIn = (g) => g.locator('[aria-checked="true"]').innerText().catch(() => '')
|
||||
// 按名字整段认:search 这种短名会撞上内置工具名里的子串
|
||||
const listItem = (name) => page.locator('nav[aria-label="工具列表"] button')
|
||||
.filter({ has: page.locator('span.mono', { hasText: new RegExp(`^${name}$`) }) }).first()
|
||||
|
||||
const detail = group('crm_lookup')
|
||||
check('自定义工具:详情区有三选一,三档是等审批 / 始终允许 · 门控把关 / 始终允许',
|
||||
(await detail.getByRole('radio').allInnerTexts()).join('|') === '等审批|始终允许 · 门控把关|始终允许',
|
||||
(await detail.getByRole('radio').allInnerTexts().catch(() => [])).join('|'))
|
||||
check('……现在选中的是后端给的「等审批」', await checkedIn(detail) === '等审批', await checkedIn(detail))
|
||||
check('……旁边说清三档的差别,写明正式运行不看这个设置', (await page.locator('[data-trust-block]').innerText()).includes('正式运行不看这个设置'))
|
||||
check('徽标按信任档:等审批挂「运行时需审批」', (await listItem('crm_lookup').innerText()).includes('运行时需审批'))
|
||||
check('……门控把关挂「门控把关」', (await listItem('search').innerText()).includes('门控把关')
|
||||
&& !(await listItem('search').innerText()).includes('运行时需审批'))
|
||||
const alwaysText = await listItem('weather').innerText()
|
||||
check('……始终允许什么都不挂', !alwaysText.includes('审批') && !alwaysText.includes('门控'), alwaysText.replace(/\s+/g, ' '))
|
||||
await shot(page, 'tool-trust-library')
|
||||
|
||||
// 改档:乐观更新,PUT 的载荷是 {key, trust}
|
||||
await detail.getByRole('radio', { name: '始终允许 · 门控把关' }).click()
|
||||
await page.waitForTimeout(40)
|
||||
check('改档先改界面(请求还没回来就已经选中、徽标跟着变)', await checkedIn(detail) === '始终允许 · 门控把关'
|
||||
&& (await listItem('crm_lookup').innerText()).includes('门控把关'), await checkedIn(detail))
|
||||
await page.waitForTimeout(300)
|
||||
const put = puts().at(-1)
|
||||
check('……PUT /tools/trust 的载荷是 {key: crm_lookup, trust: gated}', put?.body?.key === 'crm_lookup' && put?.body?.trust === 'gated'
|
||||
&& Object.keys(put?.body ?? {}).length === 2, JSON.stringify(put?.body ?? null))
|
||||
// 方向键:整组一个 Tab 位,→ 选下一档并保存
|
||||
await detail.getByRole('radio', { name: '始终允许 · 门控把关' }).focus()
|
||||
await page.keyboard.press('ArrowRight')
|
||||
await page.waitForTimeout(300)
|
||||
check('……方向键也能改(→ 到「始终允许」,同样发 PUT)', puts().at(-1)?.body?.trust === 'always' && await checkedIn(detail) === '始终允许',
|
||||
JSON.stringify(puts().map((p) => p.body?.trust)))
|
||||
check('……始终允许之后徽标不挂了', !(await listItem('crm_lookup').innerText()).includes('审批'))
|
||||
|
||||
// 失败:退回原来那一档,并说出后端给的原因
|
||||
fail = true
|
||||
await detail.getByRole('radio', { name: '等审批' }).click()
|
||||
await page.waitForTimeout(120)
|
||||
const optimistic = await checkedIn(detail)
|
||||
await page.waitForTimeout(700)
|
||||
const reverted = await checkedIn(detail)
|
||||
const alert = await page.locator('[role="alert"][aria-live="assertive"]').innerText().catch(() => '')
|
||||
check('PUT 失败:先按新的显示,回来后退回「始终允许」', optimistic === '等审批' && reverted === '始终允许', `${optimistic} → ${reverted}`)
|
||||
check('……徽标也退回去(始终允许不挂)', !(await listItem('crm_lookup').innerText()).includes('审批'))
|
||||
check('……出错提示说哪个工具没改成、为什么', alert.includes('crm_lookup') && alert.includes('没改成') && alert.includes('ask、gated、always'),
|
||||
alert.replace(/\s+/g, ' ').slice(0, 120))
|
||||
fail = false
|
||||
await page.locator('[role="alert"][aria-live="assertive"] button[aria-label*="关闭"]').first().click().catch(() => {})
|
||||
|
||||
// MCP 工具的 id 带斜杠(mcp:demo/search):以前点它落到 404,详情区里的三选一根本到不了
|
||||
await listItem('search').click()
|
||||
await page.waitForTimeout(300)
|
||||
check('MCP 工具在工具库里点得开,详情区有三选一(门控把关选中)', await checkedIn(group('mcp:demo/search')) === '始终允许 · 门控把关',
|
||||
page.url().replace(WEB, ''))
|
||||
|
||||
// 内置工具没有这个控件
|
||||
if (builtin) {
|
||||
await listItem(builtin.name).click()
|
||||
await page.waitForTimeout(300)
|
||||
check('内置工具的详情区没有三选一', await page.locator('[data-trust-control]').count() === 0 && await page.locator('[data-trust-block]').count() === 0)
|
||||
}
|
||||
|
||||
await page.getByRole('tab', { name: '自定义工具' }).click()
|
||||
await page.waitForTimeout(400)
|
||||
const row = page.locator('div.rounded-lg', { hasText: 'crm_lookup' }).last()
|
||||
check('自定义工具列表:那一行有三选一,跟着刚才改的是「始终允许」', await checkedIn(row.getByRole('radiogroup')) === '始终允许')
|
||||
check('……列表上方说清三档的差别', (await page.locator('[data-trust-explain]').innerText().catch(() => '')).includes('门控把关是每次先让一个小模型看参数'))
|
||||
await shot(page, 'tool-trust-custom')
|
||||
|
||||
await page.getByRole('tab', { name: 'MCP 接入' }).click()
|
||||
await page.waitForTimeout(400)
|
||||
const card = page.locator('article', { hasText: 'demo' })
|
||||
check('MCP 卡片列出这台服务的工具,每个一组三选一', await card.getByRole('radiogroup').count() === 2
|
||||
&& await checkedIn(group('mcp:demo/search')) === '始终允许 · 门控把关' && await checkedIn(group('mcp:demo/weather')) === '始终允许')
|
||||
await group('mcp:demo/weather').getByRole('radio', { name: '等审批' }).click()
|
||||
await page.waitForTimeout(300)
|
||||
check('……MCP 工具改档,键是 mcp:服务名/工具名', puts().at(-1)?.body?.key === 'mcp:demo/weather' && puts().at(-1)?.body?.trust === 'ask',
|
||||
JSON.stringify(puts().at(-1)?.body ?? null))
|
||||
await shot(page, 'tool-trust-mcp')
|
||||
check('改档只发 PUT /tools/trust,没有别的写请求', sent.filter((s) => s.key !== 'PUT /tools/trust' && !s.key.startsWith('GET ')).length === 0,
|
||||
sent.filter((s) => s.key !== 'PUT /tools/trust' && !s.key.startsWith('GET ')).map((s) => s.key).join(', '))
|
||||
check('没有原生对话框', natives.length === 0, natives.join(' | '))
|
||||
// 渲染崩了由错误边界接住,不一定有 pageerror:边界的兜底页也算
|
||||
check('没有运行时报错', errors.length === 0 && await page.locator('[data-error-scope]').count() === 0, errors[0] ?? '')
|
||||
await close()
|
||||
|
||||
// 老后端:工具项上没有 trust / trust_key。界面照旧:没有三选一,自定义 / MCP 写「运行时不审批」
|
||||
const strip = ({ trust, trust_key, ...rest }) => ({ ...rest, runtime_approval: false })
|
||||
const old = await open('/tools/library/crm_lookup', {
|
||||
handlers: [
|
||||
[/^GET \/tools$/, async (route) => json([...await (await route.fetch()).json(), strip(custom), strip(search)])(route)],
|
||||
[/^GET \/custom-tools$/, json([customRow])],
|
||||
[/^GET \/mcp\/servers$/, json([server])],
|
||||
],
|
||||
})
|
||||
check('老后端:详情区没有三选一', await old.page.locator('[data-trust-control]').count() === 0 && await old.page.locator('[data-trust-block]').count() === 0)
|
||||
check('……徽标照旧写「运行时不审批」', (await old.page.locator('nav[aria-label="工具列表"] button', { hasText: 'crm_lookup' }).first()
|
||||
.innerText({ timeout: 3000 }).catch(() => '')).includes('运行时不审批'))
|
||||
// 页面要是渲染崩了,后面几项照样跑完,最后那条「没有运行时报错」说出原因
|
||||
await old.page.getByRole('tab', { name: '自定义工具' }).click({ timeout: 3000 }).catch(() => {})
|
||||
await old.page.waitForTimeout(400)
|
||||
check('……自定义工具列表也没有,没有那段说明', await old.page.locator('[data-trust-control]').count() === 0 && await old.page.locator('[data-trust-explain]').count() === 0)
|
||||
await old.page.getByRole('tab', { name: 'MCP 接入' }).click({ timeout: 3000 }).catch(() => {})
|
||||
await old.page.waitForTimeout(400)
|
||||
check('……MCP 卡片不列工具', await old.page.locator('[data-trust-control]').count() === 0 && await old.page.locator('article ul').count() === 0)
|
||||
check('……没有运行时报错', old.errors.length === 0 && await old.page.locator('[data-error-scope]').count() === 0, old.errors[0] ?? '')
|
||||
await old.close()
|
||||
})
|
||||
|
||||
await section('知识库 · 撤销窗口里的轮询', async () => {
|
||||
// 有文档在处理时列表 2 秒轮询一次。以前删掉的那行 2.6 秒后又被轮询救回来
|
||||
const now = new Date().toISOString()
|
||||
|
||||
+155
-4
@@ -308,10 +308,15 @@ async function open(query, { w = 1100, h = 800, reduced = false, name } = {}) {
|
||||
}
|
||||
return { page, errors }
|
||||
}
|
||||
const scrollState = (page) => page.evaluate(() => {
|
||||
const el = document.querySelector('[data-stream-scroll]')
|
||||
return { top: Math.round(el.scrollTop), max: Math.round(el.scrollHeight - el.clientHeight) }
|
||||
})
|
||||
// 先等滚动容器挂上:open() 只固定等 400ms,并行跑、机器忙的时候组件还没渲染出来
|
||||
// (check-all 分道并行后实测挂过一次:读 null 的 scrollTop)
|
||||
const scrollState = async (page) => {
|
||||
await page.waitForSelector('[data-stream-scroll]', { timeout: 10_000 })
|
||||
return page.evaluate(() => {
|
||||
const el = document.querySelector('[data-stream-scroll]')
|
||||
return { top: Math.round(el.scrollTop), max: Math.round(el.scrollHeight - el.clientHeight) }
|
||||
})
|
||||
}
|
||||
await section('空态', async () => {
|
||||
const page = await browser.newPage({ viewport: { width: 1280, height: 900 } })
|
||||
await page.goto(`${WEB}/preview.html?case=__none__`, { waitUntil: 'networkidle' })
|
||||
@@ -357,6 +362,7 @@ await section('点开已完成轮次的执行过程:不算新进展、不拽
|
||||
// 按需取回的历史步骤以前算进「N 条新进展」:没有任何运行在跑,底下却浮出「24 条新进展」
|
||||
for (const [n, pick, label] of [[1, 0, '单轮停在顶部'], [6, 2, '6 轮点开中间一轮']]) {
|
||||
const { page, errors } = await open(`expand=${n}`, { w: 1100, h: 700, name: `expand-${n}` })
|
||||
await page.waitForSelector('[data-stream-scroll]', { timeout: 10_000 })
|
||||
const at = await page.evaluate((i) => {
|
||||
const el = document.querySelector('[data-stream-scroll]')
|
||||
const turn = document.querySelectorAll('[data-turn]')[i]
|
||||
@@ -1695,6 +1701,151 @@ await section('逐段证据:带 _evidence 的成果(可点击证据第一期
|
||||
}
|
||||
})
|
||||
|
||||
await section('审批卡:始终允许(工具信任三档)', async () => {
|
||||
// 探索运行里 MCP / 自定义工具的审批带 trust_key:「通过」旁边多一个「始终允许」,回复走
|
||||
// POST /runs/{id}/resume,response 多带 always。不带 trust_key 的审批和以前一模一样。
|
||||
// 真页面、真 store,工作流、审批、运行全用 page.route 伪造,写请求只记下来不落库
|
||||
const WF_ID = 'fx-trust'
|
||||
const RUN = 'fxtrust0001'
|
||||
const GRAPH = {
|
||||
nodes: [
|
||||
{ id: 'in', type: 'input', position: { x: 0, y: 0 }, data: { label: '问题', config: { fields: [{ name: 'q' }] } } },
|
||||
{ id: 'query', type: 'agent', position: { x: 300, y: 0 }, data: { label: '查档案', config: { tools: ['crm_lookup'] } } },
|
||||
{ id: 'out', type: 'output', position: { x: 620, y: 0 }, data: { label: '成果', config: {} } },
|
||||
],
|
||||
edges: [{ id: 'e1', source: 'in', target: 'query' }, { id: 'e2', source: 'query', target: 'out' }],
|
||||
}
|
||||
const WF = { id: WF_ID, name: '信任检查', description: '', graph: GRAPH, tags: [], version: 1, status: 'draft',
|
||||
published_version: null, run_count: 0, created_at: '2026-09-28T00:00:00Z', updated_at: '2026-09-28T00:00:00Z' }
|
||||
const approvalOf = (tool, trustKey) => ({ id: `ap-${tool}`, run_id: RUN, node_id: 'query', mode: 'approve', title: `Agent 想调用工具 ${tool}`,
|
||||
payload: { kind: 'tool_approval', node_id: 'query', tool, args: { id: 'C-1' }, title: `是否允许调用 ${tool}?`,
|
||||
...(trustKey ? { trust_key: trustKey } : {}) },
|
||||
status: 'pending', response: {}, created_at: new Date(Date.now() - 30_000).toISOString(),
|
||||
workflow_name: '信任检查', node_label: '查档案', run_class: 'exploratory' })
|
||||
|
||||
const setup = async (approval) => {
|
||||
const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 }, colorScheme: 'dark' })
|
||||
await ctx.addInitScript(() => { try { localStorage.setItem('agentlab_actor', '张工') } catch { /* 隐私窗口 */ } })
|
||||
const page = await ctx.newPage()
|
||||
const errors = []
|
||||
page.on('pageerror', (e) => errors.push(e.message))
|
||||
const json = (route, body, status = 200) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) })
|
||||
const writes = []
|
||||
const toolHits = []
|
||||
let pending = true
|
||||
await page.route(/\/api\/workflows(\?.*)?$/, async (route) => {
|
||||
if (route.request().method() !== 'GET') return json(route, { detail: '检查脚本不写库' }, 409)
|
||||
const real = await (await route.fetch()).json().catch(() => [])
|
||||
return json(route, [WF, ...(Array.isArray(real) ? real : [])])
|
||||
})
|
||||
await page.route(/\/api\/workflows\/fx-trust(\/.*)?(\?.*)?$/, (route) =>
|
||||
route.request().method() === 'GET' ? json(route, WF) : json(route, { detail: '检查脚本不写库' }, 409))
|
||||
await page.route(/\/api\/conversations(\/.*)?(\?.*)?$/, (route) =>
|
||||
route.request().method() === 'GET' ? json(route, []) : json(route, { id: 'fx-conv-trust', kind: 'canvas', title: '', turns: [] }))
|
||||
await page.route(/\/api\/tools(\?.*)?$/, (route) => { toolHits.push(Date.now()); return route.continue() })
|
||||
await page.route(/\/api\/approvals(\/.*)?(\?.*)?$/, (route) => {
|
||||
const req = route.request()
|
||||
if (req.method() === 'GET') return json(route, pending ? [approval] : [])
|
||||
writes.push({ url: new URL(req.url()).pathname, body: req.postDataJSON(), actor: req.headers()['x-actor'] ?? '' })
|
||||
return json(route, { detail: '检查脚本不写库' }, 409)
|
||||
})
|
||||
await page.route(/\/api\/runs(\/.*)?(\?.*)?$/, (route) => {
|
||||
const req = route.request()
|
||||
const path = new URL(req.url()).pathname
|
||||
if (req.method() !== 'GET') {
|
||||
writes.push({ url: path, body: req.postDataJSON(), actor: req.headers()['x-actor'] ?? '' })
|
||||
if (path.endsWith(`${RUN}/resume`)) {
|
||||
pending = false
|
||||
return json(route, { id: RUN, workflow_id: WF_ID, status: 'running', input: {}, output: {}, error: null, usage: {}, run_class: 'exploratory' })
|
||||
}
|
||||
return json(route, { detail: '检查脚本不写库' }, 409)
|
||||
}
|
||||
if (path.includes(`${RUN}/events`)) return json(route, [])
|
||||
if (path.includes(RUN)) {
|
||||
return json(route, { id: RUN, workflow_id: WF_ID, status: pending ? 'interrupted' : 'running', input: {}, output: {}, error: null, usage: {}, run_class: 'exploratory' })
|
||||
}
|
||||
return route.continue()
|
||||
})
|
||||
await page.goto(`${WEB}/studio/${WF_ID}`, { waitUntil: 'networkidle' })
|
||||
await page.waitForFunction((id) => window.__studio?.getState().workflow?.id === id, WF_ID, { timeout: 15000 })
|
||||
const now = Date.now() / 1000
|
||||
const ev = (seq, type, node_id, data = {}, t = 0) => ({ seq, type, node_id, data, ts: now - 30 + t })
|
||||
const tool = approval.payload.tool
|
||||
const WAIT = [
|
||||
ev(1, 'run.started', null, { nodes: 3 }, 0),
|
||||
ev(2, 'node.started', 'in', { node_type: 'input', label: '问题' }, 0.1),
|
||||
ev(3, 'node.finished', 'in', { duration_ms: 3, preview: { q: 'x' } }, 0.2),
|
||||
ev(4, 'node.started', 'query', { node_type: 'agent', label: '查档案' }, 0.3),
|
||||
...(approval.payload.trust_key
|
||||
? [ev(5, 'tool.gated', 'query', { tool, verdict: 'escalate', reason: '要导出整张客户表', model: 'tiny-1', duration_ms: 640 }, 0.9)] : []),
|
||||
ev(6, 'human.requested', 'query', { mode: 'approve', tool, args: { id: 'C-1' }, title: `Agent 想调用工具 ${tool}`,
|
||||
...(approval.payload.trust_key ? { trust_key: approval.payload.trust_key } : {}) }, 1.0),
|
||||
ev(7, 'run.interrupted', 'query', { payload: approval.payload }, 1.1),
|
||||
]
|
||||
await page.evaluate((run) => window.__studio.setState({ run: { id: run, workflow_id: 'fx-trust', status: 'queued', input: {},
|
||||
output: {}, error: null, usage: {}, run_class: 'exploratory', version: null }, streaming: true, unsubscribe: () => {} }), RUN)
|
||||
await page.evaluate((l) => { const s = window.__studio.getState(); for (const e of l) s.applyEvent(e) }, WAIT)
|
||||
await page.waitForTimeout(700)
|
||||
return { ctx, page, errors, writes, toolHits, card: page.locator('.sheet-in [data-approval]').first() }
|
||||
}
|
||||
|
||||
{
|
||||
const { ctx, page, errors, writes, toolHits, card } = await setup(approvalOf('crm_lookup', 'crm_lookup'))
|
||||
const buttons = await card.locator('button').allInnerTexts()
|
||||
check('带 trust_key:「通过」旁边有「始终允许」', buttons.map((b) => b.trim()).join('|').includes('通过|始终允许|驳回'), buttons.join(' | '))
|
||||
const always = card.getByRole('button', { name: '始终允许' })
|
||||
const title = await always.getAttribute('title')
|
||||
check('……悬停说明:批准这次,并把工具设为「始终允许 · 门控把关」,可以在工具页改回',
|
||||
!!title?.includes('批准这次,并把 crm_lookup 设为「始终允许 · 门控把关」') && title.includes('由门控模型逐次把关') && title.includes('可以在工具页改回'), title)
|
||||
const note = await card.innerText()
|
||||
check('……卡上常显同一句旁注,不只藏在悬停里', note.includes('始终允许 → 批准这次') && note.includes('以后的运行不再问你'))
|
||||
check('……时间线上写着门控为什么拦下', (await page.locator('.sheet-in').first().innerText()).includes('门控拦下 crm_lookup,交给人工审批:要导出整张客户表'))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(250)
|
||||
await card.screenshot({ path: `${SHOTS}/stream-approval-always-${theme}.png` })
|
||||
await page.locator('.sheet-in').first().screenshot({ path: `${SHOTS}/stream-approval-always-panel-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await card.getByLabel('备注').fill('只查一条,放行')
|
||||
const hitsBefore = toolHits.length
|
||||
await always.click()
|
||||
await page.waitForTimeout(700)
|
||||
const resume = writes.find((w) => w.url.endsWith(`/runs/${RUN}/resume`))
|
||||
check('点了发 POST /runs/{id}/resume,response 是 {approved: true, always: true, note}',
|
||||
resume?.body?.response?.approved === true && resume.body.response.always === true && resume.body.response.note === '只查一条,放行',
|
||||
JSON.stringify(resume?.body ?? writes.map((w) => w.url)))
|
||||
check('……带 approval_id 指明回复的是哪一条', resume?.body?.approval_id === 'ap-crm_lookup', resume?.body?.approval_id)
|
||||
check('……带着署名,不另走 /approvals/…/decide', decodeURIComponent(resume?.actor ?? '') === '张工'
|
||||
&& !writes.some((w) => w.url.includes('/decide')), writes.map((w) => w.url).join(', '))
|
||||
const toastText = await page.locator('[aria-live]').allInnerTexts().then((t) => t.join(' ')).catch(() => '')
|
||||
check('……回执说运行继续、以后由门控模型把关', toastText.includes('以后由门控模型把关'), toastText.replace(/\s+/g, ' ').slice(0, 120))
|
||||
check('……工具目录重拉了一次(工具页跟着显示门控把关)', toolHits.length > hitsBefore, `${hitsBefore} → ${toolHits.length}`)
|
||||
check('……审批卡收起来了', await page.locator('.sheet-in [data-approval]').count() === 0)
|
||||
check('没有运行时报错', errors.length === 0, errors.join(' | '))
|
||||
await ctx.close()
|
||||
}
|
||||
|
||||
{
|
||||
// 内置工具的审批(不带 trust_key):和以前一模一样
|
||||
const { ctx, errors, writes, card } = await setup(approvalOf('file_write', null))
|
||||
const buttons = (await card.locator('button').allInnerTexts()).map((b) => b.trim())
|
||||
check('不带 trust_key:没有「始终允许」,按钮还是通过 / 驳回', buttons.join('|').includes('通过|驳回') && !buttons.includes('始终允许'), buttons.join(' | '))
|
||||
const text = await card.innerText()
|
||||
check('……卡上不提始终允许和门控', !text.includes('始终允许') && !text.includes('门控'), text.replace(/\s+/g, ' ').slice(0, 160))
|
||||
check('……批了会怎样还是原来那句', text.includes('通过 → 接着往下跑;驳回 → 走「驳回」那条出口'))
|
||||
await card.getByRole('button', { name: '通过' }).click()
|
||||
await ctx.pages()[0].waitForTimeout(400)
|
||||
const decide = writes.find((w) => w.url.includes('/decide'))
|
||||
check('……「通过」照旧走 /approvals/…/decide,请求体里没有 always', !!decide && decide.body?.approved === true && !('always' in (decide.body ?? {}))
|
||||
&& !writes.some((w) => w.url.endsWith('/resume')), JSON.stringify(decide?.body ?? writes.map((w) => w.url)))
|
||||
check('没有运行时报错', errors.length === 0, errors.join(' | '))
|
||||
await ctx.close()
|
||||
}
|
||||
})
|
||||
|
||||
await browser.close()
|
||||
console.log(failed ? `\n✗ ${failed} 项未通过` : '\n✓ 助手流渲染全部通过')
|
||||
process.exit(failed ? 1 : 0)
|
||||
|
||||
Reference in New Issue
Block a user