mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:31:58 +00:00
可点击证据第四期：结论句由模型裁判；只引单元格的问数据图能发成受管
一、结论句裁判
- 报告节点新增 claims: judge：写完、核对完之后，把结论句和它引用的证据摘录交给裁判模型，
判为支持 / 部分支持 / 不支持 / 不是结论，附一句理由；判定只标注，不改写正文
- 送裁判的句子按固定规则预筛：带引用的、含方向词或因果词的送去判，短句、过渡句不送
- 证据摘录：指标写名称、值和原式；查询给 SQL 前 300 字和被引用的行，被引用的列一定给到；遮罩列一律不给；
规则里写明「数字已经由系统核对过，不要复算」
- 一份报告合并成一次调用，超出句数就分批；结构化输出失败时退回 JSON 解析，再失败记为未裁判
- 裁判模型的优先级：节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型；思考关闭；
和写作模型相同时给警告（等于自己审自己）
- rewrite_once（默认关）：把不支持的句子和理由交回写作者只改这几句，再裁一次；
改写稿冒出原稿没有的问题就不采用，保留原稿和原来的判定
二、成本上限
- 每份报告的句数、金额、时长，探索运行每次点击的金额，全局每日金额，每一项都可以自定义，也都可以设成不限；
默认 40 句、$0.05、30 秒，每次点击 $0.01，每天 $2
- 调用前先估算，超出上限按优先级截断：先判带引用的，再判含方向词、因果词的，其余记为未裁判并写明「已到上限」；
不报错，也不让报告失败
- 每日花费按本地日期计数，持久化在设置表里；重跑同一次运行不重复计费；价格目录里没有的模型只计次数，并写明估不出金额
三、正式运行与探索运行
- 正式运行在报告节点内裁判，判定写进报告文档、随文档封存；不支持的按 on_unsupported（degrade / withhold）降档或不予出具，
裁判没跑成记缺口，不能判完整出具
- 探索运行按需裁判：POST /api/runs/{id}/evidence/judge，点开哪句判哪句；判定作为 evidence.judged 事件追加在封存之后，
标「封存后追加」，封存核对仍然一致；只读封存范围内的工件；同一句不重复判；运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409，写明「正式运行的裁判在节点内完成」
四、治理与自动修复
- G3 放开 judge：受管级别写 judge 必须写预算（judge.max_cost_usd，或显式写 null 表示不限）
- 发布前自动修复：缺预算时给选项（默认金额 / 不限），不替人选；
从 require_citation 改成 judge 不算降低要求，judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册：受管模板可以写 judge 并带预算；问数据按需裁判，不替人打开结论句策略
五、界面
- 结论句句末放徽标：支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?；不用表示「已核对」的绿色；n / N 也跳到不支持的句子
- 证据面板里「模型的解释」：带「模型判断 · 模型名 · 非确定」，封存后追加的另外标出；
「请模型判断这句」只在探索运行、还没判过时出现，触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数；设置页新增「证据裁判」一组（裁判模型、各项上限、今日花费），设成不限时写明费用受什么约束；
报告节点可以配 judge；运行轨迹里裁判相关的事件都有中文标题
六、门禁修正
- 引用模式下，图里没有口径卡（只引查询单元格）的，出具契约不再要求 metrics_from，受管门禁也不再要求 required：
strict 加 cells 已经保证每个数都点得开、对不上就不予出具；有口径卡的照旧要求。
以前问数据的图怎么都发不成受管，Copilot 补上契约也会被判「冒出新的问题」作废
This commit is contained in:
ThearchyHelios
@@ -1313,6 +1313,103 @@ await section('agent 字段按出处核对（可点击证据第二期）：抽
|
||||
&& !odd.some((s) => /undefined|NaN/.test(`${s.title}${s.sub ?? ''}`)), odd.map((s) => s.title).join(' | '))
|
||||
})
|
||||
|
||||
await section('结论句裁判（可点击证据第四期）：裁判调用单独成行、几条警告说人话、封存后按需裁判有记录', async () => {
|
||||
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
|
||||
// 原话和后端同一个格式：engine/judge.py 的 run_request、engine/nodes/report.py 的改写一次。先核对后端源码还是这么说的
|
||||
const judgePy = readFileSync(`${root}backend/app/engine/judge.py`, 'utf8')
|
||||
const reportPy = readFileSync(`${root}backend/app/engine/nodes/report.py`, 'utf8')
|
||||
check('后端发裁判事件的格式还是夹具里这一种', judgePy.includes('purpose="judge"') && judgePy.includes('code="judge_limit"')
|
||||
&& judgePy.includes('结论句裁判已到上限（{labels}）：{n} 句没判，记为未裁判；已判的保留')
|
||||
&& judgePy.includes('"结论句裁判没跑完：" + "；".join(failed)') && judgePy.includes('code="judge_unpriced"')
|
||||
&& judgePy.includes('不在价格目录里，按令牌估不出金额：') && reportPy.includes('code="judge_same_model"')
|
||||
&& reportPy.includes('：等于自己审自己') && reportPy.includes('code="report_rewrite"') && reportPy.includes('句结论证据不支持（')
|
||||
&& reportPy.includes('已交回写作者只改这几句') && reportPy.includes('改写稿冒出 {len(fresh)} 处原稿没有的问题（')
|
||||
&& reportPy.includes('没有采用，') && reportPy.includes('保留原稿和原来的判定'),
|
||||
'改了 judge.py / report.py 的说法就同步改 decode.ts 的 explainLog 和这里的夹具')
|
||||
const LIMIT = '结论句裁判已到上限（这份报告的裁判金额上限 $0.05）：3 句没判，记为未裁判；已判的保留'
|
||||
const FAILED = '结论句裁判没跑完：有 2 句结论没裁判：裁判调用失败（ConnectError: 连不上）'
|
||||
const UNPRICED = '模型「jm-x」不在价格目录里，按令牌估不出金额：金额上限（每份报告、每次点击、每日）对它不起作用，费用只受句数、时长上限约束'
|
||||
const SAME = '裁判模型和写作模型都是「wm-1」：等于自己审自己，模型写错的地方它多半也看不出来。到设置里选一个不同的「证据裁判模型」，或者在节点的 judge.model 里指定'
|
||||
const REWRITE = '裁判认为 2 句结论证据不支持（「增长主要来自新客」「退款多数是部分退款」），已交回写作者只改这几句'
|
||||
const REJECTED = '改写稿冒出 1 处原稿没有的问题（「12」），没有采用，保留原稿和原来的判定'
|
||||
const events = [
|
||||
E(1, 'run.started', null, { total: 1 }),
|
||||
E(2, 'node.started', 'write', { node_type: 'report', label: '写周报' }),
|
||||
E(3, 'llm.start', 'write', { model: 'wm-1' }),
|
||||
E(4, 'llm.end', 'write', { model: 'wm-1', duration_ms: 900 }),
|
||||
E(5, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 4 }),
|
||||
E(6, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 2100, input_tokens: 900, output_tokens: 300, units: 4 }),
|
||||
E(7, 'log', 'write', { level: 'warn', code: 'judge_unpriced', message: UNPRICED }),
|
||||
E(8, 'log', 'write', { level: 'warn', code: 'judge_same_model', message: SAME }),
|
||||
E(9, 'log', 'write', { level: 'warn', code: 'judge_limit', limits: ['max_cost_usd'], unjudged: 3, message: LIMIT }),
|
||||
E(10, 'log', 'write', { level: 'warn', code: 'judge_failed', message: FAILED }),
|
||||
E(11, 'log', 'write', { level: 'warn', code: 'report_rewrite', message: REWRITE }),
|
||||
E(12, 'log', 'write', { level: 'warn', code: 'report_rewrite_rejected', message: REJECTED }),
|
||||
E(13, 'report.checked', 'write', { doc_artifact: 'doc-1', stats: { numbers: 3, numbers_cited: 3, uncited_numbers: 0, unresolved: 0 },
|
||||
violations: [], repairs: 0, claims: 'judge',
|
||||
judge: { mode: 'inline', counts: { supported: 1, partial: 1, unsupported: 1, not_a_claim: 1, unjudged: 1 } } }),
|
||||
E(14, 'node.finished', 'write', { duration_ms: 4000 }),
|
||||
E(15, 'run.finished', null, { status: 'succeeded' }),
|
||||
// 探索运行跑完之后有人点开结论句请模型判断：判定追加在封存之后
|
||||
E(16, 'evidence.judged', 'write', { report: 'write', doc_artifact: 'doc-1', units: ['u2'], model: 'jm-x',
|
||||
verdicts: { u2: { status: 'unsupported', rationale: '看不出', judge: 'jm-x', post_seal: true } }, limits_hit: [], gaps: [],
|
||||
post_seal: true }),
|
||||
]
|
||||
const all = flatten(mod.decodeRun(events))
|
||||
check('新事件都有翻译', !all.some((s) => s.title === '一条还没翻译的记录'), all.filter((s) => s.title === '一条还没翻译的记录').map((s) => s.raw?.type).join(','))
|
||||
const llms = all.filter((s) => s.kind === 'llm')
|
||||
check('写作照旧叫「思考并作答」，裁判那次单独起名、说判几句', llms.map((s) => s.title).join('|') === '思考并作答|请裁判模型判断 4 句结论',
|
||||
llms.map((s) => s.title).join('|'))
|
||||
check('……裁判那次跑完了、带耗时和模型', llms[1]?.status === 'done' && llms[1]?.meta === '2.1 s' && llms[1]?.code === 'judge'
|
||||
&& llms[1]?.detail === '模型：jm-x', `${llms[1]?.status} ${llms[1]?.meta} ${llms[1]?.code} ${llms[1]?.detail}`)
|
||||
const by = (code) => all.find((s) => s.code === code)
|
||||
check('judge_limit：几句没判、哪个上限、怎么调', by('judge_limit')?.title === '结论句裁判已到上限，3 句没判（记为未裁判，已判的保留）'
|
||||
&& by('judge_limit')?.sub === '这份报告的裁判金额上限 $0.05' && !!by('judge_limit')?.next?.includes('不限'),
|
||||
`${by('judge_limit')?.title} / ${by('judge_limit')?.sub} / ${by('judge_limit')?.next}`)
|
||||
check('judge_failed：没跑完、原因放副标题', by('judge_failed')?.title === '结论句裁判没跑完，没判的记为未裁判'
|
||||
&& by('judge_failed')?.sub === '有 2 句结论没裁判：裁判调用失败（ConnectError: 连不上）', `${by('judge_failed')?.title} / ${by('judge_failed')?.sub}`)
|
||||
check('judge_unpriced：点名模型、金额上限不起作用', by('judge_unpriced')?.title === '裁判模型「jm-x」估不出金额，金额上限对它不起作用',
|
||||
by('judge_unpriced')?.title)
|
||||
check('judge_same_model：自己审自己、怎么改', by('judge_same_model')?.title === '裁判模型和写作模型都是「wm-1」：等于自己审自己'
|
||||
&& !!by('judge_same_model')?.next?.includes('证据裁判'), `${by('judge_same_model')?.title} / ${by('judge_same_model')?.next}`)
|
||||
check('report_rewrite：几句不支持、交回改写，句子放副标题', by('report_rewrite')?.title === '裁判认为 2 句结论证据不支持，已交回写作者只改这几句'
|
||||
&& by('report_rewrite')?.sub === '「增长主要来自新客」「退款多数是部分退款」', `${by('report_rewrite')?.title} / ${by('report_rewrite')?.sub}`)
|
||||
check('report_rewrite_rejected：冒出几处新问题、没有采用', by('report_rewrite_rejected')?.title === '改写稿冒出 1 处原稿没有的问题，没有采用'
|
||||
&& by('report_rewrite_rejected')?.sub === '「12」' && by('report_rewrite_rejected')?.next === '保留的是原稿和原来的判定',
|
||||
`${by('report_rewrite_rejected')?.title} / ${by('report_rewrite_rejected')?.sub}`)
|
||||
check('几条警告都是提醒（warn），原话留在展开区', ['judge_limit', 'judge_failed', 'judge_unpriced', 'judge_same_model', 'report_rewrite',
|
||||
'report_rewrite_rejected'].every((c) => by(c)?.level === 'warn' && by(c)?.detail))
|
||||
const rc = all.find((s) => s.code === 'report_checked')
|
||||
check('报告核对那一行也数结论句，和出具横幅同一种说法（不是结论句的不算）',
|
||||
rc?.title === '核对报告：3 个数字都有出处 · 结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）', rc?.title)
|
||||
const ej = by('evidence_judged')
|
||||
check('封存后按需裁判：单独一行，写明封存后追加、模型判断非确定', ej?.title === '封存后按需裁判了 1 句结论'
|
||||
&& ej?.sub === '封存后追加 · 模型判断，非确定' && ej?.level === 'info', `${ej?.title} / ${ej?.sub} / ${ej?.level}`)
|
||||
|
||||
const failed = flatten(mod.decodeRun([
|
||||
E(1, 'node.started', 'write', { node_type: 'report' }),
|
||||
E(2, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 2 }),
|
||||
E(3, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 30000, units: 2, error: '已到上限（裁判时长上限 30 秒），这句没判' }),
|
||||
E(4, 'node.finished', 'write', { duration_ms: 31000 }),
|
||||
E(5, 'evidence.judged', 'write', { verdicts: { u4: { status: 'unjudged', reason: 'max_cost_usd' } }, limits_hit: ['max_cost_usd'] }),
|
||||
]))
|
||||
const jx = failed.find((s) => s.kind === 'llm')
|
||||
check('裁判调用没跑成：那一行说「没跑成」，是提醒不是失败，原因放展开区', jx?.title === '结论句裁判没跑成' && jx?.status === 'done'
|
||||
&& jx?.level === 'warn' && !!jx?.detail?.includes('裁判时长上限'), `${jx?.title} ${jx?.status} ${jx?.level}`)
|
||||
check('……报告节点照样成功', failed.find((s) => s.kind === 'node')?.status === 'done')
|
||||
const late = failed.find((s) => s.code === 'evidence_judged')
|
||||
check('按需裁判触顶：写明有的到了上限没判（提醒）', late?.title === '封存后按需裁判了 0 句结论，有的到了上限没判' && late?.level === 'warn',
|
||||
`${late?.title} ${late?.level}`)
|
||||
const odd = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report' }),
|
||||
E(2, 'log', 'w', { level: 'warn', code: 'judge_limit', message: '到上限了' }),
|
||||
E(3, 'log', 'w', { level: 'warn', code: 'report_rewrite' }),
|
||||
E(4, 'report.checked', 'w', { stats: { numbers: 1, numbers_cited: 1 }, judge: { counts: 'x' } })]))
|
||||
check('别的说法（缺字段）：标题照样说人话，不出 undefined、不瞎填数', odd.find((s) => s.code === 'judge_limit')?.title === '结论句裁判已到上限（记为未裁判，已判的保留）'
|
||||
&& !odd.some((s) => /undefined|NaN/.test(`${s.title}${s.sub ?? ''}`)), odd.map((s) => s.title).join(' | '))
|
||||
check('……裁判摘要认不出时报告核对那一行只数数字', odd.find((s) => s.code === 'report_checked')?.title === '核对报告：1 个数字都有出处',
|
||||
odd.find((s) => s.code === 'report_checked')?.title)
|
||||
})
|
||||
|
||||
await section('术语', async () => {
|
||||
// 没起名的节点退到类型名，类型名跟全站同一张表
|
||||
const h = mod.decodeRun([{ seq: 1, type: 'node.started', node_id: 'h', ts: 1, data: { node_type: 'human', label: 'h' } }])
|
||||
|
||||
+533
-11
@@ -30,6 +30,10 @@ const fxq = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-
|
||||
// 第三期：表名字段名（反引号里的、标记的、自动链接的）、可疑实体、核对不了的名字、逐字引文，
|
||||
// 以及没有契约的旧运行按数值猜的候选。文档和猜测都是后端真跑出来的（compose_doc / guess_sources）
|
||||
const fxe = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-entity.json`, 'utf8'))
|
||||
// 第四期：结论句裁判。文档由 compose_doc 真跑，候选句由 judge.prepare 真跑，判定是剧本（模拟裁判模型），
|
||||
// 经 judge 的 summarize / apply_judgement 写回：正式运行（节点里判好的）、探索运行（按需）、没开裁判的探索运行
|
||||
const fxj = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-judge.json`, 'utf8'))
|
||||
const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain]
|
||||
/** 夹具里某段文字是哪个片段：检查按文字认片段，重新生成夹具时编号变了也不用改 */
|
||||
const segOf = (doc, text) => doc.blocks.flatMap((b) => b.units.flatMap((u) => u.segments)).find((s) => s.text === text)?.id
|
||||
|
||||
@@ -52,13 +56,60 @@ async function section(key, name, fn) {
|
||||
|
||||
const browser = await chromium.launch({ executablePath: CHROME })
|
||||
|
||||
/** 伪造证据相关的 GET；片段请求记账（缓存：同一个片段点两次只取一次） */
|
||||
async function routed(page) {
|
||||
/**
|
||||
* 按需裁判的伪造答复：evidence.judged 的载荷形状（NOTES-A4 §8、E4-node 的 POST …/evidence/judge）。
|
||||
* 夹具里按句子备好了答复的用夹具的；别的句子给一条「有依据」。正式运行回 409
|
||||
*/
|
||||
function judgeReply(set, body) {
|
||||
if (set === fxj.formal) return { status: 409, json: { detail: '正式运行的裁判在节点内完成（claims: judge），判定随报告文档一起封存，不能再按需裁判', code: 'evidence_judge_formal' } }
|
||||
const unit = body?.units?.[0]
|
||||
const named = Object.entries(set.units).find(([, u]) => u === unit)?.[0]
|
||||
if (named && set.judge?.[named]) return { json: set.judge[named] }
|
||||
return { json: { report: { node_id: 'write', doc_artifact: set.doc_artifact }, verdicts: { [unit]: { status: 'supported',
|
||||
rationale: '引用的证据支持这句话', judge: fxj.model, post_seal: true, used: [] } }, model: fxj.model, limits_hit: [], post_seal: true } }
|
||||
}
|
||||
|
||||
/**
|
||||
* 伪造证据相关的 GET；片段请求记账（缓存：同一个片段点两次只取一次）。四期：按需裁判的 POST 也在这里认
|
||||
* （记进 hits 的 judge:<运行>:<句子>），别的非 GET 一律拦掉。judge 给了就用它答（改坏验证、触顶、出错）；
|
||||
* segPatch 给了就改片段接口的答复（比如 on_demand 说运行还没封存）
|
||||
*/
|
||||
async function routed(page, { judge = null, segPatch = null } = {}) {
|
||||
const hits = []
|
||||
const judged = new Set()
|
||||
await page.route((u) => new URL(u).pathname.startsWith('/api/'), (r) => {
|
||||
const req = r.request()
|
||||
const url = new URL(req.url())
|
||||
const jset = JUDGE_SETS.find((x) => url.pathname.includes(`/runs/${x.run_id}/`) || url.pathname.endsWith(`/runs/${x.run_id}`))
|
||||
if (jset && req.method() === 'POST' && /\/evidence\/judge$/.test(url.pathname)) {
|
||||
const body = req.postDataJSON()
|
||||
hits.push(`judge:${jset.run_id}:${(body?.units ?? []).join(',')}:${body?.report ?? ''}`)
|
||||
const reply = judge ? judge(jset, body) : judgeReply(jset, body)
|
||||
if (reply.status == null || reply.status < 300) for (const u of body?.units ?? []) judged.add(`${jset.run_id}:${u}`)
|
||||
return r.fulfill({ status: reply.status ?? 200, json: reply.json })
|
||||
}
|
||||
if (req.method() !== 'GET') return r.abort()
|
||||
// 四期的报告工件：问数据页的答案（Output → EvidenceField）按工件 id 取文档
|
||||
const jart = JUDGE_SETS.find((x) => url.pathname === `/api/artifacts/${x.doc_artifact}`)
|
||||
if (jart) return r.fulfill({ json: { id: jart.doc_artifact, content: jart.doc } })
|
||||
if (jset) {
|
||||
const seg = url.pathname.match(/\/evidence\/segments\/([^/]+)$/)
|
||||
if (seg) {
|
||||
const id = decodeURIComponent(seg[1])
|
||||
hits.push(`j:${id}`)
|
||||
// 判过之后片段接口按最新的 evidence.judged 叠上判定（封存后追加）：夹具里备着判过那一句的答复
|
||||
const after = jset.segments_after && judged.has(`${jset.run_id}:${jset.units.bad}`) ? jset.segments_after : jset.segments
|
||||
let body = structuredClone(after[id])
|
||||
if (body && segPatch) body = segPatch(jset, body)
|
||||
return body ? r.fulfill({ json: body })
|
||||
: r.fulfill({ status: 404, json: { detail: '报告里没有这个片段', code: 'evidence_segment_not_found' } })
|
||||
}
|
||||
if (/\/evidence$/.test(url.pathname)) return r.fulfill({ json: jset.graph })
|
||||
if (url.pathname.endsWith(`/runs/${jset.run_id}`)) {
|
||||
hits.push(`run:${jset.run_id}`)
|
||||
return r.fulfill({ json: { id: jset.run_id, run_class: jset.run_class, status: 'succeeded' } })
|
||||
}
|
||||
}
|
||||
const seg = url.pathname.match(/\/api\/runs\/([^/]+)\/evidence\/segments\/([^/]+)$/)
|
||||
// 按运行认夹具：第一期、第二期（查询链）、第三期（实体、引文）各有自己的片段
|
||||
const set = url.pathname.includes(`/runs/${fxq.run_id}/`) ? fxq : url.pathname.includes(`/runs/${fxe.run_id}/`) ? fxe : fx
|
||||
@@ -83,14 +134,14 @@ async function routed(page) {
|
||||
return hits
|
||||
}
|
||||
|
||||
async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null } = {}) {
|
||||
async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null, judge = null, segPatch = null } = {}) {
|
||||
const ctx = await browser.newContext({ viewport: { width: w, height: h }, ...(reduced ? { reducedMotion: 'reduce' } : {}) })
|
||||
const page = await ctx.newPage()
|
||||
// 找不到元素时 8 秒就报，不等默认的 30 秒：改坏验证时一节里十几处都找不到
|
||||
page.setDefaultTimeout(8000)
|
||||
const errors = []
|
||||
page.on('pageerror', (e) => errors.push(e.message))
|
||||
const hits = await routed(page)
|
||||
const hits = await routed(page, { judge, segPatch })
|
||||
// 预览页直接 import 第二期夹具（vite 把 JSON 当模块给）：要一份改过的文档时把那个模块换掉
|
||||
if (patchQuery) {
|
||||
await page.route((u) => new URL(u).pathname.endsWith('/run/__tests__/evidence-query.json'), (r) =>
|
||||
@@ -1130,7 +1181,7 @@ await section('studio', '报告撰写节点：画布认得、检查器能配（R
|
||||
check('「指标来自」只列上游的口径卡（下游那张不列）', opts.length === 1 && opts[0].includes('周报口径'), opts.join(' | '))
|
||||
const onViolation = await p.locator('[data-field="on_violation"] option').allInnerTexts()
|
||||
check('违规时的默认写明按运行类别', onViolation[0]?.includes('探索运行照常产出'), onViolation.join(' | '))
|
||||
// 三期：结论句策略 claims（off / require_citation，judge 是四期的，不出现在选项里）和表名字段名核对 entities
|
||||
// 结论句策略 claims（off / require_citation / judge，四期放开了 judge）和表名字段名核对 entities
|
||||
// 表名字段名核对收在「高级选项」里（很少要改），先展开
|
||||
await p.locator('button[aria-expanded]', { hasText: '高级选项' }).click().catch(() => {})
|
||||
await p.waitForTimeout(100)
|
||||
@@ -1138,12 +1189,14 @@ await section('studio', '报告撰写节点：画布认得、检查器能配（R
|
||||
check('检查器有结论句策略、表名字段名核对（高级选项里）两项', fields.includes('claims') && all.includes('entities')
|
||||
&& !fields.includes('entities'), all.join(','))
|
||||
const claimVals = await p.locator('[data-field="claims"] option').evaluateAll((els) => els.map((e) => e.value))
|
||||
check('结论句策略只有 off / require_citation，没有 judge', claimVals.join(',') === 'off,require_citation', claimVals.join(','))
|
||||
check('结论句策略三档：off / require_citation / judge（四期放开了模型裁判）', claimVals.join(',') === 'off,require_citation,judge',
|
||||
claimVals.join(','))
|
||||
const claimsLabel = await p.locator('[data-field="claims"] label').first().innerText().catch(() => '')
|
||||
check('结论句策略有中文标签（发布前修复的预览也用它）', /结论句/.test(claimsLabel) && !/claims/.test(claimsLabel), claimsLabel)
|
||||
const claimsHelp = await p.locator('[data-field="claims"]').innerText()
|
||||
check('……说明写明「结论句裁判在后续版本支持」、受管级别要显式选', claimsHelp.includes('结论句裁判在后续版本支持')
|
||||
&& claimsHelp.includes('受管'), claimsHelp.replace(/\s+/g, ' ').slice(0, 160))
|
||||
check('……说明写明模型逐句判断、受管级别要显式选并写金额上限，不再说「后续版本」', claimsHelp.includes('请模型逐句判断')
|
||||
&& claimsHelp.includes('受管') && claimsHelp.includes('金额上限') && !claimsHelp.includes('后续版本'),
|
||||
claimsHelp.replace(/\s+/g, ' ').slice(0, 200))
|
||||
check('没写 claims 时显示默认的 off', await p.locator('[data-field="claims"] select').inputValue() === 'off'
|
||||
|| (await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.value)) === 'off')
|
||||
const entityVals = await p.locator('[data-field="entities"] option').evaluateAll((els) => els.map((e) => e.value))
|
||||
@@ -1151,15 +1204,89 @@ await section('studio', '报告撰写节点：画布认得、检查器能配（R
|
||||
await p.locator('[data-field="claims"] select').selectOption('require_citation')
|
||||
const cfgOf = () => p.evaluate(() => window.__studio.getState().nodes.find((n) => n.id === 'write')?.data.config)
|
||||
check('选「要求挂依据」写进 config.claims', (await cfgOf())?.claims === 'require_citation')
|
||||
// 手写或别处写进来的 judge：不能悄悄显示成 off，要照实写出来并说后续版本支持
|
||||
// 手写或别处写进来的不认识的值：不能悄悄显示成 off，要照实写出来
|
||||
await p.evaluate(() => {
|
||||
const st = window.__studio.getState()
|
||||
const n = st.nodes.find((x) => x.id === 'write')
|
||||
st.updateNode('write', { config: { ...n.data.config, claims: 'judge' } })
|
||||
st.updateNode('write', { config: { ...n.data.config, claims: 'sometimes' } })
|
||||
})
|
||||
await p.waitForTimeout(150)
|
||||
const shown = await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.textContent ?? '')
|
||||
check('config 里写着 judge：下拉照实显示 judge 并说后续版本支持，不装成 off', shown.includes('judge') && shown.includes('后续版本'), shown)
|
||||
check('config 里写着不认识的值：下拉照实显示，不装成 off', shown.includes('sometimes') && shown.includes('不认识'), shown)
|
||||
|
||||
// 四期：选模型裁判，高级选项里出现结论句裁判的子配置（裁判模型、三项上限各自能设不限、改写一次、证据不支持时）
|
||||
check('没选模型裁判时不出结论句裁判那一组', await p.locator('[data-field="judge"]').count() === 0)
|
||||
await p.locator('[data-field="claims"] select').selectOption('judge')
|
||||
check('选「请模型逐句判断」写进 config.claims = judge', (await cfgOf())?.claims === 'judge')
|
||||
await p.waitForSelector('[data-field="judge"]', { timeout: 3000 }).catch(() => {})
|
||||
const jf = p.locator('[data-field="judge"]')
|
||||
const jlabel = await jf.locator('label').first().innerText().catch(() => '')
|
||||
check('高级选项里出现「结论句裁判」一组（中文标签）', await jf.count() === 1 && jlabel.includes('结论句裁判'), jlabel)
|
||||
const keys = await jf.locator('[data-judge-key]').evaluateAll((els) => els.map((e) => e.getAttribute('data-judge-key')))
|
||||
check('……裁判模型、三项上限、改写一次、证据不支持时', keys.join(',') === 'model,max_claims,max_cost_usd,timeout_s,rewrite_once,on_unsupported',
|
||||
keys.join(','))
|
||||
const labels = await jf.innerText()
|
||||
// 细节只报缺了哪几个：整段文字里有沙箱的接入名（下拉的选项），日志常被贴进报告
|
||||
const missingLabels = ['裁判模型', '最多判几句', '金额上限（美元）', '时长上限（秒）', '交回改写一次', '证据不支持时'].filter((t) => !labels.includes(t))
|
||||
check('……每一项都有中文标签', !missingLabels.length, `缺：${missingLabels.join('、')}`)
|
||||
check('……没写的上限说明跟着设置里的默认值', (await jf.locator('[data-judge-key="max_cost_usd"] [data-judge-note]').innerText()).includes('默认值'))
|
||||
const cost = jf.locator('[data-judge-key="max_cost_usd"]')
|
||||
await cost.locator('input[type="checkbox"]').click()
|
||||
check('金额上限勾「不限」：写 null（不是删掉，也不是 0）', (await cfgOf())?.judge && (await cfgOf()).judge.max_cost_usd === null
|
||||
&& 'max_cost_usd' in (await cfgOf()).judge, JSON.stringify((await cfgOf())?.judge))
|
||||
const off = await cost.locator('[data-judge-note]').innerText().catch(() => '')
|
||||
check('……写明「不设上限，费用只受……约束」，数字框变灰', off.includes('不设上限') && off.includes('约束')
|
||||
&& await cost.locator('input[type="number"]').isDisabled(), off)
|
||||
await cost.locator('input[type="checkbox"]').click()
|
||||
check('勾掉「不限」：回到没写（跟着设置里的默认值）', !((await cfgOf())?.judge && 'max_cost_usd' in (await cfgOf()).judge),
|
||||
JSON.stringify((await cfgOf())?.judge))
|
||||
await cost.locator('input[type="number"]').fill('0.2')
|
||||
await jf.locator('[data-judge-key="max_claims"] input[type="number"]').fill('20')
|
||||
await jf.locator('[data-judge-key="timeout_s"] input[type="checkbox"]').click()
|
||||
await jf.locator('[data-judge-key="rewrite_once"] input[type="checkbox"]').click()
|
||||
await jf.locator('[data-judge-key="on_unsupported"] select').selectOption('withhold')
|
||||
const j = (await cfgOf())?.judge ?? {}
|
||||
check('写数就是数、时长不限写 null、改写一次写 true、不支持时写 withhold',
|
||||
j.max_cost_usd === 0.2 && j.max_claims === 20 && j.timeout_s === null && j.rewrite_once === true && j.on_unsupported === 'withhold',
|
||||
JSON.stringify(j))
|
||||
const provFirst = () => jf.locator('[data-judge-key="model"] select').evaluate((el) => el.options[0]?.textContent ?? '').catch(() => '')
|
||||
check('接入和模型都没写：接入留空那一项叫「跟随设置」', await provFirst() === '跟随设置' && await jf.locator('[data-judge-by-model]').count() === 0)
|
||||
await jf.locator('[data-judge-key="model"] input').fill('judge-model-b')
|
||||
check('裁判模型写进 judge.model', (await cfgOf())?.judge?.model === 'judge-model-b')
|
||||
// 节点上只写了模型：节点这一级整组生效（judge_model_spec 不跨级拼），按模型名找接入——已经不是「跟随设置」
|
||||
const pf = await provFirst()
|
||||
check('只写了模型：接入留空那一项不再叫「跟随设置」，改叫「按模型名找接入」', pf === '按模型名找接入', pf)
|
||||
const byModel = jf.locator('[data-judge-by-model]')
|
||||
const byText = await byModel.innerText().catch(() => '')
|
||||
// 说明里有沙箱的接入名：细节只报认没认出来，不打印原文
|
||||
check('……写明按模型名找接入、认不出就去调默认接入，两项都没写才跟随设置', await byModel.getAttribute('data-judge-by-model').catch(() => '') === 'default'
|
||||
&& byText.includes('只填了模型') && byText.includes('默认接入') && byText.includes('接入和模型都没写时才跟随设置'),
|
||||
`data-judge-by-model=${await byModel.getAttribute('data-judge-by-model').catch(() => '（没有）')}`)
|
||||
await jf.locator('[data-judge-key="model"] input').fill('')
|
||||
check('……模型清空：又回到「跟随设置」，说明也收起', await provFirst() === '跟随设置' && await byModel.count() === 0
|
||||
&& !('model' in ((await cfgOf())?.judge ?? {})))
|
||||
await jf.locator('[data-judge-key="model"] input').fill('judge-model-b')
|
||||
const previews = await p.evaluate(async () => {
|
||||
const url = performance.getEntriesByType('resource').map((e) => e.name)
|
||||
.find((n) => { try { return new URL(n).pathname === '/src/canvas/issues.ts' } catch { return false } })
|
||||
const m = await import(url ?? '/src/canvas/issues.ts')
|
||||
return [m.fixFieldLabel('judge.max_cost_usd', 'report'), m.fixValueText(null, 'judge.max_cost_usd'),
|
||||
m.fixValueText(0.05, 'judge.max_cost_usd'), (m.fixValueLines({ max_cost_usd: null, model: 'x' }, 'judge') ?? []).join('；'),
|
||||
m.fixValueText('withhold', 'judge.on_unsupported'), m.fixFieldLabel('defaults.model', 'report')]
|
||||
}).catch((e) => [String(e)])
|
||||
check('发布前修复的预览：judge.max_cost_usd 写成「结论句裁判 · 金额上限（美元）」', previews[0] === '结论句裁判 · 金额上限（美元）', previews[0])
|
||||
check('……null 写「不限」（没写才是「（空）」）、金额带 $', previews[1] === '不限' && previews[2] === '$0.05', JSON.stringify(previews.slice(1, 3)))
|
||||
check('……补预算按整份 judge 记时一键一行', previews[3] === '裁判模型：x；金额上限（美元）：不限', previews[3])
|
||||
check('……证据不支持时写选项文字', previews[4] === '不予出具', previews[4])
|
||||
check('……别处的 model 不借用裁判的叫法', previews[5] === '全图默认 · model', previews[5])
|
||||
if (SHOTS) {
|
||||
await jf.scrollIntoViewIfNeeded()
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await p.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await p.waitForTimeout(250)
|
||||
await p.screenshot({ path: `${SHOTS}/evidence4-inspector-judge-${theme}.png` })
|
||||
}
|
||||
}
|
||||
const fixLabel = await p.evaluate(async () => {
|
||||
// 按页面自己加载时的地址 import：改过的模块地址带 ?t=，直接写 /src/… 会拿到另一份实例
|
||||
const url = performance.getEntriesByType('resource').map((e) => e.name)
|
||||
@@ -1586,6 +1713,401 @@ await section('guess', '旧运行按数值猜的候选：默认折叠，展开
|
||||
}
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 第四期：结论句裁判
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const J = '#evidence-judge'
|
||||
const JX = '#evidence-judge-explore'
|
||||
const JP = '#evidence-judge-plain'
|
||||
const badge = (pg, sel, unit) => pg.locator(`${sel} [data-ev-claim="${unit}"]`)
|
||||
/** 令牌算出来的颜色（借查询步骤那节的 tokenColor 探针），和徽标的计算色比 */
|
||||
const colorOf = (pg, token) => tokenColor(pg, 'color', `var(${token})`)
|
||||
const activeKey = (pg) => pg.evaluate(() => document.activeElement?.getAttribute('data-seg')
|
||||
?? (document.activeElement?.getAttribute('data-ev-claim') ? `claim:${document.activeElement.getAttribute('data-ev-claim')}` : null)
|
||||
?? (document.activeElement?.closest('[data-evidence-panel]') ? 'panel' : document.activeElement?.tagName ?? null))
|
||||
|
||||
await section('judge', '四期：结论句的句末徽标——四通道、不用确定性的绿、整句不画线、n / N 也跳到证据不支持的句子', async () => {
|
||||
const u = fxj.formal.units
|
||||
const look = await page.evaluate(({ sel, units }) => Object.fromEntries(Object.entries(units).map(([k, id]) => {
|
||||
const el = document.querySelector(`${sel} [data-ev-claim="${id}"]`)
|
||||
if (!el) return [k, null]
|
||||
const cs = getComputedStyle(el)
|
||||
return [k, { verdict: el.getAttribute('data-ev-verdict'), glyph: el.textContent, color: cs.color, tag: el.tagName,
|
||||
type: el.getAttribute('type'), label: el.getAttribute('aria-label'), deco: cs.textDecorationLine,
|
||||
align: cs.verticalAlign }]
|
||||
})), { sel: J, units: u })
|
||||
check('有判定的结论句句末都挂徽标（有依据、证据不支持、部分有依据、未裁判）',
|
||||
look.ok?.verdict === 'supported' && look.bad?.verdict === 'unsupported' && look.partial?.verdict === 'partial'
|
||||
&& look.limit?.verdict === 'unjudged', JSON.stringify(Object.fromEntries(Object.entries(look).map(([k, v]) => [k, v?.verdict]))))
|
||||
check('判成「不是结论句」的不挂徽标', look.not === null)
|
||||
check('字形：◆ 有依据、◇ 部分、! 不支持、? 未裁判', [look.ok, look.partial, look.bad, look.limit].map((x) => x?.glyph).join('') === '◆◇!?',
|
||||
[look.ok, look.partial, look.bad, look.limit].map((x) => x?.glyph).join(''))
|
||||
const [accent, waiting, failed, dim, done] = await Promise.all(['--accent', '--st-waiting', '--st-failed', '--text-dim', '--st-done']
|
||||
.map((t) => colorOf(page, t)))
|
||||
check('颜色令牌：有依据 --accent、部分 --st-waiting、不支持 --st-failed、未裁判 --text-dim',
|
||||
look.ok?.color === accent && look.partial?.color === waiting && look.bad?.color === failed && look.limit?.color === dim,
|
||||
JSON.stringify({ got: [look.ok?.color, look.partial?.color, look.bad?.color, look.limit?.color], want: [accent, waiting, failed, dim] }))
|
||||
check('概率性的判定一个都不用确定性的绿', [look.ok, look.partial, look.bad, look.limit].every((x) => x && x.color !== done), done)
|
||||
check('徽标是原生 button type=button、上标、不画线', [look.ok, look.bad].every((x) => x?.tag === 'BUTTON' && x.type === 'button'
|
||||
&& x.align === 'super' && x.deco === 'none'), JSON.stringify(look.ok))
|
||||
check('aria-label 写全：句子和判定', look.bad?.label === '结论句「增长主要来自新客首单，老客复购持平。」，模型判断：证据不支持', look.bad?.label)
|
||||
check('……未裁判（到上限）的说「已到上限」', look.limit?.label?.endsWith('未裁判：已到上限'), look.limit?.label)
|
||||
const titles = await page.evaluate(({ a, b, c }) => [document.querySelector(a)?.getAttribute('title') ?? '',
|
||||
document.querySelector(b)?.getAttribute('title') ?? '', document.querySelector(c)?.getAttribute('title') ?? ''],
|
||||
{ a: `${J} [data-ev-claim="${u.bad}"]`, b: `${JX} [data-ev-claim="${fxj.explore.units.ok}"]`, c: `${J} [data-ev-claim="${u.limit}"]` })
|
||||
check('悬停说明：模型判过的写明谁判的、非确定', titles[0] === `模型判断：证据不支持（${fxj.model} · 非确定）`, titles[0])
|
||||
check('……还没有模型判过的（探索运行按需）不说谁判的，只说还没判', !titles[1].includes('非确定') && titles[1].startsWith('未裁判'),
|
||||
titles[1])
|
||||
check('……到上限没判的（记着裁判模型）也不写成「（模型 · 非确定）」，说已到上限、这句没判',
|
||||
titles[2].startsWith('未裁判：已到上限') && !titles[2].includes('非确定') && !titles[2].includes(fxj.model), titles[2])
|
||||
const shade = await page.evaluate(({ sel, bad, ok }) => {
|
||||
const box = document.querySelector(`${sel} [data-ev-claim-shade="${bad}"]`)
|
||||
const other = document.querySelector(`${sel} [data-ev-claim-shade="${ok}"]`)
|
||||
const probe = document.createElement('span')
|
||||
probe.style.background = 'var(--st-failed-soft)'
|
||||
document.body.appendChild(probe)
|
||||
const soft = getComputedStyle(probe).backgroundColor
|
||||
probe.remove()
|
||||
if (!box) return { found: false }
|
||||
const texts = [...box.querySelectorAll('span')].map((el) => getComputedStyle(el).textDecorationLine)
|
||||
return { found: true, bg: getComputedStyle(box).backgroundColor, soft, deco: getComputedStyle(box).textDecorationLine,
|
||||
texts, other: !!other, text: box.textContent }
|
||||
}, { sel: J, bad: u.bad, ok: u.ok })
|
||||
check('证据不支持的整句铺浅底（--st-failed-soft）', shade.found && shade.bg === shade.soft && shade.bg !== 'rgba(0, 0, 0, 0)',
|
||||
JSON.stringify(shade))
|
||||
check('……只有不支持的那句铺底', shade.found && !shade.other)
|
||||
check('整句的文字不画下划线（浅底里的字、别的结论句的字都没有线）', shade.found && shade.deco === 'none'
|
||||
&& shade.texts.every((d) => d === 'none'), JSON.stringify(shade.texts))
|
||||
const plainText = await page.evaluate(({ sel }) => {
|
||||
const b = document.querySelector(`${sel} [data-ev-claim]`)
|
||||
const sentence = b?.previousElementSibling
|
||||
return sentence ? getComputedStyle(sentence).textDecorationLine : ''
|
||||
}, { sel: J })
|
||||
check('……有依据的那句同样不画线（只有数字自己的线）', plainText === 'none' || plainText === '', plainText)
|
||||
|
||||
// 键盘：徽标在 roving 顺序里，句尾就是它；n / N 跳到证据不支持、部分有依据的句子
|
||||
const stops = await page.locator(`${J} [data-seg][tabindex="0"], ${J} [data-ev-claim][tabindex="0"]`).count()
|
||||
check('整份报告仍只有一个 Tab 位（徽标也在 roving 里）', stops === 1, String(stops))
|
||||
const firstNum = fxj.formal.doc.blocks.flatMap((b) => b.units).find((x) => x.id === u.ok).segments.filter((s) => s.kind === 'number').at(-1).id
|
||||
await page.locator(`${J} [data-seg="${firstNum}"]`).focus()
|
||||
await page.keyboard.press('ArrowRight')
|
||||
check('→ 从句子最后一个片段走到句末徽标', await activeKey(page) === `claim:${u.ok}`, await activeKey(page))
|
||||
const jumps = []
|
||||
for (const key of ['n', 'n', 'N']) {
|
||||
await page.keyboard.press(key)
|
||||
jumps.push(await activeKey(page))
|
||||
}
|
||||
check('n 跳到证据不支持、再到部分有依据的句子，N 回来', jumps.join(',') === `claim:${u.bad},claim:${u.partial},claim:${u.bad}`, jumps.join(','))
|
||||
check('只是走动不打开面板', await panel(page).count() === 0)
|
||||
await page.keyboard.press('Enter')
|
||||
await page.waitForSelector('[data-evidence-panel] [data-ev-judge]', { timeout: 4000 })
|
||||
check('回车打开这一句的「模型的解释」', (await panel(page).locator('[data-ev-judge]').getAttribute('data-ev-judge')) === 'unsupported')
|
||||
check('徽标标着展开了面板', await badge(page, J, u.bad).getAttribute('aria-expanded') === 'true')
|
||||
check('读屏从 live region 听到这一句的判定', (await page.locator(`${J} [aria-live="polite"]`).innerText()).includes('模型判断：证据不支持'))
|
||||
await page.keyboard.press('Escape')
|
||||
await page.waitForTimeout(150)
|
||||
check('Esc 关掉，焦点回到徽标', await panel(page).count() === 0 && await activeKey(page) === `claim:${u.bad}`, await activeKey(page))
|
||||
const tally = await page.locator(`${J} [data-evidence-claims]`).innerText().catch(() => '')
|
||||
check('证据条接着数结论句，和数字那一段同一种说法', tally.includes('结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）'), tally)
|
||||
const summary = await page.locator(`${J} [data-evidence-summary]`).innerText()
|
||||
check('读屏摘要也说结论句（写明是模型判断）', summary.includes('结论 4 句') && summary.includes('模型判断'), summary)
|
||||
check('对照：一期的报告（没有判定）一个徽标都不挂', await page.locator(`${wide} [data-ev-claim], ${wide} [data-ev-claim-shade]`).count() === 0)
|
||||
})
|
||||
|
||||
await section('j-panel', '四期：面板里「模型的解释」、「请模型判断这句」只在探索运行没判过时出现、触顶、封存后追加', async () => {
|
||||
const r = await open('/ui-harness.html?evidence=1')
|
||||
const pg = r.page
|
||||
const p = pg.locator('[data-evidence-panel]')
|
||||
const u = fxj.formal.units
|
||||
await badge(pg, J, u.bad).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]')
|
||||
const chip = await p.locator('[data-ev-judge-badge]').innerText().catch(() => '')
|
||||
check('带「模型判断 · 模型名 · 非确定」徽标', chip.trim() === `模型判断 · ${fxj.model} · 非确定`, chip)
|
||||
const why = await p.locator('[data-ev-rationale]').evaluate((el) => ({ text: el.textContent, style: getComputedStyle(el).fontStyle }))
|
||||
.catch(() => null)
|
||||
check('理由用斜体', why?.style === 'italic' && why.text.includes('看不出新客、老客'), JSON.stringify(why))
|
||||
const head = await p.locator('header [data-ev-badge]').evaluate((el) => ({ code: el.getAttribute('data-ev-badge'),
|
||||
style: getComputedStyle(el).borderTopStyle, text: el.textContent })).catch(() => null)
|
||||
check('面板标题的徽标：证据不支持、虚线框（概率性）', head?.code === 'unsupported' && head.style === 'dashed', JSON.stringify(head))
|
||||
check('正式运行节点里判的：不写「封存后追加」，写明随报告封存', await p.locator('[data-ev-post-seal]').count() === 0
|
||||
&& (await p.locator('[data-ev-judge-sealed]').innerText().catch(() => '')).includes('和报告一起封存'))
|
||||
check('正式运行：没有「请模型判断这句」', await p.locator('[data-ev-judge-ask]').count() === 0)
|
||||
const cites = await p.locator('[data-ev-cites]').innerText().catch(() => '')
|
||||
check('整句的面板列出挂的依据', cites.includes('Q1') && cites.includes('db_query__shop'), cites)
|
||||
await badge(pg, J, u.limit).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]')
|
||||
const lim = await p.locator('[data-ev-judge-limit]').innerText()
|
||||
check('到上限没判：写明「已到上限」和是哪个上限', lim.includes('已到上限') && lim.includes('这份报告的裁判金额上限 $0.05'), lim)
|
||||
check('……写明怎么调（正式运行：报告撰写节点的结论句裁判，可以设成不限）', lim.includes('报告撰写节点') && lim.includes('设成不限'), lim)
|
||||
check('……正式运行也不给按钮', await p.locator('[data-ev-judge-ask]').count() === 0)
|
||||
// 到上限没判的句子也记着裁判模型（judge 字段），但模型没看过它：不能画成「模型判断 · 模型名 · 非确定」
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-seal-state="done"]', { timeout: 3000 }).catch(() => {})
|
||||
check('……模型没看过这句：不挂「模型判断 · 模型名 · 非确定」', await p.locator('[data-ev-judge-badge]').count() === 0)
|
||||
check('……也不说这条判断「和报告一起封存」', await p.locator('[data-ev-judge-sealed]').count() === 0)
|
||||
const limSeal = await p.locator('[data-ev-seal]').innerText().catch(() => '')
|
||||
check('……封存那一节只说报告文档封存了', limSeal.includes('报告文档已封存') && !limSeal.includes('这条判断随报告一起封存'), limSeal)
|
||||
const limModel = await p.locator('[data-ev-judge-model]').innerText().catch(() => '')
|
||||
check('……中性地说一句裁判模型是谁、没判这句', limModel.includes(fxj.model) && limModel.includes('没有判这句'), limModel)
|
||||
// 点开结论句里的一个数字：片段面板里也有「模型的解释」（只对结论句）
|
||||
const num = fxj.formal.doc.blocks.flatMap((b) => b.units).find((x) => x.id === u.ok).segments.find((s) => s.kind === 'number').id
|
||||
await pg.locator(`${J} [data-seg="${num}"]`).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="supported"]')
|
||||
check('结论句里的片段：面板里有这一句的模型的解释', (await p.locator('[data-ev-verdict]').innerText()).includes('模型判断：有依据'))
|
||||
// 一期的报告：没有判定、没开裁判、不知道运行类别也取不到——不画这一节
|
||||
await pg.locator(`${wide} [data-seg="s8"]`).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-substituted]')
|
||||
await pg.waitForTimeout(300)
|
||||
check('对照：没有判定的老报告不画「模型的解释」', await p.locator('[data-ev-judge]').count() === 0)
|
||||
await pg.keyboard.press('Escape')
|
||||
|
||||
// 探索运行、开了裁判（按需）：还没判过的句子给按钮
|
||||
const x = fxj.explore.units
|
||||
await badge(pg, JX, x.bad).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-ask]')
|
||||
check('探索运行、还没判过：有「请模型判断这句」', (await p.locator('[data-ev-judge-ask]').innerText()).includes('请模型判断这句'))
|
||||
check('……还没判过：没有模型徽标（还没有模型判过）', await p.locator('[data-ev-judge-badge]').count() === 0)
|
||||
await p.locator('[data-ev-judge-ask]').click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]').catch(() => {})
|
||||
const sent = r.hits.filter((h) => h.startsWith(`judge:${fxj.explore.run_id}:`))
|
||||
check('请求体带着句子和报告节点', sent.length === 1 && sent[0].endsWith(`:${x.bad}:write`), sent.join(' | '))
|
||||
check('判完：写「封存后追加」', (await p.locator('[data-ev-post-seal]').innerText().catch(() => '')).trim() === '封存后追加')
|
||||
check('……答复没带话（判成了）：不多写一行', await p.locator('[data-ev-judge-message]').count() === 0)
|
||||
check('……封存那一节也写明封存后追加（报告文档仍是封存的）', (await p.locator('[data-ev-seal-late]').innerText().catch(() => '')).includes('封存后追加')
|
||||
&& (await p.locator('[data-ev-seal]').innerText()).includes('报告文档已封存'))
|
||||
check('……判过了就不再给按钮', await p.locator('[data-ev-judge-ask]').count() === 0)
|
||||
const after = await badge(pg, JX, x.bad).evaluate((el) => ({ v: el.getAttribute('data-ev-verdict'), late: el.hasAttribute('data-ev-post-seal') }))
|
||||
check('正文的句末徽标跟着变：证据不支持、标着封存后追加', after.v === 'unsupported' && after.late, JSON.stringify(after))
|
||||
check('同一份报告在窄栏里的那份也跟着变（按运行和报告记）',
|
||||
await badge(pg, '#evidence-judge-explore-narrow', x.bad).getAttribute('data-ev-verdict') === 'unsupported')
|
||||
check('证据条的结论句计数跟着变', (await pg.locator(`${JX} [data-evidence-claims]`).innerText()).includes('不支持 1'))
|
||||
await badge(pg, JX, x.bad).click()
|
||||
await pg.waitForTimeout(100)
|
||||
await badge(pg, JX, x.bad).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]')
|
||||
check('同一句再点开：直接用已有的判定，不再请求', r.hits.filter((h) => h.startsWith(`judge:${fxj.explore.run_id}:`)).length === 1)
|
||||
// 触顶：不报错，说明「已到上限」和怎么调（每次点击的上限），还能再点一次
|
||||
await badge(pg, JX, x.limit).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-ask]')
|
||||
await p.locator('[data-ev-judge-ask]').click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]')
|
||||
const top = await p.locator('[data-ev-judge-limit]').innerText()
|
||||
check('按需裁判触顶：写「已到上限」和这次点击的上限', top.includes('已到上限') && top.includes('这次点击的裁判金额上限 $0.01'), top)
|
||||
check('……写明怎么调：设置里「每次点击的金额上限」、可以设成不限', top.includes('每次点击的金额上限') && top.includes('设成不限')
|
||||
&& await p.locator('[data-ev-judge-settings]').count() === 1, top)
|
||||
check('……不是报错', await p.locator('[data-ev-judge-error]').count() === 0)
|
||||
check('……还能再请模型判断一次', (await p.locator('[data-ev-judge-ask]').innerText().catch(() => '')).includes('再请模型判断一次'))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await pg.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await badge(pg, JX, x.bad).click()
|
||||
await pg.waitForTimeout(100)
|
||||
if (await p.count() === 0) await badge(pg, JX, x.bad).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]')
|
||||
await pg.waitForTimeout(250)
|
||||
await pg.screenshot({ path: `${SHOTS}/evidence4-explore-judged-${theme}.png` })
|
||||
await badge(pg, J, u.limit).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]')
|
||||
await pg.waitForTimeout(250)
|
||||
await pg.screenshot({ path: `${SHOTS}/evidence4-formal-limit-${theme}.png` })
|
||||
await pg.keyboard.press('Escape')
|
||||
}
|
||||
await pg.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
|
||||
// 没开裁判的探索运行：文档里不记运行类别，面板按运行 id 取一次；结论句照样能请模型判断
|
||||
const pu = fxj.plain.doc.blocks.flatMap((b) => b.units).find((x2) => x2.id === 'u3')
|
||||
const pnum = pu.segments.find((s) => s.kind === 'number').id
|
||||
await pg.locator(`${JP} [data-seg="${pnum}"]`).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-ask]').catch(() => {})
|
||||
check('没开裁判的报告：按运行 id 认出是探索运行，给按钮', r.hits.includes(`run:${fxj.plain.run_id}`)
|
||||
&& await p.locator('[data-ev-judge-ask]').count() === 1)
|
||||
check('……判之前正文没有徽标', await pg.locator(`${JP} [data-ev-claim]`).count() === 0)
|
||||
await p.locator('[data-ev-judge-ask]').click().catch(() => {})
|
||||
await pg.waitForSelector(`${JP} [data-ev-claim="u3"]`).catch(() => {})
|
||||
check('……判完正文挂上徽标（封存后追加）', await badge(pg, JP, 'u3').getAttribute('data-ev-verdict') === 'supported'
|
||||
&& await badge(pg, JP, 'u3').getAttribute('data-ev-post-seal') === '')
|
||||
const ptally = await pg.locator(`${JP} [data-evidence-claims]`).innerText().catch(() => '')
|
||||
check('……证据条多出结论句：挂了依据没判的算未裁判、没挂的算无证据', ptally.includes('结论 5 句（支持 1 · 未裁判 3 · 无证据 1）'), ptally)
|
||||
check('没有运行时报错（裁判面板）', r.errors.length === 0, r.errors.join(' | '))
|
||||
await r.ctx.close()
|
||||
|
||||
// 片段接口说暂时判不了（运行还没跑完封存）：不给按钮，照原话说为什么
|
||||
const un = await open('/ui-harness.html?evidence=1', { segPatch: (set, body) => (set === fxj.explore
|
||||
? { ...body, unit: { ...body.unit, on_demand: { available: false, reason: 'unsealed', message: '运行还没跑完封存，跑完再请模型判断' } } } : body) })
|
||||
const tnum = fxj.explore.doc.blocks.flatMap((b) => b.units).find((x2) => x2.id === x.ok).segments.find((s) => s.kind === 'number').id
|
||||
await un.page.locator(`${JX} [data-seg="${tnum}"]`).click()
|
||||
await un.page.waitForSelector('[data-evidence-panel] [data-ev-judge]')
|
||||
await un.page.waitForTimeout(300)
|
||||
check('接口说运行还没封存：不给按钮，照原话说', await un.page.locator('[data-ev-judge-ask]').count() === 0
|
||||
&& (await un.page.locator('[data-ev-judge-blocked]').innerText()).includes('跑完再请模型判断'))
|
||||
await un.ctx.close()
|
||||
// 接口 409（正式运行、封存被改过）：不白屏，照后端原话说
|
||||
const e409 = await open('/ui-harness.html?evidence=1', { judge: () => ({ status: 409, json: { detail: '封存核对没通过：不再请模型判断', code: 'evidence_seal_broken' } }) })
|
||||
await badge(e409.page, JX, x.bad).click()
|
||||
await e409.page.locator('[data-ev-judge-ask]').click()
|
||||
await e409.page.waitForSelector('[data-evidence-panel] [data-ev-judge-error]')
|
||||
check('按需裁判回 409：照后端原话说，不白屏', (await e409.page.locator('[data-ev-judge-error]').innerText()).includes('封存核对没通过'))
|
||||
check('……徽标还是未裁判', await badge(e409.page, JX, x.bad).getAttribute('data-ev-verdict') === 'unjudged')
|
||||
await e409.ctx.close()
|
||||
// 答复里的那句话（message）：判定照样交回来，但运行已经接着跑了、没记进运行记录——得照原话写出来，
|
||||
// 不能只挂一枚「封存后追加」让人以为记进去了。触顶的那句已经由上限那一框说了，不重复
|
||||
const lostText = '运行已经接着跑了，这次的判定没有记进运行记录（只在这里看得到，刷新就没了）；等它跑完封存，再请模型判断'
|
||||
const lost = await open('/ui-harness.html?evidence=1', { judge: (set, body) => {
|
||||
const unit = body?.units?.[0]
|
||||
if (set === fxj.explore && unit === x.limit) {
|
||||
return { json: { ...fxj.explore.judge.limit, message: '已到上限（这次点击的裁判金额上限 $0.01）：1 句没判，记为未裁判；已判的保留' } }
|
||||
}
|
||||
return { json: { report: { node_id: 'write', doc_artifact: set.doc_artifact }, model: fxj.model, limits_hit: [], post_seal: true,
|
||||
message: lostText, event: null, calls: 1,
|
||||
verdicts: { [unit]: { status: 'supported', rationale: '引用的证据支持这句话', judge: fxj.model, post_seal: true, used: [] } } } }
|
||||
} })
|
||||
const lp = lost.page.locator('[data-evidence-panel]')
|
||||
await badge(lost.page, JX, x.ok).click()
|
||||
await lp.locator('[data-ev-judge-ask]').click().catch(() => {})
|
||||
await lost.page.waitForSelector('[data-evidence-panel] [data-ev-judge="supported"]').catch(() => {})
|
||||
const lostMsg = await lp.locator('[data-ev-judge-message]').innerText().catch(() => '')
|
||||
check('答复带着「没有记进运行记录」：照原话写在判定下面', lostMsg.includes('没有记进运行记录') && lostMsg.includes('刷新就没了'), lostMsg || '（没写）')
|
||||
check('……判定照样显示（有依据）', (await lp.locator('[data-ev-verdict]').innerText().catch(() => '')).includes('模型判断：有依据'))
|
||||
await badge(lost.page, JX, x.limit).click()
|
||||
await lp.locator('[data-ev-judge-ask]').click().catch(() => {})
|
||||
await lost.page.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]').catch(() => {})
|
||||
check('触顶那句话已经由上限那一框说了：不再另写一遍', await lp.locator('[data-ev-judge-limit]').count() === 1
|
||||
&& await lp.locator('[data-ev-judge-message]').count() === 0)
|
||||
check('没有运行时报错（答复里的那句话）', lost.errors.length === 0, lost.errors.join(' | '))
|
||||
await lost.ctx.close()
|
||||
})
|
||||
|
||||
await section('j-narrow', '四期在 360px：徽标和「模型的解释」栏内展开不横向滚动；减少动效', async () => {
|
||||
const narrow = '#evidence-judge-narrow'
|
||||
const overflow = () => page.evaluate((sel) => {
|
||||
const el = document.querySelector(sel)
|
||||
return { sw: el.scrollWidth, cw: el.clientWidth, w: el.getBoundingClientRect().width }
|
||||
}, narrow)
|
||||
const before = await overflow()
|
||||
await badge(page, narrow, fxj.formal.units.bad).click()
|
||||
await page.waitForSelector(`${narrow} [data-evidence-panel="inline"] [data-ev-judge]`)
|
||||
const after = await overflow()
|
||||
check('窄栏里徽标照样挂在句末', await page.locator(`${narrow} [data-ev-claim]`).count() === 4)
|
||||
check('窄栏里没有横向滚动（面板打开前后）', Math.round(before.w) === 360 && before.sw <= before.cw && after.sw <= after.cw,
|
||||
JSON.stringify({ before, after }))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(200)
|
||||
await page.locator(narrow).screenshot({ path: `${SHOTS}/evidence4-360-inline-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await page.locator(`${narrow} [data-evidence-panel] button`, { hasText: '回到正文' }).click()
|
||||
await page.waitForTimeout(150)
|
||||
const small = await open('/ui-harness.html?evidence=1', { w: 360, h: 780 })
|
||||
const scroll = () => small.page.evaluate(() => ({ sw: document.documentElement.scrollWidth, vw: innerWidth }))
|
||||
const a = await scroll()
|
||||
await badge(small.page, J, fxj.formal.units.bad).click()
|
||||
await small.page.waitForSelector('[data-evidence-panel] [data-ev-judge]')
|
||||
const b = await scroll()
|
||||
check('360px 宽的屏幕：整句的面板从底部抽出、整页没有横向滚动',
|
||||
await small.page.locator('[data-evidence-panel]').getAttribute('data-evidence-panel') === 'drawer' && a.sw <= a.vw && b.sw <= b.vw,
|
||||
JSON.stringify({ a, b }))
|
||||
await small.ctx.close()
|
||||
const r = await open('/ui-harness.html?evidence=1', { reduced: true })
|
||||
await badge(r.page, J, fxj.formal.units.bad).click()
|
||||
await r.page.waitForSelector('[data-evidence-panel] [data-ev-judge]').catch(() => {})
|
||||
const still = await r.page.evaluate(() => {
|
||||
const p = document.querySelector('[data-evidence-panel]')
|
||||
return { n: p ? p.getAnimations({ subtree: true })
|
||||
.filter((a2) => !(a2 instanceof CSSTransition && Number(a2.effect?.getTiming().duration) <= 0.01)).length : -1,
|
||||
name: p ? getComputedStyle(p).animationName : '' }
|
||||
})
|
||||
check('减少动效时整句的面板没有任何动画', still.n === 0 && still.name === 'none', JSON.stringify(still))
|
||||
await r.ctx.close()
|
||||
})
|
||||
|
||||
await section('j-banner', '四期：出具横幅多一段「结论 N 句（支持 a · 无证据 b）」', async () => {
|
||||
const at = (key) => page.locator(`#issuance-claims [data-issuance-case="${key}"]`)
|
||||
const judged = await at('judged').locator('[data-evidence-line]').innerText().catch(() => '')
|
||||
check('结论句那一段：和数字那一段同一种说法', judged.includes('3 个数字都有出处') && judged.includes('结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）'), judged)
|
||||
check('有证据不支持的结论句：给「定位下一处」', await at('judged').locator('[data-evidence-next]').count() === 1)
|
||||
await page.evaluate(() => { window.__claimsNext = 0 })
|
||||
await at('judged').locator('[data-evidence-next]').click()
|
||||
check('……点了就去找下一处', await page.evaluate(() => window.__claimsNext) === 1)
|
||||
const cited = await at('cited').locator('[data-evidence-line]').innerText().catch(() => '')
|
||||
check('方案的例子：结论 4 句（支持 3 · 无证据 1）', cited.includes('结论 4 句（支持 3 · 无证据 1）'), cited)
|
||||
check('……没挂依据的算进「无证据」，不再另起「没挂依据的结论句」', await at('cited').locator('[data-evidence-line-claims]').count() === 0)
|
||||
const all = at('all').locator('[data-evidence-claims]')
|
||||
const [color, done] = [await all.evaluate((el) => getComputedStyle(el).color).catch(() => ''), await colorOf(page, '--st-done')]
|
||||
check('全都支持：不给「定位下一处」，也不用确定性的绿', await at('all').locator('[data-evidence-next]').count() === 0 && !!color && color !== done,
|
||||
`${color} / ${done}`)
|
||||
check('对照：三期那一份（没有判定）照旧写「没挂依据的结论句」', await page.locator('#issuance-line [data-issuance-case="suspect"] [data-evidence-line-claims]').count() === 1)
|
||||
// 开了裁判的文档：没挂依据的那句送了裁判、按判定数进了「支持 3」，出具照样按没挂依据记 1 句——不能被吞掉
|
||||
const ju = await at('judge-uncited').locator('[data-evidence-line]').innerText().catch(() => '')
|
||||
check('开了裁判、出具还记着没挂依据的结论句：结论句那一段旁边另起「没挂依据的结论句 1」',
|
||||
ju.includes('结论 3 句（支持 3）') && await at('judge-uncited').locator('[data-evidence-line-claims]').count() === 1
|
||||
&& ju.includes('· 没挂依据的结论句 1'), ju)
|
||||
const juTitle = await at('judge-uncited').locator('[data-evidence-line-claims]').getAttribute('title').catch(() => '')
|
||||
check('……悬停说明：判定代替不了依据，出具照样计缺口', !!juTitle && juTitle.includes('代替不了依据') && juTitle.includes('缺口'), juTitle ?? '')
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(200)
|
||||
await page.locator('#issuance-claims').screenshot({ path: `${SHOTS}/evidence4-banner-${theme}.png` })
|
||||
await page.locator(J).screenshot({ path: `${SHOTS}/evidence4-doc-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
})
|
||||
|
||||
await section('j-stream', '四期：问数据页、画布右栏的横幅（Output 这条路）——按需判过的结论句数进横幅，开了裁判的报告没挂依据的照样写出来', async () => {
|
||||
// 按需裁判把「退款 29 单，其中大多数是部分退款」判成证据不支持（这句有数字，点得开）
|
||||
const pu = fxj.plain.doc.blocks.flatMap((b) => b.units).find((x2) => x2.id === 'u3')
|
||||
const pnum = pu.segments.find((x2) => x2.kind === 'number').id
|
||||
const s = await open('/preview.html?syn=evidence-judge', { judge: (set, body) => (set === fxj.plain ? { json: {
|
||||
report: { node_id: 'write', doc_artifact: set.doc_artifact }, model: fxj.model, limits_hit: [], post_seal: true,
|
||||
verdicts: { [body.units[0]]: { status: 'unsupported', rationale: 'Q2 里部分退款 11 单，说不上「大多数」', judge: fxj.model, post_seal: true, used: ['Q2'] } },
|
||||
} } : judgeReply(set, body)) })
|
||||
const p = s.page
|
||||
const turn = (id) => p.locator(`[data-turn="${id}"]`)
|
||||
await p.waitForSelector('[data-turn="judge-plain"] [data-evidence-line]', { timeout: 6000 }).catch(() => {})
|
||||
await p.waitForSelector('[data-turn="judge-formal"] [data-evidence-line]', { timeout: 6000 }).catch(() => {})
|
||||
const plainLine = () => turn('judge-plain').locator('[data-evidence-line]').innerText().catch(() => '')
|
||||
const before = await plainLine()
|
||||
check('没开裁判、还没判过：横幅照旧写「没挂依据的结论句 1」，没有结论句那一段',
|
||||
before.includes('没挂依据的结论句 1') && await turn('judge-plain').locator('[data-evidence-claims]').count() === 0, before)
|
||||
// 点开那一句的数字，请模型判断（Output 没给运行类别：按运行 id 认出探索运行）
|
||||
await turn('judge-plain').locator(`[data-seg="${pnum}"]`).click().catch(() => {})
|
||||
const ask = p.locator('[data-evidence-panel] [data-ev-judge-ask]')
|
||||
await ask.waitFor({ timeout: 6000 }).catch(() => {})
|
||||
await ask.click().catch(() => {})
|
||||
await p.waitForSelector('[data-turn="judge-plain"] [data-ev-claim="u3"]', { timeout: 6000 }).catch(() => {})
|
||||
await p.waitForSelector('[data-turn="judge-plain"] [data-evidence-line] [data-evidence-claims]', { timeout: 3000 }).catch(() => {})
|
||||
check('……请模型判断那一下发出去了（按运行、报告、句子）', s.hits.includes(`judge:${fxj.plain.run_id}:u3:write`),
|
||||
s.hits.filter((h) => h.startsWith('judge:')).join(' '))
|
||||
const claims = await turn('judge-plain').locator('[data-evidence-line] [data-evidence-claims]').innerText().catch(() => '')
|
||||
check('按需判过一句：横幅多出结论句那一段（EvidenceField 叠好判定交出的文档，docTally 照样数）',
|
||||
claims.includes('结论 5 句（不支持 1 · 未裁判 3 · 无证据 1）'), claims || '（没有这一段）')
|
||||
const after = await plainLine()
|
||||
check('……没挂依据的那句已经数在「无证据」里，不再另起一句', !!claims && await turn('judge-plain').locator('[data-evidence-line-claims]').count() === 0, after)
|
||||
check('……有证据不支持的句子：横幅给「定位下一处」', !!claims && await turn('judge-plain').locator('[data-evidence-line] [data-evidence-next]').count() === 1, after)
|
||||
await p.keyboard.press('Escape')
|
||||
const formal = await turn('judge-formal').locator('[data-evidence-line]').innerText().catch(() => '')
|
||||
check('开了裁判的正式运行：结论句那一段照节点里的判定数', formal.includes('结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）'), formal)
|
||||
check('……没挂依据的那句（送了裁判、到上限没判）出具照样计缺口：另起「没挂依据的结论句 1」',
|
||||
await turn('judge-formal').locator('[data-evidence-line-claims]').count() === 1 && formal.includes('没挂依据的结论句 1'), formal)
|
||||
check('没有运行时报错（问数据页 · 结论句裁判）', s.errors.length === 0, s.errors.join(' | '))
|
||||
if (SHOTS) {
|
||||
// 侧面板还开着会压住横幅的右半边：先关掉
|
||||
await p.locator('[data-evidence-panel] button[aria-label="关闭证据"]').click().catch(() => {})
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await p.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await p.waitForTimeout(200)
|
||||
await turn('judge-plain').locator('[data-issuance-banner]').screenshot({ path: `${SHOTS}/evidence4-stream-plain-${theme}.png` })
|
||||
await turn('judge-formal').locator('[data-issuance-banner]').screenshot({ path: `${SHOTS}/evidence4-stream-formal-${theme}.png` })
|
||||
}
|
||||
}
|
||||
await s.ctx.close()
|
||||
})
|
||||
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
|
||||
@@ -609,6 +609,95 @@ await section('设置 · 智能体护栏', async () => {
|
||||
await t.close()
|
||||
})
|
||||
|
||||
await section('设置 · 证据裁判', async () => {
|
||||
// 结论句裁判的模型和各项上限（后端 engine/judge.py）：每一项都能设成不限（存 null），不限时写明还受什么约束。
|
||||
// 读写照运行默认值：GET 读、PUT 整组写。写请求在浏览器层拦下，不落库
|
||||
let stored = await get('/settings')
|
||||
stored = { ...stored, judge: { provider: null, model: null, report_max_claims: 40, report_max_cost_usd: null, report_timeout_s: 30,
|
||||
click_max_cost_usd: 0.01, daily_max_usd: 2 } }
|
||||
let spendGets = 0
|
||||
const { page, sent, errors, close } = await open('/settings/prefs', {
|
||||
handlers: [
|
||||
[/^GET \/settings$/, (route) => json(stored)(route)],
|
||||
[/^PUT \/settings$/, (route, { body }) => { stored = { ...stored, ...body.values }; return json(stored)(route) }],
|
||||
[/^GET \/settings\/judge\/spend$/, (route) => { spendGets += 1; return json({ date: '2026-09-29', usd: 0.0123, calls: 7, unpriced_calls: 2, daily_max_usd: 2 })(route) }],
|
||||
],
|
||||
})
|
||||
const box = page.locator('[data-judge-settings]')
|
||||
const all = (await box.innerText().catch(() => '')).replace(/\s+/g, ' ')
|
||||
check('偏好里有「证据裁判」一组，写明判断是模型给的、不是系统核对', all.includes('证据裁判') && all.includes('非确定'), all.slice(0, 80))
|
||||
check('裁判模型：接入和模型，提示建议和写作模型不同', await page.locator('#pref-judge-provider').count() === 1
|
||||
&& await page.locator('#pref-judge-model').count() === 1 && all.includes('建议和写报告的模型不同'))
|
||||
check('……接入留空时跟随 Copilot 的模型', (await page.locator('#pref-judge-provider option').first().innerText()).includes('跟随 Copilot'))
|
||||
const val = (id) => page.locator(`#${id}`).inputValue()
|
||||
check('读回各项上限：40 句、30 秒、每次点击 $0.01、每日 $2', await val('pref-judge-claims') === '40' && await val('pref-judge-timeout') === '30'
|
||||
&& await val('pref-judge-click') === '0.01' && await val('pref-judge-daily') === '2')
|
||||
const cost = page.locator('[data-judge-limit="report_max_cost_usd"]')
|
||||
check('存着 null 的每份报告金额：勾着「不限」、框变灰', await cost.locator('[data-judge-unlimited-toggle]').isChecked()
|
||||
&& await page.locator('#pref-judge-cost').isDisabled())
|
||||
const costNote = await cost.locator('[data-judge-unlimited]').innerText().catch(() => '')
|
||||
check('……写明「不设上限，费用只受……约束」', costNote === '不设上限，费用只受句数上限、时长上限、每日上限约束', costNote)
|
||||
const spend = await page.locator('[data-judge-spend]').innerText().catch(() => '')
|
||||
check('每日上限旁边写今天花了多少', spend.includes('今天已花 $0.0123（7 次调用）'), spend)
|
||||
check('……估不出金额的调用另说', spend.includes('另有 2 次调用估不出金额') && await page.locator('[data-judge-unpriced]').count() === 1, spend)
|
||||
|
||||
const daily = page.locator('[data-judge-limit="daily_max_usd"]')
|
||||
await daily.locator('[data-judge-unlimited-toggle]').check()
|
||||
const dailyNote = await daily.locator('[data-judge-unlimited]').innerText().catch(() => '')
|
||||
check('每日上限设成不限：写明每日费用还受什么约束', dailyNote.startsWith('不设上限，每日费用只受') && dailyNote.includes('每次点击的金额上限'), dailyNote)
|
||||
const costNote2 = await cost.locator('[data-judge-unlimited]').innerText().catch(() => '')
|
||||
check('……每日也不限了：每份报告金额那句不再说「每日上限」', !costNote2.includes('每日') && costNote2.startsWith('不设上限'), costNote2)
|
||||
await page.locator('#pref-judge-click').fill('0')
|
||||
const save = page.getByRole('button', { name: /保存设置/ })
|
||||
check('每次点击填 0：标红、保存按钮不可用', await save.isDisabled()
|
||||
&& (await page.locator('[data-judge-limit="click_max_cost_usd"]').innerText()).includes('填大于 0 的金额，或者勾「不限」'))
|
||||
await page.locator('#pref-judge-click').fill('0.02')
|
||||
check('……接入留空时模型框的占位也写跟随 Copilot', await page.locator('#pref-judge-model').getAttribute('placeholder') === '留空：跟随 Copilot 的模型')
|
||||
await page.locator('#pref-judge-model').fill('judge-model-b')
|
||||
// 只填了模型：设置这一级整组生效（judge_model_spec 不跨级拼），后端按模型名找接入——已经不跟随 Copilot
|
||||
const provFirst = await page.locator('#pref-judge-provider option').first().innerText().catch(() => '')
|
||||
check('只填了模型：接入留空那一项不再叫「跟随 Copilot」，改叫「按模型名找接入」', provFirst === '按模型名找接入', provFirst)
|
||||
const byModel = page.locator('[data-judge-by-model]')
|
||||
const byText = await byModel.innerText().catch(() => '')
|
||||
// 说明里有沙箱的接入名：细节只报认没认出来，不打印原文
|
||||
check('……写明按模型名找接入（认不出就去调默认接入），「跟随 Copilot」只在两项都留空时生效',
|
||||
await byModel.getAttribute('data-judge-by-model').catch(() => '') === 'default' && byText.includes('只填了模型')
|
||||
&& byText.includes('默认接入') && byText.includes('「跟随 Copilot」只在接入和模型都留空时生效'),
|
||||
`data-judge-by-model=${await byModel.getAttribute('data-judge-by-model').catch(() => '（没有）')}`)
|
||||
const bar = (await page.locator('[data-prefs-bar]').innerText().catch(() => '')).replace(/\s+/g, ' ')
|
||||
check('保存条写明改了哪几项', bar.includes('有 3 项未保存') && bar.includes('每日金额上限') && bar.includes('每次点击的金额上限')
|
||||
&& bar.includes('裁判模型'), bar)
|
||||
const gets = spendGets
|
||||
await save.click()
|
||||
await page.waitForTimeout(500)
|
||||
const put = sent.filter((s) => s.key === 'PUT /settings').at(-1)?.body?.values ?? {}
|
||||
const j = put.judge ?? {}
|
||||
check('保存：整组写 judge（七项都在），不限存 null', Object.keys(j).sort().join(',')
|
||||
=== 'click_max_cost_usd,daily_max_usd,model,provider,report_max_claims,report_max_cost_usd,report_timeout_s'
|
||||
&& j.daily_max_usd === null && j.report_max_cost_usd === null && j.click_max_cost_usd === 0.02 && j.report_max_claims === 40
|
||||
&& j.report_timeout_s === 30 && j.model === 'judge-model-b' && j.provider === null, JSON.stringify(j))
|
||||
check('……只改了裁判：不碰运行默认值那一组', !('run' in put), Object.keys(put).join(','))
|
||||
check('……存完再取一次今天的花费（每日上限变了）', spendGets > gets, `${gets} → ${spendGets}`)
|
||||
check('……存完显示「已保存」', (await page.locator('[data-prefs-bar]').innerText().catch(() => '')).includes('已保存'))
|
||||
await daily.locator('[data-judge-unlimited-toggle]').uncheck()
|
||||
check('勾掉「不限」：填回原来的数', await val('pref-judge-daily') === '2')
|
||||
await daily.locator('[data-judge-unlimited-toggle]').check()
|
||||
check('没有运行时报错', errors.length === 0, errors[0] ?? '')
|
||||
if (SHOTS) {
|
||||
await box.scrollIntoViewIfNeeded()
|
||||
await page.waitForTimeout(200)
|
||||
await box.screenshot({ path: `${SHOTS}/prefs-judge-${THEME}.png` })
|
||||
}
|
||||
await close()
|
||||
|
||||
// 老后端没有 judge 这一组：不显示，也不写
|
||||
const legacy = { ...stored }
|
||||
delete legacy.judge
|
||||
const old = await open('/settings/prefs', { handlers: [[/^GET \/settings$/, (route) => json(legacy)(route)]] })
|
||||
check('老后端没有裁判设置：这一组不出现', await old.page.locator('[data-judge-settings]').count() === 0)
|
||||
await old.close()
|
||||
})
|
||||
|
||||
await section('数据 · 数据库', async () => {
|
||||
const dbs = sources.filter((s) => !/[\\/]uploads[\\/]tables[\\/]/.test(s.database ?? ''))
|
||||
const first = dbs.find((s) => s.table_count > 0) ?? dbs[0]
|
||||
@@ -1441,8 +1530,10 @@ await section('工具', async () => {
|
||||
check('……详情区写明原因和「绑了它的节点一定失败」', (await alert.innerText().catch(() => '')).includes('一定失败'))
|
||||
await shot(page, 'tool-broken-detail')
|
||||
await alert.getByRole('button', { name: '去改参数定义' }).click()
|
||||
await page.waitForTimeout(700)
|
||||
const ed0 = page.getByRole('dialog', { name: '编辑工具「broken_demo」' })
|
||||
// 等编辑框真的出来再判：并行跑时固定等 700ms 不够，编辑框还没挂上就判成没打开（check-all 里偶发过）
|
||||
await ed0.waitFor({ timeout: 5000 }).catch(() => {})
|
||||
await page.waitForTimeout(200)
|
||||
check('……「去改参数定义」直接打开它的编辑框（/tools/custom?edit=）', await ed0.count() === 1, page.url())
|
||||
check('……读完 ?edit= 就从地址里摘掉', !page.url().includes('edit='), page.url())
|
||||
const perr = await ed0.locator('[data-params-field]').innerText().catch(() => '')
|
||||
|
||||
@@ -308,6 +308,8 @@ await section('lib/evidence：证据状态的四通道元数据、报告节点
|
||||
unknown_entities: 2, uncited_claims: 1 }, 'require_citation')), JSON.stringify({ cited: 10, none: 5 }))
|
||||
eq('……结论句策略是 off 时不算没挂依据的结论句', ev.stampCounts({ numbers: 4, numbers_cited: 3, uncited_numbers: 1, unresolved: 0,
|
||||
uncited_claims: 3 }, 'off')?.none, 1)
|
||||
eq('……judge 在挂依据上和 require_citation 一样严：没挂依据的结论句也算', ev.stampCounts({ numbers: 4, numbers_cited: 3,
|
||||
uncited_numbers: 1, unresolved: 0, uncited_claims: 3 }, 'judge')?.none, 4)
|
||||
eq('可疑实体的提示：目录键、{name, table} 都写成去掉前缀的名字，最多 3 个',
|
||||
ev.closestNames(['c:orders.amount', { alias: 't:refunds' }, { name: 'region', table: 'orders', kind: 'column' }, 'c:x']).join(','),
|
||||
'orders.amount,refunds,orders.region')
|
||||
@@ -409,6 +411,90 @@ await section('lib/evidence：证据状态的四通道元数据、报告节点
|
||||
})) check(name, ok, detail)
|
||||
})
|
||||
|
||||
await section('lib/evidence：结论句裁判（四期）——判定的徽标、计数、答复形状、改写一次', async () => {
|
||||
for (const [name, ok, detail] of await page.evaluate(() => {
|
||||
const { evidence: ev, terms: t } = window.__ui.lib
|
||||
const fj = window.__ui.evidenceJudge
|
||||
const out = []
|
||||
const eq = (name, got, want) => out.push([name, got === want, got === want ? '' : `得到 ${JSON.stringify(got)}，应为 ${JSON.stringify(want)}`])
|
||||
const ok = (name, cond, detail = '') => out.push([name, !!cond, cond ? '' : detail])
|
||||
const S = ev.EVIDENCE_STATE
|
||||
ok('概率性四种都挂在句末徽标上（line=badge），字形 ◆ ◇ ! ?', ['supported', 'partial', 'unsupported', 'unjudged']
|
||||
.map((c) => `${S[c].line}/${S[c].glyph}`).join(' ') === 'badge/◆ badge/◇ badge/! badge/?',
|
||||
['supported', 'partial', 'unsupported', 'unjudged'].map((c) => `${S[c].line}/${S[c].glyph}`).join(' '))
|
||||
// 颜色令牌照方案 6.1：支持用强调色（--st-running 就是 --accent）、部分用 --st-waiting、不支持用 --st-failed、
|
||||
// 未裁判用暗字色（--st-cancelled 就是 --text-dim）。四种都不是确定性的绿
|
||||
eq('四种的颜色令牌', ['supported', 'partial', 'unsupported', 'unjudged'].map((c) => S[c].color).join(' '),
|
||||
'var(--st-running) var(--st-waiting) var(--st-failed) var(--st-cancelled)')
|
||||
ok('概率性四种都不用确定性的绿（--st-done）', ['supported', 'partial', 'unsupported', 'unjudged']
|
||||
.every((c) => !S[c].color.includes('--st-done') && !S[c].soft.includes('--st-done')))
|
||||
ok('证据不支持、部分支持醒目（进 n / N），有依据、未裁判安静', S.unsupported.alert && S.partial.alert && !S.supported.alert && !S.unjudged.alert)
|
||||
eq('未裁判的叫法不说「有引用」（没挂依据的结论句也会没判）', S.unjudged.label, '未裁判')
|
||||
eq('判定 → 徽标：不是结论句不挂', [ev.verdictState({ status: 'supported' }), ev.verdictState({ status: 'not_a_claim' }),
|
||||
ev.verdictState({ status: 'unjudged' }), ev.verdictState({ status: 'maybe' }), ev.verdictState(undefined)].join(','),
|
||||
'supported,,unjudged,,')
|
||||
eq('封存后追加的判定盖过文档里的「未裁判 · 按需」', ev.unitVerdict({ id: 'u2', verdict: { status: 'unjudged', reason: 'on_demand' } },
|
||||
{ u2: { status: 'unsupported', post_seal: true } }).status, 'unsupported')
|
||||
eq('没有追加的用文档里的', ev.unitVerdict({ id: 'u2', verdict: { status: 'partial' } }, { u9: { status: 'supported' } }).status, 'partial')
|
||||
eq('到上限没判的认得出是哪个上限', ev.limitOf({ status: 'unjudged', reason: 'daily_max_usd' }), 'daily_max_usd')
|
||||
eq('按需、出错不是上限', `${ev.limitOf({ status: 'unjudged', reason: 'on_demand' })}|${ev.limitOf({ status: 'unjudged', reason: 'error' })}`, 'null|null')
|
||||
ok('判过的四种（不含未裁判）', ev.isJudged({ status: 'not_a_claim' }) && ev.isJudged({ status: 'partial' }) && !ev.isJudged({ status: 'unjudged' }))
|
||||
const k = (c) => (c ? `${c.total}:${c.supported}/${c.partial}/${c.unsupported}/${c.unjudged}/${c.uncited}` : 'null')
|
||||
eq('正式运行（节点里判好的）：不是结论句不算，4 句', k(ev.claimCounts(fj.formal.doc)), '4:1/1/1/1/0')
|
||||
eq('探索运行按需：5 句都未裁判', k(ev.claimCounts(fj.explore.doc)), '5:0/0/0/5/0')
|
||||
eq('没开裁判、也没按需判过：不数（横幅照旧）', k(ev.claimCounts(fj.plain.doc)), 'null')
|
||||
// 开了裁判的文档里预筛放掉的结论句（没有判定、列在 judge.screened 里）不算进结论句，也不算「无证据」
|
||||
eq('开了裁判：预筛放掉的结论句不数', k(ev.claimCounts({ judge: { mode: 'inline', screened: ['u9'] }, blocks: [{ type: 'paragraph',
|
||||
units: [{ id: 'u9', kind: 'claim', cites: [], segments: [] }, { id: 'u1', kind: 'claim', cites: ['Q1'], segments: [],
|
||||
verdict: { status: 'supported' } }] }] })), '1:1/0/0/0/0')
|
||||
eq('没开裁判、按需判过一句：挂了依据的算未裁判，没挂的算无证据', k(ev.claimCounts(fj.plain.doc,
|
||||
{ [fj.plain.units.bad]: { status: 'unsupported', post_seal: true } })), '5:0/0/1/3/1')
|
||||
eq('docTally 带上结论句计数', k(ev.docTally(fj.formal.doc).claims), '4:1/1/1/1/0')
|
||||
// 问数据页、画布右栏的横幅走的是这条路：EvidenceField 把判定叠进文档（withVerdicts）交出去，Output 再
|
||||
// docTally——这时没有单独的 overlay，判定写在 unit.verdict 上，也得认得出「按需判过」
|
||||
eq('没开裁判、按需判过一句：叠好判定的文档交给 docTally 照样数（横幅的路径）',
|
||||
k(ev.docTally(ev.withVerdicts(fj.plain.doc, { [fj.plain.units.bad]: { status: 'unsupported', post_seal: true } })).claims),
|
||||
'5:0/0/1/3/1')
|
||||
eq('叠上判定给横幅数：只换判过的那几句', k(ev.claimCounts(ev.withVerdicts(fj.explore.doc,
|
||||
{ [fj.explore.units.bad]: { status: 'unsupported', post_seal: true } }))), '5:0/0/1/4/0')
|
||||
ok('叠上判定不改原文档', ev.withVerdicts(fj.explore.doc, { [fj.explore.units.bad]: { status: 'unsupported' } }) !== fj.explore.doc
|
||||
&& fj.explore.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fj.explore.units.bad).verdict.status === 'unjudged')
|
||||
eq('计数的说法：和数字那一段同一种', t.claimTally({ total: 4, supported: 1, partial: 1, unsupported: 1, unjudged: 1, uncited: 0 }),
|
||||
'结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）')
|
||||
eq('……方案的例子', t.claimTally({ total: 4, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 1 }), '结论 4 句（支持 3 · 无证据 1）')
|
||||
eq('……没有结论句给空串', t.claimTally({ total: 0, supported: 0, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }), '')
|
||||
const unit = fj.formal.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fj.formal.units.bad)
|
||||
eq('句末徽标的 aria-label：句子、判定写全', ev.claimLabel(unit, unit.verdict), '结论句「增长主要来自新客首单，老客复购持平。」，模型判断：证据不支持')
|
||||
ok('……封存后追加的另说', ev.claimLabel(unit, { ...unit.verdict, post_seal: true }).endsWith('，封存后追加'))
|
||||
ok('……到上限没判的说「已到上限」', ev.claimLabel(unit, { status: 'unjudged', reason: 'max_cost_usd' }).endsWith('未裁判：已到上限'))
|
||||
// 按需裁判接口的答复：evidence.judged 的载荷（{verdicts: {u: …}}），也认列表；缺 post_seal 的记 true，缺 judge 的补模型
|
||||
const a = ev.judgedVerdicts({ verdicts: { u4: { status: 'supported', rationale: '对' } }, model: 'jm' })
|
||||
ok('答复 {verdicts: {u: …}}：补上模型、记封存后追加', a.u4?.status === 'supported' && a.u4.judge === 'jm' && a.u4.post_seal === true,
|
||||
JSON.stringify(a))
|
||||
const b = ev.judgedVerdicts({ verdicts: [{ unit: 'u2', verdict: 'partial', judge: 'x', post_seal: false }, { unit: 'u3' }] })
|
||||
ok('答复是列表、判定写在 verdict 里：照认；没有判定的丢掉；接口给了 post_seal 就照它', b.u2?.status === 'partial' && b.u2.post_seal === false
|
||||
&& !('u3' in b), JSON.stringify(b))
|
||||
eq('认不出的答复给空表', Object.keys(ev.judgedVerdicts('oops')).length + Object.keys(ev.judgedVerdicts({ verdicts: 3 })).length, 0)
|
||||
const g = ev.graphVerdicts({ reports: [{ node_id: 'w1', post_seal_verdicts: { u1: { status: 'supported' } } },
|
||||
{ node_id: 'w2', post_seal_verdicts: { u1: { status: 'unsupported' } } }] }, 'w2')
|
||||
ok('证据图里按报告取封存后追加的判定（reports[].post_seal_verdicts）', g.u1?.status === 'unsupported' && g.u1.post_seal === true, JSON.stringify(g))
|
||||
const applied = { judge: { rewrite: { applied: true, units: ['u1', null], sentences: ['原句甲', '原句乙'], changed: ['u5'], reason: null } } }
|
||||
ok('改写一次采用了：changed 里的句子给出交回改写的原句（units 为 null 的那几句）',
|
||||
JSON.stringify(ev.rewriteOf(applied, 'u5')) === JSON.stringify({ kind: 'changed', sentences: ['原句乙'] }), JSON.stringify(ev.rewriteOf(applied, 'u5')))
|
||||
eq('……没改的句子不标', ev.rewriteOf(applied, 'u1'), null)
|
||||
const rejected = { judge: { rewrite: { applied: false, units: ['u2'], sentences: ['原句'], changed: [], reason: '改写稿冒出 1 处原稿没有的问题' } } }
|
||||
eq('改写稿没采用：交回过的句子写明原因', ev.rewriteOf(rejected, 'u2')?.kind, 'rejected')
|
||||
eq('……没交回的不标', ev.rewriteOf(rejected, 'u3'), null)
|
||||
// 设置页：设成不限时写明还受什么约束；别的也不限时照实少说
|
||||
const all = { claims: true, cost: true, timeout: true, click: true, daily: true }
|
||||
eq('不限的说法：每日', t.judgeUnlimitedText('daily', all), '不设上限，每日费用只受每份报告的金额上限和每次点击的金额上限约束')
|
||||
eq('……每份报告的金额', t.judgeUnlimitedText('cost', all), '不设上限，费用只受句数上限、时长上限、每日上限约束')
|
||||
ok('……每日、每份、每次都不限：说到底只受调用次数约束', t.judgeUnlimitedText('daily', { ...all, cost: false, click: false }).includes('调用次数'))
|
||||
ok('……每日不限时每份报告的金额只剩句数、时长', !t.judgeUnlimitedText('cost', { ...all, daily: false }).includes('每日'))
|
||||
return out
|
||||
})) check(name, ok, detail)
|
||||
})
|
||||
|
||||
await section('lib/actor：本机署名', async () => {
|
||||
for (const [name, ok, detail] of await page.evaluate(() => {
|
||||
const a = window.__ui.lib.actor
|
||||
|
||||
Reference in New Issue
Block a user