mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 04:31:58 +00:00
可点击证据第四期:结论句由模型裁判;只引单元格的问数据图能发成受管
一、结论句裁判
- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定
二、成本上限
- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额
三、正式运行与探索运行
- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」
四、治理与自动修复
- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略
五、界面
- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题
六、门禁修正
- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
This commit is contained in:
@@ -11,6 +11,7 @@ import type { RunEvent } from '../types'
|
||||
import fixtures from '../run/__tests__/fixtures.json'
|
||||
import mdSamples from '../run/__tests__/markdown-samples.json'
|
||||
import evidenceFixture from '../run/__tests__/evidence-doc.json'
|
||||
import evidenceJudge from '../run/__tests__/evidence-judge.json'
|
||||
import {
|
||||
CALIBER_ONE_OUTPUT, CALIBERS_OUTPUT, COPILOT_STUCK, COPILOT_TOOLS_DROPPED, MIXED_OUTPUT, abandonedRun, cancelledRun, exhaustedTeam, longLoop,
|
||||
markupRun, mixedRun, pipelineRun, repairRun, teamRun, timeoutRun,
|
||||
@@ -234,11 +235,29 @@ function evidenceTurns(): StreamTurn[] {
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* 四期:结论句裁判走问数据页、画布右栏同一条路(Output → EvidenceField → 出具横幅)。
|
||||
* - judge-plain:探索运行、报告没开裁判(require_citation)。点开结论句请模型判断之后,横幅得多出「结论 N 句」那一段;
|
||||
* - judge-formal:正式运行、报告 claims=judge,节点里判好了。没挂依据的那句送了裁判(到上限没判),出具照样按没挂依据记缺口。
|
||||
* 报告工件、片段接口、按需裁判的 POST 由 check-evidence 用 page.route 伪造(夹具 evidence-judge.json)
|
||||
*/
|
||||
function evidenceJudgeTurns(): StreamTurn[] {
|
||||
const { plain, formal } = evidenceJudge
|
||||
return [
|
||||
{ id: 'judge-plain', question: '写一份本周周报', phase: 'done', status: '已完成', runClass: 'exploratory',
|
||||
steps: [], output: plain.output, runId: plain.run_id },
|
||||
{ id: 'judge-formal', question: '写一份本周周报(受管模板)', phase: 'done', status: '已完成', runClass: 'formal',
|
||||
steps: [], output: formal.output, runId: formal.run_id },
|
||||
]
|
||||
}
|
||||
|
||||
/** 合成场景:新后端才有的事件,老库里导不出来 */
|
||||
function synthetic(name: string): StreamTurn[] {
|
||||
switch (name) {
|
||||
case 'evidence':
|
||||
return evidenceTurns()
|
||||
case 'evidence-judge':
|
||||
return evidenceJudgeTurns()
|
||||
case 'team-live': {
|
||||
const ev = rebase(teamRun('members'))
|
||||
return [{ id: 'team-live', question: '比较三家供应商的交期风险', phase: 'running', status: '运行中',
|
||||
|
||||
@@ -27,6 +27,7 @@ import { Markdown } from '../run/Markdown'
|
||||
import evidenceFixture from '../run/__tests__/evidence-doc.json'
|
||||
import evidenceQuery from '../run/__tests__/evidence-query.json'
|
||||
import evidenceEntity from '../run/__tests__/evidence-entity.json'
|
||||
import evidenceJudge from '../run/__tests__/evidence-judge.json'
|
||||
import { EvidenceGuessView } from '../run/EvidenceGuess'
|
||||
import { IssuanceBanner } from '../run/AssistantStream'
|
||||
import mdPinned from '../run/__tests__/markdown-pinned.json'
|
||||
@@ -194,6 +195,10 @@ const EV_QUERY = evidenceQuery.doc as unknown as EvidenceDocData
|
||||
const EV_LEGACY = evidenceFixture.legacy
|
||||
/** 第三期:表名字段名、可疑实体、核对不了的名字、逐字引文(evidence-entity.json,compose_doc 真跑出来的) */
|
||||
const EV_ENTITY = evidenceEntity.doc as unknown as EvidenceDocData
|
||||
/** 第四期:结论句裁判。正式运行节点里判好的(inline)、探索运行按需判的(on_demand)、没开裁判的探索运行 */
|
||||
const EV_JUDGE = evidenceJudge.formal.doc as unknown as EvidenceDocData
|
||||
const EV_JUDGE_EXPLORE = evidenceJudge.explore.doc as unknown as EvidenceDocData
|
||||
const EV_JUDGE_PLAIN = evidenceJudge.plain.doc as unknown as EvidenceDocData
|
||||
/** 同一份旧契约的出具,去掉位置:老运行就是这样,只能按字符串标 */
|
||||
const LEGACY_LOOSE = {
|
||||
matched: EV_LEGACY.matched.map(({ start: _s, end: _e, ...m }) => m),
|
||||
@@ -306,6 +311,60 @@ function EvidenceOnly() {
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
{/*
|
||||
第四期:结论句裁判。片段接口、按需裁判接口由 check-evidence 用 page.route 伪造(按 runId 认出夹具)。
|
||||
探索运行没开裁判的那份不给运行类别:面板按运行 id 取一次
|
||||
*/}
|
||||
<div className="flex flex-col gap-4" id="evidence-judge-demo">
|
||||
<div id="evidence-judge" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">正式运行 · 节点里当场判的结论句(进封存)</div>
|
||||
<EvidenceDoc doc={EV_JUDGE} artifact={evidenceJudge.formal.doc_artifact} runId={evidenceJudge.formal.run_id}
|
||||
runClass="formal" label="answer" tally />
|
||||
</div>
|
||||
<div id="evidence-judge-narrow" className="rounded-lg border bg-panel p-2.5" style={{ width: 360, maxWidth: '100%' }}>
|
||||
<EvidenceDoc doc={EV_JUDGE} artifact={evidenceJudge.formal.doc_artifact} runId={evidenceJudge.formal.run_id}
|
||||
runClass="formal" label="answer" dense tally />
|
||||
</div>
|
||||
<div id="evidence-judge-explore" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">探索运行 · 按需裁判(点开哪句判哪句,封存后追加)</div>
|
||||
<EvidenceDoc doc={EV_JUDGE_EXPLORE} artifact={evidenceJudge.explore.doc_artifact} runId={evidenceJudge.explore.run_id}
|
||||
label="answer" tally />
|
||||
</div>
|
||||
<div id="evidence-judge-explore-narrow" className="rounded-lg border bg-panel p-2.5" style={{ width: 360, maxWidth: '100%' }}>
|
||||
<EvidenceDoc doc={EV_JUDGE_EXPLORE} artifact={evidenceJudge.explore.doc_artifact} runId={evidenceJudge.explore.run_id}
|
||||
label="answer" dense tally />
|
||||
</div>
|
||||
<div id="evidence-judge-plain" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">探索运行 · 没开裁判的报告(照样能点开结论句请模型判断)</div>
|
||||
<EvidenceDoc doc={EV_JUDGE_PLAIN} artifact={evidenceJudge.plain.doc_artifact} runId={evidenceJudge.plain.run_id}
|
||||
label="answer" tally />
|
||||
</div>
|
||||
{/* 出具横幅那一行多一段结论句:「结论 4 句(支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1)」 */}
|
||||
<div id="issuance-claims" className="flex flex-col gap-2 rounded-lg border p-3">
|
||||
{/*
|
||||
judge-uncited:开了裁判的文档,没挂依据的那句送了裁判、按判定数在「支持 3」里(或预筛放掉了不数),
|
||||
出具(io.py)照样按没挂依据记 1 句——横幅得另起「没挂依据的结论句 1」,不能被结论句那一段吞掉
|
||||
*/}
|
||||
{([
|
||||
['judged', { total: 4, supported: 1, partial: 1, unsupported: 1, unjudged: 1, uncited: 0 }, null],
|
||||
['cited', { total: 4, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 1 }, null],
|
||||
['all', { total: 2, supported: 2, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }, null],
|
||||
['judge-uncited', { total: 3, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 },
|
||||
{ policy: 'judge', uncited_claims: 1 }],
|
||||
] as const).map(([key, claims, issued]) => (
|
||||
<div key={key} data-issuance-case={key}>
|
||||
<IssuanceBanner
|
||||
issuance={{ tier: 'degraded', matched_numbers: 3, metrics_checked: 0,
|
||||
claims: issued ?? { policy: 'require_citation', uncited_claims: claims.uncited } }}
|
||||
evidence={{
|
||||
counts: { total: 3, cited: 3, none: 0, other: 0, hidden: 0, structural: 0, noSegment: 0, suspect: 0,
|
||||
unverified: 0, claims },
|
||||
onNext: () => { const w = window as any; w.__claimsNext = (w.__claimsNext ?? 0) + 1 },
|
||||
}} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div id="evidence-long" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">长报告:{EV_LONG.blocks.length} 块,块级 content-visibility,超过 3000 字先折叠</div>
|
||||
<EvidenceDoc doc={EV_LONG} label="长报告" tally />
|
||||
@@ -599,6 +658,7 @@ function Harness() {
|
||||
evidenceFixture,
|
||||
evidenceQuery,
|
||||
evidenceEntity,
|
||||
evidenceJudge,
|
||||
}
|
||||
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
|
||||
Reference in New Issue
Block a user