mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
交给人工审批的报告,探索运行也在节点里裁判结论句:审批人批准前就能看到判了几句、几句证据相矛盾。按需裁判要等运行封存之后才能判,封存前放开会动证据清单的完整性约束,所以这种报告改在节点里当场判,和正式运行同一条路;没有审批的探索运行照旧按需判,不在节点里花钱。审批节点审哪几份报告的认法抽成 reviewed_report_ids,审批卡和报告节点共用,两边说的「要人审」对得上。审批卡上没开裁判的探索报告改写「这份报告没有开启结论句裁判……开启后,交给审批的报告会在批准前判完」;开了裁判却没在节点里判的老运行另给一句。助手指引、出口契约注释、前端类型注释同步;审批暂停时按需裁判仍拒绝,测试改成只看点击不多一次调用
This commit is contained in:
@@ -237,13 +237,14 @@ NODE_REFERENCE = """\
|
||||
挂引用,没挂的计入缺口、出具降档)/ judge(在 require_citation 之外,再请另一个模型按每句挂的依据判断支不支持:
|
||||
正式运行在节点里判,证据相矛盾的按 judge.on_unsupported 处置——degrade 降档(默认)或 withhold 不予出具,
|
||||
证据不足的(摘录里没有判断所需的信息)只降档;
|
||||
探索运行不在节点里花钱,读的人点开哪句再按需判哪句,只标注)。judge 的子配置写在节点的 judge 里:judge.model、
|
||||
探索运行不在节点里花钱,读的人点开哪句再按需判哪句,只标注;要交给人工审批的报告例外,探索运行也在节点里判,
|
||||
审批人批准前就能看到判了几句、几句证据相矛盾)。judge 的子配置写在节点的 judge 里:judge.model、
|
||||
judge.provider 指定裁判模型(建议和写作模型不同,同一个模型等于自己审自己;不写就用设置里的「证据裁判模型」);
|
||||
judge.max_cost_usd、judge.max_claims、judge.timeout_s 是每份报告的裁判金额、句数、时长上限,写 null 表示不限,
|
||||
不写取设置里的默认;judge.rewrite_once 默认 false,打开后把证据相矛盾、证据不足的句子交回写作者只改一次。
|
||||
**多个来源汇总成的分析报告**(几个 agent 分头查库,或者几次查询的结果合在一起写成的分析、结论)默认写 claims: "judge",
|
||||
不写 judge 的预算(judge 整个不写,金额、句数、时长上限都取设置里的默认):正式运行在节点里自动判,探索运行由读的人
|
||||
点开哪句判哪句,不在节点里花钱。用户明确说不要核对结论句的不加;只回答一个问题、一次查询的不必加。要按受管级别
|
||||
点开哪句判哪句,不在节点里花钱;报告要交给人工审批的,探索运行也在节点里判。用户明确说不要核对结论句的不加;只回答一个问题、一次查询的不必加。要按受管级别
|
||||
发布的模板不适用这条默认,按下面受管级别的规则写预算
|
||||
entities 默认 link
|
||||
(核对表名、字段名),报告里根本不提表名时才写 off
|
||||
|
||||
@@ -80,9 +80,35 @@ _VAR_REF = re.compile(r"\bvars\s*\.\s*([\w-]+)")
|
||||
_JUDGE_KEYS = ("mode", "counts", "complete", "unjudged", "limits_hit")
|
||||
|
||||
|
||||
def reviewed_report_ids(spec: Any, human: Any) -> list[str]:
|
||||
"""一个人工审批节点审的是哪几份报告:上游直接连着的报告撰写节点,加上标题、说明里引用了正文的
|
||||
({{ nodes.<id>.text }},或者报告的 assign_to 变量)。只看图,不看运行状态。
|
||||
|
||||
审批卡(_reviewed_reports)和报告节点(reviewed_by_human:要交给人审的报告,探索运行也在节点里判)
|
||||
用的是同一套认法,两边说的「这份报告要人审」不会对不上。"""
|
||||
from app.engine.schema import NodeType
|
||||
|
||||
config = human.config or {}
|
||||
text = " ".join(json.dumps(config.get(key), ensure_ascii=False, default=str)
|
||||
for key in ("title", "message", "context", "draft") if config.get(key))
|
||||
# 报告节点的 assign_to 也能把正文交给下游:{{ vars.<assign_to> }} 和 {{ nodes.<id>.text }} 一样算引用
|
||||
by_var = {str(n.config.get("assign_to")): n.id for n in spec.nodes
|
||||
if n.type == NodeType.REPORT and n.config.get("assign_to")}
|
||||
nodes = spec.node_map()
|
||||
wanted = ([e.source for e in spec.edges if e.target == human.id] + _NODE_REF.findall(text)
|
||||
+ [by_var[v] for v in _VAR_REF.findall(text) if v in by_var])
|
||||
return [rid for rid in dict.fromkeys(wanted) if (n := nodes.get(rid)) is not None and n.type == NodeType.REPORT]
|
||||
|
||||
|
||||
def reviewed_by_human(spec: Any, report_id: str) -> bool:
|
||||
"""这份报告会不会交给人工审批(有审批节点审它,认法同 reviewed_report_ids)。"""
|
||||
from app.engine.schema import NodeType
|
||||
|
||||
return any(report_id in reviewed_report_ids(spec, n) for n in spec.nodes if n.type == NodeType.HUMAN)
|
||||
|
||||
|
||||
def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any]]:
|
||||
"""这次审批审的报告各自的核对概况:上游直接连着的报告撰写节点,加上标题、说明里引用了正文的
|
||||
({{ nodes.<id>.text }},或者报告的 assign_to 变量)。
|
||||
"""这次审批审的报告各自的核对概况(审的是哪几份见 reviewed_report_ids)。
|
||||
|
||||
真实运行里报告有一句结论和证据相矛盾,审批人批准时完全不知道:审批卡只有正文。这里带上报告节点产出里的
|
||||
stats(同 report.checked)、结论句策略、裁判摘要和报告文档的工件 id,审批卡据此写核对概况、给打开证据的入口。
|
||||
@@ -91,16 +117,9 @@ def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any
|
||||
|
||||
spec = ctx.run.spec
|
||||
nodes = spec.node_map()
|
||||
text = " ".join(json.dumps(ctx.config.get(key), ensure_ascii=False, default=str)
|
||||
for key in ("title", "message", "context", "draft") if ctx.config.get(key))
|
||||
# 报告节点的 assign_to 也能把正文交给下游:{{ vars.<assign_to> }} 和 {{ nodes.<id>.text }} 一样算引用
|
||||
by_var = {str(n.config.get("assign_to")): n.id for n in spec.nodes
|
||||
if n.type == NodeType.REPORT and n.config.get("assign_to")}
|
||||
wanted = ([e.source for e in spec.edges if e.target == ctx.node.id] + _NODE_REF.findall(text)
|
||||
+ [by_var[v] for v in _VAR_REF.findall(text) if v in by_var])
|
||||
outputs = state.get("nodes") or {}
|
||||
out: list[dict[str, Any]] = []
|
||||
for rid in dict.fromkeys(wanted):
|
||||
for rid in reviewed_report_ids(spec, ctx.node):
|
||||
node = nodes.get(rid)
|
||||
result = outputs.get(rid)
|
||||
if node is None or node.type != NodeType.REPORT or not isinstance(result, dict) \
|
||||
|
||||
@@ -467,10 +467,10 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
里 ignore 一律当 degrade:没有哪道门禁查契约里的 claims,不能让它把用户要的底线拉低。
|
||||
|
||||
报告节点写的是 judge 时,判定在文档里(按哈希取回的那份:判定是模型给的,出口复算不了,只能信封存的
|
||||
文档)。正式运行在节点里判过:证据相矛盾的(contradicted,旧文档里的 unsupported 同样算)按 on_unsupported
|
||||
文档)。节点里判过的(正式运行,以及交给人工审批的探索运行):证据相矛盾的(contradicted,旧文档里的 unsupported 同样算)按 on_unsupported
|
||||
判档(报告节点和契约取更严的);证据不足的(insufficient)记一条缺口——最多降档,不会不予出具,也不会
|
||||
完整出具;没判完的(裁判没跑成、触顶)把裁判摘要里的缺口照抄进来——不能判完整出具,也不当成「矛盾」。
|
||||
探索运行按需裁判,这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。
|
||||
其余探索运行按需裁判,这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。
|
||||
|
||||
数的是这里重新核对出来的 uncited,不信文档里记的 cites。升级前写的报告(节点产出里没有 claims 这个键,
|
||||
包括跨着升级还没跑完的运行)一律不管,契约写了也只记一条缺口:报告生成于升级前,结论句未核对。
|
||||
@@ -520,7 +520,7 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
inline = summary.get("mode") == "inline"
|
||||
flagged: dict[str, list[dict[str, Any]]] = {s: [] for s in ("contradicted", "insufficient", UNSUPPORTED, "partial")}
|
||||
if inline:
|
||||
# 探索运行按需裁判的判定不在文档里(在封存之后追加的 evidence.judged 事件里),这里只数正式运行的
|
||||
# 按需裁判的判定不在文档里(在封存之后追加的 evidence.judged 事件里),这里只数节点里判过的
|
||||
markdown = str((doc or {}).get("markdown") or "")
|
||||
for _, unit in iter_units(doc or {}):
|
||||
verdict = unit.get("verdict")
|
||||
|
||||
@@ -523,9 +523,13 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
|
||||
judgement: dict[str, Any] | None = None
|
||||
if claims == CLAIMS_JUDGE:
|
||||
from app.engine.nodes.human import reviewed_by_human
|
||||
|
||||
judge_cfg = report_judge(spec, ctx.node)
|
||||
if await _run_class(ctx.run.run_id) != "formal":
|
||||
# 探索运行默认按需裁判:点开哪句才判哪句,节点里不花这笔钱
|
||||
if await _run_class(ctx.run.run_id) != "formal" and not reviewed_by_human(spec, ctx.node.id):
|
||||
# 探索运行默认按需裁判:点开哪句才判哪句,节点里不花这笔钱。要交给人工审批的报告例外:审批人据此
|
||||
# 拍板,批准前就该知道哪句结论站不住;而按需裁判要等运行封存之后才能判,封存前放开会动证据清单的
|
||||
# 完整性约束,所以这种报告探索运行也在节点里当场判(和正式运行同一条路)
|
||||
judgement = judging.on_demand(doc, on_unsupported=judge_cfg["on_unsupported"],
|
||||
rewrite_once=judge_cfg["rewrite_once"], writer_model=model_id)
|
||||
elif not (_fatal(blocking) and on_violation == "fail"):
|
||||
|
||||
@@ -122,11 +122,22 @@ async def test_formal_run_carries_the_judged_claims_to_the_approval():
|
||||
assert requested.data["reports"] == approval.payload["reports"]
|
||||
|
||||
|
||||
async def test_exploratory_run_says_claims_are_judged_on_demand():
|
||||
async def test_exploratory_report_under_approval_is_judged_in_the_node():
|
||||
"""要交给人审批的报告,探索运行也在节点里判:审批人据此拍板,批准前就该知道哪句结论站不住。
|
||||
真实运行里探索运行的报告有一句和证据相矛盾,审批卡只能写「结论句尚未核对」——按需裁判要等封存
|
||||
(运行结束)之后才能判,而封存前放开按需裁判会动证据清单的完整性约束,所以改在节点里当场判。
|
||||
没有审批的探索运行照旧按需判,不在节点里花这笔钱(test_contract_judge.test_explore_runs_only_annotate)。"""
|
||||
_, approval = await paused(graph(gate={"title": "审核周报"}, claims="judge"), run_class="exploratory")
|
||||
[report] = approval.payload["reports"]
|
||||
assert report["claims"] == "judge" and report["judge"]["mode"] == "on_demand"
|
||||
assert report["judge"]["counts"]["contradicted"] == 0, "探索运行不在节点里判"
|
||||
assert report["claims"] == "judge" and report["judge"]["mode"] == "inline"
|
||||
assert report["judge"]["counts"]["contradicted"] == 1 and report["judge"]["counts"]["unjudged"] == 0
|
||||
|
||||
|
||||
async def test_report_quoted_in_the_approval_message_is_judged_in_the_node_too():
|
||||
_, approval = await paused(graph(gate={"title": "审核", "message": "请看:{{ nodes.write.text }}"},
|
||||
wire_report_to_gate=False, claims="judge"), run_class="exploratory")
|
||||
[report] = approval.payload["reports"]
|
||||
assert report["judge"]["mode"] == "inline" and report["judge"]["counts"]["contradicted"] == 1
|
||||
|
||||
|
||||
async def test_report_without_claims_policy_still_reports_numbers():
|
||||
|
||||
@@ -374,11 +374,15 @@ async def test_formal_runs_are_judged_inside_the_node_not_on_demand(client, monk
|
||||
|
||||
|
||||
async def test_a_run_paused_at_an_approval_is_not_judged(client, monkeypatch):
|
||||
"""停在审批上的运行还没封存,按需裁判不接。交给审批的报告探索运行也在节点里判过了
|
||||
(test_approval_report_check),这里只看点击:不再多一次裁判调用,也不追加判定事件。"""
|
||||
models = Models(monkeypatch)
|
||||
row = await finish(weekly(claims="judge", gate=True), statuses=("interrupted",))
|
||||
judged_in_node = list(models.judges)
|
||||
assert judged_in_node
|
||||
res = await judge(client, row.id, ["u1"])
|
||||
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_unsealed"
|
||||
assert models.judges == [] and await judged_events(row.id) == []
|
||||
assert models.judges == judged_in_node and await judged_events(row.id) == []
|
||||
|
||||
|
||||
async def test_runs_from_before_the_upgrade_are_not_judged(client, monkeypatch):
|
||||
|
||||
@@ -495,16 +495,15 @@ const APPROVAL = {
|
||||
|
||||
/**
|
||||
* ?approval=report:探索运行里审一份报告。核对概况取 evidence-doc 夹具(7/12 数字有出处、1 处引用解析不了),
|
||||
* 结论句没在节点里判;?approval=report-formal:正式运行,节点里判过,1 句证据相矛盾
|
||||
* 报告没开结论句裁判;?approval=report-formal:正式运行,节点里判过,1 句证据相矛盾
|
||||
*/
|
||||
const APPROVAL_REPORT = {
|
||||
id: 'ap-2', run_id: evidenceFixture.run_id, node_id: 'review', mode: 'approve' as const,
|
||||
title: '本周周报可以发出吗?', status: 'pending', response: {},
|
||||
payload: {
|
||||
kind: 'human_node', message: '请审核本周周报。',
|
||||
reports: [{ node_id: 'write', label: '写周报', stats: evidenceFixture.doc.stats, claims: 'judge',
|
||||
doc_artifact: evidenceFixture.doc_artifact,
|
||||
judge: { mode: 'on_demand', counts: { unjudged: 4 }, complete: false } }],
|
||||
reports: [{ node_id: 'write', label: '写周报', stats: evidenceFixture.doc.stats, claims: 'off',
|
||||
doc_artifact: evidenceFixture.doc_artifact }],
|
||||
},
|
||||
created_at: new Date(Date.now() - 120_000).toISOString().replace('Z', ''),
|
||||
workflow_name: '周报', node_label: '主管审批', run_class: 'exploratory' as const,
|
||||
@@ -513,7 +512,7 @@ const APPROVAL_REPORT_FORMAL = {
|
||||
...APPROVAL_REPORT, id: 'ap-3', run_class: 'formal' as const,
|
||||
payload: {
|
||||
...APPROVAL_REPORT.payload,
|
||||
reports: [{ ...APPROVAL_REPORT.payload.reports[0], stats: { ...evidenceFixture.doc.stats, numbers_cited: 12, uncited_numbers: 0, unresolved: 0 },
|
||||
reports: [{ ...APPROVAL_REPORT.payload.reports[0], claims: 'judge', stats: { ...evidenceFixture.doc.stats, numbers_cited: 12, uncited_numbers: 0, unresolved: 0 },
|
||||
judge: { mode: 'inline', counts: { supported: 5, contradicted: 1, unjudged: 0 }, complete: true } }],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -104,8 +104,10 @@ export const APPROVAL_TEXT = {
|
||||
+ (unjudged ? `,未裁判 ${formatNumber(unjudged)} 句` : ''),
|
||||
claimsNone: '报告中没有需要裁判的结论句',
|
||||
claimsPending: '结论句尚未核对',
|
||||
/** 探索运行停在审批上时还没封存,按需裁判要等运行结束 */
|
||||
claimsLater: '探索运行不在节点内裁判,运行结束后可在证据中逐句请模型判断',
|
||||
/** 探索运行、报告没开结论句裁判:按需裁判要等运行结束封存之后。开了裁判的报告交给审批时会在节点里当场判,不走这句 */
|
||||
claimsLater: '这份报告没有开启结论句裁判,运行结束后可在证据中逐句请模型判断;开启后,交给审批的报告会在批准前判完',
|
||||
/** 开了结论句裁判、却没在节点里判(这条规则之前的探索运行):按需裁判要等运行结束封存之后 */
|
||||
claimsAfterRun: '这次没有在节点内裁判,运行结束后可在证据中逐句请模型判断',
|
||||
/** 正式运行、报告没开结论句裁判 */
|
||||
claimsOff: '这份报告没有开启结论句裁判',
|
||||
openEvidence: '查看逐段证据',
|
||||
|
||||
@@ -40,7 +40,9 @@ function claimsLine(report: ApprovalReport, runClass: RunClass): { text: string;
|
||||
}
|
||||
return {
|
||||
text: APPROVAL_TEXT.claimsPending,
|
||||
sub: runClass === 'formal' ? (report.claims === 'judge' ? undefined : APPROVAL_TEXT.claimsOff) : APPROVAL_TEXT.claimsLater,
|
||||
sub: runClass === 'formal' ? (report.claims === 'judge' ? undefined : APPROVAL_TEXT.claimsOff)
|
||||
// 开了裁判却没在节点里判的,只会是这条规则之前的探索运行(现在交给审批的报告探索运行也当场判)
|
||||
: report.claims === 'judge' ? APPROVAL_TEXT.claimsAfterRun : APPROVAL_TEXT.claimsLater,
|
||||
warn: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,7 +176,8 @@ export interface Approval {
|
||||
/**
|
||||
* 人工审批审的报告的核对概况(human 节点请求里的 payload.reports,服务端 nodes/human._reviewed_reports):
|
||||
* 上游直接连着的、或者标题和说明里引用了正文的报告撰写节点,各一项。stats 同 report.checked,judge 是裁判摘要
|
||||
* 的几项(mode:inline 节点里当场判的 / on_demand 探索运行按需判)。老后端、审的不是报告时没有
|
||||
* 的几项(mode:inline 节点里当场判的——交给审批的报告探索运行也当场判 / on_demand 按需判,老服务端的探索运行)。
|
||||
* 老后端、审的不是报告时没有
|
||||
*/
|
||||
export interface ApprovalReport {
|
||||
node_id: string
|
||||
@@ -1797,8 +1798,8 @@ export interface EvidenceUnit {
|
||||
/** 列表项的缩进层级 */
|
||||
depth?: number
|
||||
/**
|
||||
* 结论句裁判的判定(四期):只有送了裁判的候选句有。正式运行里节点当场判的进封存(post_seal false);
|
||||
* 探索运行里候选句先标 unjudged + reason on_demand,点开再判
|
||||
* 结论句裁判的判定(四期):只有送了裁判的候选句有。正式运行(以及交给人工审批的探索运行)里节点当场判的
|
||||
* 进封存(post_seal false);其余探索运行里候选句先标 unjudged + reason on_demand,点开再判
|
||||
*/
|
||||
verdict?: EvidenceVerdict
|
||||
}
|
||||
@@ -1835,7 +1836,7 @@ export interface EvidenceVerdict {
|
||||
|
||||
/** 文档(和 report.checked、节点产出)里的裁判摘要:键的顺序同后端 judge.SUMMARY_KEYS */
|
||||
export interface EvidenceJudgeSummary {
|
||||
/** inline:正式运行节点里当场判;on_demand:探索运行按需 */
|
||||
/** inline:节点里当场判(正式运行,以及交给人工审批的探索运行);on_demand:其余探索运行按需 */
|
||||
mode?: 'inline' | 'on_demand' | string
|
||||
model?: string | null
|
||||
on_unsupported?: 'degrade' | 'withhold' | string
|
||||
|
||||
@@ -958,8 +958,9 @@ await section('审批卡:审的报告的核对概况', async () => {
|
||||
check('报告的核对概况:标题写哪份报告', (await report.innerText()).includes('报告「写周报」的核对'))
|
||||
check('数字和证据条同一种说法', (await report.locator('[data-approval-report-numbers]').innerText())
|
||||
.includes('7/12 数字有出处 · 无证据 5 · 另有 1 处引用无法解析'), await report.locator('[data-approval-report-numbers]').innerText())
|
||||
check('探索运行:结论句尚未核对,并说运行结束后才能逐句判断', (await report.locator('[data-approval-report-claims]').innerText())
|
||||
.includes('结论句尚未核对') && (await report.innerText()).includes('运行结束后可在证据中逐句请模型判断'))
|
||||
check('探索运行、报告没开裁判:结论句尚未核对,说明运行结束后可逐句判断、开启后会在批准前判完',
|
||||
(await report.locator('[data-approval-report-claims]').innerText()).includes('结论句尚未核对')
|
||||
&& (await report.innerText()).includes('没有开启结论句裁判') && (await report.innerText()).includes('交给审批的报告会在批准前判完'))
|
||||
await report.locator('[data-approval-evidence]').click()
|
||||
await page.waitForSelector('[data-approval-evidence-doc] [data-evidence-doc]', { timeout: 6000 })
|
||||
check('查看逐段证据:卡里展开可点的报告文档,自带证据条', await report.locator('[data-evidence-doc] [data-seg]').count() > 0
|
||||
|
||||
Reference in New Issue
Block a user