diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index fe0334c9..6b849d96 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -237,13 +237,14 @@ NODE_REFERENCE = """\ 挂引用,没挂的计入缺口、出具降档)/ judge(在 require_citation 之外,再请另一个模型按每句挂的依据判断支不支持: 正式运行在节点里判,证据相矛盾的按 judge.on_unsupported 处置——degrade 降档(默认)或 withhold 不予出具, 证据不足的(摘录里没有判断所需的信息)只降档; - 探索运行不在节点里花钱,读的人点开哪句再按需判哪句,只标注)。judge 的子配置写在节点的 judge 里:judge.model、 + 探索运行不在节点里花钱,读的人点开哪句再按需判哪句,只标注;要交给人工审批的报告例外,探索运行也在节点里判, + 审批人批准前就能看到判了几句、几句证据相矛盾)。judge 的子配置写在节点的 judge 里:judge.model、 judge.provider 指定裁判模型(建议和写作模型不同,同一个模型等于自己审自己;不写就用设置里的「证据裁判模型」); judge.max_cost_usd、judge.max_claims、judge.timeout_s 是每份报告的裁判金额、句数、时长上限,写 null 表示不限, 不写取设置里的默认;judge.rewrite_once 默认 false,打开后把证据相矛盾、证据不足的句子交回写作者只改一次。 **多个来源汇总成的分析报告**(几个 agent 分头查库,或者几次查询的结果合在一起写成的分析、结论)默认写 claims: "judge", 不写 judge 的预算(judge 整个不写,金额、句数、时长上限都取设置里的默认):正式运行在节点里自动判,探索运行由读的人 - 点开哪句判哪句,不在节点里花钱。用户明确说不要核对结论句的不加;只回答一个问题、一次查询的不必加。要按受管级别 + 点开哪句判哪句,不在节点里花钱;报告要交给人工审批的,探索运行也在节点里判。用户明确说不要核对结论句的不加;只回答一个问题、一次查询的不必加。要按受管级别 发布的模板不适用这条默认,按下面受管级别的规则写预算 entities 默认 link (核对表名、字段名),报告里根本不提表名时才写 off diff --git a/backend/app/engine/nodes/human.py b/backend/app/engine/nodes/human.py index 29bd3fc1..f314011b 100644 --- a/backend/app/engine/nodes/human.py +++ b/backend/app/engine/nodes/human.py @@ -80,9 +80,35 @@ _VAR_REF = re.compile(r"\bvars\s*\.\s*([\w-]+)") _JUDGE_KEYS = ("mode", "counts", "complete", "unjudged", "limits_hit") +def reviewed_report_ids(spec: Any, human: Any) -> list[str]: + """一个人工审批节点审的是哪几份报告:上游直接连着的报告撰写节点,加上标题、说明里引用了正文的 + ({{ nodes..text }},或者报告的 assign_to 变量)。只看图,不看运行状态。 + + 审批卡(_reviewed_reports)和报告节点(reviewed_by_human:要交给人审的报告,探索运行也在节点里判) + 用的是同一套认法,两边说的「这份报告要人审」不会对不上。""" + from app.engine.schema import NodeType + + config = human.config or {} + text = " ".join(json.dumps(config.get(key), ensure_ascii=False, default=str) + for key in ("title", "message", "context", "draft") if config.get(key)) + # 报告节点的 assign_to 也能把正文交给下游:{{ vars. }} 和 {{ nodes..text }} 一样算引用 + by_var = {str(n.config.get("assign_to")): n.id for n in spec.nodes + if n.type == NodeType.REPORT and n.config.get("assign_to")} + nodes = spec.node_map() + wanted = ([e.source for e in spec.edges if e.target == human.id] + _NODE_REF.findall(text) + + [by_var[v] for v in _VAR_REF.findall(text) if v in by_var]) + return [rid for rid in dict.fromkeys(wanted) if (n := nodes.get(rid)) is not None and n.type == NodeType.REPORT] + + +def reviewed_by_human(spec: Any, report_id: str) -> bool: + """这份报告会不会交给人工审批(有审批节点审它,认法同 reviewed_report_ids)。""" + from app.engine.schema import NodeType + + return any(report_id in reviewed_report_ids(spec, n) for n in spec.nodes if n.type == NodeType.HUMAN) + + def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any]]: - """这次审批审的报告各自的核对概况:上游直接连着的报告撰写节点,加上标题、说明里引用了正文的 - ({{ nodes..text }},或者报告的 assign_to 变量)。 + """这次审批审的报告各自的核对概况(审的是哪几份见 reviewed_report_ids)。 真实运行里报告有一句结论和证据相矛盾,审批人批准时完全不知道:审批卡只有正文。这里带上报告节点产出里的 stats(同 report.checked)、结论句策略、裁判摘要和报告文档的工件 id,审批卡据此写核对概况、给打开证据的入口。 @@ -91,16 +117,9 @@ def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any spec = ctx.run.spec nodes = spec.node_map() - text = " ".join(json.dumps(ctx.config.get(key), ensure_ascii=False, default=str) - for key in ("title", "message", "context", "draft") if ctx.config.get(key)) - # 报告节点的 assign_to 也能把正文交给下游:{{ vars. }} 和 {{ nodes..text }} 一样算引用 - by_var = {str(n.config.get("assign_to")): n.id for n in spec.nodes - if n.type == NodeType.REPORT and n.config.get("assign_to")} - wanted = ([e.source for e in spec.edges if e.target == ctx.node.id] + _NODE_REF.findall(text) - + [by_var[v] for v in _VAR_REF.findall(text) if v in by_var]) outputs = state.get("nodes") or {} out: list[dict[str, Any]] = [] - for rid in dict.fromkeys(wanted): + for rid in reviewed_report_ids(spec, ctx.node): node = nodes.get(rid) result = outputs.get(rid) if node is None or node.type != NodeType.REPORT or not isinstance(result, dict) \ diff --git a/backend/app/engine/nodes/io.py b/backend/app/engine/nodes/io.py index a7de8542..e289aa84 100644 --- a/backend/app/engine/nodes/io.py +++ b/backend/app/engine/nodes/io.py @@ -467,10 +467,10 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An 里 ignore 一律当 degrade:没有哪道门禁查契约里的 claims,不能让它把用户要的底线拉低。 报告节点写的是 judge 时,判定在文档里(按哈希取回的那份:判定是模型给的,出口复算不了,只能信封存的 - 文档)。正式运行在节点里判过:证据相矛盾的(contradicted,旧文档里的 unsupported 同样算)按 on_unsupported + 文档)。节点里判过的(正式运行,以及交给人工审批的探索运行):证据相矛盾的(contradicted,旧文档里的 unsupported 同样算)按 on_unsupported 判档(报告节点和契约取更严的);证据不足的(insufficient)记一条缺口——最多降档,不会不予出具,也不会 完整出具;没判完的(裁判没跑成、触顶)把裁判摘要里的缺口照抄进来——不能判完整出具,也不当成「矛盾」。 - 探索运行按需裁判,这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。 + 其余探索运行按需裁判,这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。 数的是这里重新核对出来的 uncited,不信文档里记的 cites。升级前写的报告(节点产出里没有 claims 这个键, 包括跨着升级还没跑完的运行)一律不管,契约写了也只记一条缺口:报告生成于升级前,结论句未核对。 @@ -520,7 +520,7 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An inline = summary.get("mode") == "inline" flagged: dict[str, list[dict[str, Any]]] = {s: [] for s in ("contradicted", "insufficient", UNSUPPORTED, "partial")} if inline: - # 探索运行按需裁判的判定不在文档里(在封存之后追加的 evidence.judged 事件里),这里只数正式运行的 + # 按需裁判的判定不在文档里(在封存之后追加的 evidence.judged 事件里),这里只数节点里判过的 markdown = str((doc or {}).get("markdown") or "") for _, unit in iter_units(doc or {}): verdict = unit.get("verdict") diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index c9bedf60..920930bc 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -523,9 +523,13 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: judgement: dict[str, Any] | None = None if claims == CLAIMS_JUDGE: + from app.engine.nodes.human import reviewed_by_human + judge_cfg = report_judge(spec, ctx.node) - if await _run_class(ctx.run.run_id) != "formal": - # 探索运行默认按需裁判:点开哪句才判哪句,节点里不花这笔钱 + if await _run_class(ctx.run.run_id) != "formal" and not reviewed_by_human(spec, ctx.node.id): + # 探索运行默认按需裁判:点开哪句才判哪句,节点里不花这笔钱。要交给人工审批的报告例外:审批人据此 + # 拍板,批准前就该知道哪句结论站不住;而按需裁判要等运行封存之后才能判,封存前放开会动证据清单的 + # 完整性约束,所以这种报告探索运行也在节点里当场判(和正式运行同一条路) judgement = judging.on_demand(doc, on_unsupported=judge_cfg["on_unsupported"], rewrite_once=judge_cfg["rewrite_once"], writer_model=model_id) elif not (_fatal(blocking) and on_violation == "fail"): diff --git a/backend/tests/test_approval_report_check.py b/backend/tests/test_approval_report_check.py index cd13c261..f727ff42 100644 --- a/backend/tests/test_approval_report_check.py +++ b/backend/tests/test_approval_report_check.py @@ -122,11 +122,22 @@ async def test_formal_run_carries_the_judged_claims_to_the_approval(): assert requested.data["reports"] == approval.payload["reports"] -async def test_exploratory_run_says_claims_are_judged_on_demand(): +async def test_exploratory_report_under_approval_is_judged_in_the_node(): + """要交给人审批的报告,探索运行也在节点里判:审批人据此拍板,批准前就该知道哪句结论站不住。 + 真实运行里探索运行的报告有一句和证据相矛盾,审批卡只能写「结论句尚未核对」——按需裁判要等封存 + (运行结束)之后才能判,而封存前放开按需裁判会动证据清单的完整性约束,所以改在节点里当场判。 + 没有审批的探索运行照旧按需判,不在节点里花这笔钱(test_contract_judge.test_explore_runs_only_annotate)。""" _, approval = await paused(graph(gate={"title": "审核周报"}, claims="judge"), run_class="exploratory") [report] = approval.payload["reports"] - assert report["claims"] == "judge" and report["judge"]["mode"] == "on_demand" - assert report["judge"]["counts"]["contradicted"] == 0, "探索运行不在节点里判" + assert report["claims"] == "judge" and report["judge"]["mode"] == "inline" + assert report["judge"]["counts"]["contradicted"] == 1 and report["judge"]["counts"]["unjudged"] == 0 + + +async def test_report_quoted_in_the_approval_message_is_judged_in_the_node_too(): + _, approval = await paused(graph(gate={"title": "审核", "message": "请看:{{ nodes.write.text }}"}, + wire_report_to_gate=False, claims="judge"), run_class="exploratory") + [report] = approval.payload["reports"] + assert report["judge"]["mode"] == "inline" and report["judge"]["counts"]["contradicted"] == 1 async def test_report_without_claims_policy_still_reports_numbers(): diff --git a/backend/tests/test_evidence_judge_api.py b/backend/tests/test_evidence_judge_api.py index 1f87e71e..598d5cad 100644 --- a/backend/tests/test_evidence_judge_api.py +++ b/backend/tests/test_evidence_judge_api.py @@ -374,11 +374,15 @@ async def test_formal_runs_are_judged_inside_the_node_not_on_demand(client, monk async def test_a_run_paused_at_an_approval_is_not_judged(client, monkeypatch): + """停在审批上的运行还没封存,按需裁判不接。交给审批的报告探索运行也在节点里判过了 + (test_approval_report_check),这里只看点击:不再多一次裁判调用,也不追加判定事件。""" models = Models(monkeypatch) row = await finish(weekly(claims="judge", gate=True), statuses=("interrupted",)) + judged_in_node = list(models.judges) + assert judged_in_node res = await judge(client, row.id, ["u1"]) assert res.status_code == 409 and res.json()["code"] == "evidence_judge_unsealed" - assert models.judges == [] and await judged_events(row.id) == [] + assert models.judges == judged_in_node and await judged_events(row.id) == [] async def test_runs_from_before_the_upgrade_are_not_judged(client, monkeypatch): diff --git a/frontend/src/dev/preview.tsx b/frontend/src/dev/preview.tsx index 89dc111c..67dd671c 100644 --- a/frontend/src/dev/preview.tsx +++ b/frontend/src/dev/preview.tsx @@ -495,16 +495,15 @@ const APPROVAL = { /** * ?approval=report:探索运行里审一份报告。核对概况取 evidence-doc 夹具(7/12 数字有出处、1 处引用解析不了), - * 结论句没在节点里判;?approval=report-formal:正式运行,节点里判过,1 句证据相矛盾 + * 报告没开结论句裁判;?approval=report-formal:正式运行,节点里判过,1 句证据相矛盾 */ const APPROVAL_REPORT = { id: 'ap-2', run_id: evidenceFixture.run_id, node_id: 'review', mode: 'approve' as const, title: '本周周报可以发出吗?', status: 'pending', response: {}, payload: { kind: 'human_node', message: '请审核本周周报。', - reports: [{ node_id: 'write', label: '写周报', stats: evidenceFixture.doc.stats, claims: 'judge', - doc_artifact: evidenceFixture.doc_artifact, - judge: { mode: 'on_demand', counts: { unjudged: 4 }, complete: false } }], + reports: [{ node_id: 'write', label: '写周报', stats: evidenceFixture.doc.stats, claims: 'off', + doc_artifact: evidenceFixture.doc_artifact }], }, created_at: new Date(Date.now() - 120_000).toISOString().replace('Z', ''), workflow_name: '周报', node_label: '主管审批', run_class: 'exploratory' as const, @@ -513,7 +512,7 @@ const APPROVAL_REPORT_FORMAL = { ...APPROVAL_REPORT, id: 'ap-3', run_class: 'formal' as const, payload: { ...APPROVAL_REPORT.payload, - reports: [{ ...APPROVAL_REPORT.payload.reports[0], stats: { ...evidenceFixture.doc.stats, numbers_cited: 12, uncited_numbers: 0, unresolved: 0 }, + reports: [{ ...APPROVAL_REPORT.payload.reports[0], claims: 'judge', stats: { ...evidenceFixture.doc.stats, numbers_cited: 12, uncited_numbers: 0, unresolved: 0 }, judge: { mode: 'inline', counts: { supported: 5, contradicted: 1, unjudged: 0 }, complete: true } }], }, } diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index ec222135..cb37afe9 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -104,8 +104,10 @@ export const APPROVAL_TEXT = { + (unjudged ? `,未裁判 ${formatNumber(unjudged)} 句` : ''), claimsNone: '报告中没有需要裁判的结论句', claimsPending: '结论句尚未核对', - /** 探索运行停在审批上时还没封存,按需裁判要等运行结束 */ - claimsLater: '探索运行不在节点内裁判,运行结束后可在证据中逐句请模型判断', + /** 探索运行、报告没开结论句裁判:按需裁判要等运行结束封存之后。开了裁判的报告交给审批时会在节点里当场判,不走这句 */ + claimsLater: '这份报告没有开启结论句裁判,运行结束后可在证据中逐句请模型判断;开启后,交给审批的报告会在批准前判完', + /** 开了结论句裁判、却没在节点里判(这条规则之前的探索运行):按需裁判要等运行结束封存之后 */ + claimsAfterRun: '这次没有在节点内裁判,运行结束后可在证据中逐句请模型判断', /** 正式运行、报告没开结论句裁判 */ claimsOff: '这份报告没有开启结论句裁判', openEvidence: '查看逐段证据', diff --git a/frontend/src/run/ApprovalReport.tsx b/frontend/src/run/ApprovalReport.tsx index 40fb718f..c1abb21d 100644 --- a/frontend/src/run/ApprovalReport.tsx +++ b/frontend/src/run/ApprovalReport.tsx @@ -40,7 +40,9 @@ function claimsLine(report: ApprovalReport, runClass: RunClass): { text: string; } return { text: APPROVAL_TEXT.claimsPending, - sub: runClass === 'formal' ? (report.claims === 'judge' ? undefined : APPROVAL_TEXT.claimsOff) : APPROVAL_TEXT.claimsLater, + sub: runClass === 'formal' ? (report.claims === 'judge' ? undefined : APPROVAL_TEXT.claimsOff) + // 开了裁判却没在节点里判的,只会是这条规则之前的探索运行(现在交给审批的报告探索运行也当场判) + : report.claims === 'judge' ? APPROVAL_TEXT.claimsAfterRun : APPROVAL_TEXT.claimsLater, warn: true, } } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 3d176275..0b1c6c84 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -176,7 +176,8 @@ export interface Approval { /** * 人工审批审的报告的核对概况(human 节点请求里的 payload.reports,服务端 nodes/human._reviewed_reports): * 上游直接连着的、或者标题和说明里引用了正文的报告撰写节点,各一项。stats 同 report.checked,judge 是裁判摘要 - * 的几项(mode:inline 节点里当场判的 / on_demand 探索运行按需判)。老后端、审的不是报告时没有 + * 的几项(mode:inline 节点里当场判的——交给审批的报告探索运行也当场判 / on_demand 按需判,老服务端的探索运行)。 + * 老后端、审的不是报告时没有 */ export interface ApprovalReport { node_id: string @@ -1797,8 +1798,8 @@ export interface EvidenceUnit { /** 列表项的缩进层级 */ depth?: number /** - * 结论句裁判的判定(四期):只有送了裁判的候选句有。正式运行里节点当场判的进封存(post_seal false); - * 探索运行里候选句先标 unjudged + reason on_demand,点开再判 + * 结论句裁判的判定(四期):只有送了裁判的候选句有。正式运行(以及交给人工审批的探索运行)里节点当场判的 + * 进封存(post_seal false);其余探索运行里候选句先标 unjudged + reason on_demand,点开再判 */ verdict?: EvidenceVerdict } @@ -1835,7 +1836,7 @@ export interface EvidenceVerdict { /** 文档(和 report.checked、节点产出)里的裁判摘要:键的顺序同后端 judge.SUMMARY_KEYS */ export interface EvidenceJudgeSummary { - /** inline:正式运行节点里当场判;on_demand:探索运行按需 */ + /** inline:节点里当场判(正式运行,以及交给人工审批的探索运行);on_demand:其余探索运行按需 */ mode?: 'inline' | 'on_demand' | string model?: string | null on_unsupported?: 'degrade' | 'withhold' | string diff --git a/scripts/check-stream.mjs b/scripts/check-stream.mjs index c95b0319..651db9af 100644 --- a/scripts/check-stream.mjs +++ b/scripts/check-stream.mjs @@ -958,8 +958,9 @@ await section('审批卡:审的报告的核对概况', async () => { check('报告的核对概况:标题写哪份报告', (await report.innerText()).includes('报告「写周报」的核对')) check('数字和证据条同一种说法', (await report.locator('[data-approval-report-numbers]').innerText()) .includes('7/12 数字有出处 · 无证据 5 · 另有 1 处引用无法解析'), await report.locator('[data-approval-report-numbers]').innerText()) - check('探索运行:结论句尚未核对,并说运行结束后才能逐句判断', (await report.locator('[data-approval-report-claims]').innerText()) - .includes('结论句尚未核对') && (await report.innerText()).includes('运行结束后可在证据中逐句请模型判断')) + check('探索运行、报告没开裁判:结论句尚未核对,说明运行结束后可逐句判断、开启后会在批准前判完', + (await report.locator('[data-approval-report-claims]').innerText()).includes('结论句尚未核对') + && (await report.innerText()).includes('没有开启结论句裁判') && (await report.innerText()).includes('交给审批的报告会在批准前判完')) await report.locator('[data-approval-evidence]').click() await page.waitForSelector('[data-approval-evidence-doc] [data-evidence-doc]', { timeout: 6000 }) check('查看逐段证据:卡里展开可点的报告文档,自带证据条', await report.locator('[data-evidence-doc] [data-seg]').count() > 0