From 9727c21d8dad46654ea76e763965557eabd1992c Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 29 Sep 2026 11:10:53 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=AF=E7=82=B9=E5=87=BB=E8=AF=81=E6=8D=AE?= =?UTF-8?q?=E7=AC=AC=E5=9B=9B=E6=9C=9F=EF=BC=9A=E7=BB=93=E8=AE=BA=E5=8F=A5?= =?UTF-8?q?=E7=94=B1=E6=A8=A1=E5=9E=8B=E8=A3=81=E5=88=A4=EF=BC=9B=E5=8F=AA?= =?UTF-8?q?=E5=BC=95=E5=8D=95=E5=85=83=E6=A0=BC=E7=9A=84=E9=97=AE=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=9B=BE=E8=83=BD=E5=8F=91=E6=88=90=E5=8F=97=E7=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一、结论句裁判 - 报告节点新增 claims: judge：写完、核对完之后，把结论句和它引用的证据摘录交给裁判模型， 判为支持 / 部分支持 / 不支持 / 不是结论，附一句理由；判定只标注，不改写正文 - 送裁判的句子按固定规则预筛：带引用的、含方向词或因果词的送去判，短句、过渡句不送 - 证据摘录：指标写名称、值和原式；查询给 SQL 前 300 字和被引用的行，被引用的列一定给到；遮罩列一律不给； 规则里写明「数字已经由系统核对过，不要复算」 - 一份报告合并成一次调用，超出句数就分批；结构化输出失败时退回 JSON 解析，再失败记为未裁判 - 裁判模型的优先级：节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型；思考关闭； 和写作模型相同时给警告（等于自己审自己） - rewrite_once（默认关）：把不支持的句子和理由交回写作者只改这几句，再裁一次； 改写稿冒出原稿没有的问题就不采用，保留原稿和原来的判定 二、成本上限 - 每份报告的句数、金额、时长，探索运行每次点击的金额，全局每日金额，每一项都可以自定义，也都可以设成不限； 默认 40 句、$0.05、30 秒，每次点击 $0.01，每天 $2 - 调用前先估算，超出上限按优先级截断：先判带引用的，再判含方向词、因果词的，其余记为未裁判并写明「已到上限」； 不报错，也不让报告失败 - 每日花费按本地日期计数，持久化在设置表里；重跑同一次运行不重复计费；价格目录里没有的模型只计次数，并写明估不出金额 三、正式运行与探索运行 - 正式运行在报告节点内裁判，判定写进报告文档、随文档封存；不支持的按 on_unsupported（degrade / withhold）降档或不予出具， 裁判没跑成记缺口，不能判完整出具 - 探索运行按需裁判：POST /api/runs/{id}/evidence/judge，点开哪句判哪句；判定作为 evidence.judged 事件追加在封存之后， 标「封存后追加」，封存核对仍然一致；只读封存范围内的工件；同一句不重复判；运行在裁判期间被接着跑时不记录 - 正式运行调这个接口返回 409，写明「正式运行的裁判在节点内完成」 四、治理与自动修复 - G3 放开 judge：受管级别写 judge 必须写预算（judge.max_cost_usd，或显式写 null 表示不限） - 发布前自动修复：缺预算时给选项（默认金额 / 不限），不替人选； 从 require_citation 改成 judge 不算降低要求，judge 改回 off、withhold 放宽成 degrade 算降低要求 - Copilot 手册：受管模板可以写 judge 并带预算；问数据按需裁判，不替人打开结论句策略 五、界面 - 结论句句末放徽标：支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?；不用表示「已核对」的绿色；n / N 也跳到不支持的句子 - 证据面板里「模型的解释」：带「模型判断 · 模型名 · 非确定」，封存后追加的另外标出； 「请模型判断这句」只在探索运行、还没判过时出现，触顶时写明已到上限和怎么调 - 出具横幅多一段结论句计数；设置页新增「证据裁判」一组（裁判模型、各项上限、今日花费），设成不限时写明费用受什么约束； 报告节点可以配 judge；运行轨迹里裁判相关的事件都有中文标题 六、门禁修正 - 引用模式下，图里没有口径卡（只引查询单元格）的，出具契约不再要求 metrics_from，受管门禁也不再要求 required： strict 加 cells 已经保证每个数都点得开、对不上就不予出具；有口径卡的照旧要求。 以前问数据的图怎么都发不成受管，Copilot 补上契约也会被判「冒出新的问题」作废 --- backend/app/api/copilot.py | 21 +- backend/app/api/evidence.py | 350 +- backend/app/api/settings.py | 28 +- backend/app/core/events.py | 3 + backend/app/engine/autofix.py | 165 +- backend/app/engine/governance.py | 46 +- backend/app/engine/judge.py | 1065 ++++ backend/app/engine/nodes/io.py | 89 +- backend/app/engine/nodes/report.py | 235 +- backend/app/engine/review.py | 10 + backend/app/engine/schema.py | 95 +- backend/tests/test_autofix_governance.py | 9 +- backend/tests/test_claims_prefilter.py | 111 + backend/tests/test_contract_cells.py | 71 + backend/tests/test_contract_judge.py | 323 ++ backend/tests/test_copilot_entity_guide.py | 40 +- backend/tests/test_evidence_judge.py | 554 ++ backend/tests/test_evidence_judge_api.py | 696 +++ backend/tests/test_governance_judge.py | 344 ++ backend/tests/test_governance_report_lint.py | 17 +- backend/tests/test_judge_settings.py | 262 + backend/tests/test_report_claims.py | 20 +- backend/tests/test_report_judge.py | 495 ++ frontend/src/api/client.ts | 12 +- frontend/src/canvas/Inspector.tsx | 165 +- frontend/src/canvas/NodeCard.tsx | 14 +- frontend/src/canvas/issues.ts | 37 +- frontend/src/canvas/nodeDefs.ts | 19 +- frontend/src/dev/preview.tsx | 19 + frontend/src/dev/ui-harness.tsx | 60 + frontend/src/index.css | 33 + frontend/src/lib/evidence.ts | 195 +- frontend/src/lib/terms.ts | 188 +- frontend/src/pages/SettingsPage.tsx | 228 +- frontend/src/pages/runs/EvidencePane.tsx | 2 +- frontend/src/run/AssistantStream.tsx | 27 +- frontend/src/run/EvidenceDoc.tsx | 218 +- frontend/src/run/EvidencePanel.tsx | 340 +- .../src/run/__tests__/evidence-judge.json | 4879 +++++++++++++++++ frontend/src/run/decode.ts | 105 +- frontend/src/store/catalog.ts | 15 + frontend/src/store/evidence.ts | 93 +- frontend/src/types.ts | 112 +- scripts/check-decode.mjs | 97 + scripts/check-evidence.mjs | 544 +- scripts/check-manage.mjs | 93 +- scripts/check-ui-kit.mjs | 86 + 47 files changed, 12367 insertions(+), 263 deletions(-) create mode 100644 backend/app/engine/judge.py create mode 100644 backend/tests/test_claims_prefilter.py create mode 100644 backend/tests/test_contract_judge.py create mode 100644 backend/tests/test_evidence_judge.py create mode 100644 backend/tests/test_evidence_judge_api.py create mode 100644 backend/tests/test_governance_judge.py create mode 100644 backend/tests/test_judge_settings.py create mode 100644 backend/tests/test_report_judge.py create mode 100644 frontend/src/run/__tests__/evidence-judge.json diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index dfe2971b..e0698fea 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -207,14 +207,21 @@ NODE_REFERENCE = """\ - 句末的 [[see:m:指标id,Q1,K1]]：这句结论的依据，不显示 metrics_from 不写就取所有上游口径卡；on_violation 不写时探索运行是 flag（标出来、照常产出），正式运行是 fail； numbers 默认 strict（裸数字算违规）。claims 是结论句策略：off（默认，不管）/ require_citation（每句结论都要 - 挂引用，没挂的计入缺口、出具降档）；judge（模型裁判）在后续版本支持，现在写了整张图跑不起来。entities 默认 link + 挂引用，没挂的计入缺口、出具降档）/ judge（在 require_citation 之外，再请另一个模型按每句挂的依据判断支不支持： + 正式运行在节点里判，证据不支持的按 judge.on_unsupported 处置——degrade 降档（默认）或 withhold 不予出具； + 探索运行不在节点里花钱，读的人点开哪句再按需判哪句，只标注）。judge 的子配置写在节点的 judge 里：judge.model、 + judge.provider 指定裁判模型（建议和写作模型不同，同一个模型等于自己审自己；不写就用设置里的「证据裁判模型」）； + judge.max_cost_usd、judge.max_claims、judge.timeout_s 是每份报告的裁判金额、句数、时长上限，写 null 表示不限， + 不写取设置里的默认；judge.rewrite_once 默认 false，打开后把证据不支持的句子交回写作者只改一次。entities 默认 link （核对表名、字段名），报告里根本不提表名时才写 off - output 的出具契约：config.contract = {report_from: 报告节点id, metrics_from:[口径卡id], required:[必需指标id], expected:[期望指标id], strict: true}。成果字段写 {{ nodes.报告节点id.text }}， 前后不要拼别的字——拼了就没法逐段对应证据，出具会降档。受管模板的报告要直接引用查询单元格 - （[[v:]] / [[table:]]）的，契约里写 cells: true，否则这些引用按解析不了算 + （[[v:]] / [[table:]]）的，契约里写 cells: true，否则这些引用按解析不了算；只引单元格、图里没有口径卡的 + （问数据的典型图），契约不写 metrics_from，也不写 required / expected 要按受管级别发布的模板（发布前检查不过就发不出去）：报告节点写明 numbers: "strict"、on_violation: "fail"、 - claims: "require_citation"；契约用 report_from（受管级别不收 narrative），成果字段只取报告节点的正文；给口径卡 + claims: "require_citation"（或者更严的 claims: "judge"，这时 judge 里必须写 judge.max_cost_usd——写一个金额，或者 + 写 null 表示不限，预算要用户定，拿不准就问）；契约用 report_from（受管级别不收 narrative），成果字段只取报告节点的正文；给口径卡 供数的 agent 配 output_schema 和 cite_fields: true，code 节点喂口径卡的必须是真取数（evidence_role: "source"）， 否则把计算写进口径卡；llm、agent、协作团队写的文字不能绕过报告节点直接进出口，llm 也不能喂口径卡 @@ -321,7 +328,7 @@ def _user_message(payload: GenerateIn, *, patch: bool) -> str: # 三期起报告会核对写出来的表名、字段名：instructions 里让它「列出用到的表」时，编出来的名字会被标出来 "- report 提到表名、字段名时写 [[t:表名]] / [[c:表名.字段名]] 或放进反引号，只写 agent 真查过的名字，" "编造的会被标出来；要引用知识库原话就在 report 前接 retrieve，用 [[q:K1|原话]] 逐字引用。问数是探索运行，" - "report 的 claims 不用写\n" + "report 的 claims 不用写：结论句支不支持，由读答案的人点开时按需请模型判断\n" "- 用户要的是周报、要三档出具时，按搭图规则配口径卡和 report_from 契约\n" "- 不要用 transform 解析 agent / llm 的文字：要把 agent 查到的数交给下游，就给它配 output_schema + " "cite_fields\n" @@ -1633,7 +1640,11 @@ _ASSIST_RULES = """\ 审批策略只能往严里改（每次调用都审批 > 仅危险工具需要审批 > 全部自动放行），不放宽、不删掉，更不改成 never； 不取消子工作流钉住的版本；发布级别不是你能改的。结构要动（比如换掉协作团队），输出 question 说明怎么拆，不要自己动手； 报告撰写节点的 numbers、on_violation、claims 只能往严里改（numbers: strict、on_violation: fail、 - claims: require_citation），不放宽、不删掉；契约里的 claims 也一样；不关 agent 的 cite_fields，不删它的 output_schema + claims: require_citation），不放宽、不删掉；claims 从宽到严是 off < require_citation < judge，已经是 judge 的不改回 + require_citation；judge 缺预算（judge.max_cost_usd）时输出 question 问写多少、还是写 null 不限，不替人定； + judge.on_unsupported 是 withhold 的不改回 degrade、不删掉；update_node 会把 judge 整个换掉，补预算时把原来的 + judge（包括从全图默认继承来的）整份带上，只加 max_cost_usd；契约里的 claims 也一样； + 不关 agent 的 cite_fields，不删它的 output_schema - 要人拿主意的，不替人选、不编：required 该包括哪些指标、几张口径卡该用哪张、钉哪个版本、协作团队拆成哪几个 固定步骤、沙箱代码节点是在取数还是在计算（不许自己把 evidence_role 设成 source）……每一处输出一行 {"op":"question","node_id":"节点 id","text":"要问的话，把候选列出来"}，那一处不改 diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 5a8772b0..9efce4c9 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -20,23 +20,30 @@ artifacts 表可以事后插行，run.output 是可以改写的一列，封存 查询步骤只给被引用的行和前后各 2 行，完整快照仍走 /api/artifacts/{id}。数据源 options.mask_columns 里的列换成「已遮罩」——在有身份体系之前，这只减少暴露，不是 安全边界：知道工件 id 的人照样能取到整份快照。 + +结论句的判定（四期）是模型给的，不是证据：正式运行的判定在报告文档里、随文档封存；探索运行 +按需裁判（POST …/evidence/judge），判定作为 evidence.judged 事件追加在封存之后，只填补没判过的 +句子。交给裁判的摘录同样只取封存范围内的工件。 """ from __future__ import annotations import asyncio +import contextlib import csv import io import re from collections import OrderedDict +from collections.abc import Callable from dataclasses import dataclass, field from typing import Any -from fastapi import APIRouter, Query +from fastapi import APIRouter, Body, Query from fastapi.responses import JSONResponse, Response -from sqlalchemy import select +from sqlalchemy import select, update from app.api.coded import CodedHTTPException from app.core.artifact_store import load +from app.core.events import EventType from app.data.engine import masked_columns from app.db.base import SessionLocal from app.db.models import DataSource, Run, RunEvent, Workflow @@ -54,6 +61,7 @@ from app.engine.evidence import ( guess_sources, input_eid, iter_units, + ledger_enabled, make_eid, normalize_quote, query_entry_fields, @@ -61,6 +69,7 @@ from app.engine.evidence import ( sql_tables, uncited_claims, ) +from app.engine.judge import JUDGED, VERDICTS, candidates from app.engine.toolcalls import QUERY_PREFIX router = APIRouter(prefix="/api/runs", tags=["evidence"]) @@ -120,8 +129,17 @@ class _Sealed: retrievals: dict[str, dict[str, Any]] = field(default_factory=dict) #: 封存时记下的清单哈希（没封存的是 None）：旧答案的猜测按 (run_id, 它) 缓存 manifest_hash: str | None = None + #: 运行记录上的 manifest_seq 原值（老运行是 None）：按需裁判追加判定前核对封存没换过 + run_manifest_seq: int | None = None #: verify_manifest 的完整结论（审计表照实附上） verdict: dict[str, Any] = field(default_factory=dict) + #: 运行记录上的几样：按需裁判只给跑完、封存了的探索运行，升级前发起的不给 + run_class: str | None = None + status: str | None = None + ledger_on: bool = False + #: 按需裁判的判定（evidence.judged 事件），按先后：(seq, 节点, 载荷)。不管落在封存前后都收—— + #: 失败后接着跑的运行会把早先追加的判定封进新的清单里，它仍然是按需裁判的批注 + judged: list[tuple[int, str | None, dict[str, Any]]] = field(default_factory=list) @property def sealed(self) -> bool: @@ -161,14 +179,18 @@ async def _sealed(run_id: str) -> _Sealed: select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data) .where(RunEvent.run_id == run_id).order_by(RunEvent.seq))] graph = run.graph if isinstance(run.graph, dict) else {} - manifest = run.manifest_hash + manifest, manifest_seq = run.manifest_hash, run.manifest_seq + run_class, status, ledger_on = run.run_class, run.status, ledger_enabled(run) verdict = await verify_manifest(run_id) bound = verdict.get("sealed_at") if verdict.get("sealed") else None # 没封存（还在跑、停在审批）的运行没有「封存之后」：现有的事件都算，但每件证据都标未封存 events = [r for r in rows if bound is None or r[0] <= bound] out = _Sealed(run_id=run_id, graph=graph, events=events, seal={ "sealed": bool(verdict.get("sealed")), "ok": verdict.get("ok"), "manifest_seq": bound, - "legacy": bool(verdict.get("legacy", False))}, manifest_hash=manifest, verdict=verdict) + "legacy": bool(verdict.get("legacy", False))}, manifest_hash=manifest, run_manifest_seq=manifest_seq, + verdict=verdict, run_class=run_class, status=status, ledger_on=ledger_on, + judged=[(seq, nid, data) for seq, etype, nid, data in rows + if etype == EventType.EVIDENCE_JUDGED and isinstance(data, dict)]) for _, etype, node_id, data in events: if not node_id or not isinstance(data, dict): continue @@ -306,6 +328,7 @@ async def evidence_graph(run_id: str) -> dict[str, Any]: "node_id": report.node_id, "doc_artifact": report.doc_artifact, "doc_sealed": sealed.trusted, "hash_ok": report.hash_ok, "ok": report.ok, "repairs": report.repairs, "fields": report.fields, "stats": report.stats, + **_judgement_of(report, sealed), }) if report.doc is not None: evidence, edges = _graph_of(report, sealed, inputs) @@ -666,13 +689,16 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No chain = await _chain(seg, doc, sealed, chosen.node_id) covered = sealed.trusted and all(step.get("sealed") for step in chain if "sealed" in step) masked = list(dict.fromkeys(c for step in chain if step.get("step") == "query" for c in step.get("masked") or [])) + verdict = _effective_verdicts(chosen, sealed)[0].get(str(unit.get("id"))) return { "report": {"node_id": chosen.node_id, "doc_artifact": chosen.doc_artifact}, "segment": {"id": seg["id"], "text": seg.get("text"), "kind": seg.get("kind"), "state": seg.get("state"), "span": seg.get("span"), "unit": unit.get("id"), **{k: seg[k] for k in ("ref", "issue", "strong", "cite") if k in seg}}, "unit": {"id": unit.get("id"), "kind": unit.get("kind"), "text": markdown[start:end], - "span": unit.get("span"), "cites": unit.get("cites") or []}, + "span": unit.get("span"), "cites": unit.get("cites") or [], + **({"verdict": verdict} if verdict is not None else {}), + "on_demand": _on_demand(sealed, doc, str(unit.get("id")), verdict)}, "block": {"id": block.get("id"), "type": block.get("type")}, "chain": chain, "note": _note(seg, unit), @@ -1151,8 +1177,304 @@ def _note(seg: dict[str, Any], unit: dict[str, Any]) -> str: cites = [str(c) for c in unit.get("cites") or []] if cites: return (f"这是结论句里的文字，这句挂了依据：{'、'.join(cites[:4])}{' 等' if len(cites) > 4 else ''}。" - "依据支不支持这句话，要到后续版本由模型判断") - return "这是结论句里的文字，这句没有挂依据。句子本身有没有依据要到后续版本由模型判断" + "依据支不支持这句话，系统核对不了，只能由模型判断（模型的判断不是确定性的）") + return "这是结论句里的文字，这句没有挂依据。句子本身有没有依据，系统核对不了，只能由模型判断（模型的判断不是确定性的）" + + +# -------------------------------------------------------------------------- +# 结论句的判定：封存内的（正式运行在节点里判的）和封存之后按需追加的 +# -------------------------------------------------------------------------- + +JUDGE_FORMAL = "evidence_judge_formal" +JUDGE_UNSEALED = "evidence_judge_unsealed" +JUDGE_LEGACY = "evidence_judge_legacy" +JUDGE_BAD_UNITS = "evidence_judge_bad_units" +SEAL_BROKEN_CODE = "evidence_seal_broken" +#: 一次最多点名几句。一份报告通常几十句，再多就是请求写错了 +JUDGE_MAX_UNITS = 200 +#: 跑完了的几种终态。还在跑、停在审批、正在接着跑的运行，追加的判定会被封进下一次的清单，不判 +_TERMINAL = ("succeeded", "failed", "cancelled") + +FORMAL_JUDGED = "正式运行的裁判在节点内完成（claims: judge），判定随报告文档一起封存，不能再按需裁判" +UNSEALED_JUDGE = "运行还没跑完封存（还在跑、停在人工审批，或者正在接着跑），跑完再请模型判断" +LEGACY_JUDGE = "这次运行发起于证据台账上线之前，不支持按需裁判" +BROKEN_JUDGE = "封存核对没通过：封存之后有事件被改过、删过或插过，报告和证据都不能再当真，不再请模型判断" +JUDGED_ALREADY = "这句已经判过了" +#: 裁判期间运行被接着跑了（失败的运行可以继续），或者接着跑完重新封存了：判定照样交回，不记进运行记录 +NOT_RECORDED = "运行已经接着跑了，这次的判定没有记进运行记录（只在这里看得到，刷新就没了）；等它跑完封存，再请模型判断" +NOT_A_CLAIM = "不是结论句（标题、连接性的话、表格单元格、代码），不需要模型判断" +BAD_UNITS = f'units 要写成句子编号的列表，比如 {{"units": ["u4"]}}，一次最多 {JUDGE_MAX_UNITS} 句；report 要写报告节点的 id' +#: 触顶时告诉人怎么调：设置页「证据裁判」一组的各项上限都能调高，也都能设成不限 +JUDGE_ADJUST = { + "max_cost_usd": "到设置 → 证据裁判里调高「每次点击」的金额上限，或者设成不限", + "daily_max_usd": "今天的裁判花费到了全局每日上限：到设置 → 证据裁判里调高「每日」上限或者设成不限，也可以明天再判", + "max_claims": "一次最多判这么多句：分几次点，或者到设置 → 证据裁判里调高「每份报告的句数」" + "（报告节点上写了 judge.max_claims 的按节点，写 null 是不限）", + "timeout_s": "裁判超时了：到设置 → 证据裁判里调高「时长」上限或者设成不限，再点一次", +} + + +def _verdict_ok(verdict: Any) -> bool: + return isinstance(verdict, dict) and verdict.get("status") in VERDICTS + + +def _effective_verdicts(report: _Report, sealed: _Sealed + ) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]: + """这份报告每句的判定：(生效的, 其中封存之后按需追加的)。 + + 文档里的判定随文档封存（正式运行在节点里判的；探索运行的候选句记「未裁判 · 按需」）。按需裁判的判定在 + evidence.judged 事件里，只认指向这份文档、文档里确有其句的；它只填补没判过的句子——封存内判过的，后面 + 追加什么都盖不掉，一条追加的事件改写不了封存的结论。同一句按需判过几次（先触顶、后判成），判成的为准， + 其次取最后一次。追加的判定一律标 post_seal。 + """ + doc = report.doc or {} + ids = {str(u.get("id")) for _, u in iter_units(doc)} + out = {str(u.get("id")): u["verdict"] for _, u in iter_units(doc) if _verdict_ok(u.get("verdict"))} + sealed_judged = {uid for uid, v in out.items() if v.get("status") in JUDGED} + later: dict[str, dict[str, Any]] = {} + for _, _, data in sealed.judged: + if data.get("report") != report.node_id or data.get("doc_artifact") != report.doc_artifact: + continue + verdicts = data.get("verdicts") if isinstance(data.get("verdicts"), dict) else {} + for uid, verdict in verdicts.items(): + uid = str(uid) + if uid not in ids or uid in sealed_judged or not _verdict_ok(verdict): + continue + if later.get(uid, {}).get("status") in JUDGED and verdict.get("status") not in JUDGED: + continue + later[uid] = out[uid] = {**verdict, "post_seal": True} + return out, later + + +def _judgement_of(report: _Report, sealed: _Sealed) -> dict[str, Any]: + """证据图、审计表里每份报告带的结论句情况：写作时的策略、封存的裁判摘要、封存之后按需追加的判定。""" + doc = report.doc or {} + return {"claims": report.claims, "judge": doc.get("judge") if isinstance(doc.get("judge"), dict) else None, + "post_seal_verdicts": _effective_verdicts(report, sealed)[1] if report.doc is not None else {}} + + +def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str, Any] | None) -> dict[str, Any]: + """这句能不能「请模型判断」：{available, reason, message}。和 POST …/evidence/judge 同一套条件。""" + def no(reason: str, message: str) -> dict[str, Any]: + return {"available": False, "reason": reason, "message": message} + + if sealed.run_class == "formal": + return no("formal", FORMAL_JUDGED) + if not sealed.ledger_on: + return no("legacy", LEGACY_JUDGE) + if not candidates(doc, [uid]): + return no("not_a_claim", NOT_A_CLAIM) + if isinstance(verdict, dict) and verdict.get("status") in JUDGED: + return no("judged", JUDGED_ALREADY) + if not sealed.sealed or sealed.status not in _TERMINAL: + return no("unsealed", UNSEALED_JUDGE) + if not sealed.trusted: + return no("seal_broken", BROKEN_JUDGE) + return {"available": True, "reason": None, "message": None} + + +def _sealed_loader(sealed: _Sealed) -> Callable[[str], Any]: + """给裁判取证据的 loader：只认封存范围内的事件交回过的工件（查询、表结构、检索快照和台账里记过的）， + 取回时照旧复验哈希。别的一律当取不到——人为插进工件表的行、封存之后才追加的事件引用的快照， + 都不能送进裁判的摘录。报告文档里的目录也不例外：目录写着哪件工件，不等于它在封存链上。""" + allowed = {*sealed.queries, *sealed.schemas, *sealed.retrievals, *(str(a) for _, a in sealed.ledger)} + + def loader(artifact: str) -> Any: + return load(artifact) if sealed.trusted and str(artifact) in allowed else None + return loader + + +def _node_judge(sealed: _Sealed, node_id: str) -> dict[str, Any]: + """这次运行的图上报告节点写的 judge（裁判模型、句数和时长上限）；图读不出来就当没写。""" + from app.engine.nodes.report import report_judge + from app.engine.schema import GraphSpec + + try: + spec = GraphSpec.model_validate(sealed.graph) + node = spec.node_map().get(node_id) + except Exception: # noqa: BLE001 - 老运行的图可能已经不合现在的格式 + node = None + return report_judge(spec, node) if node is not None else {} + + +class _RunLocks: + """每次运行一把锁：同一次运行的按需裁判排队，连点两下时后一下看得到前一下记下的判定，不重复花钱。""" + + def __init__(self) -> None: + self._locks: dict[str, asyncio.Lock] = {} + self._users: dict[str, int] = {} + + @contextlib.asynccontextmanager + async def hold(self, key: str): # type: ignore[no-untyped-def] + lock = self._locks.setdefault(key, asyncio.Lock()) + self._users[key] = self._users.get(key, 0) + 1 + try: + async with lock: + yield + finally: + self._users[key] -= 1 + if not self._users[key]: + del self._users[key], self._locks[key] + + +_JUDGING = _RunLocks() + + +def _judge_request(payload: Any, report: str | None) -> tuple[list[str], str | None]: + if not isinstance(payload, dict): + raise CodedHTTPException(422, BAD_UNITS, JUDGE_BAD_UNITS) + units = payload.get("units") + wanted = report if payload.get("report") is None else payload["report"] + if not isinstance(units, list) or not units or len(units) > JUDGE_MAX_UNITS \ + or not all(isinstance(u, str) and u.strip() for u in units) \ + or (wanted is not None and not isinstance(wanted, str)): + raise CodedHTTPException(422, BAD_UNITS, JUDGE_BAD_UNITS) + return list(dict.fromkeys(u.strip() for u in units)), wanted or None + + +@router.post("/{run_id}/evidence/judge") +async def judge_units(run_id: str, payload: Any = Body(None), report: str | None = None) -> dict[str, Any]: + """探索运行里按需裁判几句结论：请求体 {units: ["u4"], report?: 报告节点 id}。 + + 判过的句子直接给已有的判定，不再调用；没判过的按设置里每次点击的金额上限、每份报告的句数和时长上限、 + 全局每日上限去判，触顶照实说「已到上限」，不是报错。问过模型的这一次记一条 evidence.judged 事件， + 落在封存之后（post_seal: true）——封存核对只覆盖 manifest_seq 之前的事件，追加它 verify 照旧一致； + 封存的报告文档一个字都不改。交给裁判的证据只取封存范围内的工件，数据源遮罩的列不给。 + """ + units, wanted = _judge_request(payload, report) + async with _JUDGING.hold(run_id): + sealed = await _sealed(run_id) + if sealed.run_class == "formal": + raise CodedHTTPException(409, FORMAL_JUDGED, JUDGE_FORMAL) + if not sealed.ledger_on: + raise CodedHTTPException(409, LEGACY_JUDGE, JUDGE_LEGACY) + if not sealed.sealed or sealed.status not in _TERMINAL: + raise CodedHTTPException(409, UNSEALED_JUDGE, JUDGE_UNSEALED) + if not sealed.trusted: + raise CodedHTTPException(409, BROKEN_JUDGE, SEAL_BROKEN_CODE) + reports = _reports(sealed) + if not reports: + raise CodedHTTPException(404, "这次运行没有封存的报告文档，没有可以判断的句子", REPORT_NOT_FOUND) + chosen = _choose(reports, wanted, sealed) + if chosen.hash_ok is False: + raise CodedHTTPException(409, "报告文档和它的哈希对不上，疑似被改过，不能再当证据展示", DOC_TAMPERED) + if chosen.doc is None: + raise CodedHTTPException(404, "报告文档在工件库里取不回来", DOC_MISSING) + return await _judge_now(sealed, chosen, units) + + +async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict[str, Any]: + from app.engine import judge as judging + + doc = report.doc or {} + catalog = doc.get("catalog") or {} + known = _effective_verdicts(report, sealed)[0] + reused = [u for u in units if known.get(u, {}).get("status") in JUDGED] + pending = [u for u in units if u not in reused] + verdicts = {u: known[u] for u in reused} + node_judge = _node_judge(sealed, report.node_id) + async with SessionLocal() as session: + settings = await judging.judge_settings(session) + budget = judging.click_budget(settings, node_judge) + outcome: dict[str, Any] | None = None + skipped: dict[str, str] = {} + event = None + lost = False + if pending: + loader = _sealed_loader(sealed) + masked = await judging.source_masks(catalog, loader=loader) + request = judging.prepare(doc, catalog, units=pending, loader=loader, masked=masked) + skipped = dict(request.skipped) + if request.cands: + async with SessionLocal() as session: + spec = await judging.judge_model_spec(session, node_judge, settings=settings) + outcome = await judging.run_request(request, spec=spec, budget=budget, post_seal=True, click=True) + verdicts.update(outcome["verdicts"]) + if outcome["calls"] or outcome["cost_usd"]: + # 没问模型（触顶、模型没配好）就没有判定可记：说给点的人听，不往运行记录里追加 + event = await _append_judged(sealed, report, pending, outcome) + lost = event is None + limits = list(outcome["limits_hit"]) if outcome else [] + message = _judge_message(outcome, budget, skipped) + if lost: + message = "；".join(filter(None, [NOT_RECORDED, message])) + return { + "run_id": sealed.run_id, "report": {"node_id": report.node_id, "doc_artifact": report.doc_artifact}, + "units": units, "verdicts": {u: verdicts[u] for u in units if u in verdicts}, "reused": reused, + "judged": [u for u in pending if (outcome or {}).get("verdicts", {}).get(u, {}).get("status") in JUDGED], + "skipped": skipped, "unjudged": dict(outcome["unjudged"]) if outcome else {}, + "limits_hit": limits, "limited": bool(limits), "message": message, + "adjust": [JUDGE_ADJUST[r] for r in limits if r in JUDGE_ADJUST], + "gaps": list(outcome["gaps"]) if outcome else [], "notes": list(outcome["notes"]) if outcome else [], + "model": (outcome or {}).get("model"), "priced": (outcome or {}).get("priced"), + "cost_usd": (outcome or {}).get("cost_usd", 0.0), "calls": (outcome or {}).get("calls", 0), + "duration_ms": (outcome or {}).get("duration_ms", 0), + "budget": budget.as_dict() if outcome else None, "event": event, "post_seal": True, + "spend": {**await judging.daily_spend(), "daily_max_usd": settings.get("daily_max_usd")}, + } + + +def _judge_message(outcome: dict[str, Any] | None, budget: Any, skipped: dict[str, str]) -> str | None: + """这次点击的一句话结论：触顶、没跑成、点的不是结论句；判成了（或者全是判过的）就没什么要说的。""" + from app.engine.judge import _limit_label + + if outcome and outcome["limits_hit"]: + labels = "、".join(_limit_label(r, budget, click=True) for r in outcome["limits_hit"]) + n = sum(outcome["unjudged"].get(r, 0) for r in outcome["limits_hit"]) + return f"已到上限（{labels}）：{n} 句没判，记为未裁判；已判的保留" + if outcome and outcome["gaps"]: + return "裁判没跑完：" + "；".join(outcome["gaps"]) + parts = [] + if others := [u for u, why in skipped.items() if why == "not_a_claim"]: + parts.append(f"{'、'.join(others)} {NOT_A_CLAIM}") + if missing := [u for u, why in skipped.items() if why == "not_found"]: + parts.append(f"报告里没有 {'、'.join(missing)}") + return "；".join(parts) or None + + +async def _append_judged(sealed: _Sealed, report: _Report, units: list[str], + outcome: dict[str, Any]) -> dict[str, Any] | None: + """记一条 evidence.judged：问过模型的这一次（连同没判成的句子），接在运行记录的最后一条后面。 + + 只接在这次点击开始时的那份封存后面：裁判要花几秒（时长不限时更久），这中间有人点了「继续运行」 + （失败的运行可以接着跑），再追加就落进一次活着的运行中间，还会被封进下一份清单；接着跑完、重新封存了 + 也一样，这次的判定依据的已经不是最新的封存。这两种都不记，返回 None（判定照样交回给点的人）。 + 核对和写入是同一个事务里的条件更新，和接着跑占住运行（runner._claim）抢的是同一行：谁先提交算谁的。 + """ + from app.core.bus import bus + from app.engine.runner import _stage, run_manager + + same_seal = (Run.manifest_seq.is_(None) if sealed.run_manifest_seq is None + else Run.manifest_seq == sealed.run_manifest_seq, + Run.manifest_hash.is_(None) if sealed.manifest_hash is None + else Run.manifest_hash == sealed.manifest_hash) + async with SessionLocal() as session: + run = await session.get(Run, sealed.run_id) + if run is None: + return None + # 进程重启过的话内存里的序号从 0 起：接在库里最后一条后面，不和已有的撞号 + bus.set_seq(sealed.run_id, run.last_seq or 0) + event = run_manager._event( + sealed.run_id, EventType.EVIDENCE_JUDGED, node_id=report.node_id, data={ + "report": report.node_id, "doc_artifact": report.doc_artifact, "units": units, + "verdicts": outcome["verdicts"], + "keys": {u: outcome["keys"][u] for u in outcome["verdicts"] if u in outcome["keys"]}, + "model": outcome["model"], "priced": outcome["priced"], "cost_usd": outcome["cost_usd"], + "calls": outcome["calls"], "duration_ms": outcome["duration_ms"], "budget": outcome["budget"], + "limits_hit": outcome["limits_hit"], "unjudged": outcome["unjudged"], "gaps": outcome["gaps"], + "notes": outcome["notes"], "skipped": outcome["skipped"], "post_seal": True}) + claimed = await session.execute( + update(Run).where(Run.id == sealed.run_id, Run.status.in_(_TERMINAL), *same_seal) + .values(last_seq=event.seq)) + if claimed.rowcount != 1: + # 运行已经不是那份封存下的终态了。烧掉一个序号不要紧：不落库的事件（令牌流）本来就占序号， + # 序号只要求递增、不撞号 + await session.rollback() + return None + await _stage(session, event, run) + await session.commit() + await bus.publish(event) + return {"seq": event.seq} # -------------------------------------------------------------------------- @@ -1172,6 +1494,11 @@ UNCITED_CLAIM_COUNTED = "这句结论没有挂依据：这份报告要求结论 UNCITED_CLAIM_WITHHELD = "这句结论没有挂依据：出具契约要求结论句挂依据，没挂的按契约不予出具（on_uncited: withhold）" UNCITED_CLAIM_REQUIRED = ("这句结论没有挂依据：这份报告要求结论句挂依据（claims: require_citation），不过没有出具契约" "按它判档（没配契约，或者还没到出具那一步）") +#: claims: judge 在挂依据这件事上和 require_citation 一样严（出口按 on_uncited 计入缺口） +UNCITED_CLAIM_JUDGED = ("这句结论没有挂依据：这份报告的结论句由模型裁判（claims: judge），挂依据的要求和 " + "require_citation 一样，出具时计入缺口") +UNCITED_CLAIM_JUDGE_REQUIRED = ("这句结论没有挂依据：这份报告的结论句由模型裁判（claims: judge），要求结论句挂依据，" + "不过没有出具契约按它判档（没配契约，或者还没到出具那一步）") LEGACY_UNMATCHED = "旧版出具里这个数字对不上任何指标" #: CSV 的列：(表头, 行里的键) @@ -1239,7 +1566,7 @@ async def evidence_audit(run_id: str, fmt: str | None = Query(None, alias="forma "run_id": run_id, "schema": AUDIT_SCHEMA, "mode": mode, "seal": {**sealed.seal, "events": sealed.verdict.get("events"), "message": sealed.verdict.get("message")}, "reports": [{"node_id": r.node_id, "doc_artifact": r.doc_artifact, "hash_ok": r.hash_ok, - "doc_sealed": sealed.trusted, "fields": r.fields, "claims": r.claims, "stats": r.stats} + "doc_sealed": sealed.trusted, "fields": r.fields, "stats": r.stats, **_judgement_of(r, sealed)} for r in reports], "groups": grouped, "counts": {g["key"]: g["count"] for g in grouped}, "total": sum(g["count"] for g in grouped), @@ -1299,11 +1626,14 @@ def _claim_note(report: _Report, issuance: dict[str, Any] | None) -> str: if isinstance(issuance, dict) and issuance.get("mode") == "citations" \ and (issuance.get("report") or {}).get("node_id") == report.node_id: claims = issuance.get("claims") - if not isinstance(claims, dict) or claims.get("policy") != "require_citation": + if not isinstance(claims, dict) or claims.get("policy") not in ("require_citation", "judge"): return UNCITED_CLAIM on_uncited = claims.get("on_uncited") or "degrade" return UNCITED_CLAIM_WITHHELD if on_uncited == "withhold" else \ - UNCITED_CLAIM if on_uncited == "ignore" else UNCITED_CLAIM_COUNTED + UNCITED_CLAIM if on_uncited == "ignore" else \ + UNCITED_CLAIM_JUDGED if claims.get("policy") == "judge" else UNCITED_CLAIM_COUNTED + if report.claims == "judge": + return UNCITED_CLAIM_JUDGE_REQUIRED return UNCITED_CLAIM_REQUIRED if report.claims == "require_citation" else UNCITED_CLAIM diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 16391913..acf1124b 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -14,6 +14,9 @@ from app.core.config import settings as app_settings from app.core.crypto import encrypt, mask from app.db.base import get_session from app.db.models import Provider, Setting +from app.engine.judge import JUDGE_DEFAULTS, SETTING_GROUP as JUDGE_GROUP, SPEND_KEY as JUDGE_SPEND +from app.engine.judge import daily_spend, judge_settings +from app.engine.judge import sanitize_settings as judge_values, settings_problem as judge_problem from app.providers import catalog from app.providers.factory import ModelSpec, ProviderNotConfigured, _resolve_key, build_chat_model @@ -434,6 +437,9 @@ DEFAULT_SETTINGS: dict[str, Any] = { "max_agent_steps": app_settings.max_agent_steps, "max_run_seconds": app_settings.max_run_seconds, }, + # 结论句裁判（engine/judge.py）：证据裁判模型和各项上限，上限存 null 表示不限。不放进 limits： + # 那一组是环境变量定的、只读，这一组是用户自己定的 + JUDGE_GROUP: dict(JUDGE_DEFAULTS), } @@ -482,7 +488,11 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str # limits 是服务端配置（环境变量），只读：库里可能存着旧版设置页写进去的一份 #（见过 max_agent_steps: 25），它从来不生效，却会盖住真实的上限，界面照着错的数许诺 stored.pop("limits", None) + # 裁判的每日计数是引擎记的账，不是用户设置：设置页整组回写时带着一份旧的，会把今天的花费清零 + stored.pop(JUDGE_SPEND, None) merged = {k: {**v, **(stored.get(k) or {})} for k, v in DEFAULT_SETTINGS.items()} + # 裁判设置按引擎实际生效的样子给：存坏了的项显示默认，存了 null 的上限显示不限 + merged[JUDGE_GROUP] = judge_values(stored.get(JUDGE_GROUP)) for key, value in stored.items(): merged.setdefault(key, value) return merged @@ -498,10 +508,14 @@ async def put_settings( ) -> dict[str, Any]: from app.tools.trust import SETTING_KEY as TRUST_KEY + if JUDGE_GROUP in payload.values and (problem := judge_problem(payload.values[JUDGE_GROUP])): + # 先查后写：一组里有一项不对就整个请求不落库，免得存下半组 + raise HTTPException(422, problem) for key, value in payload.values.items(): - if key in (TRUST_KEY, "limits"): + if key in (TRUST_KEY, "limits", JUDGE_SPEND): # 信任三档只走 PUT /api/tools/trust：设置页整组回写时带着一份旧的， - # 会把刚在工具页改的档位冲掉。limits 是环境变量定的，写进库也不生效 + # 会把刚在工具页改的档位冲掉。limits 是环境变量定的，写进库也不生效。 + # 裁判的每日计数是引擎记的账，只有裁判自己写 continue row = await session.get(Setting, key) if row: @@ -512,6 +526,16 @@ async def put_settings( return await get_settings(session) +@router.get("/settings/judge/spend") +async def get_judge_spend(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """今天（本地日期）结论句裁判花了多少，设置页放在每日上限旁边。只读：计数只有裁判自己记。 + + {date, usd, calls, unpriced_calls, daily_max_usd}。unpriced_calls 是价格目录里没有的模型的调用次数， + 这些调用估不出金额、没有算进 usd。daily_max_usd 为 null 表示不限。 + """ + return {**await daily_spend(), "daily_max_usd": (await judge_settings(session))["daily_max_usd"]} + + @router.get("/system") async def system_info() -> dict[str, Any]: """给设置页顶部展示的运行环境概览。""" diff --git a/backend/app/core/events.py b/backend/app/core/events.py index 424a1304..81ed9c40 100644 --- a/backend/app/core/events.py +++ b/backend/app/core/events.py @@ -69,6 +69,9 @@ class EventType(StrEnum): # 报告撰写节点的自查结果：报告文档工件 id、统计、违规清单。落在封存范围内， # 证据接口靠它找到文档——不去翻不受封存保护的 artifacts 表 REPORT_CHECKED = "report.checked" + # 探索运行里按需裁判的结论句判定：点开哪句才判哪句，追加在封存之后（post_seal），是批注不是证据。 + # 封存核对只覆盖 manifest_seq 之前的事件，追加它不影响 verify；封存的报告文档一个字都不改 + EVIDENCE_JUDGED = "evidence.judged" # 其他 LOG = "log" diff --git a/backend/app/engine/autofix.py b/backend/app/engine/autofix.py index 419f012f..761dd695 100644 --- a/backend/app/engine/autofix.py +++ b/backend/app/engine/autofix.py @@ -23,8 +23,23 @@ from typing import Any, Callable from pydantic import BaseModel -from app.engine.governance import REPORT_POLICY, exit_text_fields, publish_issues -from app.engine.schema import REPORT_CLAIMS, GraphNode, GraphSpec, NodeType, _ancestors, claims_problem +from app.engine.governance import ( + JUDGE_UNLIMITED, + REPORT_POLICY, + REPORT_POLICY_ALSO, + exit_text_fields, + publish_issues, +) +from app.engine.schema import ( + CLAIMS_JUDGE, + JUDGE_ON_UNSUPPORTED, + REPORT_CLAIMS, + GraphNode, + GraphSpec, + NodeType, + _ancestors, + claims_problem, +) #: 被选中的修复按这个顺序应用：先补契约骨架，再补契约里的各项，最后是节点上的开关。 #: 前面的修复可能改变后面的候选（metrics_from 决定 required 从哪几张卡里挑，report_from 决定 @@ -34,7 +49,7 @@ _ORDER = ( "contract.report_from_missing", "contract.report_from_invalid", "governed.report_from_required", "governed.exit_text_source", "report.metrics_from_invalid", "contract.required_missing", "contract.strict_off", "contract.claims_ignored", "report.claims_invalid", - "governed.report_policy", + "governed.report_policy", "governed.judge_budget", "governed.caliber_agent_cite_fields", "governed.caliber_compute_input", "governed.agent_approval_never", "governed.default_approval_never", "governed.subgraph_unpinned", "governed.supervisor", "contract.not_object", "governed.text_bypass", "governed.caliber_agent_schema", "governed.caliber_model_input", @@ -210,7 +225,8 @@ def _plan_exit_fields(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) def _plan_report_policy(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: - """G3：报告撰写节点写明最严的三项。答案唯一，只往严里改；写了不支持的 claims 由 claims_invalid 管。""" + """G3：报告撰写节点写明最严的三项。答案唯一，只往严里改；写了不认识的 claims 由 claims_invalid 管。 + 已经写了更严的写法（claims: judge）的不动——改回 require_citation 就是替人放宽了。""" if node is None: return None writes: dict[str, Any] = {} @@ -218,7 +234,9 @@ def _plan_report_policy(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None for key, want, _ in REPORT_POLICY: value = node.config.get(key) effective = ctx.spec.defaults.get(key) if value in (None, "") else value - if effective != want and not (key == "claims" and claims_problem(effective)): + if effective in (want, *REPORT_POLICY_ALSO.get(key, ())): + continue + if not (key == "claims" and claims_problem(effective)): writes[key], before[key] = want, copy.deepcopy(value) if not writes: return None @@ -230,18 +248,38 @@ def _plan_report_policy(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None def _plan_claims(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: - """claims 写了不支持的值。受管级别只认 require_citation，答案唯一；已发布级别 off 和 require_citation - 都行，要人选——写的是 judge（还不支持的模型裁判）时建议 require_citation：它离裁判最近。""" + """claims 写了不认识的值。受管级别改成 require_citation：确定性的两档里只有它合规，答案唯一（judge 要花钱、 + 要写预算，不替人开）；已发布级别 off 和 require_citation 都行，要人选。""" if node is None: return None before = node.config.get("claims") if ctx.level == "governed": - return _auto(f"把「{node.title}」的 claims 改成 require_citation：受管出具要求每句结论挂引用" - + ("（结论句裁判在后续版本支持）" if before == "judge" else ""), "claims", before, "require_citation") + return _auto(f"把「{node.title}」的 claims 改成 require_citation：受管出具要求每句结论挂引用", "claims", + before, "require_citation") hints = {"off": "结论句不参与判档，和以前一样", "require_citation": "没挂引用的结论句计入缺口，按出具档位降档"} options = [{"value": v, "label": v, "hint": hints[v]} for v in REPORT_CLAIMS] - return _choice(f"选「{node.title}」的结论句策略（claims）", "claims", options, - default="require_citation" if before == "judge" else None) + return _choice(f"选「{node.title}」的结论句策略（claims）", "claims", options) + + +def _plan_judge_budget(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + """G3：claims: judge 没写每份报告的裁判预算。写多少钱只有作者能定：候选是默认金额和「不限」，不替人选、 + 不给建议。judge 原来写在图级 defaults 里的，写到节点上时带上 defaults 那一整份——节点写了 judge 就整个 + 用节点的，只补一个 max_cost_usd 会把 defaults 里选的裁判模型、上限悄悄丢掉。""" + from app.engine.judge import JUDGE_DEFAULTS + + if node is None: + return None + judge = node.config.get("judge") + if judge in (None, ""): + judge = ctx.spec.defaults.get("judge") + base = copy.deepcopy(judge) if isinstance(judge, dict) else {} + amount = JUDGE_DEFAULTS["report_max_cost_usd"] + options = [ + {"value": amount, "label": f"每份报告最多 ${amount:g}", "hint": "系统默认的每份报告上限；写进节点以后，改设置不影响这份报告"}, + {"value": None, "label": "不限（写 null）", "hint": JUDGE_UNLIMITED}, + ] + return _choice(f"给「{node.title}」的结论句裁判写每份报告的预算（judge.max_cost_usd）", "judge.max_cost_usd", + options, _judge=base) def _plan_compute_input(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: @@ -419,6 +457,7 @@ _PLANNERS: dict[str, Callable[[_Ctx, dict[str, Any], GraphNode | None], dict[str "governed.text_bypass": _plan_text_bypass, "governed.report_policy": _plan_report_policy, "report.claims_invalid": _plan_claims, + "governed.judge_budget": _plan_judge_budget, "governed.caliber_compute_input": _plan_compute_input, "governed.caliber_agent_cite_fields": _plan_cite_fields, "governed.caliber_agent_schema": _plan_agent_schema, @@ -580,12 +619,25 @@ def _edit_exit_fields(graph: dict[str, Any], fix: dict[str, Any], value: Any) -> return out +def _edit_judge_budget(graph: dict[str, Any], fix: dict[str, Any], value: Any) -> list[tuple[str, str, Any, Any]]: + """G3 的裁判预算：节点的 judge 写成（原来生效的那份 judge + 选中的 max_cost_usd）。选「不限」写显式的 null， + 和没写不是一回事。变更按整份 judge 记：前后都是 null 的 max_cost_usd 在预览里分不出写没写。""" + node = _node(graph, fix["node_id"]) + if node is None: + return [] + config = _config(node) + after = {**copy.deepcopy(fix.get("_judge") or {}), "max_cost_usd": value} + before = _write(config, "judge", after) + return [(str(node["id"]), "judge", before, copy.deepcopy(after))] + + #: 每类问题怎么改图。大多是同一种「按节点字段写值」，改全图默认、改成果字段的各走一格；留着这张表是为了 #: 某类修复要特殊处理时只换它自己的那一格（测试里也借它模拟一条出错的修复规则） _EDITORS: dict[str, Callable[[dict[str, Any], dict[str, Any], Any], list[tuple[str | None, str, Any, Any]]]] = { code: (_edit_defaults if code in _GRAPH_FIXES else _edit) for code in _PLANNERS } _EDITORS["governed.exit_text_source"] = _edit_exit_fields +_EDITORS["governed.judge_budget"] = _edit_judge_budget def _handoff(fix: dict[str, Any], value: Any) -> bool: @@ -811,25 +863,34 @@ def _contract_loosened(name: str, prior: Any, contract: Any) -> list[str]: if contract.get("cells") is True and prior.get("cells") is not True: out.append(f"替你打开了「{name}」出具契约的 cells（允许报告直接引用查询单元格），这要你自己决定") was, now = _claims_rank(prior.get("claims")), _claims_rank(contract.get("claims")) - if was is not None and now is not None and now < was: - out.append(f"把「{name}」出具契约的 claims 往宽里改了（没挂依据的结论句{_CLAIMS_WORDS[was]} → " - f"{_CLAIMS_WORDS[now]}）") + if was is not None and now is not None: + looser = [f"{what}{words[a]} → {words[b]}" for (what, words), a, b in zip(_CLAIMS_AXES, was, now) if b < a] + if looser: + out.append(f"把「{name}」出具契约的 claims 往宽里改了（{'；'.join(looser)}）") return out -#: 契约 claims 的严格程度：不查 < 查了不算缺口（ignore）< 计入缺口降档（degrade）< 不予出具（withhold） -_CLAIMS_WORDS = ("不检查", "只标出来", "计入缺口", "不予出具") +#: 契约 claims 的严格程度分两样，各自从宽到严：没挂依据的结论句（不查 < 查了不算缺口 ignore < 计入缺口降档 +#: degrade < 不予出具 withhold），证据不支持的结论句（不裁判 < 裁判、降档 < 裁判、不予出具）。judge 两样都管： +#: 挂依据的要求和 require_citation 一样，另外按裁判的判定判档 +_CLAIMS_AXES = (("没挂依据的结论句", ("不检查", "只标出来", "计入缺口", "不予出具")), + ("证据不支持的结论句", ("不裁判", "降档出具", "不予出具"))) -def _claims_rank(value: Any) -> int | None: - """契约里 claims 的严格程度，越大越严。judge 这类还不支持的写法不比较（validate 另报）。""" +def _claims_rank(value: Any) -> tuple[int, int] | None: + """契约里 claims 的严格程度：(没挂依据的, 证据不支持的)，各自越大越严，任何一样变小都算放宽。 + 认不出的写法不比较（出口按缺口处理，不会因此放松）。""" name = value.get("policy") if isinstance(value, dict) else value if name in (None, "", "off"): - return 0 - if name != "require_citation": + return 0, 0 + if name not in ("require_citation", CLAIMS_JUDGE): return None on_uncited = value.get("on_uncited") if isinstance(value, dict) else None - return {"ignore": 1, "withhold": 3}.get(on_uncited, 2) + uncited = {"ignore": 1, "withhold": 3}.get(on_uncited, 2) + if name != CLAIMS_JUDGE: + return uncited, 0 + on_unsupported = value.get("on_unsupported") if isinstance(value, dict) else None + return uncited, 2 if on_unsupported == "withhold" else 1 def forbidden_changes(before: dict[str, Any], after: dict[str, Any], @@ -844,7 +905,8 @@ def forbidden_changes(before: dict[str, Any], after: dict[str, Any], - 审批策略往宽里改（每次调用都审批 > 仅危险工具需要审批 > 全部自动放行）、删掉写明的审批策略， 把审批写成（或新加一个）「全部自动放行」；全图默认同理 - 取消子工作流钉住的版本 - - 报告撰写节点的 numbers、on_violation、claims 往宽里改；关掉 agent 的 cite_fields + - 报告撰写节点的 numbers、on_violation、claims 往宽里改（claims 改成认不出的写法也算），claims 一直是 + judge 时 judge.on_unsupported 从 withhold 退回去；关掉 agent 的 cite_fields - 替人把沙箱代码标成取数（evidence_role: source）：它在取数还是在算只有作者知道。chosen 里的 (节点 id, 字段) 是人在 choice 里亲手选的，不算 - 改发布级别——级别不在图里，修复接口也从不写工作流的 status，所以只可能以一个不认识的 @@ -928,12 +990,17 @@ def forbidden_changes(before: dict[str, Any], after: dict[str, Any], #: 报告撰写节点三项设置从严到宽的次序：没写时运行时的缺省值（on_violation 没写时探索运行按 flag） _REPORT_STRICT = {"numbers": ("strict", "strict"), "on_violation": ("fail", "flag"), "claims": ("require_citation", "off")} +#: claims 从宽到严：off < require_citation < judge（在挂引用之外再请模型逐句裁判，不支持的按 on_unsupported +#: 判档）。不在这里的写法不比较（validate 报 report.claims_invalid） +_CLAIMS_ORDER = ("off", "require_citation", CLAIMS_JUDGE) def _report_loosened(name: str, prev: dict[str, Any], cfg: dict[str, Any], old_defaults: Any, new_defaults: Any) -> list[str]: - """报告撰写节点的三项设置有没有往宽里改：原来（按节点、再按全图默认）是最严的值，改完不是了。 - claims 改成 judge 不算放宽——那是还不支持的更严的写法，validate 会报。""" + """报告撰写节点的三项设置有没有往宽里改：numbers、on_violation 原来（按节点、再按全图默认）是最严的值， + 改完不是了；claims 按 off < require_citation < judge 排，往下走一格也算（require_citation → judge 是收紧）， + 从 require_citation、judge 改成认不出的写法也算（原来就认不出的不比较）。claims 前后都是 judge 的， + 再看 judge.on_unsupported 有没有从 withhold 退回去。""" def effective(config: dict[str, Any], defaults: Any, key: str, fallback: str) -> Any: value = config.get(key) if value in (None, ""): @@ -943,12 +1010,56 @@ def _report_loosened(name: str, prev: dict[str, Any], cfg: dict[str, Any], old_d out = [] for key, (strict, fallback) in _REPORT_STRICT.items(): was, now = effective(prev, old_defaults, key, fallback), effective(cfg, new_defaults, key, fallback) - if was == strict and now != strict and not (key == "claims" and now == "judge"): - out.append(f"把「{name}」的 {key} 从 {strict} 放宽成了 {now}" - + ("（去掉了写明的值，运行时退回缺省）" if cfg.get(key) in (None, "") else "")) + if key == "claims": + if was not in _CLAIMS_ORDER or (was == "off" and now not in _CLAIMS_ORDER): + continue + if now not in _CLAIMS_ORDER: + out.append(f"把「{name}」的 claims 从 {was} 改成了不认识的 {now}") + continue + if _CLAIMS_ORDER.index(now) >= _CLAIMS_ORDER.index(was): + continue + elif not (was == strict and now != strict): + continue + out.append(f"把「{name}」的 {key} 从 {was} 放宽成了 {now}" + + ("（去掉了写明的值，运行时退回缺省）" if cfg.get(key) in (None, "") else "")) + if effective(prev, old_defaults, "claims", "off") == effective(cfg, new_defaults, "claims", "off") \ + == CLAIMS_JUDGE and (why := _judge_loosened(name, prev, cfg, old_defaults, new_defaults)): + out.append(why) return out +def _judge_loosened(name: str, prev: dict[str, Any], cfg: dict[str, Any], old_defaults: Any, + new_defaults: Any) -> str | None: + """claims 一直是 judge 时，judge.on_unsupported 从 withhold（证据不支持的结论句不予出具）退回 degrade + （只降档）。judge 和运行时（report.report_judge）同一个取法：节点上写了就整个用节点的，没写取图级 + defaults；on_unsupported 没写、写了认不出的都按 degrade。所以补预算时把 judge 整个换成 + {max_cost_usd: …}、节点另写一份不带 withhold 的 judge 盖掉全图默认，都算放宽。""" + def block(config: dict[str, Any], defaults: Any) -> tuple[Any, bool]: + own = config.get("judge") + if own not in (None, ""): + return own, False + return (defaults if isinstance(defaults, dict) else {}).get("judge"), True + + def written(judge: Any) -> Any: + return judge.get("on_unsupported") if isinstance(judge, dict) else None + + (old_block, old_inherited), (new_block, new_inherited) = block(prev, old_defaults), block(cfg, new_defaults) + if written(old_block) != "withhold" or (raw := written(new_block)) == "withhold": + return None + was = "withhold" + ("（跟随全图默认）" if old_inherited else "") + if raw in JUDGE_ON_UNSUPPORTED: + now = f"{raw}" + ("（跟随全图默认）" if new_inherited else "") + elif raw not in (None, ""): + now = f"认不出的 {raw}（运行时按 degrade）" + elif new_inherited: + now = "degrade（跟随全图默认，那里没写 on_unsupported，运行时退回缺省）" + elif old_inherited: + now = "degrade（节点上另写了 judge，整个用节点的、不再跟随全图默认，没写 on_unsupported 就退回缺省）" + else: + now = "degrade（去掉了写明的值，运行时退回缺省）" + return f"把「{name}」的 judge.on_unsupported 从 {was}{'' if old_inherited else ' '}放宽成了 {now}" + + def _type_word(node_type: Any, type_label: Callable[[Any], str]) -> str: try: return f"「{type_label(NodeType(node_type))}」" diff --git a/backend/app/engine/governance.py b/backend/app/engine/governance.py index 01c29796..db7f0d00 100644 --- a/backend/app/engine/governance.py +++ b/backend/app/engine/governance.py @@ -8,12 +8,15 @@ from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from app.engine.schema import ( + CLAIMS_JUDGE, GraphNode, GraphSpec, NodeType, ValidationResult, _caliber_inputs, claims_problem, + contract_needs_metrics, + judge_problems, type_label, validate_graph, ) @@ -158,7 +161,7 @@ def _lint_contract( if isinstance(sources, str): sources = [sources] - if not sources: + if not sources and contract_needs_metrics(contract): flag("出具契约没有声明 metrics_from（指标来自哪个「口径卡」节点），叙述里的数字无从回指", code="contract.metrics_from_missing", node_id=node_id, hard=True, field="contract.metrics_from") for src in sources: @@ -177,7 +180,11 @@ def _lint_contract( "（叙述），数字回指校验不会执行", code="contract.report_from_missing", node_id=node_id, hard=True, field="contract.report_from") - if strict and not (contract.get("required") or []): + # 引用模式下整张图没有口径卡（只引查询单元格的问数据图）：没有指标可列，strict + cells 已经保证 + # 每个数点得开出处、对不上就不予出具（用户 2026-09-29 拍板）。有口径卡的照旧要列必需指标—— + # 按图里有没有口径卡判，不按契约写没写 metrics_from，免得不写 metrics_from 就绕过去 + cells_only = contract.get("report_from") not in (None, "") and not metric_nodes + if strict and not (contract.get("required") or []) and not cells_only: flag( "受管模板的出具契约必须声明 required（必需指标），否则「不予出具」这一档永远触发不了", code="contract.required_missing", node_id=node_id, hard=True, field="contract.required", @@ -202,7 +209,7 @@ def _lint_contract( # # G1 带契约的出口核对报告撰写节点的文档，成果字段的文字只能来自报告撰写节点 # G2 模型写的文字不绕过报告撰写节点直接流进出口（从图上追） -# G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation +# G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation（或 judge，带上预算） # G4 口径卡的输入不来自计算角色的沙箱代码 # G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields（模型调用节点不能给口径卡供数） # @@ -216,6 +223,11 @@ REPORT_POLICY = ( ("on_violation", "fail", "重写之后还违规就让节点失败，不带着问题出具"), ("claims", "require_citation", "没挂引用的结论句计入缺口，按出具档位降档"), ) +#: 除了 REPORT_POLICY 里写的那个值，还认的更严的写法。claims: judge 在挂引用之外再请另一个模型按证据逐句 +#: 裁判，不支持的按 on_unsupported 判档——比 require_citation 严，但要写预算（见 judge_budget_missing） +REPORT_POLICY_ALSO: dict[str, tuple[str, ...]] = {"claims": (CLAIMS_JUDGE,)} +#: 不限的写法：judge.max_cost_usd 写 null。说给人听的时候照 engine/judge 设置项的说法 +JUDGE_UNLIMITED = "不设上限，费用只受句数、时长上限和每日上限约束" #: 产出本身就是证据、或者由系统确定地算出来的节点。成果字段取它们的值，不算「别的节点写的字」 _TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE) @@ -289,6 +301,21 @@ def _effective(spec: GraphSpec, node: GraphNode, key: str, default: Any = None) return spec.defaults.get(key, default) if value in (None, "") else value +def judge_budget_missing(spec: GraphSpec, node: GraphNode) -> bool: + """claims 是 judge、生效的 judge 配置里却没有 max_cost_usd 这个键（G3）。 + + 写 null 是显式的不限，算写了；没写的键运行时取设置里的默认——受管出具要作者自己定。judge 本身写坏了 + （不是对象、max_cost_usd 写成了别的东西）由 validate 的 report.judge_invalid 报，这里不重复。 + judge 和 claims 一样按运行时的规则取：节点上写了就整个用节点的，不和图级 defaults 拼。 + """ + if _effective(spec, node, "claims") != CLAIMS_JUDGE: + return False + judge = _effective(spec, node, "judge") + if any(field in ("judge", "judge.max_cost_usd") for field, _ in judge_problems(judge)): + return False + return not (isinstance(judge, dict) and "max_cost_usd" in judge) + + class _Graph: """一张图上反复要用的查找：谁产出了某个值、最后一条消息是谁写的。""" @@ -489,13 +516,20 @@ def _lint_evidence(spec: GraphSpec, flag: Callable[..., None], *, strict: bool) for key, want, why in REPORT_POLICY: value = _effective(spec, node, key) if key == "claims" and claims_problem(value): - continue # 写了不支持的值（比如 judge），validate 的 report.claims_invalid 报 - if value != want: + continue # 写了不认识的值，validate 的 report.claims_invalid 报 + also = REPORT_POLICY_ALSO.get(key, ()) + if value != want and value not in also: now = "现在没写" if value in (None, "") else f"现在是 {value}" - loose.append((key, f"{key}: {want}（{why}；{now}）")) + more = "".join(f"，或者更严的 {v}" for v in also) + loose.append((key, f"{key}: {want}（{why}{more}；{now}）")) if loose: flag(f"{_who(node)}要写明 {'；'.join(text for _, text in loose)}。受管模板的报告撰写节点按最严的规则自查", code="governed.report_policy", node_id=node.id, hard=True, field=loose[0][0]) + if judge_budget_missing(spec, node): + flag(f"{_who(node)}的结论句由模型裁判（claims: judge），却没有写每份报告的裁判预算：在 judge 里写 " + "max_cost_usd——写一个金额（美元），或者显式写 null 表示不限（" + JUDGE_UNLIMITED + "）。" + "受管出具不让裁判的花费悄悄按设置里的默认值走", + code="governed.judge_budget", node_id=node.id, hard=True, field="judge.max_cost_usd") # G4、G5 feeders: dict[str, tuple[GraphNode, list[str]]] = {} diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py new file mode 100644 index 00000000..c14a551a --- /dev/null +++ b/backend/app/engine/judge.py @@ -0,0 +1,1065 @@ +"""结论句裁判：另一个模型按证据逐句判断报告里的话站不站得住（可点击证据第四期）。 + +数字、实体、引文是确定性的，系统自己取值、自己核对。可「增长主要来自华东」说得对不对，规则判断不了， +只能交给模型——这是概率性的判断，所以这里每一步都说实话、留余地： + +1. 预筛（claim_priority）：只把结论句送去判。挂了依据的排第一，有方向词、因果词的排第二，其余有数字、 + 实体这类实质内容的排第三；标题、表格单元格、代码、连接性的话、只有过渡词的、短句不送 +2. 证据摘录（prepare）：句子挂的每件证据摘一段。指标写「名称 = 值」加原式；查询给 SQL 前 300 字和被 + 引用的行（最多 20 行，数据源遮罩的列不给；宽表先给被引用的列，再补满 12 列）；知识库片段最多 600 字。 + 规则里写明数字已经由系统核对过 +3. 调用（judge_doc）：一份报告合并成一次调用，超过每批的句数就分批；结构化输出失败退回 JSON 解析， + 再失败就记未裁判。thinking 关掉，max_tokens 4096 +4. 预算：每份报告的句数、金额、时长和全局每日金额，每一项都能是 None（不限）。调用前按目录价估算， + 超了按优先级截断；触顶就停，已判的保留，没判的记未裁判并说明是哪个上限——不报错 +5. 每日计数落在设置表（SPEND_KEY），按本地日期滚动。目录里没有价格的模型估不出金额，金额上限对它 + 不起作用，照实说「按令牌估不出金额」 + +判定只标注、不改写正文。报告节点在正式运行里调它（结果写进文档、随文档封存）；探索运行按需裁判的 +接口也调它（post_seal=True，结果落在封存之后）。所有模型调用都经 get_chat_model，测试换成剧本。 +""" +from __future__ import annotations + +import asyncio +import json +import math +import re +import time +from collections.abc import Callable, Iterable +from dataclasses import asdict, dataclass +from datetime import datetime +from typing import Any + +from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage +from sqlalchemy import select + +from app.core.artifact_store import canonical_json, content_hash +from app.core.artifact_store import load as load_artifact +from app.core.errors import describe_exception +from app.core.events import EventType +from app.db.base import SessionLocal +from app.db.models import Setting +from app.engine.evidence import ( + _DIRECTIONAL, + MISSING, + RenderError, + _auto_entity, + _clean, + _mask_names, + iter_units, + render_cell, +) +from app.engine.expressions import CellError, column_kind, locate_cell +from app.engine.nodes.llm import _split_structured, _usage_of +from app.engine.state import message_text +from app.providers import catalog as pricing +from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model + +# -------------------------------------------------------------------------- +# 常量 +# -------------------------------------------------------------------------- + +#: 模型给的四种判定；unjudged 是系统记的（没判成），不是模型说的 +JUDGED = ("supported", "partial", "unsupported", "not_a_claim") +VERDICTS = (*JUDGED, "unjudged") +#: 模型的理由最多几个字 +RATIONALE_MAX = 60 +#: 一次调用最多判几句：再多，4096 个输出令牌装不下每句一条判定加理由 +BATCH_SIZE = 40 +MAX_TOKENS = 4096 +#: 证据摘录：SQL 前几个字、每次查询最多几行几列、每段知识库片段最多几个字、整份引用的检索给几段。 +#: 列数上限只管没被引用的列：句子引用的格子所在的列一律给（遮罩的除外），再用别的列补满到这么多列 +SQL_CHARS = 300 +EXCERPT_ROWS = 20 +EXCERPT_COLS = 12 +CHUNK_CHARS = 600 +EXCERPT_HITS = 3 +#: 短句：看得见的字（不算标点、空白）少于这么多，又没挂依据、没有方向词的，不送裁判 +SHORT_CHARS = 6 +#: 估算输出令牌：每句一条判定（理由 60 字上下，加上 JSON 的键） +_OUT_PER_UNIT = 90 +_OUT_BASE = 20 + +#: 未裁判的原因。前四个是上限，报告里写「已到上限」 +LIMITS = ("max_claims", "max_cost_usd", "daily_max_usd", "timeout_s") +REASONS = (*LIMITS, "error", "format", "missing", "on_demand") + +JUDGE_ROLE = "你是报告核查员：逐句判断报告里的话，能不能被它挂的证据支持。你不写报告，也不改报告。" + +JUDGE_RULES = """判定（verdict）四选一： +- supported：证据足以支持这句话的说法——方向、比较、范围、因果都对得上 +- partial：一部分说法有证据支持，另一部分（原因、范围、程度）证据里没有 +- unsupported：证据不支持、和证据矛盾，或者这句在陈述数据事实却没有任何证据 +- not_a_claim：这句不陈述数据事实（过渡、背景、建议、计划），不需要证据 +规则： +1. 数字已经由系统核对过，不要复算：每个数都是系统从证据里取出来、按固定规则渲染的，不要因为数字本身或它的格式质疑这句话；你只判断句子的说法有没有证据支持。 +2. 只看下面给的证据摘录，不用常识、行业经验或外部知识补证据；摘录里没有的就是没有。 +3. rationale 用中文写，不超过 60 个字，说清依据是什么，或者缺了什么。 +4. used 写你实际用到的证据编号（摘录里【】中的那个，比如 m:gmv、Q1），没用到就写空数组。 +5. 每句都要给判定，unit 照抄句子前面方括号里的编号。""" + +#: 报告节点 claims 为 judge 时拼在写作规则后面：写作者要知道还有一道按证据的核对 +JUDGE_RULE = ("这份报告的结论句还会由另一个模型逐句核对：它只看你挂的依据，判断这句话站不站得住。" + "说法不要超出证据——证据里没有的原因、范围、趋势，不要写成结论。") + +JUDGE_SCHEMA: dict[str, Any] = { + # 走工具调用的结构化输出要求函数名只含字母、数字、下划线 + "title": "claim_verdicts", + "description": "逐句判断报告里的话能不能被它挂的证据支持", + "type": "object", + "properties": {"items": {"type": "array", "items": { + "type": "object", + "properties": { + "unit": {"type": "string", "description": "句子编号，照抄 [u3] 里的 u3"}, + "verdict": {"type": "string", "enum": list(JUDGED)}, + "rationale": {"type": "string", "description": f"判定的理由，中文，不超过 {RATIONALE_MAX} 个字"}, + "used": {"type": "array", "items": {"type": "string"}, "description": "实际用到的证据编号"}, + }, + "required": ["unit", "verdict", "rationale", "used"], + }}}, + "required": ["items"], +} + +#: 只有过渡作用的说法。一句话去掉它们和标点之后什么都不剩，就是连接性的话 +_TRANSITIONS = re.compile( + r"综上所述|综上|总的来说|总体来看|总体而言|整体来看|整体而言|具体来看|具体如下|具体而言|如下所示|" + r"详见下表|见下表|主要原因如下|原因如下|原因有以下几点|情况如下|结论如下|分析如下|" + r"下面(?:来)?(?:看看|看|是|分析|说明|展开|列出)?|接下来(?:看)?|首先|其次|再次|最后|另外|此外|同时|" + r"因此|所以|总之|如下|以下|(? int | None: + """这句结论送不送裁判、排第几：1 / 2 / 3，不送是 None。只看文档本身，同一份文档永远判得一样。 + + - 不是结论句（连接性的话、标题、代码）、或者在表格里（一格值不是一句话）：不送 + - 写作者挂了依据（行内引用或 [[see:]]，自动链接的名字不算）：1 + - 去掉过渡词和标点什么都不剩（「原因如下：」）：不送 + - 有方向词、因果词：2 + - 有数字、值、实体、引文，看得见的字又不少于 SHORT_CHARS 个：3 + - 其余（短句）：不送 + """ + if unit.get("kind") != "claim" or block.get("type") in ("table", "code", "heading"): + return None + if unit.get("cites"): + return 1 + segments = unit.get("segments") or [] + words = "".join(str(s.get("text") or "") for s in segments if s.get("kind") == "text" or _auto_entity(s)) + body = _INVISIBLE.sub("", _TRANSITIONS.sub("", words)) + substance = [s for s in segments if s.get("kind") in ("number", "value", "entity", "quote") and not _auto_entity(s)] + shown = body + "".join(_INVISIBLE.sub("", str(s.get("text") or "")) for s in substance) + if not shown: + return None + if _DIRECTIONAL.search(body): + return 2 + if (substance or unit.get("see")) and len(shown) >= SHORT_CHARS: + return 3 + return None + + +def candidates(doc: dict[str, Any], units: Iterable[str] | None = None) -> list[Candidate]: + """送裁判的句子，按优先级、再按文档里的先后排好。 + + units 给了就是有人点名要判这几句（按需裁判）：预筛放掉的结论句照判（按最低优先级），不是结论句的 + （标题、连接性的话、代码、表格单元格）和不存在的编号不判。 + """ + wanted = None if units is None else {str(u) for u in units} + markdown = str(doc.get("markdown") or "") + out: list[Candidate] = [] + section = "" + for order, (block, unit) in enumerate(iter_units(doc)): + text = _unit_text(unit, markdown) + if block.get("type") == "heading": + section = text + priority = claim_priority(unit, block, markdown) + if wanted is not None: + if unit.get("id") not in wanted or unit.get("kind") != "claim" or block.get("type") in ("table", "code"): + continue + priority = priority or 3 + if priority is None or not text: + continue + out.append(_candidate(unit, priority, order, text, section)) + return sorted(out, key=lambda c: (c.priority, c.order)) + + +def _unit_text(unit: dict[str, Any], markdown: str) -> str: + span = unit.get("span") + if not (isinstance(span, list) and len(span) == 2 and all(isinstance(x, int) for x in span)): + return "" + return _clean(markdown[span[0]:span[1]]) + + +def unit_texts(doc: dict[str, Any]) -> dict[str, str]: + """{unit id: 这句的正文（压成一行）}，按文档里的先后；没有正文的不列。改写前后按句子认编号时用。""" + markdown = str(doc.get("markdown") or "") + out: dict[str, str] = {} + for _, unit in iter_units(doc): + if text := _unit_text(unit, markdown): + out[str(unit.get("id"))] = text + return out + + +def _candidate(unit: dict[str, Any], priority: int, order: int, text: str, section: str) -> Candidate: + rows: list[tuple[str, int]] = [] + cols: list[tuple[str, str]] = [] + whole: list[str] = [] + hits: list[tuple[str, int]] = [] + cites = [s["cite"] for s in unit.get("segments") or [] if isinstance(s.get("cite"), dict) and not s.get("auto")] + for cite in [*cites, *(unit.get("see") or [])]: + if not isinstance(cite, dict) or cite.get("status") != "resolved" or not cite.get("alias"): + continue + alias, kind, locator = str(cite["alias"]), cite.get("kind"), cite.get("locator") or {} + if kind == "cell" or (kind == "query" and isinstance(locator.get("row"), int)): + rows.append((alias, int(locator["row"]))) + if kind == "cell" and locator.get("column") not in (None, ""): + cols.append((alias, str(locator["column"]))) + elif kind in ("query", "retrieval"): + whole.append(alias) + elif kind == "quote" and isinstance(locator.get("hit"), int): + hits.append((alias, int(locator["hit"]))) + return Candidate(unit=str(unit.get("id")), priority=priority, order=order, text=text, section=section, + aliases=tuple(dict.fromkeys(str(a) for a in unit.get("cites") or [])), + rows=tuple(dict.fromkeys(rows)), cols=tuple(dict.fromkeys(cols)), + whole=tuple(dict.fromkeys(whole)), + hits=tuple(dict.fromkeys(hits))) + + +# -------------------------------------------------------------------------- +# 证据摘录与请求 +# -------------------------------------------------------------------------- + + +class _Fetch: + """这一次请求里取过的工件（取回时复验哈希；取不到、被改过的一律当没有）。""" + + def __init__(self, loader: Callable[[str], Any] | None) -> None: + self.loader = loader or load_artifact + self.memo: dict[str, Any] = {} + + def __call__(self, artifact: Any) -> Any: + key = str(artifact or "") + if key not in self.memo: + try: + self.memo[key] = self.loader(key) if key else None + except Exception: # noqa: BLE001 - 取不回来、哈希不符：这段摘录不给，裁判照实看不到 + self.memo[key] = None + return self.memo[key] + + +def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None) -> set[str]: + """要遮住的列（小写）：查询当时记下的，加上数据源现在设的。""" + names = _mask_names(snapshot.get("mask_columns")) + for name, cols in (masked or {}).items(): + if str(name) == str(source or ""): + names |= _mask_names(cols) + return names + + +def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]: + """摘录给哪几列：被引用的列一律给（超过上限也给），再按原来的顺序用别的列补满到 EXCERPT_COLS 列。 + + columns 已经去掉了遮罩的列：遮罩的列被引用了也不给。列名不分大小写。 + """ + wanted = {str(c).lower() for c in cited} + first = [c for c in columns if c.lower() in wanted] + rest = [c for c in columns if c.lower() not in wanted] + return first + rest[:max(0, EXCERPT_COLS - len(first))] + + +def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool, + fetch: _Fetch, masked: dict[str, Any] | None) -> str | None: + snapshot = fetch(entry.get("artifact")) + if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list): + return None + total = len(snapshot["rows"]) + # 旧形状的台账条目没记数据源：按快照自己记的找遮罩（证据面板同一个口径） + source = entry.get("source") or snapshot.get("source") + head = f"【{alias}】查询结果（{entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行）" + sql = _clean(str(snapshot.get("sql") or "")) + lines = [head, f"SQL：{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"] + hidden = _hidden(snapshot, source, masked) + columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden] + shown_cols = _shown_columns(columns, cols) + # 点名引用的行先给，整份引用的再从头补满，一共不超过 EXCERPT_ROWS 行 + picked = sorted(r for r in rows if 0 <= r < total)[:EXCERPT_ROWS] + if whole: + picked += [r for r in range(total) if r not in picked][:EXCERPT_ROWS - len(picked)] + omitted = len(columns) - len(shown_cols) + lines.append("列：" + " | ".join(shown_cols) + (f" | …另有 {omitted} 列" if omitted > 0 else "")) + for r in sorted(picked): + cells = [] + for c in shown_cols: + try: + raw, _ = locate_cell(snapshot, r, c) + cells.append(render_cell(raw, kind=column_kind(snapshot, c))) + except (CellError, RenderError): + cells.append(MISSING) + lines.append(f"r{r}：" + " | ".join(cells)) + if hidden & {str(c).lower() for c in snapshot.get("columns") or []}: + lines.append("（有的列按数据源的设置遮罩了，没有给出）") + return "\n".join(lines) + + +def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch) -> str: + name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias + rendered = entry.get("rendered") + where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "", + str(entry.get("version") or "")) if v) + value = f"{name}：这次没有值" if entry.get("value") is None or rendered in (None, MISSING) \ + else f"{name} = {rendered}" + lines = [f"【{alias}】口径卡指标：{value}" + (f"（{where}）" if where else "")] + card = fetch(entry.get("artifact")) + wanted = (entry.get("locator") or {}).get("metric") + for metric in (card.get("metrics") if isinstance(card, dict) else None) or []: + if isinstance(metric, dict) and metric.get("id") == wanted and metric.get("expression"): + lines.append(f"原式：{_clean(str(metric['expression']))}") + break + return "\n".join(lines) + + +def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch) -> str | None: + content = fetch(entry.get("artifact")) + hits = content.get("hits") if isinstance(content, dict) else None + if not isinstance(hits, list): + return None + chosen = [h for h in dict.fromkeys(picked) if 0 <= h < len(hits)] + if whole: + chosen += [h for h in range(len(hits)) if h not in chosen][:max(0, EXCERPT_HITS - len(chosen))] + lines = [f"【{alias}】知识库检索（{entry.get('source') or '—'} · 共 {len(hits)} 段）"] + for h in sorted(chosen): + hit = hits[h] + if not isinstance(hit, dict) or not isinstance(hit.get("content"), str): + continue + body = _clean(hit["content"]) + title = f"《{_clean(str(hit['title']))}》" if hit.get("title") else "" + lines.append(f"片段 {h + 1}{title}：{body[:CHUNK_CHARS]}{'…' if len(body) > CHUNK_CHARS else ''}") + return "\n".join(lines) + + +def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch, + masked: dict[str, Any] | None) -> str | None: + kind = entry.get("kind") + if kind == "metric": + return _metric_excerpt(alias, entry, fetch) + if kind == "input": + field_name = (entry.get("locator") or {}).get("field") or alias + return f"【{alias}】运行输入：{field_name} = {entry.get('rendered')}" + if kind == "query": + rows = {r for c in cands for a, r in c.rows if a == alias} + cols = [col for c in cands for a, col in c.cols if a == alias] + return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked) + if kind == "retrieval": + picked = [h for c in cands for a, h in c.hits if a == alias] + return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch) + if kind == "table": + return f"【{alias}】表 {entry.get('name')}" + (f"（数据源 {entry['source']}）" if entry.get("source") else "") + if kind == "column": + table = (entry.get("locator") or {}).get("table") + return f"【{alias}】字段 {f'{table}.' if table else ''}{(entry.get('locator') or {}).get('column') or entry.get('name')}" + return None + + +def _tokens(text: str) -> int: + """估算令牌数：中日韩的字一个字算一个，其余大约 3.5 个字符一个。宁可估多，不估少。""" + wide = sum(1 for ch in text if ord(ch) > 0x2E7F) + return wide + math.ceil((len(text) - wide) / 3.5) + + +@dataclass +class JudgeRequest: + """一次裁判的全部输入：候选句、每件证据的摘录、预筛放掉的结论句、点名了却不能判的句子。""" + + cands: list[Candidate] + excerpts: dict[str, str] + screened: list[str] + skipped: dict[str, str] + + def unit_line(self, c: Candidate) -> str: + cited = "、".join(a for a in c.aliases if a in self.excerpts) or "无" + where = f" ｜ 小节：{c.section}" if c.section else "" + return f"[{c.unit}] {c.text}\n 依据：{cited}{where}" + + def messages(self, units: list[Candidate]) -> list[BaseMessage]: + aliases = list(dict.fromkeys(a for c in units for a in c.aliases if a in self.excerpts)) + evidence = "\n\n".join(self.excerpts[a] for a in aliases) or "（这些句子都没有挂依据）" + ordered = sorted(units, key=lambda c: c.order) + return [ + SystemMessage(content=f"{JUDGE_ROLE}\n\n{JUDGE_RULES}"), + HumanMessage(content=( + f"证据摘录（【】里是编号，判定时 used 写它）：\n{evidence}\n\n" + "要判断的句子（每句都要给判定）：\n" + "\n".join(self.unit_line(c) for c in ordered) + "\n\n" + '只输出 JSON：{"items": [{"unit": "u1", "verdict": "supported", "rationale": "…", "used": ["m:gmv"]}]}' + )), + ] + + def cost(self, model_id: str, units: list[Candidate]) -> float | None: + """这几句合成一次调用，按目录价估多少钱；目录里没有这个模型的价格时是 None（估不出金额）。""" + if pricing.find_pricing(model_id) is None: + return None + prompt = "\n".join(str(m.content) for m in self.messages(units)) + out = min(MAX_TOKENS, _OUT_BASE + _OUT_PER_UNIT * len(units)) + return pricing.estimate_cost(model_id, _tokens(prompt), out) + + def unit_key(self, c: Candidate, model_id: str) -> str: + """同一个模型、同一句话、同样的证据摘录：判定可以直接复用（改写之后没动的句子不再花钱判）。""" + evidence = [self.excerpts.get(a) for a in c.aliases] + return content_hash(canonical_json({"model": model_id, "text": c.text, "section": c.section, + "evidence": evidence}))[:16] + + +def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units: Iterable[str] | None = None, + loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None) -> JudgeRequest: + """裁判的输入：候选句和证据摘录。纯函数（快照经 loader 取，缺省读工件库），不调模型。 + + catalog 缺省用文档自己存的目录；masked = {数据源名: 要遮的列}，和快照里记下的 mask_columns 合在一起。 + """ + catalog = catalog if catalog is not None else (doc.get("catalog") or {}) + cands = candidates(doc, units) + fetch = _Fetch(loader) + excerpts: dict[str, str] = {} + for alias in dict.fromkeys(a for c in cands for a in c.aliases): + entry = catalog.get(alias) + if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked)): + excerpts[alias] = text + picked = {c.unit for c in cands} + skipped: dict[str, str] = {} + screened: list[str] = [] + ids = set() + for _, unit in iter_units(doc): + ids.add(unit.get("id")) + if units is None and unit.get("kind") == "claim" and unit.get("id") not in picked: + screened.append(str(unit.get("id"))) + if units is not None: + for uid in dict.fromkeys(str(u) for u in units): + if uid not in picked: + skipped[uid] = "not_a_claim" if uid in ids else "not_found" + return JudgeRequest(cands=cands, excerpts=excerpts, screened=screened, skipped=skipped) + + +def request_key(request: JudgeRequest, spec: ModelSpec, budget: Budget, known: dict[str, Any] | None = None) -> str: + """一次裁判的输入指纹：报告节点拿它认 checkpoint 里缓存的判定是不是这一稿的。""" + return content_hash(canonical_json({ + "units": [[c.unit, c.priority, c.text, c.section, list(c.aliases)] for c in request.cands], + "excerpts": request.excerpts, "model": [spec.provider, spec.model], "budget": budget.as_dict(), + "known": sorted(known or {}), + })) + + +# -------------------------------------------------------------------------- +# 预算与设置 +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Budget: + """一次裁判的上限。每一项都可以是 None：不限。""" + + max_claims: int | None = 40 + max_cost_usd: float | None = 0.05 + timeout_s: float | None = 30 + daily_max_usd: float | None = 2.0 + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +#: 设置里 judge 这一组（DEFAULT_SETTINGS 引用这一份）。上限写 null 表示不限 +SETTING_GROUP = "judge" +JUDGE_DEFAULTS: dict[str, Any] = { + "provider": None, # 证据裁判模型：空着依次用 Copilot 模型、默认 provider + "model": None, + "report_max_claims": 40, # 每份报告最多判几句 + "report_max_cost_usd": 0.05, # 每份报告的裁判金额上限 + "report_timeout_s": 30, # 每份报告的裁判时长上限（每次调用的超时就是剩下的时长） + "click_max_cost_usd": 0.01, # 探索运行里每点一次「请模型判断」的金额上限 + "daily_max_usd": 2.0, # 全局每日裁判金额上限（按本地日期） +} +#: 每日计数存在设置表的这一行：{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出 +SPEND_KEY = "judge_spend" +_COUNTS = frozenset({"report_max_claims"}) +_SECONDS = frozenset({"report_timeout_s"}) + + +def _limit_ok(key: str, value: Any) -> bool: + if value is None: + return True + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: + return False + return not (key in _COUNTS and int(value) != value) + + +def _limit_words(key: str) -> str: + if key in _COUNTS: + return "正整数（句数）" + if key in _SECONDS: + return "大于 0 的秒数" + return "大于 0 的金额（美元）" + + +def settings_problem(value: Any) -> str | None: + """设置页写进来的 judge 这一组有什么问题，没问题返回 None。""" + if not isinstance(value, dict): + return f"裁判设置（judge）要写成一个对象，写的是 {value!r}" + for key, item in value.items(): + if key not in JUDGE_DEFAULTS: + return f"裁判设置里不认识「{key}」：能写的是 {'、'.join(JUDGE_DEFAULTS)}" + if key in ("provider", "model"): + if item is not None and not isinstance(item, str): + return f"{key} 要写模型接入的名字 / 模型 id（字符串），写 null 表示不指定，写的是 {item!r}" + elif not _limit_ok(key, item): + return f"{key} 要写{_limit_words(key)}，写 null 表示不限，写的是 {item!r}" + return None + + +def sanitize_settings(stored: Any) -> dict[str, Any]: + """库里存的 judge 一组 → 补齐默认的完整设置。存坏了的项（旧版本、手改）按默认，存了 null 的上限就是不限。""" + stored = stored if isinstance(stored, dict) else {} + out = dict(JUDGE_DEFAULTS) + for key in JUDGE_DEFAULTS: + if key not in stored: + continue + value = stored[key] + if key in ("provider", "model"): + out[key] = value.strip() or None if isinstance(value, str) else None + elif _limit_ok(key, value): + out[key] = value + return out + + +async def judge_settings(session: Any) -> dict[str, Any]: + """设置里的 judge 一组，没存过的键补默认（读法同 run_defaults）。""" + row = await session.get(Setting, SETTING_GROUP) + return sanitize_settings(row.value if row else None) + + +def _pick(node: dict[str, Any], key: str, settings: dict[str, Any], setting_key: str) -> Any: + # 节点上写了（包括写 null：不限）就按节点，没写才取设置 + return node[key] if key in node else settings.get(setting_key) + + +def report_budget(settings: dict[str, Any], node_judge: dict[str, Any] | None = None) -> Budget: + """一份报告的裁判上限：节点 judge 里写了的键盖过设置；每日上限只在设置里。""" + node = node_judge if isinstance(node_judge, dict) else {} + return Budget(max_claims=_pick(node, "max_claims", settings, "report_max_claims"), + max_cost_usd=_pick(node, "max_cost_usd", settings, "report_max_cost_usd"), + timeout_s=_pick(node, "timeout_s", settings, "report_timeout_s"), + daily_max_usd=settings.get("daily_max_usd")) + + +def click_budget(settings: dict[str, Any], node_judge: dict[str, Any] | None = None) -> Budget: + """探索运行里点一次「请模型判断」的上限：金额按每次点击的上限，句数、时长按每份报告的。""" + node = node_judge if isinstance(node_judge, dict) else {} + return Budget(max_claims=_pick(node, "max_claims", settings, "report_max_claims"), + max_cost_usd=settings.get("click_max_cost_usd"), + timeout_s=_pick(node, "timeout_s", settings, "report_timeout_s"), + daily_max_usd=settings.get("daily_max_usd")) + + +def _text(value: Any) -> str | None: + return value.strip() or None if isinstance(value, str) else None + + +async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = None, *, + settings: dict[str, Any] | None = None) -> ModelSpec: + """裁判用哪个模型：节点上的 judge.model（连同 provider）> 设置里的证据裁判模型 > Copilot 模型 > + 默认 provider。哪一级写了就整组用哪一级，不跨级拼 provider 和模型。thinking 关、max_tokens 4096。 + + 建议和写作模型不同，否则等于自己审自己（报告节点发现相同会发 judge_same_model 警告）。 + """ + from app.api.copilot import COPILOT_SETTING_KEY + + node = node_judge if isinstance(node_judge, dict) else {} + provider, model = _text(node.get("provider")), _text(node.get("model")) + if not (provider or model): + chosen = settings if settings is not None else await judge_settings(session) + provider, model = _text(chosen.get("provider")), _text(chosen.get("model")) + if not (provider or model): + row = await session.get(Setting, COPILOT_SETTING_KEY) + saved = row.value if row is not None and isinstance(row.value, dict) else {} + provider, model = _text(saved.get("provider")), _text(saved.get("model")) + return ModelSpec(provider=provider, model=model, max_tokens=MAX_TOKENS, thinking="off") + + +async def source_masks(catalog: dict[str, Any], *, loader: Callable[[str], Any] | None = None + ) -> dict[str, list[str]]: + """目录里的查询来自的数据源现在设的遮罩：{数据源名: 列名}。和证据面板同一个口径。 + + 旧形状的台账条目没记数据源的，按快照自己记的数据源查（快照经 loader 取，缺省读工件库； + 按需裁判的接口要传只认封存工件的 loader，和 prepare 同一个）。 + """ + from app.data.engine import masked_columns + from app.db.models import DataSource + + fetch = _Fetch(loader) + found: set[str] = set() + for entry in catalog.values(): + if not isinstance(entry, dict) or entry.get("kind") != "query": + continue + source = entry.get("source") + if not source: + snapshot = fetch(entry.get("artifact")) + source = snapshot.get("source") if isinstance(snapshot, dict) else None + if source: + found.add(str(source)) + names = sorted(found) + if not names: + return {} + async with SessionLocal() as session: + rows = (await session.execute(select(DataSource.name, DataSource.options) + .where(DataSource.name.in_(names)))).all() + return {str(name): cols for name, options in rows if (cols := masked_columns(options))} + + +# -------------------------------------------------------------------------- +# 每日计数 +# -------------------------------------------------------------------------- + +_spend_lock = asyncio.Lock() + + +def _today() -> str: + """本地日期：每日上限按它滚动。""" + return datetime.now().astimezone().date().isoformat() + + +def _spend_of(row: Any) -> dict[str, Any]: + value = row.value if row is not None and isinstance(row.value, dict) else {} + today = _today() + if value.get("date") != today: + return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0} + return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0), + "unpriced_calls": int(value.get("unpriced_calls") or 0)} + + +async def daily_spend() -> dict[str, Any]: + """今天（本地日期）裁判已经花了多少：{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。""" + async with SessionLocal() as session: + return _spend_of(await session.get(Setting, SPEND_KEY)) + + +async def _charge(amount: float, *, limit: float | None, priced: bool) -> str | None: + """记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记，返回 None；记下了返回记在哪天。""" + async with _spend_lock: + async with SessionLocal() as session: + row = await session.get(Setting, SPEND_KEY) + spend = _spend_of(row) + if limit is not None and spend["usd"] + amount > limit + 1e-12: + return None + spend["usd"] = round(spend["usd"] + amount, 8) + spend["calls"] += 1 + spend["unpriced_calls"] += 0 if priced else 1 + if row is None: + session.add(Setting(key=SPEND_KEY, value=spend)) + else: + row.value = spend + await session.commit() + return spend["date"] + + +async def _settle(date: str, delta: float) -> None: + """调用结束后按实际用量找补预留的差额。跨了天就不找补：预留记在前一天，今天从零算。""" + if not delta: + return + async with _spend_lock: + async with SessionLocal() as session: + row = await session.get(Setting, SPEND_KEY) + spend = _spend_of(row) + if row is None or spend["date"] != date: + return + spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) + row.value = spend + await session.commit() + + +# -------------------------------------------------------------------------- +# 调用 +# -------------------------------------------------------------------------- + +Emit = Callable[..., None] + + +def _quiet(*_args: Any, **_data: Any) -> None: + return None + + +def _json_in(text: str) -> Any: + from app.api.copilot import _extract_json + + try: + return json.loads(text) + except ValueError: + pass + try: + return _extract_json(text) + except ValueError: + return None + + +def _items_of(value: Any) -> list[Any] | None: + """判定清单：{"items": [...]}，或者直接一个数组。有的模型走工具调用时把嵌套的数组写成一段 JSON 文本 + （{"items": "[...]"}），也有把整个对象写进正文的：都认。""" + if isinstance(value, str): + value = _json_in(value) + if isinstance(value, dict) and isinstance(value.get("items"), str): + value = {**value, "items": _json_in(value["items"])} + if isinstance(value, dict) and isinstance(value.get("items"), list): + return value["items"] + return value if isinstance(value, list) else None + + +async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | None, list[BaseMessage], str | None, + int]: + """问一次裁判：(判定清单, 模型的原始回复, 没成的原因, 调了几次模型)。 + + 先走结构化输出；不成（模型不支持、给的不是合法结构）先从原始回复里找 JSON，再退回纯文本调用、 + 从回复里抠 JSON。原因以 error: 开头的是调用本身失败，format 是格式不对。超时由调用方掐。 + """ + raws: list[BaseMessage] = [] + try: + got = await model.with_structured_output(JUDGE_SCHEMA, include_raw=True).ainvoke(messages) + raw, parsed = _split_structured(got) + if raw is not None: + raws.append(raw) + if (items := _items_of(parsed)) is not None: + return items, raws, None, 1 + if raw is not None: + # 有的网关把 JSON 写在正文里、或者塞进了一个没按 schema 解析出来的工具调用 + for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: + if (items := _items_of(found)) is not None: + return items, raws, None, 1 + except Exception: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析 + pass + try: + reply = await model.ainvoke(messages) + except Exception as e: # noqa: BLE001 + return None, raws, f"error:{describe_exception(e)}", 2 + raws.append(reply) + items = _items_of(message_text(reply)) + return items, raws, None if items is not None else "format", 2 + + +def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]: + return {"status": "unjudged", "rationale": rationale, "judge": judge, "post_seal": post_seal, "reason": reason} + + +def _limit_label(reason: str, budget: Budget, *, click: bool = False) -> str: + if reason == "max_claims": + return f"每{'次' if click else '份报告'}最多判 {budget.max_claims} 句" + if reason == "max_cost_usd": + return f"{'这次点击' if click else '这份报告'}的裁判金额上限 ${budget.max_cost_usd:g}" + if reason == "daily_max_usd": + return f"全局每日裁判金额上限 ${budget.daily_max_usd:g}" + return f"裁判时长上限 {budget.timeout_s:g} 秒" + + +def _why_unjudged(reason: str, budget: Budget, detail: str = "", *, click: bool = False) -> str: + if reason in LIMITS: + return f"已到上限（{_limit_label(reason, budget, click=click)}），这句没判" + if reason == "error": + return f"裁判没跑成：{detail}"[:120] + if reason == "format": + return detail or "裁判给的结果格式不对，这句没判" + if reason == "missing": + return "裁判没有给这句判定" + return "探索运行按需裁判：点开这句、请模型判断时才判" + + +async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, known: dict[str, Any] | None = None, + post_seal: bool = False, emit: Emit | None = None, click: bool = False, + shown: Budget | None = None) -> dict[str, Any]: + """按预算判一份请求。永远不抛：模型没配好、调用失败、格式不对、触顶，都落成未裁判和一条缺口。 + + 返回 outcome：{model, priced, verdicts:{unit: verdict}, keys:{unit: 复用键}, candidates, screened, + skipped, counts, unjudged:{原因: 句数}, limits_hit, cost_usd, estimated_usd, calls, reused, duration_ms, + usage, budget, gaps, notes}。click=True 时上限的说法按「这次点击」写。 + + shown 只管说给人看的上限：同一份报告分几轮判时（rewrite_once 的第二轮），budget 是这一轮剩下的钱和 + 时间，未裁判的理由、缺口、judge_limit 事件里写的却该是配置的上限（「$0.05」，不是剩下的零头）。 + """ + emit = emit or _quiet + label = shown or budget + started = time.monotonic() + spec = spec.model_copy(update={"thinking": "off", "max_tokens": MAX_TOKENS, + "timeout": math.ceil(budget.timeout_s) if budget.timeout_s else spec.timeout}) + verdicts: dict[str, dict[str, Any]] = {} + details: dict[str, str] = {} + usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_usd": 0.0, "calls": 0} + notes: list[str] = [] + estimated = 0.0 + reused = 0 + model = None + model_id = spec.model or "" + try: + async with SessionLocal() as session: + model, model_id = await get_chat_model(session, spec) + except Exception as e: # noqa: BLE001 - 裁判模型没配好：这回判不了，报告照常 + why = str(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) + details["error"] = f"裁判模型用不了（{why}）" + judge = model_id or None + priced = model is not None and pricing.find_pricing(model_id) is not None + keys = {c.unit: request.unit_key(c, model_id) for c in request.cands} + + def stop(units: Iterable[Candidate], reason: str, detail: str = "") -> None: + for c in units: + verdicts.setdefault(c.unit, _unjudged(reason, _why_unjudged(reason, label, detail, click=click), + judge=judge, post_seal=post_seal)) + + ranked = list(request.cands) + if budget.max_claims is not None: + stop(ranked[budget.max_claims:], "max_claims") + ranked = ranked[:budget.max_claims] + pending: list[Candidate] = [] + for c in ranked: + hit = (known or {}).get(keys[c.unit]) + if isinstance(hit, dict) and hit.get("status") in JUDGED: + verdicts[c.unit] = dict(hit) + reused += 1 + else: + pending.append(c) + + if model is None: + stop(pending, "error", details["error"]) + pending = [] + elif pending and not priced and (budget.max_cost_usd is not None or budget.daily_max_usd is not None): + note = (f"模型「{model_id}」不在价格目录里，按令牌估不出金额：金额上限（每份报告、每次点击、每日）对它不起作用，" + "费用只受句数、时长上限约束") + notes.append(note) + emit(EventType.LOG, level="warn", code="judge_unpriced", message=note) + + spent = 0.0 + while pending: + remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started) + if remaining is not None and remaining <= 0: + stop(pending, "timeout_s") + break + batch = pending[:BATCH_SIZE] + cost = request.cost(model_id, batch) if priced else None + if cost is not None: + caps = [] + if budget.max_cost_usd is not None: + caps.append((budget.max_cost_usd - spent, "max_cost_usd")) + if budget.daily_max_usd is not None: + caps.append((budget.daily_max_usd - (await daily_spend())["usd"], "daily_max_usd")) + if caps: + cap, binding = min(caps, key=lambda x: x[0]) + trimmed = False + while batch and (cost := request.cost(model_id, batch) or 0.0) > cap + 1e-12: + batch, trimmed = batch[:-1], True # 按优先级从后往前丢：先判挂了依据的 + if not batch: + stop(pending, binding) + break + if trimmed: + # 这一批已经是剩下的钱装得下的最多句子，后面的句子更装不下 + stop(pending[len(batch):], binding) + pending = pending[:len(batch)] + pending = pending[len(batch):] + charged = cost or 0.0 + day = await _charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced) + if day is None: # 估算之后、记账之前别的裁判先花掉了今天剩下的钱 + stop([*batch, *pending], "daily_max_usd") + break + estimated += charged + began = time.perf_counter() + emit(EventType.LLM_START, model=model_id, structured=True, purpose="judge", message_count=2, + units=len(batch)) + items: list[Any] | None = None + raws: list[BaseMessage] = [] + problem: str | None = None + calls = 1 + try: + async with asyncio.timeout(remaining): + items, raws, problem, calls = await _ask(model, request.messages(batch)) + except TimeoutError: + problem = "timeout_s" + tokens = [_usage_of(r, model_id) for r in raws] + inp, out = sum(t["input_tokens"] for t in tokens), sum(t["output_tokens"] for t in tokens) + # 回复没带用量（有的网关不给、调用超时）就按估算记：宁可多记，不能让每日上限漏掉 + actual = (pricing.estimate_cost(model_id, inp, out) if inp or out else charged) if priced else 0.0 + await _settle(day, actual - charged) + spent += actual + usage["input_tokens"] += inp + usage["output_tokens"] += out + usage["calls"] += calls + usage["cost_usd"] += actual + end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000), + "input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, "cost_usd": round(actual, 6), + "calls": calls, "units": len(batch)} + if problem is not None and problem != "format": + reason = "timeout_s" if problem == "timeout_s" else "error" + detail = problem.partition(":")[2] + emit(EventType.LLM_END, **end, error=detail or _why_unjudged(reason, label, click=click)) + if reason == "error": + # 调用本身失败（鉴权、网关挂了）：后面几批多半一样，不再一次次去撞 + details["error"] = detail + stop([*batch, *pending], reason, detail) + break + stop(batch, reason, detail) + continue + emit(EventType.LLM_END, **end) + if items is None: + stop(batch, "format") + continue + _take(items, batch, request, verdicts, judge=judge, post_seal=post_seal) + stop(batch, "missing") + + for c in request.cands: # 兜底：每个候选都有判定 + verdicts.setdefault(c.unit, _unjudged("missing", _why_unjudged("missing", label), judge=judge, + post_seal=post_seal)) + usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"] + usage["cost_usd"] = round(usage["cost_usd"], 6) + outcome = _outcome(request, verdicts, budget, details, click=click, shown=label) + outcome.update({ + "model": judge, "priced": priced, "keys": keys, "cost_usd": usage["cost_usd"], + "estimated_usd": round(estimated, 6), "calls": usage["calls"], "reused": reused, + "duration_ms": int((time.monotonic() - started) * 1000), "usage": usage, "notes": notes, + }) + if hit := outcome["limits_hit"]: + labels = "、".join(_limit_label(r, label, click=click) for r in hit) + n = sum(outcome["unjudged"].get(r, 0) for r in hit) + emit(EventType.LOG, level="warn", code="judge_limit", limits=hit, unjudged=n, + message=f"结论句裁判已到上限（{labels}）：{n} 句没判，记为未裁判；已判的保留") + if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS]: + emit(EventType.LOG, level="warn", code="judge_failed", message="结论句裁判没跑完：" + "；".join(failed)) + outcome.pop("_gaps") + return outcome + + +def _take(items: list[Any], batch: list[Candidate], request: JudgeRequest, verdicts: dict[str, dict[str, Any]], *, + judge: str | None, post_seal: bool) -> None: + """模型给的判定逐条收下：不在这一批的编号不认，判定不认识的记格式不对，理由截到 60 字， + used 只留这一批摘录里真有的编号。""" + ids = {c.unit for c in batch} + aliases = {a for c in batch for a in c.aliases if a in request.excerpts} + for item in items: + if not isinstance(item, dict): + continue + uid = str(item.get("unit") or "").strip().strip("[]") + if uid not in ids or uid in verdicts: + continue + status = str(item.get("verdict") or "").strip().lower() + if status not in JUDGED: + verdicts[uid] = _unjudged("format", f"裁判给的判定「{str(item.get('verdict'))[:20]}」不认识，这句没判", + judge=judge, post_seal=post_seal) + continue + used = item.get("used") if isinstance(item.get("used"), list) else [] + verdicts[uid] = {"status": status, "rationale": _clean(str(item.get("rationale") or ""))[:RATIONALE_MAX], + "judge": judge, "post_seal": post_seal, + "used": list(dict.fromkeys(u.strip() for u in used if isinstance(u, str) + and u.strip() in aliases))} + + +def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget: Budget, details: dict[str, str], *, + click: bool = False, shown: Budget | None = None) -> dict[str, Any]: + """汇总判定。outcome.budget 是这一轮实际用的上限；缺口里写的上限按 shown（缺省同 budget）。""" + label = shown or budget + counts = {s: 0 for s in VERDICTS} + why: dict[str, int] = {} + for c in request.cands: + v = verdicts[c.unit] + counts[v["status"]] += 1 + if v["status"] == "unjudged": + why[v["reason"]] = why.get(v["reason"], 0) + 1 + gaps: list[tuple[str, str]] = [] + for reason in REASONS: + n = why.get(reason) + if not n or reason == "on_demand": + continue + if reason in LIMITS: + text = f"有 {n} 句结论没裁判：已到上限（{_limit_label(reason, label, click=click)}）" + elif reason == "error": + text = f"有 {n} 句结论没裁判：裁判调用失败（{details.get('error') or '原因不明'}）" + elif reason == "format": + text = f"有 {n} 句结论没裁判：裁判给的结果格式不对" + else: + text = f"有 {n} 句结论裁判漏判了" + gaps.append((reason, text)) + return { + "verdicts": {c.unit: verdicts[c.unit] for c in sorted(request.cands, key=lambda c: c.order)}, + "candidates": len(request.cands), "screened": list(request.screened), "skipped": dict(request.skipped), + "counts": counts, "unjudged": {r: why[r] for r in REASONS if r in why}, + "limits_hit": [r for r in LIMITS if r in why], "budget": budget.as_dict(), + "gaps": [text for _, text in gaps], "_gaps": gaps, + } + + +async def judge_doc(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, spec: ModelSpec, budget: Budget, + units: Iterable[str] | None = None, loader: Callable[[str], Any] | None = None, + masked: dict[str, Any] | None = None, known: dict[str, Any] | None = None, + post_seal: bool = False, emit: Emit | None = None, click: bool = False) -> dict[str, Any]: + """判一份报告文档（units 给了就只判这几句）。见 prepare 和 run_request。""" + request = prepare(doc, catalog, units=units, loader=loader, masked=masked) + return await run_request(request, spec=spec, budget=budget, known=known, post_seal=post_seal, emit=emit, + click=click) + + +# -------------------------------------------------------------------------- +# 写回文档 +# -------------------------------------------------------------------------- + +#: 文档的 judge 摘要里、report.checked 和节点产出的 judge 里都有的键 +SUMMARY_KEYS = ("mode", "model", "on_unsupported", "rewrite_once", "limits", "candidates", "screened", "counts", + "unjudged", "limits_hit", "complete", "priced", "cost_usd", "calls", "reused", "duration_ms", + "gaps", "notes", "same_model", "rewrite") + + +def summarize(outcome: dict[str, Any], *, mode: str, on_unsupported: str = "degrade", rewrite_once: bool = False, + same_model: bool = False, rewrite: dict[str, Any] | None = None) -> dict[str, Any]: + """裁判摘要：写进文档的 judge，也原样放进 report.checked 和节点产出。""" + counts = outcome["counts"] + return { + "mode": mode, "model": outcome.get("model"), "on_unsupported": on_unsupported, "rewrite_once": rewrite_once, + "limits": outcome.get("budget"), "candidates": outcome["candidates"], "screened": outcome["screened"], + "counts": counts, "unjudged": outcome["unjudged"], "limits_hit": outcome["limits_hit"], + "complete": mode == "inline" and counts["unjudged"] == 0, + "priced": outcome.get("priced"), "cost_usd": outcome.get("cost_usd", 0.0), "calls": outcome.get("calls", 0), + "reused": outcome.get("reused", 0), "duration_ms": outcome.get("duration_ms", 0), + "gaps": outcome["gaps"] if mode == "inline" else [], "notes": outcome.get("notes") or [], + "same_model": same_model, "rewrite": rewrite, + } + + +def apply_judgement(doc: dict[str, Any], outcome: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]: + """判定写进文档：每句的 unit.verdict、文档的 judge 摘要、stats 里各判定的句数。返回同一份文档。""" + units = {u.get("id"): u for _, u in iter_units(doc)} + for uid, verdict in outcome["verdicts"].items(): + if uid in units: + units[uid]["verdict"] = verdict + doc["judge"] = summary + doc["stats"] = {**(doc.get("stats") or {}), **summary["counts"]} + return doc + + +def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_once: bool = False) -> dict[str, Any]: + """探索运行：节点里不花这笔钱。候选句标「未裁判 · 按需」，点开哪句再判哪句。返回摘要（已写进文档）。""" + request = prepare(doc, {}) + budget = Budget() + verdicts = {c.unit: _unjudged("on_demand", _why_unjudged("on_demand", budget), judge=None, post_seal=False) + for c in request.cands} + outcome = _outcome(request, verdicts, budget, {}) + outcome.pop("_gaps") + outcome["budget"] = None + summary = summarize(outcome, mode="on_demand", on_unsupported=on_unsupported, rewrite_once=rewrite_once) + apply_judgement(doc, outcome, summary) + return summary diff --git a/backend/app/engine/nodes/io.py b/backend/app/engine/nodes/io.py index e20d0da0..e3e5ca01 100644 --- a/backend/app/engine/nodes/io.py +++ b/backend/app/engine/nodes/io.py @@ -237,6 +237,7 @@ def _apply_contract( strict=bool(contract.get("strict")), gaps=gaps, unresolved=cited.get("unresolved"), + unsupported=cited.get("unsupported"), uncited_claims=cited.get("uncited"), claims_policy=cited.get("claims_policy"), ) @@ -298,7 +299,7 @@ def _check_citations( reports: list[dict[str, Any]], *, cells: bool = True, governed: bool = False, ) -> dict[str, Any]: """独立复核报告文档：{matched, unmatched, unresolved, gaps, doc_artifact, stats, claims_policy, uncited, - claims, entities}。 + unsupported, claims, entities}。 不信报告节点自己的统计：文档按 id 从工件库取回（取回时复验哈希），目录按报告节点 同一套参数从状态里重建（不用文档里存的那份），每个带引用的片段重新解析、重新渲染、 @@ -313,7 +314,7 @@ def _check_citations( out: dict[str, Any] = {"matched": [], "unmatched": [], "unresolved": [], "gaps": [], "doc_artifact": None, "stats": None, "claims_policy": None, "uncited": None, - "claims": None, "entities": None} + "unsupported": None, "claims": None, "entities": None} gaps = out["gaps"] spec = ctx.run.spec node = spec.node_map().get(report_from) @@ -403,7 +404,7 @@ def _check_citations( "span": [start, end], "eid": cite["eid"]}) _claims(out, contract, payload, evidence_on=ledger_enabled(ctx.run), governed=governed, - uncited=checked.get("uncited") or []) + uncited=checked.get("uncited") or [], doc=doc, title=node.title) mine = next((r for r in reports if r["node"].id == report_from), None) edited = mine["edited"] if mine else [] @@ -416,20 +417,29 @@ def _check_citations( #: 没挂依据的结论句怎么处置，从松到严。认不出的写法按 degrade 算：写错一个词不能让缺口悄悄消失 _ON_UNCITED = ("ignore", "degrade", "withhold") +#: 证据不支持的结论句怎么处置（claims: judge），从松到严。认不出的写法同样按 degrade 算 +_ON_UNSUPPORTED = ("degrade", "withhold") +#: 要求结论句挂依据的策略：judge 在这一点上和 require_citation 一样严，另外再按裁判的判定判档 +_CITING = ("require_citation", "judge") def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, Any], *, evidence_on: bool, - governed: bool = False, uncited: list[dict[str, Any]]) -> None: - """结论句策略：没挂依据的结论句按 claims_policy 交给 decide_tier 判档。 + governed: bool = False, uncited: list[dict[str, Any]], doc: dict[str, Any] | None = None, + title: str = "") -> None: + """结论句策略：没挂依据的、裁判判为证据不支持的结论句按 claims_policy 交给 decide_tier 判档。 两处可以要求挂依据：报告节点写作时记下的 claims（节点产出里的，写作提示按它要求过），和契约自己写的 - require_citation（对象形式可以另写 on_uncited）。两处都写了取更严的那个——契约只能收紧、不能放松： + require_citation / judge（对象形式可以另写 on_uncited）。两处都写了取更严的那个——契约只能收紧、不能放松： 报告节点要求了，契约写 ignore 或 off 也照旧计入缺口，写 withhold 就收紧成不予出具。受管级别的正式运行 里 ignore 一律当 degrade：没有哪道门禁查契约里的 claims，不能让它把用户要的底线拉低。 + 报告节点写的是 judge 时，判定在文档里（按哈希取回的那份：判定是模型给的，出口复算不了，只能信封存的 + 文档）。正式运行在节点里判过：证据不支持的按 on_unsupported 判档（报告节点和契约取更严的），没判完的 + （裁判没跑成、触顶）把裁判摘要里的缺口照抄进来——不能判完整出具，也不当成「不支持」。探索运行按需裁判， + 这里只标注。契约要 judge、报告节点却没开的，照实记缺口：出口替不了报告节点去裁判。 + 数的是这里重新核对出来的 uncited，不信文档里记的 cites。升级前写的报告（节点产出里没有 claims 这个键， - 包括跨着升级还没跑完的运行）一律不管，契约写了也照旧记「这一版还不支持」。judge（裁判模型）是后续版本 - 的事，契约写了照实记缺口。 + 包括跨着升级还没跑完的运行）一律不管，契约写了也照旧记「这一版还不支持」。 """ written = payload.get("claims") if isinstance(payload, dict) else None # 报告节点升级后才在产出里记 claims（off 也记）：没有这个键就是升级前写的报告 @@ -437,23 +447,66 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An declared = contract.get("claims") name = declared.get("policy") if isinstance(declared, dict) else declared asks: list[str] = [] - if upgraded and written == "require_citation": + if upgraded and written in _CITING: asks.append("degrade") - if upgraded and name == "require_citation": + if upgraded and name in _CITING: wanted = declared.get("on_uncited") if isinstance(declared, dict) else None asks.append(wanted if wanted in _ON_UNCITED else "degrade") elif name not in (None, "", "off"): out["gaps"].append("契约声明了结论句检查（claims），这一版还不支持，结论句没有核对") + judged = upgraded and written == "judge" + if upgraded and name == "judge" and not judged: + out["gaps"].append(f"契约要求结论句由模型裁判（claims: judge），报告「{title}」没有开 claims: judge，" + "结论句没有裁判") if not asks: return on_uncited = max(asks, key=_ON_UNCITED.index) if governed and on_uncited == "ignore": on_uncited = "degrade" - policy: Any = "require_citation" if on_uncited == "degrade" else \ - {"policy": "require_citation", "on_uncited": on_uncited} - out["claims_policy"], out["uncited"] = policy, uncited - out["claims"] = {"policy": "require_citation", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED], - **({"on_uncited": on_uncited} if on_uncited != "degrade" else {})} + extra = {"on_uncited": on_uncited} if on_uncited != "degrade" else {} + if not judged: + policy: Any = "require_citation" if on_uncited == "degrade" else \ + {"policy": "require_citation", "on_uncited": on_uncited} + out["claims_policy"], out["uncited"] = policy, uncited + out["claims"] = {"policy": "require_citation", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED], + **extra} + return + + from app.engine.evidence import iter_units + + summary = (doc or {}).get("judge") + if not isinstance(summary, dict): + # 报告节点开了 judge、文档里却没有裁判摘要（不该发生）：当作没判，不能判完整出具 + summary = {"mode": "inline", "counts": {}, "unjudged": {}, "limits_hit": [], "complete": False, + "gaps": ["开了 claims: judge，文档里却没有裁判结果，结论句没有裁判"]} + asked = [summary.get("on_unsupported")] + if name == "judge" and isinstance(declared, dict): + asked.append(declared.get("on_unsupported")) + on_unsupported = max((a if a in _ON_UNSUPPORTED else "degrade" for a in asked), key=_ON_UNSUPPORTED.index) + inline = summary.get("mode") == "inline" + flagged: dict[str, list[dict[str, Any]]] = {"unsupported": [], "partial": []} + if inline: + # 探索运行按需裁判的判定不在文档里（在封存之后追加的 evidence.judged 事件里），这里只数正式运行的 + markdown = str((doc or {}).get("markdown") or "") + for _, unit in iter_units(doc or {}): + verdict = unit.get("verdict") + status = verdict.get("status") if isinstance(verdict, dict) else None + if status in flagged: + span = unit.get("span") or [0, 0] + flagged[status].append({"unit": unit.get("id"), "span": span, "text": markdown[span[0]:span[1]], + "rationale": verdict.get("rationale") or ""}) + out["gaps"].extend(f"报告「{title}」{gap}" for gap in summary.get("gaps") or []) + counts = {k: int((summary.get("counts") or {}).get(k) or 0) + for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")} + out["claims_policy"] = {"policy": "judge", "on_unsupported": on_unsupported, "on_uncited": on_uncited} + out["uncited"], out["unsupported"] = uncited, flagged["unsupported"] + out["claims"] = { + "policy": "judge", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED], **extra, + "on_unsupported": on_unsupported, "mode": "inline" if inline else "on_demand", "counts": counts, + "unsupported": flagged["unsupported"][:_LISTED], "partial": flagged["partial"][:_LISTED], + "unjudged": dict(summary.get("unjudged") or {}), "limits_hit": list(summary.get("limits_hit") or []), + "complete": bool(summary.get("complete")), "model": summary.get("model"), + } def _declare_citations( @@ -497,8 +550,10 @@ def _declare_citations( metrics_checked=len(metrics), matched_numbers=len(cited["matched"]), matched=cited["matched"][:50], - # on_uncited 只在不是默认的 degrade 时才有（收紧成 withhold、探索运行的契约写了 ignore） - **({"claims": {k: cited["claims"][k] for k in ("policy", "uncited_claims", "on_uncited") + # on_uncited 只在不是默认的 degrade 时才有（收紧成 withhold、探索运行的契约写了 ignore）； + # judge 另带不支持时怎么判档、在哪判的和各判定的句数（名单只在出具声明里） + **({"claims": {k: cited["claims"][k] for k in ("policy", "uncited_claims", "on_uncited", "on_unsupported", + "mode", "counts") if k in cited["claims"]}} if cited.get("claims") else {}), # 计数取复核的统计（声明里的名单最多列 _LISTED 个） **({"entities": {"unknown": (cited["stats"] or {}).get("unknown_entities", len(entities["unknown"])), diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index 8881d6e8..d24b887f 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -15,20 +15,32 @@ 写了个哪里都找不到的名字）只标注：不重写、不失败，受管级别的正式出具按缺口降档；表结构快照 不全时核对不了的名字（unverified_entity）只标注。entities: off 整层不管表名、字段名。 结论句策略 claims：off（默认）不管；require_citation 在写作提示里要求结论句挂依据（自动链接的 -名字不算依据），没挂的计入缺口、出口降档；judge（模型裁判）是后续版本的事，现在写了直接报错。 +名字不算依据），没挂的计入缺口、出口降档；judge 另外请一个模型按证据逐句判断（engine/judge.py）： + +- 正式运行在节点里判：写完、核对完之后判，判定写进文档的 unit.verdict，随文档一起封存。判定只标注、 + 不改写正文；裁判没跑成、触顶了照样产出，没判的记未裁判并留下缺口，出口按 on_unsupported 判档 +- 探索运行不在节点里花这笔钱：候选句标「未裁判 · 按需」，点开哪句再判哪句（证据接口） +- rewrite_once（默认关）：证据不支持的句子连同理由交回写作者只改这几句，再判一次，最多一轮； + 改写稿冒出原稿没有的违规（只标注的可疑实体也算）就不采用 +- 裁判调用包在 @task 里，排在节点所有已有调用之后：「接着跑」时直接取 checkpoint 里的判定， + 不重复调用、不重复计费。写作调用不进 checkpoint，接着跑时会重写一稿——所以判定带着输入指纹， + 对不上这一稿的不用 出口契约复核（io.py）用 report_catalog 按同一套参数重建目录、独立重算，不信这里的自查。 """ from __future__ import annotations import time +from collections import Counter from typing import Any from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage +from langgraph.func import task from app.core.errors import describe_exception from app.core.events import EventType from app.db.base import SessionLocal +from app.engine import judge as judging from app.engine.context import NodeContext, NodeError from app.engine.evidence import ( CELLS_REASON, @@ -43,7 +55,17 @@ from app.engine.evidence import ( ledger_enabled, ) from app.engine.nodes.llm import _invoke_streaming, _model_spec, _report_call -from app.engine.schema import REPORT_CLAIMS, GraphNode, GraphSpec, NodeType, _ancestors, claims_problem +from app.engine.schema import ( + CLAIMS_JUDGE, + CLAIMS_VALUES, + JUDGE_ON_UNSUPPORTED, + GraphNode, + GraphSpec, + NodeType, + _ancestors, + claims_problem, + judge_problems, +) from app.engine.state import GraphState, message_text, thinking_text from app.engine.toolcalls import leaked_markup, markup_warning from app.providers.factory import ProviderNotConfigured, get_chat_model @@ -52,8 +74,8 @@ ON_VIOLATION = ("fail", "flag") NUMBERS = ("strict", "off") #: 实体层：link（默认）核对表名、字段名并自动链接；off 整层不管（目录里不收表和字段） ENTITIES = ("link", "off") -#: 结论句策略，和画布校验同一份（schema.REPORT_CLAIMS）；judge 是后续版本的事 -CLAIMS = REPORT_CLAIMS +#: 结论句策略，和画布校验同一份（schema.CLAIMS_VALUES）：off / require_citation / judge +CLAIMS = CLAIMS_VALUES #: 只标注、不让写作者重写也不判失败的违规：可疑实体（受管的正式出具按缺口降档）、表结构快照不全时 #: 核对不了的名字（出口只标注） SOFT = frozenset({"unknown_entity", "unverified_entity"}) @@ -67,6 +89,8 @@ MARKUP_NUDGE = ("你上一条回复把工具调用写成了文字。报告撰写 "证据目录里，直接用引用标记写报告正文。") MARKUP_ERROR = ("模型输出了工具调用的原始标记，而报告撰写节点不调用工具——数据已经在证据目录里。" "换一个模型再试") +#: 接着跑时 checkpoint 里缓存的判定最多翻几条去找这一稿的（每条都是先前某一稿的，对不上就往后翻） +_STALE_LIMIT = 16 # -------------------------------------------------------------------------- @@ -120,14 +144,26 @@ def report_entities(spec: GraphSpec, node: GraphNode) -> str: def report_claims(spec: GraphSpec, node: GraphNode) -> str: - """报告节点的结论句策略：off / require_citation。出口判档按它（和节点执行时同一个取值规则）。 + """报告节点的结论句策略：off / require_citation / judge。出口判档按它（和节点执行时同一个取值规则）。 - 写错的值（包括 judge）画布校验就拦了，执行时节点也会报错，这里只把没写当成 off。 + 写错的值画布校验就拦了，执行时节点也会报错，这里只把没写当成 off。 """ value = _setting(spec, node, "claims", "off") return value if isinstance(value, str) and value in CLAIMS else "off" +def report_judge(spec: GraphSpec, node: GraphNode) -> dict[str, Any]: + """claims: judge 的子配置：节点上的 judge（没写取图级 defaults），写了的键原样保留——上限写 null 就是 + 不限，没写的键运行时取设置里的默认（judge.report_budget）。rewrite_once、on_unsupported 补上缺省 + （false、degrade）：出口判档读 on_unsupported。写错的值画布校验就拦了，执行时节点也会报错。 + """ + raw = _setting(spec, node, "judge") + raw = raw if isinstance(raw, dict) else {} + return {**raw, "rewrite_once": raw.get("rewrite_once") is True, + "on_unsupported": raw.get("on_unsupported") if raw.get("on_unsupported") in JUDGE_ON_UNSUPPORTED + else "degrade"} + + def report_allowance(spec: GraphSpec, node: GraphNode) -> list[Any]: """报告自查放行的字面量：出口契约 report_from 指向它时，契约的 allow_numbers。 @@ -209,13 +245,18 @@ class _RenderedStream: self._ctx.emit(EventType.LLM_TOKEN, delta=tail) -async def _default_on_violation(run_id: str) -> str: - """探索运行默认标注、照常产出；正式运行默认失败——封存的发布版不能带着裸数字出具。""" +async def _run_class(run_id: str) -> str | None: + """按运行记录上的 run_class 取：子图、审批之后续跑都是另起的执行上下文，只有运行记录靠得住。""" from app.db.models import Run async with SessionLocal() as session: run = await session.get(Run, run_id) - return "fail" if run is not None and run.run_class == "formal" else "flag" + return run.run_class if run is not None else None + + +async def _default_on_violation(run_id: str) -> str: + """探索运行默认标注、照常产出；正式运行默认失败——封存的发布版不能带着裸数字出具。""" + return "fail" if await _run_class(run_id) == "formal" else "flag" def _options(ctx: NodeContext) -> tuple[str, int]: @@ -225,6 +266,8 @@ def _options(ctx: NodeContext) -> tuple[str, int]: # 画布校验也拦这两项；执行时再守一道，报的是同一句话，免得写错的值被悄悄当成默认 if problem := claims_problem(ctx.cfg("claims")): raise NodeError(ctx.node.id, problem) + if ctx.cfg("claims") == CLAIMS_JUDGE and (problems := judge_problems(ctx.cfg("judge"))): + raise NodeError(ctx.node.id, "；".join(message for _, message in problems)) entities = ctx.cfg("entities", "link") or "link" if entities not in ENTITIES: raise NodeError(ctx.node.id, f"entities 只能是 {' / '.join(ENTITIES)}，写的是 {entities!r}") @@ -248,6 +291,29 @@ def _blocking(violations: list[dict[str, Any]], numbers: str) -> list[dict[str, and not (numbers == "off" and v.get("code") == "uncited_number")] +def _violation_key(violation: dict[str, Any]) -> tuple[str, str]: + """认一处违规：(code, 点名的字)。点名的字按 text、ref、message 的顺序取第一个有的。""" + named = violation.get("text") or violation.get("ref") or violation.get("message") or "" + return str(violation.get("code") or ""), str(named) + + +def _new_violations(before: list[dict[str, Any]], after: list[dict[str, Any]]) -> list[dict[str, Any]]: + """after 里 before 没有的违规，按 (code, 点名的字) 逐个抵消（同一个裸数字多写一处也算新的）。 + + 全部违规都比，包括只标注、不让重写的那几类（SOFT）：改写稿冒出一个编造的表名，受管的正式出具就会 + 因此降档；去掉一处、又冒出另一处，总数没变，冒出来的也还是新的。 + """ + left = Counter(_violation_key(v) for v in before) + fresh: list[dict[str, Any]] = [] + for violation in after: + key = _violation_key(violation) + if left[key] > 0: + left[key] -= 1 + else: + fresh.append(violation) + return fresh + + def _named(violations: list[dict[str, Any]], limit: int = 3) -> str: """违规里点名的那几个字：「45678」「m:nope」。""" names = [str(v.get("text") or v.get("ref") or "") for v in violations] @@ -269,7 +335,10 @@ def _messages(ctx: NodeContext, state: GraphState, catalog: dict[str, Any], cell system = ctx.render_str(ctx.cfg("system", ""), state) instructions = ctx.render_str(ctx.cfg("instructions", ""), state).strip() \ or "根据下面的证据写一份简洁的报告，先总后分。" - rules = f"{MARKER_RULES}\n{CLAIMS_RULE}" if claims == "require_citation" else MARKER_RULES + rules = MARKER_RULES + if claims in ("require_citation", CLAIMS_JUDGE): + # 裁判只看写作者挂的依据：要判，就更得每句都挂上 + rules = f"{rules}\n{CLAIMS_RULE}" + (f"\n{judging.JUDGE_RULE}" if claims == CLAIMS_JUDGE else "") return [ SystemMessage(content="\n\n".join(p for p in (system, ROLE) if p)), HumanMessage(content=( @@ -362,11 +431,26 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: response, raw = await draft(messages) doc = compose(raw) + judgement: dict[str, Any] | None = None + if claims == CLAIMS_JUDGE: + judge_cfg = report_judge(spec, ctx.node) + if await _run_class(ctx.run.run_id) != "formal": + # 探索运行默认按需裁判：点开哪句才判哪句，节点里不花这笔钱 + judgement = judging.on_demand(doc, on_unsupported=judge_cfg["on_unsupported"], + rewrite_once=judge_cfg["rewrite_once"]) + elif not (blocking and on_violation == "fail"): + # 反正要判失败的报告不值得再花钱判（节点紧接着就报错） + doc, response, blocking, judgement, usages = await _judge_inline( + ctx, doc, catalog, judge_cfg, model_id=model_id, messages=[*messages, response], + draft=draft, compose=compose, numbers=numbers, blocking=blocking) + spent.extend(usages) + doc_artifact = await _store(doc, ctx) ctx.emit(EventType.REPORT_CHECKED, doc_artifact=doc_artifact, ok=not doc["violations"], stats=doc["stats"], violations=doc["violations"][:20], repairs=repairs, on_violation=on_violation, failed=bool(blocking) and on_violation == "fail", - **({"claims": claims} if evidence_on else {})) + **({"claims": claims} if evidence_on else {}), + **({"judge": judgement} if judgement is not None else {})) if blocking and on_violation == "fail": head = "；".join(v["message"] for v in blocking[:5]) more = f"；…另有 {len(blocking) - 5} 处" if len(blocking) > 5 else "" @@ -388,6 +472,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: "model": model_id, "duration_ms": int((time.perf_counter() - started) * 1000), **({"claims": claims} if evidence_on else {}), + **({"judge": judgement} if judgement is not None else {}), } updates: dict[str, Any] = {"nodes": {ctx.node.id: result}, "usage": usage} if ctx.cfg("emit_message", True): @@ -396,3 +481,131 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: if var_name: updates["vars"] = {var_name: text} return updates + + +# -------------------------------------------------------------------------- +# claims: judge——正式运行在节点里裁判 +# -------------------------------------------------------------------------- + + +def _rewrite_request(sentences: list[tuple[str, str]]) -> str: + listed = "\n".join(f"{i}.「{text}」——{why or '证据不支持这句话'}" for i, (text, why) in enumerate(sentences, 1)) + return ( + "另一个模型按证据逐句核对了你的报告，下面这几句它认为证据不支持：\n" + f"{listed}\n\n" + "请只改这几句：补上证据目录里真有的依据 [[see:…]]，或者把说法收窄到证据能支持的程度，或者删掉；" + "其余部分一字不改。数字仍然只能用引用标记写。不要解释改了什么，只输出改好的整篇报告正文。" + ) + + +def _merge(first: dict[str, Any], second: dict[str, Any], budget: judging.Budget) -> dict[str, Any]: + """改写前后两轮裁判合成一份：判定、句数、缺口按改写后这一稿，花费、调用、用时两轮相加。""" + merged = dict(second) + for key in ("cost_usd", "estimated_usd"): + merged[key] = round(first[key] + second[key], 6) + for key in ("calls", "duration_ms"): + merged[key] = first[key] + second[key] + merged["notes"] = list(dict.fromkeys([*first["notes"], *second["notes"]])) + merged["budget"] = budget.as_dict() + merged["model"] = second.get("model") or first.get("model") + merged["priced"] = first.get("priced") + return merged + + +async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str, Any], judge_cfg: dict[str, Any], *, + model_id: str, messages: list[BaseMessage], draft: Any, compose: Any, numbers: str, + blocking: list[dict[str, Any]] + ) -> tuple[dict[str, Any], BaseMessage, list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]: + """写完、核对完之后请裁判判一遍，判定写进文档。返回 (文档, 写作者最后一稿, 要拦的违规, 裁判摘要, 用量)。 + + messages 是写作的对话、最后一条是写作者这一稿。rewrite_once 时把证据不支持的句子交回去只改这几句， + 再判一次（没改的句子直接用第一次的判定），最多一轮；改写稿冒出原稿没有的违规就不采用，保留原稿。 + + 摘要里的 rewrite = {units, sentences, applied, reason, changed}： + - sentences：交回去的句子（原稿里的原文） + - units：这几句在封存文档里的编号。没采用时封存的是原稿，就是原稿的编号；采用了封存的是改写稿， + 编号按改写稿重排，按原文认：原句原样留在改写稿里的给新编号，改掉了的是 null + - changed：采用时改写稿里新出来的句子（原稿里没有这段原文）在封存文档里的编号；没采用是 [] + """ + from app.engine.replay import _in_graph + + async with SessionLocal() as session: + settings = await judging.judge_settings(session) + judge_spec = await judging.judge_model_spec(session, judge_cfg, settings=settings) + budget = judging.report_budget(settings, judge_cfg) + masks = await judging.source_masks(catalog) + + @task + async def judge_step(request: judging.JudgeRequest, key: str, limits: judging.Budget, known: dict[str, Any], + first: bool) -> dict[str, Any]: + # 裁判进 checkpoint：「接着跑」时直接取这里的判定，不再调用、不再计费（每日计数也记在这里面）。 + # 事件都在这里面发：取缓存时不会再发一遍。第二轮 limits 是剩下的钱和时间，说给人看的仍是配置的上限 + outcome = await judging.run_request(request, spec=judge_spec, budget=limits, known=known, emit=ctx.emit, + shown=budget) + if first and outcome.get("model") and outcome["model"] == model_id: + outcome["same_model"] = True + ctx.emit(EventType.LOG, level="warn", code="judge_same_model", + message=f"裁判模型和写作模型都是「{model_id}」：等于自己审自己，模型写错的地方它多半也看不出来。" + "到设置里选一个不同的「证据裁判模型」，或者在节点的 judge.model 里指定") + return {**outcome, "key": key} + + async def judge_round(round_doc: dict[str, Any], limits: judging.Budget, known: dict[str, Any], + first: bool) -> dict[str, Any]: + request = judging.prepare(round_doc, catalog, masked=masks) + key = judging.request_key(request, judge_spec, limits, known) + if _in_graph(): + # 写作不进 checkpoint，接着跑时写作者会重写一稿：缓存里先前那一稿的判定对不上指纹，往后翻 + for _ in range(_STALE_LIMIT): + outcome = await judge_step(request, key, limits, known, first) + if outcome.get("key") == key: + return outcome + return await judging.run_request(request, spec=judge_spec, budget=limits, known=known, emit=ctx.emit, + shown=budget) + + outcome = await judge_round(doc, budget, {}, True) + same_model = bool(outcome.get("same_model")) + usages = [outcome["usage"]] + response = messages[-1] + rewrite: dict[str, Any] | None = None + bad = [uid for uid, v in outcome["verdicts"].items() if v["status"] == "unsupported"] + if judge_cfg["rewrite_once"] and bad: + texts = judging.unit_texts(doc) + sentences = [(texts.get(uid, ""), outcome["verdicts"][uid].get("rationale") or "") for uid in bad] + ctx.emit(EventType.LOG, level="warn", code="report_rewrite", + message=f"裁判认为 {len(bad)} 句结论证据不支持（{_named([{'text': t} for t, _ in sentences])}），" + "已交回写作者只改这几句") + again, raw = await draft([*messages, HumanMessage(content=_rewrite_request(sentences))]) + rewritten = compose(raw) + worse = _blocking(rewritten["violations"], numbers) + fresh = _new_violations(doc["violations"], rewritten["violations"]) + rewrite = {"units": bad, "sentences": [t for t, _ in sentences], "applied": False, "reason": None, + "changed": []} + if fresh: + rewrite["reason"] = (f"改写稿冒出 {len(fresh)} 处原稿没有的问题（{_named(fresh)}），没有采用，" + "保留原稿和原来的判定") + ctx.emit(EventType.LOG, level="warn", code="report_rewrite_rejected", message=rewrite["reason"]) + else: + known = {outcome["keys"][u]: v for u, v in outcome["verdicts"].items() if v["status"] in judging.JUDGED} + # 第二轮用第一轮剩下的钱和时间：每份报告的上限管的是整份报告 + left = judging.Budget( + max_claims=budget.max_claims, + max_cost_usd=None if budget.max_cost_usd is None else max(0.0, budget.max_cost_usd - outcome["cost_usd"]), + timeout_s=None if budget.timeout_s is None else max(0.0, budget.timeout_s - outcome["duration_ms"] / 1000), + daily_max_usd=budget.daily_max_usd) + second = await judge_round(rewritten, left, known, False) + usages.append(second["usage"]) + outcome = _merge(outcome, second, budget) + doc, response, blocking = rewritten, again, worse + # 封存的是改写稿，编号按它重排：按原文认回去，改掉了的句子在改写稿里没有编号 + now = judging.unit_texts(rewritten) + at: dict[str, str] = {} + for uid, text in now.items(): + at.setdefault(text, uid) + before = set(texts.values()) + rewrite.update(applied=True, units=[at.get(texts.get(uid, "")) for uid in bad], + changed=[uid for uid, text in now.items() if text not in before]) + summary = judging.summarize(outcome, mode="inline", on_unsupported=judge_cfg["on_unsupported"], + rewrite_once=judge_cfg["rewrite_once"], same_model=same_model, rewrite=rewrite) + judging.apply_judgement(doc, outcome, summary) + return doc, response, blocking, summary, usages + diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 224b25ba..2615766c 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -95,6 +95,16 @@ WARN_CODES: dict[str, tuple[str, str]] = { # （记成空值，不兜底成 0）。答案能用，缺口要说清是哪一种 "agent_field_mismatch": ("agent_field_mismatch", DEGRADED), "agent_field_unverified": ("agent_field_unverified", DEGRADED), + # 结论句裁判（claims: judge）：触顶、没跑成的句子记为未裁判，出具不能判完整——是缺口，要说明 + "judge_limit": ("judge_limit", DEGRADED), + "judge_failed": ("judge_failed", DEGRADED), + # 证据不支持的句子交回写作者改过一次（和 report_repair 一样只记缺口）；改写稿冒出新问题没采用的， + # 原稿里那几句仍是证据不支持 + "report_rewrite": ("report_rewrite", DEGRADED), + "report_rewrite_rejected": ("report_rewrite_rejected", DEGRADED), + # 估不出裁判的金额、裁判和写作是同一个模型：配置上的提醒，答案本身没受影响 + "judge_unpriced": ("judge_unpriced", BENIGN), + "judge_same_model": ("judge_same_model", BENIGN), } # 这些 kind 重跑一次大概率能好：原因说得清，且不是「问题本身没问明白」。 diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index 93a00aab..4adc80b6 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math import re from collections import defaultdict from enum import StrEnum @@ -423,31 +424,101 @@ def _check_report_sources(node: GraphNode, spec: GraphSpec, result: ValidationRe #: 报告撰写节点的结论句策略（claims）。off：结论句不参与判档（默认，保持以前的行为）； -#: require_citation：没挂引用的结论句计入缺口、按出具档位降档。judge（模型裁判）是后续版本的 +#: require_citation：没挂引用的结论句计入缺口、按出具档位降档。这两档是确定性的（发布前自动修复 +#: 给人选的也只有这两档） REPORT_CLAIMS = ("off", "require_citation") -CLAIMS_LATER = "结论句裁判在后续版本支持" +#: judge：结论句由另一个模型按证据逐句判断（engine/judge.py），子配置写在节点的 judge 里 +CLAIMS_JUDGE = "judge" +CLAIMS_VALUES = (*REPORT_CLAIMS, CLAIMS_JUDGE) +#: judge 子配置能写的键。三个上限写 null 表示不限；没写的键运行时取设置里的默认（设置 → judge 一组） +JUDGE_KEYS = ("provider", "model", "max_claims", "max_cost_usd", "timeout_s", "rewrite_once", "on_unsupported") +#: 证据不支持时正式运行怎么判档：degrade 降档（默认）/ withhold 不予出具。探索运行只标注 +JUDGE_ON_UNSUPPORTED = ("degrade", "withhold") def claims_problem(value: Any) -> str | None: """claims 的取值有什么问题，没问题返回 None。校验和报告撰写节点执行时报的是同一句话。""" - if value in (None, "") or (isinstance(value, str) and value in REPORT_CLAIMS): + if value in (None, "") or (isinstance(value, str) and value in CLAIMS_VALUES): return None - if value == "judge": - return f"claims 写的是 judge：{CLAIMS_LATER}，现在只能写 off 或 require_citation" - return f"claims 只能是 {' / '.join(REPORT_CLAIMS)}，写的是 {value!r}" + return f"claims 只能是 {' / '.join(CLAIMS_VALUES)}，写的是 {value!r}" + + +def _positive(value: Any, *, integer: bool = False) -> bool: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: + return False + return not integer or int(value) == value + + +def judge_problems(value: Any) -> list[tuple[str, str]]: + """claims: judge 的子配置有什么问题：[(字段, 人话)]，没问题返回 []。校验和节点执行时报的是同一句话。 + + 上限写 null 就是不限（和没写不一样：没写取设置里的默认）。键写错了也报：写成 max_cost 的预算会被 + 悄悄当成没写、按默认值花钱。 + """ + if value in (None, ""): + return [] + if not isinstance(value, dict): + return [("judge", f'judge 要写成一个对象，比如 {{"max_cost_usd": 0.05, "max_claims": 40}}，写的是 {value!r}')] + out: list[tuple[str, str]] = [] + for key, item in value.items(): + field = f"judge.{key}" + if key not in JUDGE_KEYS: + out.append((field, f"judge 里不认识「{key}」：能写的是 {'、'.join(JUDGE_KEYS)}")) + elif key in ("provider", "model") and item is not None and not isinstance(item, str): + what = "模型接入的名字" if key == "provider" else "模型 id" + out.append((field, f"judge.{key} 要写{what}（字符串），不写或写 null 就用设置里的「证据裁判模型」，" + f"写的是 {item!r}")) + elif key == "max_claims" and item is not None and not _positive(item, integer=True): + out.append((field, f"judge.max_claims 要写正整数（每份报告最多判几句），写 null 表示不限，写的是 {item!r}")) + elif key == "max_cost_usd" and item is not None and not _positive(item): + out.append((field, "judge.max_cost_usd 要写大于 0 的金额（美元，每份报告的裁判花费上限），" + f"写 null 表示不限，写的是 {item!r}")) + elif key == "timeout_s" and item is not None and not _positive(item): + out.append((field, f"judge.timeout_s 要写大于 0 的秒数（每份报告的裁判时长上限），写 null 表示不限，" + f"写的是 {item!r}")) + elif key == "rewrite_once" and not isinstance(item, bool): + out.append((field, f"judge.rewrite_once 只能是 true 或 false，写的是 {item!r}")) + elif key == "on_unsupported" and item not in JUDGE_ON_UNSUPPORTED: + out.append((field, f"judge.on_unsupported 只能是 {' / '.join(JUDGE_ON_UNSUPPORTED)}，写的是 {item!r}")) + return out def _check_report_claims(node: GraphNode, spec: GraphSpec, result: ValidationResult) -> None: - """claims 写错了（包括还不支持的 judge）挡住运行：执行时报告节点照样会报这一句， - 与其跑完取数、算完口径卡才死在写报告那一步，不如在画图时说破。 + """claims 写错了挡住运行：执行时报告节点照样会报这一句，与其跑完取数、算完口径卡才死在写报告 + 那一步，不如在画图时说破。claims 是 judge 时再查 judge 子配置的形状；裁判模型和写作模型写成 + 同一个的给警告——等于自己审自己。 和运行时同一个取值规则：节点上没写就取图级 defaults（NodeContext.cfg）。 """ - value = node.config.get("claims") - if value in (None, ""): - value = spec.defaults.get("claims") + def effective(key: str) -> Any: + value = node.config.get(key) + return spec.defaults.get(key) if value in (None, "") else value + + value = effective("claims") if problem := claims_problem(value): result.add(problem, node_id=node.id, field="claims", code="report.claims_invalid") + if value != CLAIMS_JUDGE: + return + judge = effective("judge") + for field, message in judge_problems(judge): + result.add(message, node_id=node.id, field=field, code="report.judge_invalid") + judge_model = judge.get("model") if isinstance(judge, dict) else None + if isinstance(judge_model, str) and judge_model and judge_model == effective("model"): + result.add(f"裁判模型和写作模型都是「{judge_model}」：等于自己审自己，模型写错的地方它多半也看不出来。" + "给 judge.model 换一个不同的模型", level="warning", node_id=node.id, field="judge.model", + code="report.judge_same_model") + + +def contract_needs_metrics(contract: dict[str, Any]) -> bool: + """出具契约要不要写 metrics_from（指标来自哪几张口径卡）。 + + 旧写法按数值回指叙述，没有口径卡就无从回指，一定要写。引用模式（report_from）下，数字回指靠 + 报告里的引用标记：只引查询单元格（cells）的问数据图根本没有口径卡，不该逼它写；契约里列了 + required / expected 指标的，指标总得有个来处，照样要写。运行时 io.py 也是这么判的。 + """ + if contract.get("report_from") in (None, ""): + return True + return bool(contract.get("required") or contract.get("expected")) def _check_report_from(node: GraphNode, report_from: Any, spec: GraphSpec, @@ -542,7 +613,7 @@ def validate_graph(spec: GraphSpec) -> ValidationResult: # 报成 error，两档发布和运行前都挡住 result.add("出具契约必须是一个 JSON 对象", node_id=node.id, field="contract", code="contract.not_object") - elif contract and not contract.get("metrics_from"): + elif contract and not contract.get("metrics_from") and contract_needs_metrics(contract): result.add("出具契约缺 metrics_from（指标来自哪个「口径卡」节点）", node_id=node.id, field="contract.metrics_from", code="contract.metrics_from_missing") if isinstance(contract, dict) and contract.get("report_from") not in (None, ""): diff --git a/backend/tests/test_autofix_governance.py b/backend/tests/test_autofix_governance.py index 97e34897..a6ff682d 100644 --- a/backend/tests/test_autofix_governance.py +++ b/backend/tests/test_autofix_governance.py @@ -118,22 +118,21 @@ def test_published_level_warnings_get_no_fix(): # -------------------------------------------------------------------------- -def test_judge_becomes_require_citation_under_governed(): - graph = traceable(report={**STRICT, "claims": "judge"}) +def test_an_invalid_claims_value_becomes_require_citation_under_governed(): + graph = traceable(report={**STRICT, "claims": "sometimes"}) fid = "report.claims_invalid:write" fix = plan(graph)[fid] - assert fix["kind"] == "auto" and fix["preview"] == {"field": "claims", "before": "judge", + assert fix["kind"] == "auto" and fix["preview"] == {"field": "claims", "before": "sometimes", "after": "require_citation"} out = apply_fixes(graph, [fid], level="governed") assert out["ok"] is True and config(out["graph"], "write")["claims"] == "require_citation" def test_an_invalid_claims_value_is_a_choice_under_published(): - graph = traceable(report={"claims": "judge"}) + graph = traceable(report={"claims": "sometimes"}) fid = "report.claims_invalid:write" fix = plan(graph, "published")[fid] assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == ["off", "require_citation"] - assert fix["default"] == "require_citation" # 离裁判最近的是「每句结论挂引用」，仍然要人确认 assert apply_fixes(graph, [fid], level="published")["applied"] == [] out = apply_fixes(graph, [fid], {fid: "off"}, level="published") assert out["applied"] == [fid] and config(out["graph"], "write")["claims"] == "off" and out["ok"] is True diff --git a/backend/tests/test_claims_prefilter.py b/backend/tests/test_claims_prefilter.py new file mode 100644 index 00000000..d4353fd1 --- /dev/null +++ b/backend/tests/test_claims_prefilter.py @@ -0,0 +1,111 @@ +"""结论句预筛：哪些句子送裁判模型、按什么优先级，哪些直接判为连接性。 + +裁判按调用计费，一份报告里的过渡话、标题、表格里的一格、代码都不值得花钱判。预筛是确定性的 +规则（同一份文档永远得到同一个结果），而且只看文档本身： + +- 优先级 1：写作者给这句挂了依据（行内引用或 [[see:…]]） +- 优先级 2：没挂依据，但有方向词、因果词（增长、主要来自、导致…） +- 优先级 3：其余有数字、实体这类实质内容、又不是短句的 +- 不送：连接性的话、标题、代码、表格单元格；只有过渡词的（「原因如下：」）；短句 +预算不够时按优先级截断：先判 1，再判 2，剩下的记未裁判。 +""" +from __future__ import annotations + +from app.engine.evidence import build_catalog, compose_doc, iter_units +from app.engine.judge import SHORT_CHARS, candidates, claim_priority + + +def catalog(): + return build_catalog(nodes={"caliber": { + "kind": "metric_set", "caliber": "周报口径", "caliber_version": "v2", "artifact": "f" * 64, + "metrics": [{"id": "gmv", "name": "销售额", "value": 45678.5, "unit": "元", "decimals": 1}, + {"id": "orders", "name": "订单数", "value": 1234, "unit": "单"}]}}) + + +RAW = """## 本周概览 + +本周销售额 [[m:gmv]]。增长主要来自新客首单。[[see:m:gmv]]原因如下： +下面看细节。订单一共是 1234 单，整体平稳。共 3 单。增长明显。 + +- 列表里订单 [[m:orders]] +- 过渡说明 + +| 区域 | 销售额 | +|---|---| +| 华东 | [[m:gmv]] | + +```sql +SELECT 1 +``` +""" + + +def by_text(doc): + """{句子原文: unit}，测试按字找句子，不依赖编号。""" + md = doc["markdown"] + return {md[u["span"][0]:u["span"][1]].strip(): (b, u) for b, u in iter_units(doc)} + + +def priorities(doc): + return {text: claim_priority(u, b, doc["markdown"]) for text, (b, u) in by_text(doc).items()} + + +def test_the_rules(): + got = priorities(compose_doc(RAW, catalog())) + assert got["本周销售额 45,678.5元。"] == 1 # 行内引用 + assert got["增长主要来自新客首单。"] == 1 # 句末 [[see:]] 挂的依据 + assert got["列表里订单 1,234单"] == 1 # 列表项也是句子 + assert got["增长明显。"] == 2 # 没挂依据，有方向词：短也要判 + assert got["订单一共是 1234 单，整体平稳。"] == 3 # 有数字、不是短句 + assert got["原因如下："] is None # 有「原因」但只是过渡 + assert got["共 3 单。"] is None # 短句 + assert got["下面看细节。"] is None # 连接性 + assert got["过渡说明"] is None + assert got["本周概览"] is None # 标题 + assert got["45,678.5元"] is None # 表格单元格：一格值不是一句结论 + assert got["SELECT 1"] is None # 代码 + + +def test_it_is_deterministic(): + one, two = compose_doc(RAW, catalog()), compose_doc(RAW, catalog()) + assert priorities(one) == priorities(two) + assert [c.unit for c in candidates(one)] == [c.unit for c in candidates(two)] + + +def test_candidates_come_in_priority_order_then_document_order(): + doc = compose_doc(RAW, catalog()) + texts = {u["id"]: t for t, (_, u) in by_text(doc).items()} + got = [(c.priority, texts[c.unit]) for c in candidates(doc)] + assert got == [ + (1, "本周销售额 45,678.5元。"), (1, "增长主要来自新客首单。"), (1, "列表里订单 1,234单"), + (2, "增长明显。"), + (3, "订单一共是 1234 单，整体平稳。"), + ] + + +def test_short_means_visible_characters_not_bytes(): + """短句按看得见的字算（标点、空白不算），数字的字也算进去。""" + assert SHORT_CHARS == 6 + doc = compose_doc("订单 1234 单。订单总数 1234 单。", catalog()) + got = priorities(doc) + assert got["订单 1234 单。"] == 3 and got["订单总数 1234 单。"] == 3 + doc = compose_doc("共 12 单。", catalog()) + assert priorities(doc)["共 12 单。"] is None + + +def test_an_explicit_request_overrides_the_screen_but_not_the_kind(): + """有人点开一句、明确要求判它：预筛放掉的结论句照判（按最低优先级）；标题、连接性的话不是结论，不判。""" + doc = compose_doc(RAW, catalog()) + units = {t: u["id"] for t, (_, u) in by_text(doc).items()} + picked = candidates(doc, units=[units["共 3 单。"], units["下面看细节。"], units["本周概览"], "u999"]) + assert [(c.unit, c.priority) for c in picked] == [(units["共 3 单。"], 3)] + + +def test_cites_are_the_authors_own(): + """自动链接的名字不算依据（三期 D2）：只提到表名的句子不会因此排进优先级 1。""" + ledger = [{"kind": "schema", "node_id": "fetch", "exec": 1, "artifact": "a" * 64, "source": "shop", + "schema": None, "synced_at": None, "tables": {"orders": ["amount", "region"]}}] + cat = build_catalog(nodes={}, ledger=ledger) + doc = compose_doc("明细在 `orders` 表里，按区域汇总之后东区最高。", cat) + [(text, prio)] = [(t, p) for t, p in priorities(doc).items()] + assert prio == 2, (text, prio) diff --git a/backend/tests/test_contract_cells.py b/backend/tests/test_contract_cells.py index 132af5b0..cd70a762 100644 --- a/backend/tests/test_contract_cells.py +++ b/backend/tests/test_contract_cells.py @@ -326,3 +326,74 @@ async def test_the_report_node_and_the_exit_check_agree_when_the_status_changes_ assert seen == [True, True], seen issuance = row.output["_issuance"] assert issuance["tier"] == "withheld" and CELLS_REASON in issuance["unresolved"][0]["message"], issuance + + +# -------------------------------------------------------------------------- +# 只引单元格、没有口径卡：契约不用写 metrics_from +# +# 问数据的典型图是 input → 取数 → 报告 → 出口，报告直接引用查过的单元格。受管级别在契约里 +# 写 cells: true 就够了（用户拍板的第 5 条），以前校验和门禁无条件要 metrics_from， +# 这张图怎么都发不成受管，Copilot 补上契约也会被判「冒出新的问题」作废 +# -------------------------------------------------------------------------- + + +def cells_only(**contract): + contract = {"report_from": "write", "strict": True, "cells": True, **contract} + nodes = [ + node("start", "input"), + node("fetch", "tool", tool="db_query__shop", + args={"sql": "SELECT SUM(amount) AS gmv, COUNT(*) AS orders FROM orders"}), + node("write", "report", instructions="写周报", numbers="strict", on_violation="fail", + claims="require_citation"), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract=contract), + ] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def contract_codes(graph: dict, level: str = "governed") -> tuple[list[str], list[str]]: + from app.engine.governance import lint_for_publish + from app.engine.schema import GraphSpec, validate_graph + + spec = GraphSpec.model_validate(graph) + return ([i.code for i in validate_graph(spec).issues if i.level == "error"], + [i.code for i in lint_for_publish(spec, level=level).issues if i.level == "error"]) + + +def test_a_cells_only_contract_needs_no_metrics_from(): + validate, gate = contract_codes(cells_only()) + assert validate == [], validate + assert gate == [], gate # 受管门禁整张图一条 error 都没有 + + +def test_a_cited_contract_that_lists_required_metrics_still_needs_metrics_from(): + validate, gate = contract_codes(cells_only(required=["gmv"])) + assert "contract.metrics_from_missing" in validate and "contract.metrics_from_missing" in gate + validate, gate = contract_codes(cells_only(expected=["gmv"])) + assert "contract.metrics_from_missing" in validate and "contract.metrics_from_missing" in gate + + +def test_a_narrative_contract_still_needs_metrics_from(): + graph = cells_only(report_from=None, narrative="{{ nodes.write.text }}") + graph["nodes"][-1]["data"]["config"]["contract"].pop("report_from") + validate, _ = contract_codes(graph, level="published") # 受管级别不收 narrative，只看校验 + assert "contract.metrics_from_missing" in validate + + +async def test_a_governed_formal_run_over_cells_only_issues_formally(client, monkeypatch): + Writer(monkeypatch, text="本周销售额 [[v:Q1.r0.gmv]]。") + row = await formal(client, cells_only(), status="governed") + assert row.status == "succeeded", row.error + issuance = row.output["_issuance"] + assert issuance["mode"] == "citations" and issuance["tier"] == "formal", issuance + assert issuance["unresolved"] == [] and issuance["gaps"] == [], issuance + [cell] = [m for m in issuance["matched"] if m.get("cell")] + assert cell["cell"] == "Q1.r0.gmv" and cell["token"] == "300.5" + + +def test_a_graph_with_a_caliber_card_still_needs_required_even_without_metrics_from(): + graph = weekly(cells=True) + contract = graph["nodes"][-1]["data"]["config"]["contract"] + contract.pop("metrics_from"), contract.pop("required") + validate, gate = contract_codes(graph) + assert "contract.required_missing" in gate, gate # 不写 metrics_from 绕不过必需指标 + assert validate == [], validate diff --git a/backend/tests/test_contract_judge.py b/backend/tests/test_contract_judge.py new file mode 100644 index 00000000..281704f9 --- /dev/null +++ b/backend/tests/test_contract_judge.py @@ -0,0 +1,323 @@ +"""出口契约（引用模式）按结论句裁判的判定判档（四期）。 + +- 报告节点 claims: judge 在正式运行里当场裁判，判定随文档封存。出口按文档里的判定判档：证据不支持的结论句 + 按 on_unsupported 降档（degrade，默认）或不予出具（withhold）；裁判没跑成、触顶了没判完的，裁判摘要里的缺口 + 照抄进出具声明——不能判完整出具，也不因此不予出具 +- 判定是模型给的，出口复算不了：信按哈希取回的文档 +- judge 在挂引用这件事上和 require_citation 一样严：没挂依据的结论句照样计入缺口 +- 探索运行按需裁判：出口只标注（候选句都是「未裁判 · 按需」），不因判定改档位 +- 契约也能写 claims: judge：只能收紧（on_unsupported 取更严的）；报告节点没开 judge 的，照实记缺口 +- 升级前的运行照旧：契约写了也记「这一版还不支持」 +""" +from __future__ import annotations + +import asyncio +import json +import re +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import delete, select + +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion +from app.engine import judge as judge_mod +from app.engine import runner as runner_mod +from app.engine.judge import JUDGE_ROLE, SPEND_KEY +from app.engine.runner import run_manager +from app.main import app +from app.providers.mock_model import MockChatModel + +JUDGE_MODEL = "claude-sonnet-5" +METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] +#: 两句结论都挂了依据、一句连接性的话 +CITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]下面看细节。" +#: 第二句结论没挂依据 +UNCITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。下面看细节。" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"]))) + await session.commit() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def weekly(*, report=None, contract=None) -> dict: + nodes = [ + node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]), + node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS), + node("write", "report", instructions="写周报", **{"claims": "judge", "judge": {"max_cost_usd": 0.05}, + **(report or {})}), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], + contract={"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True, + **(contract or {})}), + ] + ids = [n["id"] for n in nodes] + return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} + + +class Models: + """写作者回 text；裁判按句子里的字给判定（verdicts：{句中片段: 判定}，没对上的是 supported）。""" + + def __init__(self, monkeypatch, text=CITED, verdicts=None, *, fail_judge=None): + self.judges: list[str] = [] + verdicts = verdicts or {} + + def decide(model, messages): + if JUDGE_ROLE in str(messages[0].content): + request = str(messages[-1].content) + self.judges.append(request) + if fail_judge: + raise RuntimeError(fail_judge) + items = [{"unit": uid, + "verdict": next((s for frag, s in verdicts.items() if frag in line), "supported"), + "rationale": "摘录里有对应的指标" if "销售额" in line else "证据里没有新客维度", "used": []} + for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)] + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False)) + return AIMessage(content=text) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + + async def judge_model_of(session, spec): + return MockChatModel(model_name=JUDGE_MODEL), JUDGE_MODEL + + monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of) + + +async def wait(run_id: str) -> Run: + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run_id} 没跑完：{row.status}") + + +async def issued(graph, *, run_class="formal") -> dict: + run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class) + row = await wait(run.id) + assert row.status == "succeeded", row.error + return row.output["_issuance"] + + +async def governed(client, graph) -> Run: + """把这张图发布成受管级别，从发布版本发起正式运行。""" + async with SessionLocal() as session: + wf = Workflow(name=f"judge-{uuid.uuid4().hex[:6]}", graph=graph, status="governed", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph)) + await session.commit() + wf_id = wf.id + res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"}) + assert res.status_code == 201, res.text + row = await wait(res.json()["id"]) + assert row.status == "succeeded", row.error + return row + + +async def issuance_event(run_id: str) -> dict: + async with SessionLocal() as session: + [event] = (await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars() + return event.data + + +# -------------------------------------------------------------------------- +# 正式运行：按封存的判定判档 +# -------------------------------------------------------------------------- + + +async def test_every_claim_supported_is_a_full_issuance(monkeypatch): + Models(monkeypatch) + issuance = await issued(weekly()) + assert issuance["tier"] == "formal" and issuance["gaps"] == [], issuance + claims = issuance["claims"] + assert claims["policy"] == "judge" and claims["mode"] == "inline" and claims["complete"] is True + assert claims["counts"] == {"supported": 2, "partial": 0, "unsupported": 0, "not_a_claim": 0, "unjudged": 0} + assert claims["on_unsupported"] == "degrade" and claims["uncited_claims"] == 0 + assert claims["unsupported"] == [] and claims["model"] == JUDGE_MODEL + + +async def test_an_unsupported_claim_degrades_by_default(monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + issuance = await issued(weekly()) + assert issuance["tier"] == "degraded", "不支持按缺口降档，不是不予出具（契约是 strict 的）" + assert issuance["gaps"] == [], "判了就不是「没核」：档位来自判定，不来自缺口" + [flagged] = issuance["claims"]["unsupported"] + assert flagged["text"] == "增长主要来自新客首单。" and flagged["rationale"] == "证据里没有新客维度" + assert flagged["unit"] and len(flagged["span"]) == 2 + assert issuance["claims"]["counts"]["unsupported"] == 1 + + +async def test_withhold_withholds(monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}})) + assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold" + + +async def test_partial_support_is_listed_but_does_not_cost_a_grade(monkeypatch): + Models(monkeypatch, verdicts={"新客": "partial"}) + issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}})) + assert issuance["tier"] == "formal" + [item] = issuance["claims"]["partial"] + assert item["text"] == "增长主要来自新客首单。" and issuance["claims"]["counts"]["partial"] == 1 + + +async def test_a_failed_judge_is_a_gap_not_a_withheld_report(monkeypatch): + Models(monkeypatch, fail_judge="网关 502") + issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}})) + assert issuance["tier"] == "degraded", "没判成不能盖完整出具，也不能当成「不支持」不予出具" + [gap] = [g for g in issuance["gaps"] if "裁判" in g] + assert gap.startswith("报告「write」") and "没裁判" in gap and "网关 502" in gap + assert issuance["claims"]["complete"] is False and issuance["claims"]["counts"]["unjudged"] == 2 + + +async def test_hitting_a_limit_is_a_gap(monkeypatch): + Models(monkeypatch) + issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "max_claims": 1}})) + assert issuance["tier"] == "degraded" + [gap] = [g for g in issuance["gaps"] if "裁判" in g] + assert "已到上限" in gap and "1 句" in gap + assert issuance["claims"]["limits_hit"] == ["max_claims"] and issuance["claims"]["counts"]["unjudged"] == 1 + + +async def test_an_uncited_claim_still_counts_under_judge(monkeypatch): + """judge 在挂引用这件事上和 require_citation 一样严：从 require_citation 换成 judge 不能让没挂依据的句子过关。""" + Models(monkeypatch, text=UNCITED) + issuance = await issued(weekly()) + assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1 + assert issuance["claims"]["uncited"][0]["text"] == "增长主要来自新客首单。" + + +async def test_the_issuance_event_carries_the_counts(monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + run = await run_manager.start(graph=weekly(), input_payload={}, run_class="formal") + row = await wait(run.id) + event = await issuance_event(row.id) + assert event["tier"] == "degraded" + assert event["claims"] == {"policy": "judge", "uncited_claims": 0, "on_unsupported": "degrade", + "mode": "inline", "counts": {"supported": 1, "partial": 0, "unsupported": 1, + "not_a_claim": 0, "unjudged": 0}} + + +async def test_a_governed_formal_run_grades_by_the_verdicts(client, monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + graph = weekly(report={"numbers": "strict", "on_violation": "fail", + "judge": {"max_cost_usd": None, "on_unsupported": "withhold"}}) + row = await governed(client, graph) + issuance = row.output["_issuance"] + assert issuance["tier"] == "withheld" and issuance["claims"]["unsupported"][0]["text"] == "增长主要来自新客首单。" + + +# -------------------------------------------------------------------------- +# 探索运行：只标注 +# -------------------------------------------------------------------------- + + +async def test_explore_runs_only_annotate(monkeypatch): + models = Models(monkeypatch, verdicts={"新客": "unsupported"}) + issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}), + run_class="exploratory") + assert models.judges == [], "探索运行节点里不花这笔钱" + assert issuance["tier"] == "formal" and issuance["gaps"] == [] + claims = issuance["claims"] + assert claims["mode"] == "on_demand" and claims["complete"] is False + assert claims["counts"]["unjudged"] == 2 and claims["unjudged"] == {"on_demand": 2} + assert claims["unsupported"] == [] + + +async def test_explore_runs_still_count_uncited_claims(monkeypatch): + Models(monkeypatch, text=UNCITED) + issuance = await issued(weekly(), run_class="exploratory") + assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1 + + +# -------------------------------------------------------------------------- +# 契约里的 claims: judge +# -------------------------------------------------------------------------- + + +async def test_the_contract_can_tighten_on_unsupported(monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + issuance = await issued(weekly(contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}})) + assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold" + assert not any("不支持" in g and "这一版" in g for g in issuance["gaps"]) + + +async def test_the_contract_cannot_loosen_on_unsupported(monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}, + contract={"claims": {"policy": "judge", "on_unsupported": "degrade"}})) + assert issuance["tier"] == "withheld" + + +async def test_a_contract_asking_for_judge_on_a_report_without_it_is_a_gap(monkeypatch): + Models(monkeypatch) + issuance = await issued(weekly(report={"claims": "require_citation"}, contract={"claims": "judge"})) + assert issuance["tier"] == "degraded" + [gap] = [g for g in issuance["gaps"] if "judge" in g] + assert "「write」" in gap and "没有开" in gap and "这一版" not in gap + assert issuance["claims"]["policy"] == "require_citation" + + +def test_a_run_in_flight_across_the_upgrade_keeps_the_old_gap(): + """升级前写的报告（产出里没有 claims）碰上写了 judge 的契约：照旧记「这一版还不支持」，不判档。""" + from app.engine.nodes.io import _claims + + out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None, "unsupported": None} + _claims(out, {"claims": {"policy": "judge", "on_unsupported": "withhold"}}, {"text": "x"}, evidence_on=True, + uncited=[{"unit": "u1", "span": [0, 4], "text": "增长很快"}]) + assert out["claims_policy"] is None and out["claims"] is None and out["unsupported"] is None + assert len(out["gaps"]) == 1 and "这一版还不支持" in out["gaps"][0] + + +async def test_runs_from_before_the_upgrade_ignore_judge(monkeypatch): + async def no_snapshot(session): + return None + + monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) + models = Models(monkeypatch, text=UNCITED, verdicts={"新客": "unsupported"}) + issuance = await issued(weekly()) + assert models.judges == [] and issuance["tier"] == "formal" and "claims" not in issuance + + +# -------------------------------------------------------------------------- +# 答案复核（review.scan）认得裁判的几条警告 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("code", "shown"), [ + ("judge_limit", True), ("judge_failed", True), ("report_rewrite", True), ("report_rewrite_rejected", True), + ("judge_unpriced", False), ("judge_same_model", False), +]) +def test_review_knows_the_judge_warnings(code, shown): + """没判完、没跑成、改写过的是缺口（要说明）；估不出金额、自己审自己只是配置上的提醒，轨迹里有就够了， + 不值得为它再叫一次复核模型。""" + from app.engine.review import DEGRADED, scan + + signals = scan([{"type": "log", "data": {"level": "warn", "code": code, "message": f"{code} 的原话"}}], + {"answer": "本周销售额 45,678.5元。"}) + if shown: + [signal] = signals + assert signal.kind == code and signal.severity == DEGRADED and signal.detail == f"{code} 的原话" + else: + assert signals == [] diff --git a/backend/tests/test_copilot_entity_guide.py b/backend/tests/test_copilot_entity_guide.py index fbbfb63b..02e4fc54 100644 --- a/backend/tests/test_copilot_entity_guide.py +++ b/backend/tests/test_copilot_entity_guide.py @@ -1,4 +1,5 @@ -"""Copilot 的节点手册要教三期的写法：表名字段名、知识库原话、结论句策略，以及受管模板的报告节点怎么配。 +"""Copilot 的节点手册要教三期、四期的写法：表名字段名、知识库原话、结论句策略（含模型裁判 judge），以及受管模板的 +报告节点怎么配。 手册里没写的配置，Copilot 就不会用；写错的写法，报告节点会一直被打回。所以这里不只查关键词， 还把手册里举的标记例子交给真正的解析器，确认写法和系统认的是同一套。 @@ -7,9 +8,9 @@ from __future__ import annotations import re -from app.api.copilot import NODE_REFERENCE, GenerateIn, _user_message +from app.api.copilot import _ASSIST_RULES, NODE_REFERENCE, GenerateIn, _user_message from app.engine.evidence import QUOTE_MIN, SUPPORTED_KINDS, parse_markers -from app.engine.schema import REPORT_CLAIMS +from app.engine.schema import CLAIMS_VALUES, JUDGE_KEYS, JUDGE_ON_UNSUPPORTED def _report_section() -> str: @@ -22,12 +23,22 @@ def test_the_report_entry_teaches_entities_quotes_and_claims(): for words in ( ["[[t:", "[[c:", "反引号", "可能是编造的名字"], # 表名、字段名 ["[[q:K1|", "逐字", f"至少 {QUOTE_MIN} 个字", "[[see:K1]]"], # 知识库原话 - ["claims", "require_citation", "off", "judge", "后续版本"], # 结论句策略 + ["claims", "require_citation", "off", "judge"], # 结论句策略 + ["judge.max_cost_usd", "null", "不限", "rewrite_once", "on_unsupported", "按需"], # 模型裁判怎么配 ["entities", "link"], # 实体层开关 ): assert all(w in report for w in words), [w for w in words if w not in report] - # 结论句策略的取值和校验认的是同一份 - assert all(value in report for value in REPORT_CLAIMS) + # 结论句策略的取值和校验认的是同一份；judge 已经能用，不能再说「后续版本」「跑不起来」 + assert all(value in report for value in CLAIMS_VALUES) + assert all(value in report for value in JUDGE_ON_UNSUPPORTED) + assert "后续版本" not in report and "跑不起来" not in report + # 裁判模型建议和写作模型不同：否则等于自己审自己 + assert "不同" in report and "自己审自己" in report + + +def test_the_judge_keys_in_the_guide_are_the_ones_the_validator_knows(): + named = set(re.findall(r"judge\.(\w+)", NODE_REFERENCE)) + assert named and named <= set(JUDGE_KEYS), named - set(JUDGE_KEYS) def test_the_markers_in_the_guide_parse_as_the_system_reads_them(): @@ -40,15 +51,26 @@ def test_the_markers_in_the_guide_parse_as_the_system_reads_them(): def test_governed_templates_spell_out_the_report_settings(): ref = NODE_REFERENCE - governed = ref[ref.index("受管"):] + governed = ref[ref.index("要按受管级别发布的模板"):] for words in (['numbers: "strict"', 'on_violation: "fail"', 'claims: "require_citation"'], + ['claims: "judge"', "judge.max_cost_usd", "null"], ["report_from", "narrative"], ["cite_fields: true"], ["evidence_role", "source"]): assert all(w in governed for w in words), [w for w in words if w not in governed] +def test_the_publish_assist_knows_judge_is_the_stricter_one(): + """发布前的 Copilot 兜底只许往严里改：它要知道 judge 比 require_citation 严，不能为了过门禁把 judge 改回去。""" + assert "judge" in _ASSIST_RULES and "max_cost_usd" in _ASSIST_RULES + # update_node 把 judge 整个换掉：补预算只写 {max_cost_usd} 会把 withhold 连同裁判模型一起丢了 + rules = " ".join(_ASSIST_RULES.split()) + assert re.search(r"on_unsupported.{0,20}withhold.{0,20}不改回\s*degrade", rules), "withhold 不许改回 degrade" + assert re.search(r"补预算.{0,40}整份", rules), "补预算时 judge 要整份带上原来的键" + + def test_the_answer_branch_mentions_names_and_quotes(): text = _user_message(GenerateIn(instruction="上周哪张表的退款最多", intent="answer"), patch=False) assert "[[t:" in text and "反引号" in text and "编造" in text assert "[[q:K1|" in text and "逐字" in text - # 问数是探索运行：不要替人打开结论句策略 - assert not re.search(r"claims\s*[:=]\s*['\"]?require_citation", text) + # 问数是探索运行：不要替人打开结论句策略（结论句由读的人点开时按需请模型判断） + assert not re.search(r"claims\s*[:=]\s*['\"]?(require_citation|judge)", text) + assert "按需" in text diff --git a/backend/tests/test_evidence_judge.py b/backend/tests/test_evidence_judge.py new file mode 100644 index 00000000..944ac797 --- /dev/null +++ b/backend/tests/test_evidence_judge.py @@ -0,0 +1,554 @@ +"""结论句裁判的内核（engine/judge.py）：证据摘录、裁判调用、结构化输出兜底、预算与每日计数。 + +数字、实体是确定性的，系统自己能核对；「增长主要来自华东」这种话说得对不对，只能交给另一个模型按 +证据判断。这是概率性的判断，所以每一步都要留余地、说实话： + +- 一份报告合并成一次调用（超过每批的句数就分批），每句给 supported / partial / unsupported / not_a_claim +- 模型不支持结构化输出、或者给的不是合法结构时退回 JSON 解析，再失败就记「未裁判」，不编判定 +- 裁判模型关掉 thinking，max_tokens 4096 +- 预算（每份报告的句数、金额、时长，全局每日金额）每一项都能写 null 表示不限；触顶就停、已判的保留、 + 没判的记未裁判并说明是哪个上限——不报错、不让报告失败。金额超了按优先级截断：先判挂了依据的， + 再判有方向词、因果词的 +- 每日计数落库，按本地日期滚动 +- 给裁判的证据摘录里没有数据源遮罩的列 + +所有模型调用都是 mock 剧本，不碰真接口。 +""" +from __future__ import annotations + +import json +import re + +import pytest +from langchain_core.messages import AIMessage +from sqlalchemy import delete + +from app.core import artifact_store +from app.core.events import EventType +from app.db.base import SessionLocal +from app.db.models import Setting +from app.engine import judge as judge_mod +from app.engine.evidence import build_catalog, compose_doc, iter_units +from app.engine.judge import ( + BATCH_SIZE, + CHUNK_CHARS, + EXCERPT_COLS, + EXCERPT_ROWS, + JUDGE_ROLE, + JUDGE_RULES, + MAX_TOKENS, + RATIONALE_MAX, + SPEND_KEY, + SQL_CHARS, + Budget, + daily_spend, + judge_doc, + prepare, + source_masks, +) +from app.providers.factory import ModelSpec +from app.providers.mock_model import MockChatModel + +#: 目录里有价格的模型：金额上限对它起作用 +PRICED = "claude-sonnet-5" +UNLIMITED = Budget(max_claims=None, max_cost_usd=None, timeout_s=None, daily_max_usd=None) +SPEC = ModelSpec(model=PRICED) + + +# -------------------------------------------------------------------------- +# 夹具：一张口径卡、一次查询（带手机号列）、一次知识库检索、一个运行输入 +# -------------------------------------------------------------------------- + + +def _store(content) -> str: + """按 put_json 的落盘方式写进测试用的临时工件库：取回走真实的 load，复验哈希。""" + text = artifact_store.canonical_json(content) + artifact = artifact_store.content_hash(text) + path = artifact_store._path_of(artifact) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return artifact + + +CARD = {"kind": "metric_set", "caliber": "周报口径", "caliber_version": "v2", "metrics": [ + {"id": "gmv", "name": "销售额", "unit": "元", "value": 45678.5, "decimals": 1, "format": "thousands", + "status": "ok", "expression": "vars.kpi.gmv", "rendered": "45,678.5元"}, + {"id": "wow", "name": "环比增幅", "unit": "%", "value": 8.7, "decimals": 1, "format": "plain", "status": "ok", + "expression": "round((vars.kpi.gmv - vars.kpi.gmv_prev) / vars.kpi.gmv_prev * 100, 1)", "rendered": "8.7%"}, +]} +LONG_SQL = "SELECT region, SUM(amount) AS amount, MAX(phone) AS phone FROM orders WHERE " + " AND ".join( + f"flag_{i} = 1" for i in range(60)) + " GROUP BY region ORDER BY amount DESC" +ROWS = [[f"区域{i}", 1000.0 - i, f"PHONE-{i}"] for i in range(30)] +SNAP = {"columns": ["region", "amount", "phone"], "rows": ROWS, "row_count": len(ROWS), "truncated": False, + "elapsed_ms": 3, "sql": LONG_SQL, "source": "shop"} +HITS = {"query": "退款", "collection": "docs", "hits": [ + {"content": "退款主要集中在物流延误的订单。" * 60, "title": "售后周报", "document": "doc-1", "chunk": 0}, + {"content": "第二段原文。", "title": "售后周报", "document": "doc-1", "chunk": 1}]} + + +@pytest.fixture +def stored(): + card = {**CARD} + art = _store(card) + return {"card": art, "snap": _store(SNAP), "masked": _store({**SNAP, "mask_columns": ["phone"]}), + "hits": _store(HITS)} + + +def make_catalog(stored, snap="snap"): + ledger = [ + {"kind": "metric_set", "node_id": "caliber", "exec": 1, "artifact": stored["card"], "caliber": "周报口径", + "version": "v2", "metrics": ["gmv", "wow"]}, + {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": stored[snap], "via": "b" * 64, + "call_id": "call_1", "tool": "db_query__shop", "source": "shop", "columns": SNAP["columns"], + "rows": len(ROWS), "truncated": False}, + {"kind": "retrieval", "node_id": "kb", "exec": 1, "artifact": stored["hits"], "source": "docs", "rows": 2}, + ] + return build_catalog(nodes={"caliber": {**CARD, "artifact": stored["card"]}}, ledger=ledger, + inputs={"week": "2026-W37"}) + + +RAW = ("## 本周概览\n\n" + "本周销售额 [[m:gmv]]。[[see:m:gmv]]" + "环比 [[m:wow]]，增长主要来自华东。[[see:m:wow,Q1]]" + "增长主要来自新客首单。" + "订单一共是 1234 单，整体平稳。" + "下面看细节。") + + +def units_of(doc) -> dict[str, str]: + """{句子原文: unit id}""" + md = doc["markdown"] + return {md[u["span"][0]:u["span"][1]].strip(): u["id"] for _, u in iter_units(doc)} + + +CITED, CITED2, CAUSAL, NUMBER = ("本周销售额 45,678.5元。", "环比 8.7%，增长主要来自华东。", "增长主要来自新客首单。", + "订单一共是 1234 单，整体平稳。") + + +@pytest.fixture(autouse=True) +async def clean_settings(): + """每日计数和裁判设置都落在设置表里：每个测试从干净的一天开始。""" + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"]))) + await session.commit() + yield + + +class Judge: + """裁判模型的剧本。结构化调用和退回的纯文本调用分开记；verdicts 按 unit id 给判定，没写的都是 supported。""" + + def __init__(self, monkeypatch, verdicts=None, *, model_id=PRICED, structured=None, plain=None, + latency=0.0, fail=None): + self.calls: list[dict] = [] + self.specs: list[ModelSpec] = [] + verdicts = verdicts or {} + + def answer(units): + return json.dumps({"items": [{"unit": u, "verdict": verdicts.get(u, "supported"), + "rationale": f"{u} 的依据", "used": ["m:gmv"]} for u in units]}, + ensure_ascii=False) + + def decide(model, messages): + assert JUDGE_ROLE in str(messages[0].content), "这里只该有裁判调用" + units = re.findall(r"^\[(u\d+)\]", str(messages[-1].content), re.M) + kind = "structured" if model.response_format else "plain" + self.calls.append({"kind": kind, "units": units, "messages": list(messages)}) + if fail: + raise RuntimeError(fail) + custom = structured if kind == "structured" else plain + if custom is not None: + return AIMessage(content=custom(units) if callable(custom) else custom) + return AIMessage(content=answer(units)) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + + async def fake_model(session, spec): + self.specs.append(spec) + return MockChatModel(model_name=model_id, latency=latency), model_id + + monkeypatch.setattr(judge_mod, "get_chat_model", fake_model) + + @property + def judged(self) -> list[list[str]]: + return [c["units"] for c in self.calls if c["kind"] == "structured"] + + +def status_of(outcome, doc, text): + return outcome["verdicts"][units_of(doc)[text]] + + +# -------------------------------------------------------------------------- +# 调用与输出 +# -------------------------------------------------------------------------- + + +async def test_one_call_for_the_whole_report_and_the_verdict_shape(monkeypatch, stored): + judge = Judge(monkeypatch, {"u3": "unsupported"}) + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert len(judge.calls) == 1, "一份报告合并成一次调用" + assert sorted(judge.judged[0]) == sorted(units_of(doc)[t] for t in (CITED, CITED2, CAUSAL, NUMBER)) + v = status_of(outcome, doc, CITED) + assert v["status"] == "supported" and v["judge"] == PRICED and v["post_seal"] is False + assert v["rationale"] and v["used"] == ["m:gmv"] + assert outcome["counts"] == {"supported": 3, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0} + assert outcome["model"] == PRICED and outcome["calls"] == 1 and outcome["gaps"] == [] + assert units_of(doc)["下面看细节。"] not in outcome["verdicts"], "连接性的话不送裁判" + + +async def test_the_judge_model_runs_without_thinking(monkeypatch, stored): + judge = Judge(monkeypatch) + catalog = make_catalog(stored) + await judge_doc(compose_doc(RAW, catalog), catalog, spec=ModelSpec(model=PRICED, thinking="adaptive", + max_tokens=64000), + budget=Budget(max_claims=None, max_cost_usd=None, timeout_s=30, daily_max_usd=None)) + [spec] = judge.specs + assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 4096 + assert spec.model == PRICED and spec.timeout and spec.timeout <= 30 + + +async def test_structured_output_failure_falls_back_to_json(monkeypatch, stored): + """结构化输出失败（模型不支持、给的不是 JSON）：退回纯文本调用，从回复里把 JSON 抠出来。""" + def fenced(units): + body = {"items": [{"unit": u, "verdict": "partial", "rationale": "只对了一半", "used": []} for u in units]} + return "判定如下：\n```json\n" + json.dumps(body, ensure_ascii=False) + "\n```" + + judge = Judge(monkeypatch, structured="这不是 JSON", plain=fenced) + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert [c["kind"] for c in judge.calls] == ["structured", "plain"] + assert {v["status"] for v in outcome["verdicts"].values()} == {"partial"} + assert outcome["gaps"] == [] + + +async def test_an_array_written_as_json_text_is_still_read(monkeypatch, stored): + """有的模型走工具调用时把嵌套的数组写成一段 JSON 文本（{"items": "[…]"}）：照样认，不为它再花一次调用。""" + def stringified(units): + items = [{"unit": u, "verdict": "unsupported", "rationale": "对不上", "used": []} for u in units] + return json.dumps({"items": json.dumps(items, ensure_ascii=False)}, ensure_ascii=False) + + judge = Judge(monkeypatch, structured=stringified) + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert [c["kind"] for c in judge.calls] == ["structured"] + assert {v["status"] for v in outcome["verdicts"].values()} == {"unsupported"} + + +async def test_a_broken_provider_is_not_called_batch_after_batch(monkeypatch, stored): + """第一批就调不通（鉴权、网关挂了）：后面几批不再一次次去撞，统一记未裁判。""" + judge = Judge(monkeypatch, fail="401 鉴权失败") + catalog = make_catalog(stored) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 5)) + doc = compose_doc(raw, catalog) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert len(judge.calls) == 2, "结构化一次、退回纯文本一次，第二批不再调用" + assert outcome["unjudged"] == {"error": BATCH_SIZE + 5} and "401 鉴权失败" in outcome["gaps"][0] + + +async def test_both_paths_failing_leaves_the_claims_unjudged(monkeypatch, stored): + Judge(monkeypatch, structured="不是 JSON", plain="还是不是") + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert {(v["status"], v["reason"]) for v in outcome["verdicts"].values()} == {("unjudged", "format")} + assert outcome["counts"]["unjudged"] == 4 and outcome["unjudged"] == {"format": 4} + assert outcome["gaps"] and "4 句" in outcome["gaps"][0] + + +async def test_bad_items_are_not_trusted(monkeypatch, stored): + """判定不认识、理由太长、编出目录里没有的依据、漏判的句子：各按各的处理，不编判定。""" + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + ids = units_of(doc) + + def messy(units): + return json.dumps({"items": [ + {"unit": ids[CITED], "verdict": "supported", "rationale": "长" * 100, "used": ["m:gmv", "m:nope", "Q9"]}, + {"unit": ids[CITED2], "verdict": "maybe", "rationale": "不认识的判定", "used": []}, + {"unit": ids[CAUSAL], "verdict": "unsupported", "rationale": "没有新客维度", "used": []}, + {"unit": "u999", "verdict": "supported", "rationale": "不存在的句子", "used": []}, + ]}, ensure_ascii=False) + + Judge(monkeypatch, structured=messy) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + v = outcome["verdicts"] + assert v[ids[CITED]]["rationale"] == "长" * RATIONALE_MAX and v[ids[CITED]]["used"] == ["m:gmv"] + assert (v[ids[CITED2]]["status"], v[ids[CITED2]]["reason"]) == ("unjudged", "format") + assert v[ids[CAUSAL]]["status"] == "unsupported" + assert (v[ids[NUMBER]]["status"], v[ids[NUMBER]]["reason"]) == ("unjudged", "missing") + assert "u999" not in v + + +async def test_a_failed_call_is_unjudged_not_an_error(monkeypatch, stored): + Judge(monkeypatch, fail="网关超时") + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + events: list = [] + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED, + emit=lambda kind, **data: events.append((str(kind), data))) + assert {v["reason"] for v in outcome["verdicts"].values()} == {"error"} + assert "网关超时" in outcome["gaps"][0] + ends = [d for k, d in events if k == EventType.LLM_END] + assert ends and ends[-1]["purpose"] == "judge" and "网关超时" in ends[-1]["error"] + assert any(k == EventType.LOG and d["code"] == "judge_failed" for k, d in events) + + +# -------------------------------------------------------------------------- +# 证据摘录 +# -------------------------------------------------------------------------- + + +def request_text(judge) -> str: + return "\n".join(str(m.content) for m in judge.calls[0]["messages"]) + + +async def test_excerpts_follow_the_rules(monkeypatch, stored): + judge = Judge(monkeypatch) + catalog = make_catalog(stored) + raw = ("本周销售额 [[m:gmv]]。[[see:m:gmv]]" + "第 25 行的区域排在后面。[[see:Q1.r25]]" + "整体看华东领先。[[see:Q1]]" + "退款集中在物流延误的订单。[[see:K1]]" + "本周是 [[i:week]]，增长来自新客。") + doc = compose_doc(raw, catalog) + await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + text = request_text(judge) + assert "数字已经由系统核对过，不要复算" in JUDGE_RULES and JUDGE_RULES in text + # 指标：名称 = 值，加原式 + assert "销售额 = 45,678.5元" in text and "vars.kpi.gmv" in text + # 查询：SQL 前 300 字 + assert LONG_SQL[:SQL_CHARS] in text and LONG_SQL[:SQL_CHARS + 20] not in text + # 被引用的行最多 20 行：点名的第 25 行先给，整份引用的再从头补满 + rows = re.findall(r"^r(\d+)：", text, re.M) + assert len(rows) == EXCERPT_ROWS and "25" in rows and "0" in rows and "22" not in rows + # 片段：最多 600 字 + first = HITS["hits"][0]["content"] + assert first[:CHUNK_CHARS] in text and first[:CHUNK_CHARS + 1] not in text + assert "week = 2026-W37" in text + + +async def test_masked_columns_never_reach_the_judge(monkeypatch, stored): + judge = Judge(monkeypatch) + catalog = make_catalog(stored) + doc = compose_doc("整体看华东领先。[[see:Q1]]", catalog) + await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert "PHONE-0" in request_text(judge), "对照：没设遮罩时原值在摘录里" + + # 数据源现在设的遮罩（列名不分大小写） + judge = Judge(monkeypatch) + await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED, masked={"shop": ["Phone"]}) + text = request_text(judge) + assert "PHONE-" not in text and "区域0" in text + [header] = re.findall(r"^列：(.*)$", text, re.M) + assert "phone" not in header.lower() and "region" in header, "表头里也不给遮罩的列" + + # 查询当时记下的遮罩（快照里的 mask_columns） + judge = Judge(monkeypatch) + catalog = make_catalog(stored, snap="masked") + doc = compose_doc("整体看华东领先。[[see:Q1]]", catalog) + await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert "PHONE-" not in request_text(judge) + + +#: 宽表：前 13 列是填充，被引用的列排在第 14、15 列，最后一列是手机号 +WIDE_COLS = [f"c{i}" for i in range(13)] + ["refund_rate", "region", "phone"] +WIDE = {"columns": WIDE_COLS, "row_count": 5, "truncated": False, "elapsed_ms": 3, "source": "shop", + "sql": "SELECT * FROM wide_orders", + "rows": [[i * 100 + j for j in range(13)] + [f"RATE-{i}", f"区域{i}", f"PHONE-{i}"] for i in range(5)]} + + +def query_catalog(artifact: str, columns: list[str], *, source: str | None = "shop"): + ledger = [{"kind": "query", "node_id": "fetch", "exec": 1, "artifact": artifact, "tool": "db_query__shop", + **({"source": source} if source else {}), "columns": columns, "rows": 5, "truncated": False}] + return build_catalog(nodes={}, ledger=ledger) + + +def header_of(excerpt: str) -> tuple[list[str], str]: + """(摘录给出的列, 表头那一整行)""" + [header] = re.findall(r"^列：(.*)$", excerpt, re.M) + return [c for c in header.split(" | ") if not c.startswith("…")], header + + +async def test_cited_columns_of_a_wide_table_always_reach_the_judge(): + """宽表只给前几列时，句子引用的列排在后面也一定给：裁判看不到被引用的格子，就只能判不支持。""" + catalog = query_catalog(_store(WIDE), WIDE_COLS) + doc = compose_doc("[[v:Q1.r2.region]] 的退款率最高，达到 [[v:Q1.r2.refund_rate]]。" + "手机尾号 [[v:Q1.r2.phone]] 的客户下单最多。", catalog) + excerpt = prepare(doc, catalog, masked={"shop": ["phone"]}).excerpts["Q1"] + shown, header = header_of(excerpt) + assert "region" in shown and "refund_rate" in shown, header + [row] = re.findall(r"^r2：(.*)$", excerpt, re.M) + cells = dict(zip(shown, row.split(" | "), strict=True)) + assert cells["region"] == "区域2" and cells["refund_rate"] == "RATE-2" + # 遮罩的列被引用了也不给 + assert "phone" not in header.lower() and "PHONE-" not in excerpt + # 没遮罩的 15 列给了 EXCERPT_COLS 列，「另有」按实际没给出的列数计 + assert len(shown) == EXCERPT_COLS and header.endswith(f"…另有 {15 - EXCERPT_COLS} 列") + + # 被引用的列比上限还多：一律保留 + cited = "".join(f"[[v:Q1.r1.{c}]]、" for c in WIDE_COLS[:-1]) + doc = compose_doc(f"这一行各列依次是 {cited}都比上周高。", catalog) + shown, header = header_of(prepare(doc, catalog).excerpts["Q1"]) + assert set(shown) == set(WIDE_COLS[:-1]) and "另有 1 列" in header + + +async def test_current_masks_apply_when_the_entry_does_not_name_its_source(stored): + """旧形状的台账条目没记数据源：按快照自己记的数据源找遮罩（和证据面板同一个口径），摘录里照样不给。""" + catalog = query_catalog(stored["snap"], SNAP["columns"], source=None) + doc = compose_doc("整体看华东领先。[[see:Q1]]", catalog) + excerpt = prepare(doc, catalog, masked={"shop": ["phone"]}).excerpts["Q1"] + assert "PHONE-" not in excerpt and "区域0" in excerpt and "数据源 shop" in excerpt + + # 数据源现在设的遮罩也按快照里的数据源去查 + from app.db.models import DataSource + + catalog = query_catalog(_store({**SNAP, "source": "judge_mask_src"}), SNAP["columns"], source=None) + async with SessionLocal() as session: + session.add(DataSource(name="judge_mask_src", kind="sqlite", database="unused.db", + options={"mask_columns": "Phone"})) + await session.commit() + try: + assert await source_masks(catalog) == {"judge_mask_src": ["Phone"]} + finally: + async with SessionLocal() as session: + await session.execute(delete(DataSource).where(DataSource.name == "judge_mask_src")) + await session.commit() + + +# -------------------------------------------------------------------------- +# 预算 +# -------------------------------------------------------------------------- + + +async def test_claims_over_max_claims_are_unjudged(monkeypatch, stored): + judge = Judge(monkeypatch) + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + outcome = await judge_doc(doc, catalog, spec=SPEC, + budget=Budget(max_claims=2, max_cost_usd=None, timeout_s=None, daily_max_usd=None)) + ids = units_of(doc) + assert sorted(judge.judged[0]) == sorted([ids[CITED], ids[CITED2]]), "先判挂了依据的" + for text in (CAUSAL, NUMBER): + v = status_of(outcome, doc, text) + assert (v["status"], v["reason"]) == ("unjudged", "max_claims") and "已到上限" in v["rationale"] + assert outcome["limits_hit"] == ["max_claims"] and outcome["unjudged"] == {"max_claims": 2} + assert outcome["gaps"] and "已到上限" in outcome["gaps"][0] + + +async def test_cost_truncation_follows_priority(monkeypatch, stored): + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + request = prepare(doc, catalog) + by_text = {t: c for c in request.cands for t, u in units_of(doc).items() if u == c.unit} + without_p3 = request.cost(PRICED, [by_text[CITED], by_text[CITED2], by_text[CAUSAL]]) + everything = request.cost(PRICED, request.cands) + assert 0 < without_p3 < everything + + judge = Judge(monkeypatch) + budget = Budget(max_claims=None, max_cost_usd=(without_p3 + everything) / 2, timeout_s=None, daily_max_usd=None) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=budget) + assert status_of(outcome, doc, NUMBER)["reason"] == "max_cost_usd" + assert all(status_of(outcome, doc, t)["status"] == "supported" for t in (CITED, CITED2, CAUSAL)) + assert outcome["limits_hit"] == ["max_cost_usd"] and len(judge.judged[0]) == 3 + + cited_only = request.cost(PRICED, [by_text[CITED], by_text[CITED2]]) + judge = Judge(monkeypatch) + budget = Budget(max_claims=None, max_cost_usd=cited_only * 1.001, timeout_s=None, daily_max_usd=None) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=budget) + assert {status_of(outcome, doc, t)["reason"] for t in (CAUSAL, NUMBER)} == {"max_cost_usd"} + assert {status_of(outcome, doc, t)["status"] for t in (CITED, CITED2)} == {"supported"} + + +async def test_unlimited_never_truncates(monkeypatch, stored): + """每一项都写 null：句数再多也全判（按每批上限分批），金额、时长、每日都不拦。""" + judge = Judge(monkeypatch) + catalog = make_catalog(stored) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 5)) + doc = compose_doc(raw, catalog) + async with SessionLocal() as session: # 今天已经花掉很多：不限就是不限 + session.add(Setting(key=SPEND_KEY, value={"date": judge_mod._today(), "usd": 999.0, "calls": 9})) + await session.commit() + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + assert [len(units) for units in judge.judged] == [BATCH_SIZE, 5] + assert outcome["counts"]["supported"] == BATCH_SIZE + 5 and outcome["counts"]["unjudged"] == 0 + assert outcome["limits_hit"] == [] and outcome["gaps"] == [] + + +async def test_the_daily_cap_resets_the_next_day(monkeypatch, stored): + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + cost = prepare(doc, catalog).cost(PRICED, prepare(doc, catalog).cands) + budget = Budget(max_claims=None, max_cost_usd=None, timeout_s=None, daily_max_usd=cost * 1.05) + day = {"today": "2026-09-28"} + monkeypatch.setattr(judge_mod, "_today", lambda: day["today"]) + + judge = Judge(monkeypatch) + first = await judge_doc(doc, catalog, spec=SPEC, budget=budget) + assert first["counts"]["unjudged"] == 0 + spent = await daily_spend() + assert spent["date"] == "2026-09-28" and spent["usd"] == pytest.approx(cost) and spent["calls"] == 1 + + second = await judge_doc(doc, catalog, spec=SPEC, budget=budget) + assert {v["reason"] for v in second["verdicts"].values()} == {"daily_max_usd"} + assert second["limits_hit"] == ["daily_max_usd"] and len(judge.calls) == 1, "触顶就不再调用" + + day["today"] = "2026-09-29" + third = await judge_doc(doc, catalog, spec=SPEC, budget=budget) + assert third["counts"]["unjudged"] == 0 and len(judge.calls) == 2 + assert (await daily_spend())["date"] == "2026-09-29" + + +async def test_the_time_limit_stops_a_slow_judge(monkeypatch, stored): + Judge(monkeypatch, latency=2.0) + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + outcome = await judge_doc(doc, catalog, spec=SPEC, + budget=Budget(max_claims=None, max_cost_usd=None, timeout_s=0.2, daily_max_usd=None)) + assert {v["reason"] for v in outcome["verdicts"].values()} == {"timeout_s"} + assert outcome["limits_hit"] == ["timeout_s"] and outcome["gaps"] + + +async def test_a_model_without_a_price_ignores_money_limits(monkeypatch, stored): + """目录里没有价格的模型估不出金额：金额上限对它不起作用，并照实说「按令牌估不出金额」。""" + judge = Judge(monkeypatch, model_id="house-judge-model") + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + events: list = [] + outcome = await judge_doc(doc, catalog, spec=ModelSpec(model="house-judge-model"), + budget=Budget(max_claims=None, max_cost_usd=1e-9, timeout_s=None, daily_max_usd=1e-9), + emit=lambda kind, **data: events.append((str(kind), data))) + assert len(judge.calls) == 1 and outcome["counts"]["unjudged"] == 0 + assert outcome["priced"] is False and any("按令牌估不出金额" in n for n in outcome["notes"]) + [log] = [d for k, d in events if k == EventType.LOG and d.get("code") == "judge_unpriced"] + assert "按令牌估不出金额" in log["message"] + + +# -------------------------------------------------------------------------- +# 复用与按需 +# -------------------------------------------------------------------------- + + +async def test_known_sentences_are_not_judged_again(monkeypatch, stored): + judge = Judge(monkeypatch, {"u2": "unsupported"}) + catalog = make_catalog(stored) + doc = compose_doc(RAW, catalog) + first = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED) + known = {first["keys"][u]: v for u, v in first["verdicts"].items()} + again = await judge_doc(compose_doc(RAW, catalog), catalog, spec=SPEC, budget=UNLIMITED, known=known) + assert len(judge.calls) == 1 and again["reused"] == 4 and again["calls"] == 0 + assert again["verdicts"] == first["verdicts"] + + +async def test_on_demand_judges_just_the_requested_sentence_after_the_seal(monkeypatch, stored): + judge = Judge(monkeypatch) + catalog = make_catalog(stored) + doc = compose_doc(RAW + "共 3 单。", catalog) + ids = units_of(doc) + outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED, post_seal=True, + units=[ids[CAUSAL], ids["共 3 单。"], ids["本周概览"]]) + assert sorted(judge.judged[0]) == sorted([ids[CAUSAL], ids["共 3 单。"]]), "点名要判的短句照判" + assert all(v["post_seal"] is True for v in outcome["verdicts"].values()) + assert outcome["skipped"] == {ids["本周概览"]: "not_a_claim"} diff --git a/backend/tests/test_evidence_judge_api.py b/backend/tests/test_evidence_judge_api.py new file mode 100644 index 00000000..1524cbab --- /dev/null +++ b/backend/tests/test_evidence_judge_api.py @@ -0,0 +1,696 @@ +"""按需裁判接口 POST /api/runs/{id}/evidence/judge（四期）。 + +探索运行不在节点里花裁判的钱：点开哪句才判哪句。 + +- 判定作为批注追加在封存之后（evidence.judged 事件，post_seal: true）：封存核对只覆盖 manifest_seq 之前的 + 事件，所以追加之后 verify 照旧一致；封存的报告文档一个字都不改 +- 只在探索运行里可用：正式运行的裁判在节点内完成，按需裁判返回 409；还没封存的、升级前发起的、封存链断了的 + 也不判 +- 每次点击的上限、全局每日上限按设置控制；触顶不是报错，照实说「已到上限」和怎么调 +- 同一次运行里同一句判过就直接给已有的判定，不再调用、不再花钱（连点两下也一样） +- 裁判只看得到封存范围内的证据：人为插进工件表的、封存之后才追加事件引用的快照，一行都不进摘录； + 数据源遮罩的列同样不给 +- 片段接口和证据图带上判定，封存内的和封存后追加的分得开 + +所有裁判都用 mock 模型给剧本，不调真接口。 +""" +from __future__ import annotations + +import asyncio +import json +import re +import sqlite3 +from types import SimpleNamespace + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import delete, select, update + +from app.core import artifact_store +from app.core.events import EventType +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent, Setting +from app.engine import judge as judge_mod +from app.engine import runner as runner_mod +from app.engine.evidence import iter_segments, iter_units +from app.engine.judge import JUDGE_ROLE, SPEND_KEY, daily_spend +from app.engine.runner import run_manager, verify_manifest +from app.main import app +from app.providers.mock_model import MockChatModel + +PRICED = "claude-sonnet-5" +METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] +#: 一句挂了依据的结论、一句没挂依据的归因、一句连接性的话 +TEXT = "## 本周\n\n本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。下面看细节。" +CITED, CAUSAL, LINK = "本周销售额 45,678.5元。", "增长主要来自新客首单。", "下面看细节。" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"]))) + await session.commit() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes) -> dict: + ids = [n["id"] for n in nodes] + return {"nodes": list(nodes), "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} + + +def weekly(*, gate=False, **report) -> dict: + return chain( + node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]), + node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS), + node("write", "report", instructions="写周报", **report), + *([node("gate", "human", mode="approve", title="确认")] if gate else []), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + ) + + +class Models: + """写作者回 text；裁判按句子里的字给判定（verdicts：{句中片段: 判定}，没对上的是 supported）。 + judges 记下每次交给裁判的请求原文，specs 记下每次要的裁判模型。""" + + def __init__(self, monkeypatch, text=TEXT, verdicts=None, *, model=PRICED, fail_judge=None, delay=0.0): + self.judges: list[str] = [] + self.specs: list = [] + verdicts = verdicts or {} + + def decide(chat, messages): + if JUDGE_ROLE in str(messages[0].content): + request = str(messages[-1].content) + self.judges.append(request) + if fail_judge: + raise RuntimeError(fail_judge) + items = [{"unit": uid, + "verdict": next((s for frag, s in verdicts.items() if frag in line), "supported"), + "rationale": f"判的是：{line.strip()[:16]}", "used": []} + for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)] + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False)) + return AIMessage(content=text) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + + async def judge_model_of(session, spec): + self.specs.append(spec) + if delay: + await asyncio.sleep(delay) + return MockChatModel(model_name=spec.model or model), spec.model or model + + monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of) + + +async def wait(run_id: str, statuses=("succeeded", "failed")) -> Run: + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + return row + raise AssertionError(f"run {run_id} 没跑完：{row.status}") + + +async def finish(graph, *, run_class="exploratory", statuses=("succeeded", "failed")) -> Run: + run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class) + row = await wait(run.id, statuses) + return row + + +def doc_of(row: Run) -> dict: + return artifact_store.load(row.output["_evidence"]["doc_artifact"]) + + +def unit_ids(doc: dict) -> dict[str, str]: + md = doc["markdown"] + return {md[u["span"][0]:u["span"][1]].strip(): u["id"] for _, u in iter_units(doc)} + + +def unit_starting(doc: dict, prefix: str) -> str: + return next(uid for text, uid in unit_ids(doc).items() if text.startswith(prefix)) + + +def seg_in(doc: dict, uid: str) -> str: + return next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"] if s["kind"] == "text") + + +async def judged_events(run_id: str) -> list[RunEvent]: + async with SessionLocal() as session: + return list((await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED).order_by(RunEvent.seq))).scalars()) + + +async def judge(client, run_id: str, units, **extra): + return await client.post(f"/api/runs/{run_id}/evidence/judge", json={"units": units, **extra}) + + +async def save_setting(key: str, value) -> None: + async with SessionLocal() as session: + row = await session.get(Setting, key) + if row is None: + session.add(Setting(key=key, value=value)) + else: + row.value = value + await session.commit() + + +# -------------------------------------------------------------------------- +# 判定落在封存之后，封存核对照旧一致 +# -------------------------------------------------------------------------- + + +async def test_an_on_demand_verdict_lands_after_the_seal_and_the_seal_still_holds(client, monkeypatch): + models = Models(monkeypatch, verdicts={"新客": "unsupported"}) + row = await finish(weekly(claims="judge")) + assert row.status == "succeeded" and models.judges == [], "探索运行节点里不判" + doc = doc_of(row) + ids = unit_ids(doc) + + res = await judge(client, row.id, [ids[CAUSAL]]) + assert res.status_code == 200, res.text + body = res.json() + verdict = body["verdicts"][ids[CAUSAL]] + assert verdict["status"] == "unsupported" and verdict["post_seal"] is True and verdict["judge"] == PRICED + assert verdict["rationale"] and body["judged"] == [ids[CAUSAL]] and body["reused"] == [] + assert body["report"] == {"node_id": "write", "doc_artifact": row.output["_evidence"]["doc_artifact"]} + assert body["post_seal"] is True and body["limited"] is False and body["calls"] == 1 + assert len(models.judges) == 1 and CAUSAL in models.judges[0] and CITED not in models.judges[0] + + [event] = await judged_events(row.id) + assert event.seq > row.manifest_seq and event.node_id == "write" and body["event"] == {"seq": event.seq} + data = event.data + assert data["post_seal"] is True and data["report"] == "write" + assert data["doc_artifact"] == body["report"]["doc_artifact"] + assert data["verdicts"] == {ids[CAUSAL]: verdict} and data["model"] == PRICED and data["calls"] == 1 + assert set(data["keys"]) == {ids[CAUSAL]} and data["units"] == [ids[CAUSAL]] + + check = await verify_manifest(row.id) + assert check["ok"] is True and check["sealed_at"] == row.manifest_seq + assert (await client.get(f"/api/runs/{row.id}/verify")).json()["ok"] is True + async with SessionLocal() as session: + after = await session.get(Run, row.id) + assert after.manifest_seq == row.manifest_seq and after.last_seq == event.seq + assert doc_of(row) == doc, "封存的文档一个字都不改" + csv = await client.get(f"/api/runs/{row.id}/evidence/audit", params={"format": "csv"}) + assert csv.headers["x-evidence-seal"] == "ok" + + +async def test_any_claim_in_an_explore_report_can_be_judged(client, monkeypatch): + """报告节点没开 claims: judge 也能按需判：结论句都是候选。""" + models = Models(monkeypatch) + row = await finish(weekly()) + ids = unit_ids(doc_of(row)) + body = (await judge(client, row.id, [ids[CITED], ids[CAUSAL]])).json() + assert {u: v["status"] for u, v in body["verdicts"].items()} == {ids[CITED]: "supported", ids[CAUSAL]: "supported"} + assert len(models.judges) == 1, "一次点击的几句合成一次调用" + assert "【m:gmv】" in models.judges[0], "挂的依据摘给裁判" + + +async def test_sequence_numbers_continue_after_a_restart(client, monkeypatch): + """进程重启后内存里的序号从 0 起：追加的事件要接在库里最后一条后面，不能和已有的撞号。""" + from app.core.bus import bus + + Models(monkeypatch) + row = await finish(weekly()) + ids = unit_ids(doc_of(row)) + bus._seq.pop(row.id, None) + await judge(client, row.id, [ids[CAUSAL]]) + [event] = await judged_events(row.id) + async with SessionLocal() as session: + seqs = [s for (s,) in await session.execute(select(RunEvent.seq).where(RunEvent.run_id == row.id))] + assert event.seq == max(seqs) and seqs.count(event.seq) == 1 and event.seq > row.manifest_seq + + +# -------------------------------------------------------------------------- +# 同一句不重复判 +# -------------------------------------------------------------------------- + + +async def test_asking_again_returns_the_verdict_without_another_call(client, monkeypatch): + models = Models(monkeypatch, verdicts={"新客": "partial"}) + row = await finish(weekly(claims="judge")) + ids = unit_ids(doc_of(row)) + first = (await judge(client, row.id, [ids[CAUSAL]])).json() + spent = (await daily_spend())["calls"] + again = (await judge(client, row.id, [ids[CAUSAL]])).json() + assert len(models.judges) == 1 and (await daily_spend())["calls"] == spent, "不再调用、不再计费" + assert again["verdicts"] == first["verdicts"] and again["reused"] == [ids[CAUSAL]] and again["judged"] == [] + assert again["calls"] == 0 and again["event"] is None and again["message"] is None + assert len(await judged_events(row.id)) == 1 + + # 判过的和没判过的一起点：只判没判过的 + both = (await judge(client, row.id, [ids[CAUSAL], ids[CITED]])).json() + assert both["reused"] == [ids[CAUSAL]] and both["judged"] == [ids[CITED]] + assert len(models.judges) == 2 and CAUSAL not in models.judges[1] + + +async def test_a_double_click_is_judged_once(client, monkeypatch): + models = Models(monkeypatch, delay=0.05) + row = await finish(weekly(claims="judge")) + uid = unit_ids(doc_of(row))[CAUSAL] + one, two = await asyncio.gather(judge(client, row.id, [uid]), judge(client, row.id, [uid])) + assert one.status_code == two.status_code == 200 + assert len(models.judges) == 1 and len(await judged_events(row.id)) == 1 + assert sorted([one.json()["reused"], two.json()["reused"]]) == [[], [uid]] + + +async def test_a_failed_attempt_can_be_retried(client, monkeypatch): + Models(monkeypatch, fail_judge="网关 502") + row = await finish(weekly(claims="judge")) + uid = unit_ids(doc_of(row))[CAUSAL] + failed = (await judge(client, row.id, [uid])).json() + assert failed["verdicts"][uid]["status"] == "unjudged" and failed["verdicts"][uid]["reason"] == "error" + assert "网关 502" in failed["message"] and failed["limited"] is False + models = Models(monkeypatch, verdicts={"新客": "unsupported"}) + retried = (await judge(client, row.id, [uid])).json() + assert retried["verdicts"][uid]["status"] == "unsupported" and retried["reused"] == [] and len(models.judges) == 1 + + +# -------------------------------------------------------------------------- +# 上限 +# -------------------------------------------------------------------------- + + +async def test_hitting_the_click_limit_is_a_result_not_an_error(client, monkeypatch): + models = Models(monkeypatch) + await save_setting("judge", {"click_max_cost_usd": 1e-9}) + row = await finish(weekly(claims="judge")) + uid = unit_ids(doc_of(row))[CAUSAL] + res = await judge(client, row.id, [uid]) + assert res.status_code == 200 + body = res.json() + assert body["limited"] is True and body["limits_hit"] == ["max_cost_usd"] + assert body["unjudged"] == {"max_cost_usd": 1} + assert body["message"].startswith("已到上限") and "这次点击" in body["message"] + assert any("设置" in how and "不限" in how for how in body["adjust"]) + verdict = body["verdicts"][uid] + assert verdict["status"] == "unjudged" and verdict["reason"] == "max_cost_usd" and verdict["post_seal"] is True + assert models.judges == [] and body["calls"] == 0 and body["budget"]["max_cost_usd"] == 1e-9 + assert body["event"] is None and await judged_events(row.id) == [], "没问模型就没有判定可记" + + +async def test_the_daily_limit_applies_to_clicks_too(client, monkeypatch): + models = Models(monkeypatch) + today = (await daily_spend())["date"] + await save_setting(SPEND_KEY, {"date": today, "usd": 2.0, "calls": 9, "unpriced_calls": 0}) + row = await finish(weekly(claims="judge")) + body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json() + assert body["limits_hit"] == ["daily_max_usd"] and "每日" in body["message"] and models.judges == [] + assert body["spend"]["usd"] == 2.0 and body["spend"]["daily_max_usd"] == 2.0 + + +async def test_unlimited_click_budget_judges(client, monkeypatch): + models = Models(monkeypatch) + await save_setting("judge", {"click_max_cost_usd": None, "daily_max_usd": None}) + row = await finish(weekly(claims="judge")) + body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json() + assert body["limited"] is False and len(models.judges) == 1 + assert body["budget"]["max_cost_usd"] is None and body["budget"]["daily_max_usd"] is None + assert body["spend"]["calls"] == 1 and body["spend"]["daily_max_usd"] is None + + +async def test_the_sentence_limit_keeps_what_was_judged(client, monkeypatch): + models = Models(monkeypatch) + row = await finish(weekly(claims="judge", judge={"max_claims": 1})) + ids = unit_ids(doc_of(row)) + body = (await judge(client, row.id, [ids[CITED], ids[CAUSAL]])).json() + statuses = {u: v["status"] for u, v in body["verdicts"].items()} + assert statuses == {ids[CITED]: "supported", ids[CAUSAL]: "unjudged"} and len(models.judges) == 1 + assert body["limits_hit"] == ["max_claims"] and body["verdicts"][ids[CAUSAL]]["reason"] == "max_claims" + [event] = await judged_events(row.id) + assert set(event.data["verdicts"]) == {ids[CITED], ids[CAUSAL]}, "问过模型的这一次连同没判的一起记下" + # 没判的那句还能再点 + again = (await judge(client, row.id, [ids[CAUSAL]])).json() + assert again["judged"] == [ids[CAUSAL]] and again["reused"] == [] + + +# -------------------------------------------------------------------------- +# 用哪个模型 +# -------------------------------------------------------------------------- + + +async def test_the_judge_model_follows_node_then_settings(client, monkeypatch): + models = Models(monkeypatch) + await save_setting("judge", {"model": "judge-from-settings"}) + row = await finish(weekly(claims="judge")) + await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]]) + [spec] = models.specs + assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 4096 + + row = await finish(weekly(claims="judge", judge={"model": "judge-from-node"})) + body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json() + assert models.specs[-1].model == "judge-from-node" and body["model"] == "judge-from-node" + + +# -------------------------------------------------------------------------- +# 什么时候不判 +# -------------------------------------------------------------------------- + + +async def test_formal_runs_are_judged_inside_the_node_not_on_demand(client, monkeypatch): + models = Models(monkeypatch) + row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal") + assert row.status == "succeeded", row.error + calls = len(models.judges) + res = await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]]) + assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal" + assert "正式运行的裁判在节点内完成" in res.json()["detail"] + assert len(models.judges) == calls and await judged_events(row.id) == [] + + +async def test_a_run_paused_at_an_approval_is_not_judged(client, monkeypatch): + models = Models(monkeypatch) + row = await finish(weekly(claims="judge", gate=True), statuses=("interrupted",)) + res = await judge(client, row.id, ["u1"]) + assert res.status_code == 409 and res.json()["code"] == "evidence_judge_unsealed" + assert models.judges == [] and await judged_events(row.id) == [] + + +async def test_runs_from_before_the_upgrade_are_not_judged(client, monkeypatch): + async def no_snapshot(session): + return None + + monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) + models = Models(monkeypatch) + row = await finish(weekly()) + res = await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]]) + assert res.status_code == 409 and res.json()["code"] == "evidence_judge_legacy" + assert models.judges == [] and await judged_events(row.id) == [] + + +async def test_a_broken_seal_is_not_judged(client, monkeypatch): + models = Models(monkeypatch) + row = await finish(weekly(claims="judge")) + async with SessionLocal() as session: + event = (await session.execute(select(RunEvent).where( + RunEvent.run_id == row.id, RunEvent.type == "node.finished", RunEvent.node_id == "fetch"))).scalar_one() + event.data = {**event.data, "tampered": True} + await session.commit() + res = await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]]) + assert res.status_code == 409 and res.json()["code"] == "evidence_seal_broken" and models.judges == [] + + +@pytest.mark.parametrize("meanwhile", [ + # 裁判的这几秒里有人点了「继续运行」（失败的运行可以接着跑）：运行又活了 + {"status": "running", "error": None}, + # 接着跑完、重新封存了：manifest_seq 和清单哈希都换了，这次点击依据的那份封存已经不是最新的 + {"manifest_seq": 10_000, "manifest_hash": "0" * 64}, +]) +async def test_a_run_continued_during_the_judge_call_records_no_event(client, monkeypatch, meanwhile): + """判定只接在点击开始时的那份封存后面。运行在裁判期间被接着跑了，再追加就落进一次活着的运行中间， + 还会被封进下一份清单：不记进运行记录，判定照样交回给点的人，并说明为什么没记。""" + models = Models(monkeypatch, verdicts={"新客": "unsupported"}) + row = await finish(weekly(claims="judge")) + uid = unit_ids(doc_of(row))[CAUSAL] + ask_model = judge_mod.get_chat_model + + async def continued_while_judging(session, spec): + async with SessionLocal() as other: + await other.execute(update(Run).where(Run.id == row.id).values(**meanwhile)) + await other.commit() + return await ask_model(session, spec) + + monkeypatch.setattr(judge_mod, "get_chat_model", continued_while_judging) + res = await judge(client, row.id, [uid]) + assert res.status_code == 200, res.text + body = res.json() + assert len(models.judges) == 1 and body["calls"] == 1, "模型问过了" + assert body["verdicts"][uid]["status"] == "unsupported" and body["judged"] == [uid], "判定照样交回" + assert body["event"] is None and "接着跑" in body["message"] and "没有记进运行记录" in body["message"] + assert await judged_events(row.id) == [] + async with SessionLocal() as session: + after = await session.get(Run, row.id) + assert after.last_seq == row.last_seq, "运行记录一条都没多" + + +async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_event(client, monkeypatch): + """核对过状态、还没写入的那一下，接着跑的条件更新（runner._claim）先提交了：追加判定的条件更新落空， + 照样不记——核对和写入是同一行上的条件更新，不是先读后写。""" + from app.core.bus import bus + from app.core.config import settings + + Models(monkeypatch, verdicts={"新客": "unsupported"}) + row = await finish(weekly(claims="judge")) + uid = unit_ids(doc_of(row))[CAUSAL] + set_seq = bus.set_seq + + def claimed_in_between(run_id, value): + # _append_judged 读完运行、分配序号之前正好调用它：在这里用另一条连接把运行占成 running + if run_id == row.id: + db = sqlite3.connect(settings.db_path, timeout=5) + try: + db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (row.id,)) + db.commit() + finally: + db.close() + set_seq(run_id, value) + + monkeypatch.setattr(bus, "set_seq", claimed_in_between) + body = (await judge(client, row.id, [uid])).json() + assert body["verdicts"][uid]["status"] == "unsupported" and body["event"] is None + assert "没有记进运行记录" in body["message"] and await judged_events(row.id) == [] + async with SessionLocal() as session: + after = await session.get(Run, row.id) + assert after.status == "running" and after.last_seq == row.last_seq + + +async def test_missing_runs_and_reports(client, monkeypatch): + Models(monkeypatch) + res = await judge(client, "nope", ["u1"]) + assert res.status_code == 404 and res.json()["code"] == "run_not_found" + row = await finish(chain(node("start", "input"), node("out", "output", fields=[{"name": "a", "value": "x"}]))) + res = await judge(client, row.id, ["u1"]) + assert res.status_code == 404 and res.json()["code"] == "evidence_report_not_found" + + +@pytest.mark.parametrize("payload", [{}, {"units": []}, {"units": "u1"}, {"units": [1]}, {"units": [""]}, + {"units": [f"u{i}" for i in range(201)]}, {"units": ["u1"], "report": 3}]) +async def test_bad_requests_are_422(client, monkeypatch, payload): + Models(monkeypatch) + row = await finish(weekly()) + res = await client.post(f"/api/runs/{row.id}/evidence/judge", json=payload) + assert res.status_code == 422 and res.json()["code"] == "evidence_judge_bad_units" + + +async def test_sentences_that_are_not_claims_are_skipped(client, monkeypatch): + models = Models(monkeypatch) + row = await finish(weekly(claims="judge")) + ids = unit_ids(doc_of(row)) + body = (await judge(client, row.id, [ids["本周"], ids[LINK], "u999"])).json() + assert body["skipped"] == {ids["本周"]: "not_a_claim", ids[LINK]: "not_a_claim", "u999": "not_found"} + assert body["verdicts"] == {} and models.judges == [] and body["event"] is None + assert "不是结论句" in body["message"] + + +# -------------------------------------------------------------------------- +# 片段接口、证据图带上判定 +# -------------------------------------------------------------------------- + + +async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(client, monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + row = await finish(weekly(claims="judge")) + doc = doc_of(row) + ids = unit_ids(doc) + before = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CAUSAL])}")).json() + assert before["unit"]["verdict"]["status"] == "unjudged" and before["unit"]["verdict"]["reason"] == "on_demand" + assert before["unit"]["on_demand"] == {"available": True, "reason": None, "message": None} + + await judge(client, row.id, [ids[CAUSAL]]) + after = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CAUSAL])}")).json() + assert after["unit"]["verdict"]["status"] == "unsupported" and after["unit"]["verdict"]["post_seal"] is True + assert after["unit"]["on_demand"]["available"] is False and after["unit"]["on_demand"]["reason"] == "judged" + assert "模型" in after["note"] and "后续版本" not in after["note"] + other = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CITED])}")).json() + assert other["unit"]["on_demand"]["available"] is True + assert "挂了依据" in other["note"] and "模型" in other["note"] and "后续版本" not in other["note"] + link = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[LINK])}")).json() + assert link["unit"]["on_demand"]["reason"] == "not_a_claim" and "verdict" not in link["unit"] + + graph = (await client.get(f"/api/runs/{row.id}/evidence")).json() + [report] = graph["reports"] + assert report["claims"] == "judge" and report["judge"]["mode"] == "on_demand" + assert report["post_seal_verdicts"] == {ids[CAUSAL]: after["unit"]["verdict"]} + audit = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + assert audit["reports"][0]["post_seal_verdicts"] == report["post_seal_verdicts"] + + +async def test_sealed_verdicts_of_a_formal_run_are_shown_as_sealed(client, monkeypatch): + Models(monkeypatch, verdicts={"新客": "unsupported"}) + row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal") + doc = doc_of(row) + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, unit_ids(doc)[CAUSAL])}")).json() + assert body["unit"]["verdict"]["status"] == "unsupported" and body["unit"]["verdict"]["post_seal"] is False + assert body["unit"]["on_demand"]["reason"] == "formal" + graph = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert graph["reports"][0]["post_seal_verdicts"] == {} and graph["reports"][0]["judge"]["mode"] == "inline" + + +async def test_a_forged_event_cannot_overwrite_a_sealed_verdict(client, monkeypatch): + """封存后追加的判定只填补没判过的句子：封存内判过的，后面追加什么都盖不掉。""" + Models(monkeypatch, verdicts={"新客": "unsupported"}) + row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal") + doc = doc_of(row) + uid = unit_ids(doc)[CAUSAL] + doc_artifact = row.output["_evidence"]["doc_artifact"] + await run_manager.note(row.id, EventType.EVIDENCE_JUDGED, node_id="write", report="write", + doc_artifact=doc_artifact, post_seal=True, + verdicts={uid: {"status": "supported", "rationale": "改好了", "judge": "x", "post_seal": True}}) + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, uid)}")).json() + assert body["unit"]["verdict"]["status"] == "unsupported" + + +async def test_appended_verdicts_always_read_as_post_seal(client, monkeypatch): + """追加的判定不管载荷里怎么写，展示出来都是「封存后追加」：只有随文档封存的判定才算封存内的。""" + Models(monkeypatch) + row = await finish(weekly(claims="judge")) + doc = doc_of(row) + uid = unit_ids(doc)[CAUSAL] + await run_manager.note(row.id, EventType.EVIDENCE_JUDGED, node_id="write", report="write", + doc_artifact=row.output["_evidence"]["doc_artifact"], + verdicts={uid: {"status": "supported", "rationale": "x", "judge": "x", "post_seal": False}, + "u999": {"status": "supported", "rationale": "x", "judge": "x"}}) + body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, uid)}")).json() + assert body["unit"]["verdict"]["status"] == "supported" and body["unit"]["verdict"]["post_seal"] is True + graph = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert set(graph["reports"][0]["post_seal_verdicts"]) == {uid}, "文档里没有的句子不认" + + +# -------------------------------------------------------------------------- +# 裁判只看得到封存范围内的证据 +# -------------------------------------------------------------------------- + +SOURCE = "warehouse" +TOOL = f"db_query__{SOURCE}" +SQL = "SELECT region, SUM(amount) AS gmv, COUNT(*) AS order_cnt FROM orders GROUP BY region ORDER BY gmv DESC" +EAST, WEST = "东区销售额 [[v:Q1.r0.gmv]]。[[see:Q1]]", "西区销售额 [[v:Q1.r1.gmv]]。[[see:Q1]]" + + +@pytest.fixture +async def warehouse(tmp_path): + from app.data.introspect import introspect + + path = tmp_path / "warehouse.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL);") + db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(1, "east", 100.5), (2, "west", 200.0), (3, "east", 300.0)]) + db.commit() + db.close() + probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None, + port=None, username=None, password=None, options={}, readonly=True, schema_cache={}) + cache = await introspect(probe) + await engines.invalidate(probe.id) + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none() + if row is None: + row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache + row.options = {} + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + yield source_id + await engines.invalidate(source_id) + + +def queried() -> dict: + return chain(node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}), + node("write", "report", instructions="写周报"), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])) + + +async def reseal(run_id: str) -> None: + from app.core.artifact_store import manifest_hash + + async with SessionLocal() as session: + run = await session.get(Run, run_id) + rows = [tuple(r) for r in await session.execute( + select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data) + .where(RunEvent.run_id == run_id, RunEvent.seq <= run.manifest_seq).order_by(RunEvent.seq))] + run.manifest_hash = manifest_hash(rows) + await session.commit() + + +async def unseal_queries(run_id: str) -> str: + """把查询快照从封存的事件里摘掉、按改过的事件重新封存：快照还在工件库和工件表里，封存链完好， + 只是没有哪条封存的事件引用它（等于「人为插进工件表」）。返回那件快照的 id。""" + async with SessionLocal() as session: + rows = list((await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type.in_(["tool.end", "node.finished"])))).scalars()) + artifact = None + for event in rows: + data = dict(event.data or {}) + if event.type == "tool.end": + artifact = data.pop("query_artifact", None) or artifact + data.pop("schema_artifact", None) + elif data.get("evidence"): + data["evidence"] = [e for e in data["evidence"] if e.get("kind") not in ("query", "schema")] + event.data = data + await session.commit() + await reseal(run_id) + return artifact + + +def excerpt_lines(request: str) -> tuple[str | None, list[str]]: + header = re.search(r"^列：(.*)$", request, re.M) + return (header.group(1) if header else None), re.findall(r"^r\d+：.*$", request, re.M) + + +async def test_only_sealed_snapshots_reach_the_judge(client, monkeypatch, warehouse): + models = Models(monkeypatch, text=EAST + WEST) + control = await finish(queried()) + assert control.status == "succeeded", control.error + await judge(client, control.id, [unit_starting(doc_of(control), "东区")]) + assert "【Q1】查询结果" in models.judges[0] and "SQL：SELECT region" in models.judges[0] + assert excerpt_lines(models.judges[0])[1], "封存链完好时照常摘被引用的行" + + row = await finish(queried()) + doc = doc_of(row) + artifact = await unseal_queries(row.id) + assert artifact and artifact_store.load(artifact) is not None, "快照本身还在工件库里" + assert (await verify_manifest(row.id))["ok"] is True + res = await judge(client, row.id, [unit_starting(doc, "东区")]) + assert res.status_code == 200, res.text + assert "【Q1】" not in models.judges[1] and "SELECT" not in models.judges[1] + assert excerpt_lines(models.judges[1]) == (None, []) + + # 封存之后才追加的事件引用了它，也不算 + async with SessionLocal() as session: + run = await session.get(Run, row.id) + seq = run.last_seq + from app.core.bus import bus + + bus.set_seq(row.id, seq) + await run_manager.note(row.id, "tool.end", node_id="fetch", tool=TOOL, query_artifact=artifact) + await judge(client, row.id, [unit_starting(doc, "西区")]) + assert "【Q1】" not in models.judges[2] and "SELECT" not in models.judges[2] + + +async def test_masked_columns_never_reach_the_judge(client, monkeypatch, warehouse): + models = Models(monkeypatch, text=EAST) + row = await finish(queried()) + async with SessionLocal() as session: + source = await session.get(DataSource, warehouse) + source.options = {"mask_columns": ["order_cnt"]} + await session.commit() + await judge(client, row.id, [unit_starting(doc_of(row), "东区")]) + header, lines = excerpt_lines(models.judges[0]) + assert header is not None and "order_cnt" not in header and "gmv" in header + assert lines and all(line.count("|") == 1 for line in lines), lines + assert "遮罩" in models.judges[0] diff --git a/backend/tests/test_governance_judge.py b/backend/tests/test_governance_judge.py new file mode 100644 index 00000000..4052d3ee --- /dev/null +++ b/backend/tests/test_governance_judge.py @@ -0,0 +1,344 @@ +"""四期：受管模板放开 claims: judge（结论句由另一个模型按证据逐句裁判），但要写明预算。 + +- G3：报告撰写节点的 claims 在受管级别认 require_citation 或 judge；写 judge 的，生效的 judge 配置里 + 必须有 max_cost_usd 这个键——写金额，或者显式写 null 表示不限。没写就是 error（已发布级别只给警告）： + 受管出具不能让每份报告的裁判花费悄悄按默认值走，也不能悄悄不设上限 +- 发布前自动修复：缺预算是 choice，候选是默认金额和「不限」，不替人选；G3 另外那几项的一键修复不再把 + judge 改回 require_citation +- 降低要求的判断：require_citation → judge 是往严里改；judge → require_citation / off / 删掉都算放宽， + 报告节点上和出具契约里一样 +""" +from __future__ import annotations + +import copy + +import pytest + +from app.engine.autofix import NEED_CHOICE, apply_fixes, forbidden_changes, plan_fixes, public +from app.engine.governance import lint_for_publish, publish_issues +from app.engine.judge import JUDGE_DEFAULTS +from app.engine.schema import GraphSpec, validate_graph + +STRICT = {"numbers": "strict", "on_violation": "fail", "claims": "require_citation"} +JUDGED = {**STRICT, "claims": "judge", "judge": {"max_cost_usd": 0.05}} +BUDGET = "governed.judge_budget" + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 10, "y": 20}, + "data": {"label": label or nid, "config": config}} + + +def traceable(*, report=None, defaults=None, **contract_overrides) -> dict: + """input → 查库 → 口径卡 → 报告撰写 → 出具：受管要求（含 G1–G5）全部满足。""" + contract = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True} + contract.update(contract_overrides) + nodes = [ + node("start", "input", "周期"), + node("fetch", "tool", "取数", tool="db_query__shop", + args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}), + node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[ + {"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]), + node("write", "report", "报告撰写", instructions="写周报", **(JUDGED if report is None else report)), + node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], + contract={k: v for k, v in contract.items() if v is not None}), + ] + edges = [{"id": f"e{i}", "source": a["id"], "target": b["id"]} for i, (a, b) in enumerate(zip(nodes, nodes[1:]))] + graph = {"nodes": nodes, "edges": edges} + if defaults is not None: + graph["defaults"] = defaults + return graph + + +def config(graph: dict, nid: str) -> dict: + return next(n for n in graph["nodes"] if n["id"] == nid)["data"]["config"] + + +def by_code(graph: dict, code: str, level: str = "governed") -> list: + return [i for i in lint_for_publish(GraphSpec.model_validate(graph), level=level).issues if i.code == code] + + +def errors(graph: dict, level: str = "governed") -> list: + return [i for i in publish_issues(GraphSpec.model_validate(graph), level=level) if i.level == "error"] + + +def plan(graph: dict, level: str = "governed") -> dict[str, dict]: + spec = GraphSpec.model_validate(graph) + return {f["id"]: f for f in plan_fixes(spec, publish_issues(spec, level=level), level=level)} + + +# -------------------------------------------------------------------------- +# G3：judge 可以发布成受管，但要写预算 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("budget", [0.05, 1, None]) +def test_judge_with_a_budget_publishes_as_governed(budget): + graph = traceable(report={**JUDGED, "judge": {"max_cost_usd": budget, "max_claims": 20}}) + assert errors(graph) == [], [i.message for i in errors(graph)] + assert by_code(graph, "governed.report_policy") == [] and by_code(graph, BUDGET) == [] + + +@pytest.mark.parametrize("judge", [None, {}, {"max_claims": 20, "model": "judge-model"}]) +def test_judge_without_a_budget_is_an_error_at_governed(judge): + report = {**STRICT, "claims": "judge", **({"judge": judge} if judge is not None else {})} + graph = traceable(report=report) + [issue] = by_code(graph, BUDGET) + assert issue.level == "error" and issue.node_id == "write" and issue.field == "judge.max_cost_usd" + assert "max_cost_usd" in issue.message and "不限" in issue.message and "null" in issue.message + assert by_code(graph, "governed.report_policy") == [], "judge 本身合规，只缺预算" + assert [i.level for i in by_code(graph, BUDGET, "published")] == ["warning"] + + +def test_the_budget_follows_the_same_config_rule_as_run_time(): + """和运行时同一个取值规则：节点没写 judge 取图级 defaults；节点写了就整个用节点的（不和 defaults 拼）。""" + graph = traceable(report={**STRICT, "claims": "judge"}, defaults={"judge": {"max_cost_usd": 0.02}}) + assert by_code(graph, BUDGET) == [] + graph = traceable(report={**STRICT, "claims": "judge", "judge": {"max_claims": 5}}, + defaults={"judge": {"max_cost_usd": 0.02}}) + assert [i.field for i in by_code(graph, BUDGET)] == ["judge.max_cost_usd"] + # claims 本身也可以来自图级 defaults + graph = traceable(report={"numbers": "strict", "on_violation": "fail"}, + defaults={"claims": "judge", "judge": {"max_cost_usd": None}}) + assert by_code(graph, BUDGET) == [] and by_code(graph, "governed.report_policy") == [] + + +def test_a_malformed_budget_is_left_to_validate(): + graph = traceable(report={**STRICT, "claims": "judge", "judge": {"max_cost_usd": "cheap"}}) + assert by_code(graph, BUDGET) == [], "写错的值由 validate 的 report.judge_invalid 报，同一处不说两遍" + [bad] = [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.code == "report.judge_invalid"] + assert bad.field == "judge.max_cost_usd" and bad.level == "error" + + +@pytest.mark.parametrize("claims", [None, "off"]) +def test_a_loose_claims_setting_names_both_accepted_values(claims): + report = {"numbers": "strict", "on_violation": "fail", **({"claims": claims} if claims else {})} + [issue] = by_code(traceable(report=report), "governed.report_policy") + assert issue.field == "claims" and "claims: require_citation" in issue.message and "judge" in issue.message + + +def test_judge_needs_no_other_report_policy_fix(): + graph = traceable(report={**JUDGED, "numbers": "off"}) + [issue] = by_code(graph, "governed.report_policy") + assert issue.field == "numbers" and "claims" not in issue.message + + +# -------------------------------------------------------------------------- +# 发布前自动修复 +# -------------------------------------------------------------------------- + + +def test_the_report_policy_fix_leaves_judge_alone(): + graph = traceable(report={**JUDGED, "numbers": "off", "on_violation": "flag"}) + fix = plan(graph)["governed.report_policy:write"] + assert fix["kind"] == "auto" and fix["preview"]["after"] == {"numbers": "strict", "on_violation": "fail"} + out = apply_fixes(graph, [fix["id"]], level="governed") + assert out["ok"] is True and config(out["graph"], "write")["claims"] == "judge" + assert config(out["graph"], "write")["judge"] == {"max_cost_usd": 0.05} + + +def test_a_missing_budget_is_a_choice_between_the_default_and_unlimited(): + graph = traceable(report={**STRICT, "claims": "judge", "judge": {"model": "judge-model"}}) + fid = f"{BUDGET}:write" + fix = public(plan(graph)[fid]) + assert fix["kind"] == "choice" and fix["node_id"] == "write" and fix["field"] == "judge.max_cost_usd" + assert [o["value"] for o in fix["options"]] == [JUDGE_DEFAULTS["report_max_cost_usd"], None] + assert "default" not in fix, "花多少钱要人定：不替人选，也不给建议" + unlimited = fix["options"][1] + assert "不限" in unlimited["label"] and "不设上限" in unlimited["hint"] and "约束" in unlimited["hint"] + assert all(not k.startswith("_") for k in fix), "内部键不交给前端" + + [refused] = apply_fixes(graph, [fid], level="governed")["rejected"] + assert refused["reason"] == NEED_CHOICE.format(label=fix["label"]) + + out = apply_fixes(graph, [fid], {fid: 0.05}, level="governed") + assert out["applied"] == [fid] and out["ok"] is True + assert config(out["graph"], "write")["judge"] == {"model": "judge-model", "max_cost_usd": 0.05} + [change] = out["changes"] + assert change["field"] == "judge" and change["before"] == {"model": "judge-model"} + assert out["ops"] == [{"op": "update_node", "id": "write", + "config": {"judge": {"model": "judge-model", "max_cost_usd": 0.05}}}] + + out = apply_fixes(graph, [fid], {fid: None}, level="governed") + judge = config(out["graph"], "write")["judge"] + assert out["ok"] is True and "max_cost_usd" in judge and judge["max_cost_usd"] is None, "不限要显式写成 null" + + +def test_the_budget_fix_keeps_what_the_graph_defaults_said(): + """judge 写在图级 defaults 里：修复写到节点上时带上 defaults 里的整份 judge，节点写了就不再跟随 defaults， + 不能因为补一个预算把 defaults 里选的裁判模型丢了。""" + graph = traceable(report={**STRICT, "claims": "judge"}, + defaults={"judge": {"model": "judge-model", "rewrite_once": True}}) + fid = f"{BUDGET}:write" + out = apply_fixes(graph, [fid], {fid: None}, level="governed") + assert out["ok"] is True + assert config(out["graph"], "write")["judge"] == {"model": "judge-model", "rewrite_once": True, + "max_cost_usd": None} + assert out["graph"]["defaults"] == {"judge": {"model": "judge-model", "rewrite_once": True}} + + +def test_a_choice_outside_the_options_is_refused(): + graph = traceable(report={**STRICT, "claims": "judge"}) + fid = f"{BUDGET}:write" + out = apply_fixes(graph, [fid], {fid: 99}, level="governed") + assert out["applied"] == [] and "不在候选里" in out["rejected"][0]["reason"] + + +def test_published_level_budget_warnings_get_no_fix(): + assert f"{BUDGET}:write" not in plan(traceable(report={**STRICT, "claims": "judge"}), "published") + + +def test_invalid_claims_fixes_no_longer_mention_a_later_version(): + graph = traceable(report={**STRICT, "claims": "sometimes"}) + governed = plan(graph)["report.claims_invalid:write"] + published = plan(graph, "published")["report.claims_invalid:write"] + assert "后续版本" not in governed["label"] and "default" not in published + assert [o["value"] for o in published["options"]] == ["off", "require_citation"] + + +# -------------------------------------------------------------------------- +# 降低要求的判断 +# -------------------------------------------------------------------------- + + +def _changed(report_before: dict, report_after: dict | None, **graph_kw) -> list[str]: + old = traceable(report=report_before, **graph_kw) + new = copy.deepcopy(old) + if report_after is None: + config(new, "write").pop("claims") + else: + config(new, "write").clear() + config(new, "write").update({"instructions": "写周报", **report_after}) + return forbidden_changes(old, new) + + +def test_moving_up_to_judge_is_not_loosening(): + assert _changed(STRICT, JUDGED) == [] + assert _changed({**STRICT, "claims": "off"}, JUDGED) == [] + assert _changed({"numbers": "strict", "on_violation": "fail"}, JUDGED) == [] + + +@pytest.mark.parametrize("after", ["off", "require_citation", None]) +def test_moving_down_from_judge_is_loosening(after): + report_after = None if after is None else {**JUDGED, "claims": after} + [reason] = _changed(JUDGED, report_after) + assert "「报告撰写」" in reason and "claims" in reason and "judge" in reason + + +def test_dropping_a_judge_set_in_the_graph_defaults_is_loosening(): + [reason] = _changed({"numbers": "strict", "on_violation": "fail"}, {**STRICT, "claims": "off"}, + defaults={"claims": "judge", "judge": {"max_cost_usd": 0.05}}) + assert "claims" in reason and "judge" in reason + + +def test_changing_the_budget_is_not_loosening(): + assert _changed(JUDGED, {**JUDGED, "judge": {"max_cost_usd": None}}) == [] + + +# claims 一直是 judge 时，judge.on_unsupported 从 withhold 往回改也是放宽：不支持的结论句从「不予出具」变成 +# 「降档出具」。Copilot 改图的 update_node 会把 judge 整个盖掉，补预算时写 judge={max_cost_usd: 0.05} +# 就悄悄丢了 withhold——这一类要和契约里的 on_unsupported 一样拒掉 +WITHHOLD = {**JUDGED, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}} + + +@pytest.mark.parametrize("judge, dropped", [ + ({"max_cost_usd": 0.05, "on_unsupported": "degrade"}, False), + ({"max_cost_usd": 0.05}, True), + ({"max_cost_usd": 0.05, "on_unsupported": "sometimes"}, False), +]) +def test_relaxing_on_unsupported_on_the_node_is_loosening(judge, dropped): + [reason] = _changed(WITHHOLD, {**WITHHOLD, "judge": judge}) + assert "「报告撰写」" in reason and "judge.on_unsupported" in reason + assert "withhold" in reason and "degrade" in reason + assert ("去掉了写明的值" in reason) is dropped, reason + + +def test_replacing_the_whole_judge_block_to_add_a_budget_is_loosening(): + """补预算把 judge 整个换成 {max_cost_usd}：连同裁判模型、withhold 一起丢了。""" + before = {**STRICT, "claims": "judge", "judge": {"model": "judge-model", "on_unsupported": "withhold"}} + [reason] = _changed(before, {**before, "judge": {"max_cost_usd": 0.05}}) + assert "judge.on_unsupported" in reason and "withhold" in reason + assert _changed(before, {**before, "judge": {**before["judge"], "max_cost_usd": 0.05}}) == [], \ + "带着原来的键补预算不算放宽" + + +def test_a_withhold_from_the_graph_defaults_lost_by_a_node_judge_is_loosening(): + """judge 按运行时的规则取：节点写了自己的 judge 就整个用节点的，图级 defaults 里的 withhold 不再生效。""" + defaults = {"judge": {"on_unsupported": "withhold", "max_cost_usd": 0.05}} + report = {**STRICT, "claims": "judge"} + [reason] = _changed(report, {**report, "judge": {"max_cost_usd": 0.05}}, defaults=defaults) + assert "judge.on_unsupported" in reason and "从 withhold（跟随全图默认）" in reason, reason + assert "不再跟随全图默认" in reason, "说清楚为什么全图默认里的 withhold 不生效了" + assert _changed(report, {**report, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}, + defaults=defaults) == [] + + +def test_relaxing_on_unsupported_in_the_graph_defaults_is_loosening(): + old = traceable(report={**STRICT, "claims": "judge"}, + defaults={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}) + new = copy.deepcopy(old) + new["defaults"]["judge"]["on_unsupported"] = "degrade" + [reason] = forbidden_changes(old, new) + assert "「报告撰写」" in reason and "judge.on_unsupported" in reason + + +@pytest.mark.parametrize("before, after", [ + (JUDGED, {**JUDGED, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}), + (WITHHOLD, {**WITHHOLD, "judge": {**WITHHOLD["judge"], "max_cost_usd": None, "rewrite_once": True}}), + (WITHHOLD, WITHHOLD), + # 升到 judge 本身是收紧：不管带不带 withhold + ({**STRICT, "judge": {"on_unsupported": "withhold"}}, JUDGED), +]) +def test_tightening_or_keeping_on_unsupported_is_fine(before, after): + assert _changed(before, after) == [] + + +def test_dropping_withhold_along_with_judge_is_one_reason(): + """从 judge 退回 require_citation：claims 那一条已经说了，不再为 on_unsupported 另记一条。""" + [reason] = _changed(WITHHOLD, {**STRICT, "judge": {"max_cost_usd": 0.05}}) + assert "claims" in reason and "on_unsupported" not in reason + + +@pytest.mark.parametrize("before", ["require_citation", "judge"]) +def test_moving_claims_to_an_unknown_value_is_loosening(before): + """require_citation、judge 改成认不出的写法：出口按认不出的走，要求没了——照样算放宽，别只落个 + 「没让图变好」。原来就认不出的不比较。""" + report = {**STRICT, "claims": before, **({"judge": {"max_cost_usd": 0.05}} if before == "judge" else {})} + [reason] = _changed(report, {**report, "claims": "sometimes"}) + assert "「报告撰写」" in reason and before in reason and "sometimes" in reason and "不认识" in reason + assert _changed({**STRICT, "claims": "sometimes"}, {**STRICT, "claims": "off"}) == [] + assert _changed({**STRICT, "claims": "off"}, {**STRICT, "claims": "sometimes"}) == [] + + +def _contract_changed(before, after) -> list[str]: + old = traceable(claims=before) + new = copy.deepcopy(old) + contract = config(new, "done")["contract"] + if after is None: + contract.pop("claims", None) + else: + contract["claims"] = after + return forbidden_changes(old, new) + + +@pytest.mark.parametrize("before, after", [ + ("judge", "off"), + ("judge", None), + ("judge", "require_citation"), + ({"policy": "judge", "on_unsupported": "withhold"}, "judge"), + ({"policy": "judge", "on_unsupported": "withhold"}, {"policy": "judge", "on_unsupported": "degrade"}), + ({"policy": "require_citation", "on_uncited": "withhold"}, "judge"), +]) +def test_loosening_a_judge_contract_is_refused(before, after): + [reason] = _contract_changed(before, after) + assert "「出具」" in reason and "claims" in reason + + +@pytest.mark.parametrize("before, after", [ + ("require_citation", "judge"), + ("off", {"policy": "judge", "on_unsupported": "withhold"}), + ("judge", {"policy": "judge", "on_unsupported": "withhold"}), + ({"policy": "require_citation", "on_uncited": "withhold"}, {"policy": "judge", "on_uncited": "withhold"}), +]) +def test_tightening_a_contract_towards_judge_is_fine(before, after): + assert _contract_changed(before, after) == [] diff --git a/backend/tests/test_governance_report_lint.py b/backend/tests/test_governance_report_lint.py index fc336a65..8b8ba76f 100644 --- a/backend/tests/test_governance_report_lint.py +++ b/backend/tests/test_governance_report_lint.py @@ -20,7 +20,7 @@ from app.db.base import SessionLocal from app.db.models import Run, Workflow, WorkflowVersion from app.engine.governance import lint_for_publish, publish_issues from app.engine.runner import run_manager -from app.engine.schema import CLAIMS_LATER, GraphSpec, validate_graph +from app.engine.schema import GraphSpec, validate_graph from app.main import app from app.seed import TEMPLATES @@ -281,10 +281,10 @@ def test_graph_defaults_count_like_they_do_at_run_time(): def test_an_unsupported_claims_value_is_left_to_validate(): - graph = traceable(report={**STRICT, "claims": "judge"}) + graph = traceable(report={**STRICT, "claims": "sometimes"}) assert by_code(graph, "governed.report_policy") == [] [bad] = [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.code == "report.claims_invalid"] - assert bad.level == "error" and CLAIMS_LATER in bad.message + assert bad.level == "error" and "off / require_citation" in bad.message # -------------------------------------------------------------------------- @@ -442,7 +442,7 @@ async def test_an_already_governed_version_keeps_running_formally(client, engine # -------------------------------------------------------------------------- -# validate：报告撰写节点的 claims 只接受 off / require_citation +# validate：报告撰写节点的 claims 只接受 off / require_citation / judge（四期放开 judge） # -------------------------------------------------------------------------- @@ -458,10 +458,9 @@ def test_supported_claims_values_pass(value): assert claims_issues(value) == [] -def test_judge_is_a_later_version(): - [issue] = claims_issues("judge") - assert issue.level == "error" and issue.node_id == "write" and issue.field == "claims" - assert CLAIMS_LATER in issue.message and "off" in issue.message and "require_citation" in issue.message +def test_judge_is_supported_now(): + assert claims_issues("judge") == [] + assert claims_issues("judge", on_defaults=True) == [] @pytest.mark.parametrize("value", ["strict", "Require_Citation", 1, ["off"], {"policy": "judge"}]) @@ -471,5 +470,5 @@ def test_other_claims_values_are_errors(value): def test_claims_from_graph_defaults_are_checked_too(): - [issue] = claims_issues("judge", on_defaults=True) + [issue] = claims_issues("sometimes", on_defaults=True) assert issue.node_id == "write" diff --git a/backend/tests/test_judge_settings.py b/backend/tests/test_judge_settings.py new file mode 100644 index 00000000..82603175 --- /dev/null +++ b/backend/tests/test_judge_settings.py @@ -0,0 +1,262 @@ +"""结论句裁判的设置与画布校验。 + +- 设置里单开一组 judge：证据裁判模型、每份报告的句数 / 金额 / 时长上限、探索运行每次点击的金额上限、 + 全局每日金额上限。每一项都能存成 null，表示不限——存了 null 就是不限，不会被默认值悄悄盖回去 +- limits 组是环境变量定的、只读，裁判设置不放进去；每日计数存在设置表里，但不从设置接口进出 +- 裁判模型的优先级：节点上的 judge.model > 设置里的证据裁判模型 > Copilot 模型 > 默认 provider +- 画布校验放开 claims: judge，并校验 judge 子配置的形状：上限可以写 null（不限），rewrite_once 是布尔值， + on_unsupported 取 degrade / withhold +""" +from __future__ import annotations + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.db.base import SessionLocal +from app.db.models import Setting +from app.engine.judge import ( + JUDGE_DEFAULTS, + SETTING_GROUP, + SPEND_KEY, + Budget, + click_budget, + judge_model_spec, + judge_settings, + report_budget, +) +from app.engine.schema import CLAIMS_VALUES, JUDGE_KEYS, GraphSpec, claims_problem, validate_graph +from app.main import app + + +@pytest.fixture(autouse=True) +async def clean(): + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, SETTING_GROUP, "copilot"]))) + await session.commit() + yield + + +def client() -> AsyncClient: + return AsyncClient(transport=ASGITransport(app=app), base_url="http://test") + + +# -------------------------------------------------------------------------- +# 设置 +# -------------------------------------------------------------------------- + + +def test_the_defaults_are_the_agreed_ones(): + assert SETTING_GROUP == "judge" and SETTING_GROUP != "limits" + assert JUDGE_DEFAULTS == { + "provider": None, "model": None, + "report_max_claims": 40, "report_max_cost_usd": 0.05, "report_timeout_s": 30, + "click_max_cost_usd": 0.01, "daily_max_usd": 2.0, + } + + +async def test_settings_expose_the_judge_group_with_defaults(): + async with client() as c: + got = (await c.get("/api/settings")).json() + assert got["judge"] == JUDGE_DEFAULTS + assert SPEND_KEY not in got + + +async def test_null_is_stored_as_unlimited_and_stays_unlimited(): + async with client() as c: + r = await c.put("/api/settings", json={"values": {"judge": { + **JUDGE_DEFAULTS, "model": "judge-model", "report_max_cost_usd": None, "daily_max_usd": None, + "report_timeout_s": None, "report_max_claims": None, "click_max_cost_usd": None}}}) + assert r.status_code == 200, r.text + got = (await c.get("/api/settings")).json()["judge"] + assert got["model"] == "judge-model" + assert all(got[k] is None for k in ("report_max_cost_usd", "daily_max_usd", "report_timeout_s", + "report_max_claims", "click_max_cost_usd")) + async with SessionLocal() as session: + settings = await judge_settings(session) + assert report_budget(settings, None) == Budget(max_claims=None, max_cost_usd=None, timeout_s=None, + daily_max_usd=None) + assert click_budget(settings).max_cost_usd is None + + +async def test_a_partial_group_keeps_the_other_defaults(): + async with client() as c: + await c.put("/api/settings", json={"values": {"judge": {"daily_max_usd": 5}}}) + got = (await c.get("/api/settings")).json()["judge"] + assert got == {**JUDGE_DEFAULTS, "daily_max_usd": 5} + + +@pytest.mark.parametrize("key,value", [ + ("daily_max_usd", -1), ("report_max_cost_usd", 0), ("report_max_cost_usd", "0.05"), + ("report_max_claims", 2.5), ("report_max_claims", True), ("report_timeout_s", 0), + ("click_max_cost_usd", float("nan")), ("model", 3), ("unknown_key", 1), +]) +async def test_bad_values_are_refused_with_a_reason(key, value): + async with client() as c: + r = await c.put("/api/settings", json={"values": {"judge": {key: value}}}) if value == value else \ + await c.put("/api/settings", content='{"values": {"judge": {"%s": NaN}}}' % key, + headers={"content-type": "application/json"}) + assert r.status_code == 422, r.text + assert key in r.json()["detail"] + assert (await c.get("/api/settings")).json()["judge"] == JUDGE_DEFAULTS, "整组都没写进去" + + +async def test_the_daily_counter_does_not_travel_through_the_settings_api(): + async with SessionLocal() as session: + session.add(Setting(key=SPEND_KEY, value={"date": "2026-09-29", "usd": 1.5, "calls": 3})) + await session.commit() + async with client() as c: + assert SPEND_KEY not in (await c.get("/api/settings")).json() + await c.put("/api/settings", json={"values": {SPEND_KEY: {"date": "2026-09-29", "usd": 0, "calls": 0}}}) + async with SessionLocal() as session: + row = await session.get(Setting, SPEND_KEY) + assert row.value["usd"] == 1.5, "设置页整组回写不能把今天的花费清零" + + +async def test_todays_spend_is_readable_next_to_the_daily_cap(monkeypatch): + from app.engine import judge as judge_mod + + monkeypatch.setattr(judge_mod, "_today", lambda: "2026-09-29") + async with SessionLocal() as session: + session.add(Setting(key=SPEND_KEY, value={"date": "2026-09-28", "usd": 1.9, "calls": 7})) + await session.commit() + async with client() as c: + got = (await c.get("/api/settings/judge/spend")).json() + assert got == {"date": "2026-09-29", "usd": 0.0, "calls": 0, "unpriced_calls": 0, "daily_max_usd": 2.0}, \ + "昨天的花费不算今天的" + async with SessionLocal() as session: + row = await session.get(Setting, SPEND_KEY) + row.value = {"date": "2026-09-29", "usd": 0.42, "calls": 3, "unpriced_calls": 1} + await session.commit() + await c.put("/api/settings", json={"values": {"judge": {"daily_max_usd": None}}}) + got = (await c.get("/api/settings/judge/spend")).json() + assert got == {"date": "2026-09-29", "usd": 0.42, "calls": 3, "unpriced_calls": 1, "daily_max_usd": None} + + +async def test_bad_rows_already_in_the_database_fall_back_to_defaults(): + """库里存坏了的值（旧版本、手改）读出来按默认，不让裁判拿着一个负数上限跑。""" + async with SessionLocal() as session: + session.add(Setting(key=SETTING_GROUP, value={"daily_max_usd": -3, "report_max_claims": "many", + "report_timeout_s": None})) + await session.commit() + settings = await judge_settings(session) + assert settings["daily_max_usd"] == JUDGE_DEFAULTS["daily_max_usd"] + assert settings["report_max_claims"] == JUDGE_DEFAULTS["report_max_claims"] + assert settings["report_timeout_s"] is None + + +def test_the_node_overrides_key_by_key(): + settings = {**JUDGE_DEFAULTS, "report_max_claims": 10} + assert report_budget(settings, None) == Budget(max_claims=10, max_cost_usd=0.05, timeout_s=30, daily_max_usd=2.0) + assert report_budget(settings, {"max_cost_usd": None, "timeout_s": 5}) == \ + Budget(max_claims=10, max_cost_usd=None, timeout_s=5, daily_max_usd=2.0) + # 探索运行每次点击：金额按点击的上限，句数、时长按每份报告的 + assert click_budget(settings) == Budget(max_claims=10, max_cost_usd=0.01, timeout_s=30, daily_max_usd=2.0) + + +# -------------------------------------------------------------------------- +# 裁判模型的优先级 +# -------------------------------------------------------------------------- + + +async def test_the_judge_model_priority(): + async with SessionLocal() as session: + spec = await judge_model_spec(session, None) + assert (spec.provider, spec.model) == (None, None), "什么都没配：默认 provider" + assert spec.thinking == "off" and spec.max_tokens == 4096 + + session.add(Setting(key="copilot", value={"provider": "", "model": "copilot-model"})) + await session.commit() + assert (await judge_model_spec(session, None)).model == "copilot-model" + + session.add(Setting(key=SETTING_GROUP, value={"provider": "house", "model": "judge-model"})) + await session.commit() + spec = await judge_model_spec(session, None) + assert (spec.provider, spec.model) == ("house", "judge-model") + + spec = await judge_model_spec(session, {"model": "node-model"}) + assert (spec.provider, spec.model) == (None, "node-model"), "节点上的写法整组生效，不和设置里的拼" + spec = await judge_model_spec(session, {"model": None, "provider": None, "max_claims": 3}) + assert spec.model == "judge-model" + + +# -------------------------------------------------------------------------- +# 画布校验 +# -------------------------------------------------------------------------- + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def graph(report: dict, defaults: dict | None = None) -> dict: + nodes = [node("start", "input"), + node("caliber", "metrics", caliber="周报口径", metrics=[{"id": "gmv", "expression": "1"}]), + node("write", "report", instructions="写周报", **report), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])] + ids = [n["id"] for n in nodes] + g = {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} + if defaults: + g["defaults"] = defaults + return g + + +def issues(g, code=None): + return [i for i in validate_graph(GraphSpec.model_validate(g)).issues + if i.code in ((code,) if code else ("report.claims_invalid", "report.judge_invalid"))] + + +def test_judge_is_a_supported_policy_now(): + assert CLAIMS_VALUES == ("off", "require_citation", "judge") + assert claims_problem("judge") is None + assert issues(graph({"claims": "judge"})) == [] + assert issues(graph({}, defaults={"claims": "judge"})) == [] + + +def test_every_limit_can_be_unlimited(): + judge = {"provider": None, "model": None, "max_claims": None, "max_cost_usd": None, "timeout_s": None, + "rewrite_once": True, "on_unsupported": "withhold"} + assert set(judge) == set(JUDGE_KEYS) + assert issues(graph({"claims": "judge", "judge": judge})) == [] + assert issues(graph({"claims": "judge", "judge": {"max_claims": 40, "max_cost_usd": 0.05, "timeout_s": 30, + "model": "judge-model", "on_unsupported": "degrade"}})) == [] + + +@pytest.mark.parametrize("judge,field,words", [ + ("fast", "judge", "对象"), + ({"max_cost": 1}, "judge.max_cost", "max_cost_usd"), + ({"max_claims": 0}, "judge.max_claims", "正整数"), + ({"max_claims": 2.5}, "judge.max_claims", "正整数"), + ({"max_claims": True}, "judge.max_claims", "正整数"), + ({"max_cost_usd": -1}, "judge.max_cost_usd", "大于 0"), + ({"max_cost_usd": "0.05"}, "judge.max_cost_usd", "大于 0"), + ({"timeout_s": 0}, "judge.timeout_s", "大于 0"), + ({"rewrite_once": "yes"}, "judge.rewrite_once", "true 或 false"), + ({"on_unsupported": "ignore"}, "judge.on_unsupported", "degrade / withhold"), + ({"model": 3}, "judge.model", "模型 id"), +]) +def test_a_malformed_judge_block_is_an_error(judge, field, words): + [issue] = issues(graph({"claims": "judge", "judge": judge}), "report.judge_invalid") + assert issue.level == "error" and issue.node_id == "write" and issue.field == field + assert words in issue.message, issue.message + if field in ("judge.max_claims", "judge.max_cost_usd", "judge.timeout_s", "judge.model"): + assert "null" in issue.message, "上限、模型都说清写 null 是什么意思" + + +def test_graph_defaults_are_checked_like_at_run_time(): + [issue] = issues(graph({"claims": "judge"}, defaults={"judge": {"max_claims": -1}}), "report.judge_invalid") + assert issue.field == "judge.max_claims" + + +def test_the_judge_block_only_matters_when_claims_is_judge(): + assert issues(graph({"claims": "require_citation", "judge": {"max_claims": -1}})) == [] + + +def test_judging_with_the_writing_model_is_a_warning(): + [warn] = [i for i in validate_graph(GraphSpec.model_validate(graph( + {"claims": "judge", "model": "writer-x", "judge": {"model": "writer-x"}}))).issues + if i.code == "report.judge_same_model"] + assert warn.level == "warning" and warn.field == "judge.model" and "自己审自己" in warn.message + assert not [i for i in validate_graph(GraphSpec.model_validate(graph( + {"claims": "judge", "model": "writer-x", "judge": {"model": "judge-y"}}))).issues + if i.code == "report.judge_same_model"] diff --git a/backend/tests/test_report_claims.py b/backend/tests/test_report_claims.py index 22a6ce64..0dfd8f8b 100644 --- a/backend/tests/test_report_claims.py +++ b/backend/tests/test_report_claims.py @@ -1,10 +1,10 @@ """报告节点的结论句策略 claims：off（默认，和以前一样）/ require_citation（没挂依据的结论句计入缺口）。 -结论句要靠裁判模型判断支不支持，那是后续版本的事；这一版只做确定性的一半： +这里测确定性的一半（模型裁判 judge 见 test_report_judge.py）： - require_citation 时写作提示明确要求每句结论挂 [[see:…]]；没挂的照常产出、计数， 出口按出具档位降档（io.py），报告节点自己不为它失败、不为它重写 - report.checked 事件和节点产出都带上 claims，出口据此判档 -- 写 judge 时节点直接报错「结论句裁判在后续版本支持」，不能悄悄当成 off +- 写了不认识的值节点直接报错，不能悄悄当成 off - 升级前发起的运行一个字都不变 """ from __future__ import annotations @@ -21,7 +21,7 @@ from app.db.models import Run, RunEvent from app.engine import runner as runner_mod from app.engine.evidence import CLAIMS_RULE, build_catalog, compose_doc, uncited_claims from app.engine.runner import run_manager -from app.engine.schema import CLAIMS_LATER, REPORT_CLAIMS, claims_problem +from app.engine.schema import CLAIMS_VALUES, claims_problem @pytest.fixture(autouse=True) @@ -131,14 +131,14 @@ async def test_require_citation_asks_for_support_but_does_not_block(monkeypatch) assert flagged["text"] == "增长主要来自新客。" and doc["markdown"][slice(*flagged["span"])] == flagged["text"] -async def test_judge_is_refused_until_it_exists(monkeypatch): +async def test_a_bad_value_is_refused_before_any_model_call(monkeypatch): seen = script(monkeypatch) - with pytest.raises(ValueError) as info: # 画布校验先拦（E3-gov 的 validate） - await run_manager.start(graph=weekly(claims="judge"), input_payload={}) - assert CLAIMS_LATER in str(info.value) + with pytest.raises(ValueError) as info: # 画布校验先拦 + await run_manager.start(graph=weekly(claims="sometimes"), input_payload={}) + assert claims_problem("sometimes") in str(info.value) lenient(monkeypatch) - row = await finish(weekly(claims="judge")) - assert row.status == "failed" and claims_problem("judge") in (row.error or ""), row.error + row = await finish(weekly(claims="sometimes")) + assert row.status == "failed" and claims_problem("sometimes") in (row.error or ""), row.error assert seen == [], "不支持的策略不该先花一次模型调用" @@ -173,7 +173,7 @@ def test_the_supported_policies_match_the_validator(): from app.engine.nodes.report import CLAIMS, report_claims from app.engine.schema import GraphSpec - assert CLAIMS == REPORT_CLAIMS == ("off", "require_citation") + assert CLAIMS == CLAIMS_VALUES == ("off", "require_citation", "judge") spec = GraphSpec.model_validate(weekly(claims="require_citation")) assert report_claims(spec, spec.node_map()["write"]) == "require_citation" spec = GraphSpec.model_validate(weekly()) diff --git a/backend/tests/test_report_judge.py b/backend/tests/test_report_judge.py new file mode 100644 index 00000000..e5db6f89 --- /dev/null +++ b/backend/tests/test_report_judge.py @@ -0,0 +1,495 @@ +"""报告撰写节点的 claims: judge：结论句由另一个模型按证据逐句判断（四期）。 + +- 正式运行在节点里裁判：写完、核对完之后判，判定写进报告文档的 unit.verdict，随文档一起封存 +- 探索运行不在节点里花这笔钱：候选句标「未裁判 · 按需」，点开哪句再判哪句（接口在 B 段） +- 判定只标注、不改写正文；裁判没跑成、触顶了，照样产出报告，没判的记未裁判并留下缺口 + （出口据此不能判完整出具，接线在 B 段） +- rewrite_once（默认关）：把证据不支持的句子和理由交回写作者只改这几句，再判一次，最多一轮； + 改出了新的违规就不采用 +- 裁判调用包在 @task 里、追加在节点已有的调用之后：节点「接着跑」时不重复调用、不重复计费； + 升级前发起的运行一样都不加，停在审批上的运行恢复后照常 +""" +from __future__ import annotations + +import asyncio +import json +import re + +import pytest +from langchain_core.messages import AIMessage +from sqlalchemy import delete, select + +from app.core import artifact_store +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent, Setting +from app.engine import judge as judge_mod +from app.engine import replay +from app.engine import runner as runner_mod +from app.engine.evidence import CLAIMS_RULE, iter_units +from app.engine.judge import JUDGE_ROLE, JUDGE_RULE, SPEND_KEY, SUMMARY_KEYS, daily_spend +from app.engine.nodes import report as report_mod +from app.engine.runner import run_manager, verify_manifest +from app.providers.mock_model import MockChatModel + +PRICED = "claude-sonnet-5" +METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] +#: 一句挂了依据的结论、一句没挂依据的归因、一句连接性的话 +TEXT = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。下面看细节。" +CITED, CAUSAL, LINK = "本周销售额 45,678.5元。", "增长主要来自新客首单。", "下面看细节。" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"]))) + await session.commit() + yield + await run_manager.shutdown() + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def weekly(*, gate=False, **report): + nodes = [ + node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]), + node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS), + *([node("gate", "human", mode="approve", title="确认口径")] if gate else []), + node("write", "report", instructions="写周报", **report), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + ] + ids = [n["id"] for n in nodes] + return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} + + +class Models: + """写作者和裁判两份剧本。写作者依次回 writer 里的几段（用完了一直回最后一段）；裁判按句子里的字给判定 + （verdicts：{句中片段: 判定}，没对上的都是 supported）。""" + + def __init__(self, monkeypatch, writer=(TEXT,), verdicts=None, *, judge_model=PRICED, fail_judge=None): + self.writes: list[list] = [] + self.judges: list[str] = [] + verdicts = verdicts or {} + + def decide(model, messages): + if JUDGE_ROLE in str(messages[0].content): + request = str(messages[-1].content) + self.judges.append(request) + if fail_judge: + raise RuntimeError(fail_judge) + items = [] + for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M): + status = next((s for frag, s in verdicts.items() if frag in line), "supported") + items.append({"unit": uid, "verdict": status, "rationale": f"判的是：{line[:20]}", "used": []}) + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False)) + self.writes.append(list(messages)) + return AIMessage(content=writer[min(len(self.writes), len(writer)) - 1]) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + + async def judge_model_of(session, spec): + return MockChatModel(model_name=judge_model), judge_model + + monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of) + + +def spy_tasks(monkeypatch) -> list[str]: + """报告节点里每调用一次 @task 记一笔名字：证明加进来的 task 只在该出现的地方出现。""" + from langgraph.func import task as real + + calls: list[str] = [] + + def spying(func=None, **kwargs): + def wrap(fn): + inner = real(fn, **kwargs) + + def call(*args, **kw): + calls.append(fn.__name__) + return inner(*args, **kw) + + return call + return wrap(func) if func is not None else wrap + + monkeypatch.setattr(report_mod, "task", spying) + return calls + + +async def finish(graph, *, run_class="formal", statuses=("succeeded", "failed")) -> Run: + run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class) + return await wait(run.id, statuses) + + +async def wait(run_id, statuses=("succeeded", "failed")) -> Run: + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + return row + raise AssertionError(f"run {run_id} 没跑完：{row.status}") + + +async def events(run_id: str, etype: str, node_id: str | None = "write") -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype) + if node_id: + q = q.where(RunEvent.node_id == node_id) + return list((await session.execute(q.order_by(RunEvent.seq))).scalars()) + + +async def report_of(run_id: str) -> tuple[dict, dict, dict]: + """(report.checked 的载荷, 报告文档, 节点产出)""" + [checked] = await events(run_id, "report.checked") + [finished] = await events(run_id, "node.finished") + return checked.data, artifact_store.load(checked.data["doc_artifact"]), artifact_store.load(finished.data["artifact"]) + + +def verdicts(doc) -> dict[str, dict | None]: + md = doc["markdown"] + return {md[u["span"][0]:u["span"][1]].strip(): u.get("verdict") for _, u in iter_units(doc)} + + +def unit_ids(doc) -> dict[str, str]: + md = doc["markdown"] + return {md[u["span"][0]:u["span"][1]].strip(): u["id"] for _, u in iter_units(doc)} + + +async def judge_ends(run_id: str) -> list[dict]: + return [e.data for e in await events(run_id, "llm.end") if e.data.get("purpose") == "judge"] + + +# -------------------------------------------------------------------------- +# 正式运行：节点内裁判，判定随文档封存 +# -------------------------------------------------------------------------- + + +async def test_a_formal_run_judges_inside_the_node_and_seals_the_verdicts(monkeypatch): + models = Models(monkeypatch, verdicts={"新客": "unsupported"}) + row = await finish(weekly(claims="judge", judge={"on_unsupported": "withhold"})) + assert row.status == "succeeded", row.error + assert len(models.writes) == 1 and len(models.judges) == 1 + + checked, doc, output = await report_of(row.id) + got = verdicts(doc) + assert got[CITED]["status"] == "supported" and got[CITED]["judge"] == PRICED + assert got[CITED]["post_seal"] is False and got[CITED]["rationale"] + assert got[CAUSAL]["status"] == "unsupported" + assert got[LINK] is None, "连接性的话不送裁判" + assert CITED in models.judges[0] and CAUSAL in models.judges[0] and LINK not in models.judges[0] + + summary = checked["judge"] + assert tuple(summary) == SUMMARY_KEYS, "B 段照着这份键读：不能悄悄少一个、多一个" + assert summary["mode"] == "inline" and summary["model"] == PRICED and summary["on_unsupported"] == "withhold" + assert summary["counts"] == {"supported": 1, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0} + assert summary["complete"] is True and summary["gaps"] == [] + assert checked["claims"] == output["claims"] == "judge" and output["judge"] == summary + assert doc["judge"]["mode"] == "inline" and doc["judge"]["counts"] == summary["counts"] + assert {k: doc["stats"][k] for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")} \ + == summary["counts"] + + [end] = await judge_ends(row.id) + assert end["model"] == PRICED and end["cost_usd"] >= 0 + verdict = await verify_manifest(row.id) + assert verdict["ok"] is True + [event] = await events(row.id, "report.checked") + assert event.seq <= row.manifest_seq, "判定在封存范围内" + + +async def test_the_writer_is_told_that_claims_will_be_judged(monkeypatch): + models = Models(monkeypatch) + await finish(weekly(claims="judge")) + prompt = "\n".join(str(m.content) for m in models.writes[0]) + assert CLAIMS_RULE in prompt and JUDGE_RULE in prompt + + +async def test_explore_runs_leave_it_to_the_reader(monkeypatch): + """探索运行默认按需裁判：节点里不调用，候选句标「未裁判 · 按需」，连接性的话什么都不标。""" + models = Models(monkeypatch) + row = await finish(weekly(claims="judge"), run_class="exploratory") + assert row.status == "succeeded", row.error + assert models.judges == [] + checked, doc, output = await report_of(row.id) + got = verdicts(doc) + for text in (CITED, CAUSAL): + assert got[text]["status"] == "unjudged" and got[text]["reason"] == "on_demand" + assert got[text]["judge"] is None and got[text]["post_seal"] is False + assert got[LINK] is None + assert checked["judge"]["mode"] == "on_demand" and checked["judge"]["gaps"] == [] + assert tuple(checked["judge"]) == SUMMARY_KEYS and checked["judge"]["limits"] is None + assert output["judge"] == checked["judge"] and await judge_ends(row.id) == [] + + +async def test_a_failed_judge_leaves_a_gap_not_a_failed_report(monkeypatch): + Models(monkeypatch, fail_judge="网关超时") + row = await finish(weekly(claims="judge")) + assert row.status == "succeeded", row.error + checked, doc, output = await report_of(row.id) + assert {verdicts(doc)[t]["reason"] for t in (CITED, CAUSAL)} == {"error"} + summary = checked["judge"] + assert summary["complete"] is False and summary["counts"]["unjudged"] == 2 + assert summary["gaps"] and "网关超时" in summary["gaps"][0] + assert output["judge"]["gaps"] == summary["gaps"] + [end] = await judge_ends(row.id) + assert "网关超时" in end["error"] + + +async def test_hitting_a_limit_is_reported_not_raised(monkeypatch): + models = Models(monkeypatch) + row = await finish(weekly(claims="judge", judge={"max_claims": 1})) + assert row.status == "succeeded", row.error + checked, doc, _ = await report_of(row.id) + assert verdicts(doc)[CITED]["status"] == "supported", "先判挂了依据的" + assert (verdicts(doc)[CAUSAL]["status"], verdicts(doc)[CAUSAL]["reason"]) == ("unjudged", "max_claims") + assert checked["judge"]["limits_hit"] == ["max_claims"] and "已到上限" in checked["judge"]["gaps"][0] + assert CAUSAL not in models.judges[0] + + +async def test_the_node_budget_overrides_the_settings_and_null_means_unlimited(monkeypatch): + """节点 judge 里写了的键盖过设置；写 null 就是不限（不是「没写」）。""" + async with SessionLocal() as session: + session.add(Setting(key="judge", value={"report_max_claims": 1, "report_max_cost_usd": 1e-9})) + await session.commit() + Models(monkeypatch) + row = await finish(weekly(claims="judge", judge={"max_cost_usd": None})) + checked, doc, _ = await report_of(row.id) + assert verdicts(doc)[CITED]["status"] == "supported" + assert verdicts(doc)[CAUSAL]["reason"] == "max_claims", "max_claims 没写，取设置里的 1" + assert checked["judge"]["limits"]["max_cost_usd"] is None and checked["judge"]["limits"]["max_claims"] == 1 + + +async def test_judging_yourself_is_called_out(monkeypatch): + Models(monkeypatch, judge_model="mock-fast") + row = await finish(weekly(claims="judge")) + assert row.status == "succeeded", row.error + [warn] = [e.data for e in await events(row.id, "log") if e.data.get("code") == "judge_same_model"] + assert "mock-fast" in warn["message"] and "自己审自己" in warn["message"] + + +# -------------------------------------------------------------------------- +# rewrite_once +# -------------------------------------------------------------------------- + +REWRITTEN = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单的贡献要等新客维度的数据再看。下面看细节。" + + +async def test_rewrite_once_hands_back_only_the_unsupported_sentences(monkeypatch): + models = Models(monkeypatch, writer=(TEXT, REWRITTEN), + verdicts={"主要来自新客": "unsupported", "要等新客维度": "not_a_claim"}) + row = await finish(weekly(claims="judge", judge={"rewrite_once": True})) + assert row.status == "succeeded", row.error + assert len(models.writes) == 2 and len(models.judges) == 2 + + request = str(models.writes[1][-1].content) + assert CAUSAL in request and "判的是：" in request and CITED not in request, "只交回不支持的那几句和理由" + # 第二次只判改过的句子：没改的那句直接用第一次的判定 + assert "要等新客维度" in models.judges[1] and CITED not in models.judges[1] + + checked, doc, output = await report_of(row.id) + assert "新客首单的贡献要等新客维度的数据再看。" in doc["markdown"] and CAUSAL not in doc["markdown"] + got = verdicts(doc) + assert got[CITED]["status"] == "supported" and got["新客首单的贡献要等新客维度的数据再看。"]["status"] == "not_a_claim" + rewrite = checked["judge"]["rewrite"] + assert rewrite["applied"] is True and rewrite["sentences"] == [CAUSAL] and checked["judge"]["reused"] == 1 + # 封存的是改写稿：units 按句子认封存文档里的编号，交回去的那句已经改掉了，对不上就是 null—— + # 不能拿原稿的编号去指改写稿里碰巧同号的另一句；改写稿里新出来的句子在 changed 里 + ids = unit_ids(doc) + assert rewrite["units"] == [None] + assert rewrite["changed"] == [ids["新客首单的贡献要等新客维度的数据再看。"]] + assert output["text"] == doc["markdown"] + [log] = [e.data for e in await events(row.id, "log") if e.data.get("code") == "report_rewrite"] + assert "1 句" in log["message"] + assert len(await judge_ends(row.id)) == 2 + + +async def test_rewrite_once_is_off_by_default(monkeypatch): + models = Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "unsupported"}) + row = await finish(weekly(claims="judge")) + assert row.status == "succeeded", row.error + assert len(models.writes) == 1 and len(models.judges) == 1 + checked, doc, _ = await report_of(row.id) + assert verdicts(doc)[CAUSAL]["status"] == "unsupported" and checked["judge"].get("rewrite") is None + + +async def test_a_rewrite_that_breaks_the_numbers_is_not_used(monkeypatch): + models = Models(monkeypatch, writer=(TEXT, "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客贡献了 45678 元。下面看细节。"), + verdicts={"主要来自新客": "unsupported"}) + row = await finish(weekly(claims="judge", judge={"rewrite_once": True})) + assert row.status == "succeeded", row.error + assert len(models.writes) == 2 and len(models.judges) == 1, "改坏了的那一稿不再花钱判" + checked, doc, output = await report_of(row.id) + assert CAUSAL in doc["markdown"] and "45678" not in doc["markdown"] + assert verdicts(doc)[CAUSAL]["status"] == "unsupported" + rewrite = checked["judge"]["rewrite"] + assert rewrite["applied"] is False and "45678" in rewrite["reason"] + # 封存的是原稿：units 就是原稿（也就是封存文档）里的编号 + assert rewrite["units"] == [unit_ids(doc)[CAUSAL]] and rewrite["changed"] == [] + + +def with_schema(monkeypatch): + """报告的目录里加一张表：实体层有了核对的底，反引号里的名字才会被认出是编造的。""" + from app.engine.evidence import _entity_entries + + real = report_mod.report_catalog + + def catalog_with_schema(state, spec, node): + return {**real(state, spec, node), **_entity_entries( + [{"artifact": "d" * 64, "source": "shop", "tables": {"orders": ["order_id", "amount"]}}], [])} + + monkeypatch.setattr(report_mod, "report_catalog", catalog_with_schema) + + +async def test_a_rewrite_that_makes_up_a_name_is_not_used(monkeypatch): + """改写稿冒出一个编造的表名（只标注、不拦的那类违规）：同样是原稿没有的新违规，不采用。""" + with_schema(monkeypatch) + models = Models(monkeypatch, writer=(TEXT, "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单的贡献要看 `refund_detail` 表。" + "下面看细节。"), + verdicts={"主要来自新客": "unsupported"}) + row = await finish(weekly(claims="judge", judge={"rewrite_once": True})) + assert row.status == "succeeded", row.error + assert len(models.writes) == 2 and len(models.judges) == 1 + checked, doc, _ = await report_of(row.id) + assert CAUSAL in doc["markdown"] and "refund_detail" not in doc["markdown"] + rewrite = checked["judge"]["rewrite"] + assert rewrite["applied"] is False and "refund_detail" in rewrite["reason"] + [log] = [e.data for e in await events(row.id, "log") if e.data.get("code") == "report_rewrite_rejected"] + assert "refund_detail" in log["message"] + + +async def test_a_rewrite_that_trades_one_violation_for_another_is_not_used(monkeypatch): + """去掉一处违规、又冒出另一处：总数没变，可冒出来的是原稿没有的，一样不采用。""" + draft = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单，共 1200 单。下面看细节。" + swapped = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单约占 3500 单。下面看细节。" + models = Models(monkeypatch, writer=(draft, draft, swapped), verdicts={"主要来自新客": "unsupported"}) + row = await finish(weekly(claims="judge", on_violation="flag", max_repairs=1, judge={"rewrite_once": True})) + assert row.status == "succeeded", row.error + assert len(models.writes) == 3 and len(models.judges) == 1 + checked, doc, _ = await report_of(row.id) + assert "1200" in doc["markdown"] and "3500" not in doc["markdown"] + rewrite = checked["judge"]["rewrite"] + assert rewrite["applied"] is False and "3500" in rewrite["reason"] and "1200" not in rewrite["reason"] + + +def test_new_violations_are_counted_one_by_one(): + """改写稿的违规逐个和原稿抵消：原样留着的不算新的，同一个裸数字多写一处、只标注的那类，都算新的。""" + number = {"code": "uncited_number", "text": "1200", "message": "数字「1200」没有出处"} + name = {"code": "unknown_entity", "text": "refund_detail", "message": "可能是编造的名字"} + see = {"code": "unresolved_ref", "ref": "Q9", "message": "依据 [[see:Q9]] 解析不了"} + assert report_mod._new_violations([number, name, see], [see, number, name]) == [] + assert report_mod._new_violations([number], [number, number]) == [number] + assert report_mod._new_violations([number], [name]) == [name] + assert report_mod._new_violations([number, see], []) == [] + + +async def test_a_second_round_that_hits_a_limit_names_the_configured_limit(monkeypatch): + """第二轮用的是第一轮剩下的钱，可说给人看的上限是节点上配的那个数，不是剩下的零头。""" + Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "unsupported"}) + # 每句估 1 分钱、回复不带用量（实际就按估算记）：第一轮两句花掉 2 分，第二轮只剩半分，一句也装不下 + monkeypatch.setattr(judge_mod.JudgeRequest, "cost", lambda self, model_id, units: 0.01 * len(units)) + monkeypatch.setattr(judge_mod, "_usage_of", lambda message, model_id: {"input_tokens": 0, "output_tokens": 0}) + row = await finish(weekly(claims="judge", judge={"rewrite_once": True, "max_cost_usd": 0.025})) + assert row.status == "succeeded", row.error + checked, doc, _ = await report_of(row.id) + v = verdicts(doc)["新客首单的贡献要等新客维度的数据再看。"] + assert (v["status"], v["reason"]) == ("unjudged", "max_cost_usd") + assert "$0.025" in v["rationale"] and "0.005" not in v["rationale"], v["rationale"] + summary = checked["judge"] + assert summary["rewrite"]["applied"] is True and summary["limits"]["max_cost_usd"] == 0.025 + assert "$0.025" in summary["gaps"][0] and "0.005" not in summary["gaps"][0], summary["gaps"] + [log] = [e.data for e in await events(row.id, "log") if e.data.get("code") == "judge_limit"] + assert "$0.025" in log["message"] and "0.005" not in log["message"] + + +# -------------------------------------------------------------------------- +# 重放安全 +# -------------------------------------------------------------------------- + + +async def test_continuing_after_a_failure_does_not_judge_or_charge_twice(monkeypatch): + models = Models(monkeypatch) + real = report_mod._store + broken = {"on": True} + + async def flaky(doc, ctx): + if broken["on"]: + raise RuntimeError("落盘时断了") + return await real(doc, ctx) + + monkeypatch.setattr(report_mod, "_store", flaky) + run = await run_manager.start(graph=weekly(claims="judge"), input_payload={}, run_class="formal") + row = await wait(run.id) + assert row.status == "failed" and "落盘时断了" in (row.error or ""), row.error + assert len(models.judges) == 1 + before = await daily_spend() + await run_manager.wait_idle(run.id) + + broken["on"] = False + await run_manager.continue_failed(run.id) + row = await wait(run.id) + assert row.status == "succeeded", row.error + assert len(models.judges) == 1, "接着跑时裁判又调了一次：裁判的结果没进 checkpoint" + assert await daily_spend() == before, "接着跑时每日计数又记了一次" + assert len(await judge_ends(row.id)) == 1 + _, doc, _ = await report_of(row.id) + assert verdicts(doc)[CITED]["status"] == "supported" + + +async def test_a_changed_draft_is_judged_afresh_when_continuing(monkeypatch): + """接着跑时写作者重写了一稿（写作调用不进 checkpoint），缓存的判定是上一稿的：不能拿来用。""" + models = Models(monkeypatch, writer=(TEXT, "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自老客复购。下面看细节。")) + real = report_mod._store + broken = {"on": True} + + async def flaky(doc, ctx): + if broken["on"]: + raise RuntimeError("落盘时断了") + return await real(doc, ctx) + + monkeypatch.setattr(report_mod, "_store", flaky) + run = await run_manager.start(graph=weekly(claims="judge"), input_payload={}, run_class="formal") + assert (await wait(run.id)).status == "failed" + await run_manager.wait_idle(run.id) + broken["on"] = False + await run_manager.continue_failed(run.id) + row = await wait(run.id) + assert row.status == "succeeded", row.error + assert len(models.judges) == 2 and "老客复购" in models.judges[1] + _, doc, _ = await report_of(row.id) + assert verdicts(doc)["增长主要来自老客复购。"]["status"] == "supported" + + +async def test_runs_from_before_the_upgrade_never_judge(monkeypatch): + """升级前发起的运行（没有护栏快照）：claims 一律按 off，节点里一个 task 都不多调用，产出一个键都不多。""" + async def no_snapshot(session): + return None + + monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) + models = Models(monkeypatch) + calls = spy_tasks(monkeypatch) + row = await finish(weekly(claims="judge")) + assert row.status == "succeeded", row.error + assert models.judges == [] and calls == [] + checked, doc, output = await report_of(row.id) + assert "judge" not in checked and "claims" not in checked and "judge" not in output and "judge" not in doc + assert all(v is None for v in verdicts(doc).values()) + + +async def test_runs_paused_at_an_approval_resume_with_tasks_in_place(monkeypatch): + """停在审批上的运行恢复：审批节点照旧只问一次；报告节点在恢复之后才第一次执行，裁判 task 只调一次、 + 排在节点所有已有调用之后。没开裁判的报告节点一个 task 都不调用（和升级前一样）。""" + assert replay.PROTOCOL == 2, "没有改动已有 task 的顺序，重放协议不升版" + for claims, expected in (("require_citation", []), ("judge", ["judge_step"])): + models = Models(monkeypatch) + calls = spy_tasks(monkeypatch) + paused = await finish(weekly(gate=True, claims=claims), statuses=("interrupted", "failed")) + assert paused.status == "interrupted", paused.error + await run_manager.resume(paused.id, {"approved": True}) + row = await wait(paused.id) + assert row.status == "succeeded", row.error + assert calls == expected, (claims, calls) + assert len(models.judges) == len(expected) + assert len(await events(row.id, "human.requested", "gate")) == 1 + checked, _, _ = await report_of(row.id) + assert ("judge" in checked) is (claims == "judge") diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index b88453bf..dfb870b3 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,6 +1,6 @@ import type { Approval, AutofixResult, Conversation, ConversationDetail, ConversationTurn, CustomTool, DataSource, EvidenceGraph, - EvidenceAudit, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run, + EvidenceAudit, EvidenceJudgeResult, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run, RunEvent, RunStatus, Skill, ToolChange, ToolInfo, ToolTrust, ValidationIssue, VarIssue, Variable, Workflow, WorkflowVersion, } from '../types' @@ -545,6 +545,9 @@ export const api = { settings: { get: () => get>('/settings'), put: (values: Record) => put>('/settings', { values }), + /** 今天（本地日期）结论句裁判花了多少：设置页放在每日上限旁边。只读，计数只有裁判自己记 */ + judgeSpend: () => get<{ date?: string; usd?: number; calls?: number; unpriced_calls?: number + daily_max_usd?: number | null }>('/settings/judge/spend'), }, // ---- 工具 ---- @@ -708,6 +711,13 @@ export const api = { */ audit: (runId: string, opts?: RequestOptions & { groups?: string[] }) => get(`/runs/${encodeURIComponent(runId)}/evidence/audit${qs({ groups: opts?.groups?.join(',') || undefined })}`, opts), + /** + * 按需裁判（四期，只在探索运行里有）：请模型判断这几句结论。答复是 evidence.judged 事件的载荷 + * （判定、花费、触顶的上限），判定标 post_seal：落在封存之后，不改封存的文档。正式运行回 409。 + * 一次运行里有好几份报告时 report 给报告节点 id + */ + judge: (runId: string, body: { units: string[]; report?: string }) => + post(`/runs/${encodeURIComponent(runId)}/evidence/judge`, body), /** * 导出审计表：?format=json / csv 作为附件下载。回来的是文件本身和后端给的文件名 * （Content-Disposition；没给就用 evidence-<运行号前 8 位>.<格式>） diff --git a/frontend/src/canvas/Inspector.tsx b/frontend/src/canvas/Inspector.tsx index f2051b29..c6106427 100644 --- a/frontend/src/canvas/Inspector.tsx +++ b/frontend/src/canvas/Inspector.tsx @@ -8,11 +8,11 @@ import { fieldOfIssue, unboundToolOf, withToolBound, type FieldRef } from './iss import { hintOf } from './shortcuts' import { isActivePhase } from '../run/trace' import { EDIT_LOCK_TEXT, useEditLock, useStudio } from '../store/studio' -import { datasourceTools, modelOptions, useCatalog, useDatasources } from '../store/catalog' +import { datasourceTools, modelOptions, providerOfModel, useCatalog, useDatasources } from '../store/catalog' import { api } from '../api/client' import { IconButton, JsonInput, Modal, isComposing, useRadioGroup } from '../components/ui' import { formatShortcut } from '../lib/keys' -import { SUBGRAPH_UPGRADE_HELP, upgradeNewerText } from '../lib/terms' +import { JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_LABEL, JUDGE_SETTING_TEXT, SUBGRAPH_UPGRADE_HELP, judgeByModelText, upgradeNewerText } from '../lib/terms' import { TemplateText } from './TemplateText' import type { NodeType, ValidationIssue, WorkflowVersion } from '../types' @@ -403,9 +403,10 @@ function Field({ field, nodeId, value, config, issues, onChange }: { const id = `f-${nodeId}-${field.key}` const syntax = syntaxOf(field) const composite = field.type === 'cases' || field.type === 'agents' || field.type === 'ioFields' - || field.type === 'metricsList' || field.type === 'caliberFrom' - // 复合字段自己把问题落到第几项；落不到具体某一项的，和普通字段一样挂在下面 - const own = composite ? issues.filter((i) => i.at?.index == null) : issues + || field.type === 'metricsList' || field.type === 'caliberFrom' || field.type === 'judge' + // 复合字段自己把问题落到第几项；落不到具体某一项的，和普通字段一样挂在下面。结论句裁判按子键（judge.max_cost_usd）落 + const own = field.type === 'judge' ? issues.filter((i) => !i.at?.sub || !JUDGE_SUB_KEYS.has(i.at.sub)) + : composite ? issues.filter((i) => i.at?.index == null) : issues const bad = own.some((i) => i.level === 'error') const errorId = own.length ? `${id}-issues` : undefined // 长文本在 360px 宽的栏里没法写：提示词、代码可以展开到大编辑器里（同一份值，边写边存） @@ -608,6 +609,9 @@ function FieldInput({ field, id, nodeId, syntax, value, config, invalid, describ case 'caliberFrom': return + case 'judge': + return + case 'agents': return @@ -616,6 +620,157 @@ function FieldInput({ field, id, nodeId, syntax, value, config, invalid, describ } } +/** judge 子配置里各有自己一行的键：落在这些键上的问题挂在那一行下面 */ +const JUDGE_SUB_KEYS = new Set(['provider', 'model', 'max_claims', 'max_cost_usd', 'timeout_s', 'rewrite_once', 'on_unsupported']) + +/** + * 结论句裁判（claims: judge）的子配置：裁判模型、三项上限（各自可以设成不限）、改写一次、证据不支持时怎么判档。 + * + * 上限有三种写法，得分得清：没写（跟随设置里「证据裁判」的默认值）、写数、写 null（不限）。勾「不限」写 null， + * 清空数字框回到没写。全都没写时整个 judge 去掉，不留一个空对象。受管级别发布要求写了 max_cost_usd（数或 null） + */ +function JudgeConfig({ id, value, config, issues, onChange }: { + id: string; value: any; config: Record; issues: FieldIssue[]; onChange: (v: any) => void +}) { + const providers = useCatalog((s) => s.providers) + const judge: Record = value && typeof value === 'object' && !Array.isArray(value) ? value : {} + const set = (key: string, v: unknown) => { + const next = { ...judge } + if (v === undefined) delete next[key] + else next[key] = v + onChange(Object.keys(next).length ? next : undefined) + } + const issuesOf = (key: string) => issues.filter((i) => i.at?.sub === key) + const lines = (key: string) => issuesOf(key).map((i, k) => ) + const provider = typeof judge.provider === 'string' ? judge.provider : '' + const pv = provider ? providers.find((p) => p.name === provider) : undefined + const model = typeof judge.model === 'string' ? judge.model : '' + // 和写报告的模型是同一个：等于自己审自己。校验也会报（report.judge_same_model），报了就不重复说 + const same = !!model && model === config.model && !issuesOf('model').length + // 节点上只写了模型：这一级写了就整组用节点上的（judge_model_spec 不跨级拼），后端按模型名找接入—— + // 不再跟随设置，接入留空那一项不能还叫「跟随设置」 + const byModel = !provider && !!model.trim() + const owner = byModel ? providerOfModel(providers, model.trim()) : undefined + const enabled = providers.filter((p) => p.enabled) + const fallback = enabled.find((p) => p.kind !== 'mock') ?? enabled[0] + const byModelHint = byModel && providers.length > 0 + ? judgeByModelText('node', model.trim(), owner && { name: owner.provider.name, enabled: owner.enabled }, fallback?.name) + : '' + const models = pv ? (pv.models ?? []).map((m) => m.id) : [...new Set(modelOptions(providers).map((o) => o.value))] + return ( +
+
+
+
+ + +
+
+ + set('model', e.target.value.trim() ? e.target.value : undefined)} /> + {models.map((m) => +
+
+
+ 接入和模型都没写就用设置里的「证据裁判模型」；写了哪一项就整组用节点上的，不和设置拼。建议和写报告的模型不同：同一个模型审自己写的，写错的地方它多半也看不出来 +
+ {byModelHint && ( +
+ {byModelHint} +
+ )} + {same && ( +
+ 裁判模型和写报告的模型都是「{model}」：等于自己审自己 +
+ )} + {lines('provider')}{lines('model')} +
+ {(['max_claims', 'max_cost_usd', 'timeout_s'] as const).map((key) => ( + set(key, v)} /> + ))} +
+ + {lines('rewrite_once')} +
+
+ + +
只管正式运行；探索运行只标注，不拦
+ {lines('on_unsupported')} +
+
+ ) +} + +/** 一项裁判上限：数字框加「不限」。没写、写数、写 null（不限）三种各有一句说明 */ +function JudgeLimit({ id, name, value, issues, onChange }: { + id: string; name: 'max_claims' | 'max_cost_usd' | 'timeout_s'; value: unknown; issues: React.ReactNode + onChange: (v: number | null | undefined) => void +}) { + const off = value === null + const shown = typeof value === 'number' || typeof value === 'string' ? String(value) : '' + const note = off ? JUDGE_LIMIT_OFF[name] + : value === undefined ? '没写：用设置里「证据裁判」的默认值' : '' + return ( +
+ +
+ onChange(e.target.value === '' ? undefined : Number(e.target.value))} /> + +
+ {note && ( +
+ {note} +
+ )} + {issues} +
+ ) +} + +/** 节点上某项设成不限时写明：费用还受什么约束（同设置页，节点上的只说这一份报告） */ +const JUDGE_LIMIT_OFF: Record<'max_claims' | 'max_cost_usd' | 'timeout_s', string> = { + max_claims: '不设上限，句数只受报告里结论句多少约束', + max_cost_usd: '不设上限，这份报告的裁判费用只受句数、时长上限和每日上限约束', + timeout_s: '不设上限，时长只受模型接口自身的超时约束', +} + /** * 值里能写 {{ }} 的 JSON（工具参数、子工作流入参、出具契约）。 * diff --git a/frontend/src/canvas/NodeCard.tsx b/frontend/src/canvas/NodeCard.tsx index 6d756e2d..a84cf969 100644 --- a/frontend/src/canvas/NodeCard.tsx +++ b/frontend/src/canvas/NodeCard.tsx @@ -16,7 +16,7 @@ import { matchedSource } from '../lib/evidence' import { explainRunError } from '../lib/explain' import { formatDuration, formatLapse, formatNumber, formatTokens, NONE, shortId } from '../lib/format' import { statusMeta } from '../lib/status' -import { issuanceLabel, reportStampText, RUN_CLASS_LABEL } from '../lib/terms' +import { claimTally, issuanceLabel, reportStampText, RUN_CLASS_LABEL } from '../lib/terms' import { useReportStamp, type ReportStamp } from './reportStamp' import { ApprovalCard } from '../run/RunPanel' import type { NodeState, NodeTrace } from '../run/trace' @@ -188,6 +188,13 @@ function matchedLines(matched: unknown): string { return `回指明细：\n${lines.join('\n')}${more}` } +/** stats 里的裁判句数（claims 为 judge 时才有）→「结论 4 句（支持 1 · 不支持 1 · 未裁判 2）」 */ +function claimsLine(st: ReportStamp['stats']): string { + const n = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) ? v : 0) + const c = { supported: n(st.supported), partial: n(st.partial), unsupported: n(st.unsupported), unjudged: n(st.unjudged), uncited: 0 } + return claimTally({ ...c, total: c.supported + c.partial + c.unsupported + c.unjudged }) +} + /** 报告节点章的悬停说明：两个数怎么来的，缺口里各有什么 */ function reportStampTitle(r: ReportStamp): string { const st = r.stats @@ -200,7 +207,10 @@ function reportStampTitle(r: ReportStamp): string { n(st.unresolved) ? `引用解析不了 ${formatNumber(n(st.unresolved))} 处` : '', n(st.unknown_entities) ? `可能是编造的名字 ${formatNumber(n(st.unknown_entities))} 个` : '', n(st.unverified_entities) ? `核对不了的名字 ${formatNumber(n(st.unverified_entities))} 个（只标注，不计入）` : '', - uncited ? `没挂依据的结论句 ${formatNumber(uncited)} 句${r.claims === 'require_citation' ? '（计入缺口）' : '（结论句策略不要求，不计入）'}` : '', + uncited ? `没挂依据的结论句 ${formatNumber(uncited)} 句${r.claims === 'require_citation' || r.claims === 'judge' + ? '（计入缺口）' : '（结论句策略不要求，不计入）'}` : '', + // 结论句裁判（四期）：模型的判断，不计入章上的两个数 + r.claims === 'judge' && claimsLine(st) ? `${claimsLine(st)}（模型判断，非确定，不计入）` : '', '完整清单在记录页这次运行的「证据」页签', ].filter(Boolean).join('\n') } diff --git a/frontend/src/canvas/issues.ts b/frontend/src/canvas/issues.ts index bea9ca82..f1ba3cbc 100644 --- a/frontend/src/canvas/issues.ts +++ b/frontend/src/canvas/issues.ts @@ -11,7 +11,7 @@ * 那一栏，定位落到工具上（见 toolBindingOf）。 */ import { ApiError } from '../api/client' -import { APPROVAL_POLICY_LABEL } from '../lib/terms' +import { APPROVAL_POLICY_LABEL, JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_LABEL } from '../lib/terms' import { NODE_DEFS } from './nodeDefs' import type { FlowNode } from '../store/studio' import type { AutofixResult, GraphSpec, PublishCheck, PublishFix, PublishLevel, ValidationIssue } from '../types' @@ -370,14 +370,33 @@ export type NodeNameOf = (id: string) => string | undefined const isRecord = (v: unknown): v is Record => !!v && typeof v === 'object' && !Array.isArray(v) -/** 修复改的是哪个字段：顶层键用节点定义里的叫法，契约里的键另有一张表，其余原样 */ +/** 修复改的是哪个字段：顶层键用节点定义里的叫法，契约里的键另有一张表，结论句裁判（judge.*）另一张，其余原样 */ export function fixFieldLabel(field: string | null | undefined, nodeType?: string): string { if (!field) return '' const [head, ...rest] = field.split('.') if (head === 'label') return '节点名称' const def = nodeType ? NODE_DEFS[nodeType as keyof typeof NODE_DEFS] : undefined const base = def?.fields.find((f) => f.key === head)?.label ?? EXTRA_FIELDS[head] ?? head - return [base, ...rest.map((k) => SUB_KEYS[k] ?? k)].join(' · ') + const sub = head === 'judge' ? JUDGE_FIELD_LABEL : SUB_KEYS + return [base, ...rest.map((k) => sub[k] ?? k)].join(' · ') +} + +/** judge 里写 null 表示不限的三项上限：预览里 null 写「不限」，不写「（空）」——没写和不限是两回事 */ +const JUDGE_LIMITS = new Set(['max_claims', 'max_cost_usd', 'timeout_s']) + +/** judge.* 一项的值怎么写：上限带单位，null 是不限，判档写选项文字。不是 judge 的键返回 null */ +function judgeValueText(value: unknown, key: string): string | null { + if (JUDGE_LIMITS.has(key)) { + if (value === null) return '不限' + if (typeof value === 'number' && Number.isFinite(value)) { + return key === 'max_cost_usd' ? `$${value}` : key === 'timeout_s' ? `${value} 秒` : `${value} 句` + } + return null + } + if (key === 'on_unsupported' && typeof value === 'string' && value in JUDGE_ON_UNSUPPORTED_LABEL) { + return JUDGE_ON_UNSUPPORTED_LABEL[value] + } + return null } /** @@ -386,6 +405,10 @@ export function fixFieldLabel(field: string | null | undefined, nodeType?: strin * 其余压成一行 */ export function fixValueText(value: unknown, field?: string | null, nameOf?: NodeNameOf): string { + if (field?.startsWith('judge.')) { + const said = judgeValueText(value, field.slice(6)) + if (said != null) return said + } if (value === undefined || value === null || value === '' || (Array.isArray(value) && !value.length) || (isRecord(value) && !Object.keys(value).length)) return '（空）' const key = field?.split('.').pop() @@ -414,6 +437,14 @@ export function fixValueText(value: unknown, field?: string | null, nameOf?: Nod * 不是整份契约的返回 null */ export function fixValueLines(value: unknown, field?: string | null, nameOf?: NodeNameOf): string[] | null { + // 整份结论句裁判（补预算那条修复按整份 judge 记）：一键一行，「金额上限（美元）：不限」 + if (field === 'judge' && isRecord(value) && Object.keys(value).length) { + const order = Object.keys(JUDGE_FIELD_LABEL) + return Object.keys(value).sort((a, b) => { + const [x, y] = [order.indexOf(a), order.indexOf(b)] + return (x < 0 ? order.length : x) - (y < 0 ? order.length : y) + }).map((k) => `${JUDGE_FIELD_LABEL[k] ?? k}：${fixValueText(value[k], `judge.${k}`, nameOf)}`) + } if (field !== 'contract' || !isRecord(value) || !Object.keys(value).length) return null const order = Object.keys(CONTRACT_KEYS) const keys = Object.keys(value).sort((a, b) => { diff --git a/frontend/src/canvas/nodeDefs.ts b/frontend/src/canvas/nodeDefs.ts index b31ff9d8..825f5bd7 100644 --- a/frontend/src/canvas/nodeDefs.ts +++ b/frontend/src/canvas/nodeDefs.ts @@ -8,7 +8,7 @@ import type { NodeType } from '../types' export type FieldType = | 'text' | 'textarea' | 'prompt' | 'code' | 'number' | 'select' | 'switch' | 'json' | 'model' | 'tools' | 'skills' | 'collection' - | 'ioFields' | 'cases' | 'agents' | 'metricsList' | 'nodeRefs' | 'caliberFrom' + | 'ioFields' | 'cases' | 'agents' | 'metricsList' | 'nodeRefs' | 'caliberFrom' | 'judge' /** * 这个字段里写的是什么语法。 @@ -564,17 +564,24 @@ export const NODE_DEFS: Record = { help: '有违规时把清单交回去让模型重写。每次都是一整篇的调用，写不对的模型多给几次也大多写不对', }, { - // 后端只认 off / require_citation（schema.REPORT_CLAIMS）；judge 是四期的模型裁判，写了会被校验拦下。 - // 没写等于 off：select 显示第一项，正好是默认 + // 后端认 off / require_citation / judge（schema.CLAIMS_VALUES）。没写等于 off：select 显示第一项，正好是默认 key: 'claims', label: '没挂依据的结论句', type: 'select', options: [ { value: 'off', label: '不管（默认）：不参与出具判档' }, { value: 'require_citation', label: '计入缺口：出具按档位降档' }, + { value: 'judge', label: '计入缺口，并请模型逐句判断证据支不支持' }, ], help: '结论句是带数字、方向词或因果词的句子。选「计入缺口」时，写作提示要求每句结论挂上 [[see:…]] 依据，' - + '没挂的记为缺口、出具降档（系统自动链接的表名字段名不算依据）。受管级别发布必须显式选它。' - + '结论句裁判在后续版本支持', - unknownLabel: (v) => (v === 'judge' ? 'judge：结论句裁判在后续版本支持，现在不能用' : `${v}：不认识的值，只能选上面两项`), + + '没挂的记为缺口、出具降档（系统自动链接的表名字段名不算依据）。选「请模型逐句判断」时，另一个模型按证据' + + '判断每句结论：正式运行里当场判、结果随报告封存，探索运行点开哪句判哪句；判断是模型给的，不是系统核对。' + + '受管级别发布必须显式选后两项之一，选模型判断的还要在高级选项里写金额上限（或写不限）', + unknownLabel: (v) => `${v}：不认识的值，只能选上面三项`, + }, + { + // claims 为 judge 时的子配置（schema.JUDGE_KEYS）。没写的上限用设置里「证据裁判」的默认值；写 null 是不限 + key: 'judge', label: '结论句裁判', type: 'judge', advanced: true, + when: (c) => c.claims === 'judge' || (!!c.judge && typeof c.judge === 'object'), + help: '没写的上限用设置里「证据裁判」的默认值；勾「不限」写 null，只受其余上限约束。裁判模型建议和写报告的模型不同', }, { // 后端只认 link / off（report.py 的 ENTITIES） diff --git a/frontend/src/dev/preview.tsx b/frontend/src/dev/preview.tsx index 1e5032c6..c5320ae2 100644 --- a/frontend/src/dev/preview.tsx +++ b/frontend/src/dev/preview.tsx @@ -11,6 +11,7 @@ import type { RunEvent } from '../types' import fixtures from '../run/__tests__/fixtures.json' import mdSamples from '../run/__tests__/markdown-samples.json' import evidenceFixture from '../run/__tests__/evidence-doc.json' +import evidenceJudge from '../run/__tests__/evidence-judge.json' import { CALIBER_ONE_OUTPUT, CALIBERS_OUTPUT, COPILOT_STUCK, COPILOT_TOOLS_DROPPED, MIXED_OUTPUT, abandonedRun, cancelledRun, exhaustedTeam, longLoop, markupRun, mixedRun, pipelineRun, repairRun, teamRun, timeoutRun, @@ -234,11 +235,29 @@ function evidenceTurns(): StreamTurn[] { ] } +/** + * 四期：结论句裁判走问数据页、画布右栏同一条路（Output → EvidenceField → 出具横幅）。 + * - judge-plain：探索运行、报告没开裁判（require_citation）。点开结论句请模型判断之后，横幅得多出「结论 N 句」那一段； + * - judge-formal：正式运行、报告 claims=judge，节点里判好了。没挂依据的那句送了裁判（到上限没判），出具照样按没挂依据记缺口。 + * 报告工件、片段接口、按需裁判的 POST 由 check-evidence 用 page.route 伪造（夹具 evidence-judge.json） + */ +function evidenceJudgeTurns(): StreamTurn[] { + const { plain, formal } = evidenceJudge + return [ + { id: 'judge-plain', question: '写一份本周周报', phase: 'done', status: '已完成', runClass: 'exploratory', + steps: [], output: plain.output, runId: plain.run_id }, + { id: 'judge-formal', question: '写一份本周周报（受管模板）', phase: 'done', status: '已完成', runClass: 'formal', + steps: [], output: formal.output, runId: formal.run_id }, + ] +} + /** 合成场景：新后端才有的事件，老库里导不出来 */ function synthetic(name: string): StreamTurn[] { switch (name) { case 'evidence': return evidenceTurns() + case 'evidence-judge': + return evidenceJudgeTurns() case 'team-live': { const ev = rebase(teamRun('members')) return [{ id: 'team-live', question: '比较三家供应商的交期风险', phase: 'running', status: '运行中', diff --git a/frontend/src/dev/ui-harness.tsx b/frontend/src/dev/ui-harness.tsx index 0d9279d1..1ca7cf3a 100644 --- a/frontend/src/dev/ui-harness.tsx +++ b/frontend/src/dev/ui-harness.tsx @@ -27,6 +27,7 @@ import { Markdown } from '../run/Markdown' import evidenceFixture from '../run/__tests__/evidence-doc.json' import evidenceQuery from '../run/__tests__/evidence-query.json' import evidenceEntity from '../run/__tests__/evidence-entity.json' +import evidenceJudge from '../run/__tests__/evidence-judge.json' import { EvidenceGuessView } from '../run/EvidenceGuess' import { IssuanceBanner } from '../run/AssistantStream' import mdPinned from '../run/__tests__/markdown-pinned.json' @@ -194,6 +195,10 @@ const EV_QUERY = evidenceQuery.doc as unknown as EvidenceDocData const EV_LEGACY = evidenceFixture.legacy /** 第三期：表名字段名、可疑实体、核对不了的名字、逐字引文（evidence-entity.json，compose_doc 真跑出来的） */ const EV_ENTITY = evidenceEntity.doc as unknown as EvidenceDocData +/** 第四期：结论句裁判。正式运行节点里判好的（inline）、探索运行按需判的（on_demand）、没开裁判的探索运行 */ +const EV_JUDGE = evidenceJudge.formal.doc as unknown as EvidenceDocData +const EV_JUDGE_EXPLORE = evidenceJudge.explore.doc as unknown as EvidenceDocData +const EV_JUDGE_PLAIN = evidenceJudge.plain.doc as unknown as EvidenceDocData /** 同一份旧契约的出具，去掉位置：老运行就是这样，只能按字符串标 */ const LEGACY_LOOSE = { matched: EV_LEGACY.matched.map(({ start: _s, end: _e, ...m }) => m), @@ -306,6 +311,60 @@ function EvidenceOnly() { ))} + {/* + 第四期：结论句裁判。片段接口、按需裁判接口由 check-evidence 用 page.route 伪造（按 runId 认出夹具）。 + 探索运行没开裁判的那份不给运行类别：面板按运行 id 取一次 + */} +
+
+
正式运行 · 节点里当场判的结论句（进封存）
+ +
+
+ +
+
+
探索运行 · 按需裁判（点开哪句判哪句，封存后追加）
+ +
+
+ +
+
+
探索运行 · 没开裁判的报告（照样能点开结论句请模型判断）
+ +
+ {/* 出具横幅那一行多一段结论句：「结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）」 */} +
+ {/* + judge-uncited：开了裁判的文档，没挂依据的那句送了裁判、按判定数在「支持 3」里（或预筛放掉了不数）， + 出具（io.py）照样按没挂依据记 1 句——横幅得另起「没挂依据的结论句 1」，不能被结论句那一段吞掉 + */} + {([ + ['judged', { total: 4, supported: 1, partial: 1, unsupported: 1, unjudged: 1, uncited: 0 }, null], + ['cited', { total: 4, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 1 }, null], + ['all', { total: 2, supported: 2, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }, null], + ['judge-uncited', { total: 3, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }, + { policy: 'judge', uncited_claims: 1 }], + ] as const).map(([key, claims, issued]) => ( +
+ { const w = window as any; w.__claimsNext = (w.__claimsNext ?? 0) + 1 }, + }} /> +
+ ))} +
+
长报告：{EV_LONG.blocks.length} 块，块级 content-visibility，超过 3000 字先折叠
@@ -599,6 +658,7 @@ function Harness() { evidenceFixture, evidenceQuery, evidenceEntity, + evidenceJudge, } createRoot(document.getElementById('root')!).render( diff --git a/frontend/src/index.css b/frontend/src/index.css index 406a1cce..1c44f094 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1639,6 +1639,39 @@ vertical-align: 1px; color: var(--st-waiting); } +/* 结论句的判定（四期）：句末上标的小字形，是按钮但不像按钮。颜色随判定（组件写成 --ev-glyph）， + 绝不用确定性的绿；整句的字不画线——判定是模型给的，不能长得像系统核对过的数字 */ +.ev-badge { + display: inline-block; + margin: 0 1px 0 2px; + padding: 0 3px; + border: 0; + border-radius: 3px; + background: none; + font: inherit; + font-size: 0.72em; + font-weight: 700; + line-height: 1; + vertical-align: super; + color: var(--ev-glyph, currentColor); + text-decoration: none; + cursor: pointer; +} +.ev-badge:hover, +.ev-badge[aria-expanded="true"] { background: var(--ev-soft, transparent); } +.ev-badge:focus-visible, +.ev-badge[data-flash] { + outline: 2px solid var(--accent); + outline-offset: 1px; +} +/* 证据不支持：整句铺一层浅底，折行时每一行各自带圆角；不画线、不改字色 */ +.ev-claim { + background: var(--ev-soft, transparent); + border-radius: 2px; + text-decoration: none; + -webkit-box-decoration-break: clone; + box-decoration-break: clone; +} .ev-panel-side { animation: sheet-in var(--dur-base) var(--ease-out); } .ev-panel-rise { animation: fade-up var(--dur-base) var(--ease-out); } diff --git a/frontend/src/lib/evidence.ts b/frontend/src/lib/evidence.ts index 7ac7e060..d7d6af5d 100644 --- a/frontend/src/lib/evidence.ts +++ b/frontend/src/lib/evidence.ts @@ -11,17 +11,22 @@ * 三期多了表名字段名、逐字引文：有出处的实体、引文和数字同一套「有出处」线型（状态就是 * 确定性，另有 EVIDENCE_KIND_STYLE 给它们各自的字形和说法）；可疑实体（可能是编造的 * 名字）和核对不了的名字是无证据的两个变体，线型同无证据、字形和文字各不相同；旧运行 - * 按数值猜的候选用最淡的点状线。概率性的几种先把外观定下，四期接上时不用再各处补。 + * 按数值猜的候选用最淡的点状线。 + * + * 四期接上了概率性的几种：结论句的判定是模型给的，只在句末挂一枚小徽标（◆ 有依据、◇ 部分有依据、 + * ! 证据不支持且整句浅底、? 未裁判），整句的字不画线。判定有两个来源：封存的文档里节点当场判的， + * 和探索运行里点开再判、封存之后追加的（后者盖过前者，另写「封存后追加」）。 */ import { - EVIDENCE_AUDIT_TEXT, EVIDENCE_KIND_LABEL, EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, UPGRADE_POLICY_LABEL, evidenceTally, + EVIDENCE_AUDIT_TEXT, EVIDENCE_KIND_LABEL, EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, JUDGE_TEXT, UPGRADE_POLICY_LABEL, + claimTally, evidenceTally, type ClaimTallyCounts, } from './terms' import { NONE, formatNumber, shortId } from './format' import type { EvidenceBlock, EvidenceCaliberSource, EvidenceCaliberUpgrade, EvidenceCandidate, EvidenceDocData, EvidenceFieldRef, EvidenceGraph, EvidenceGuess, EvidenceInput, EvidenceLocator, EvidenceSeal, EvidenceSegment, EvidenceSegmentDetail, - EvidenceStats, EvidenceStep, EvidenceUnit, EvidenceViolation, ReviewResult, + EvidenceStats, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation, ReviewResult, } from '../types' import type { MarkSpec } from '../run/Markdown' @@ -85,7 +90,7 @@ export const EVIDENCE_STATE: Record = { code: 'unjudged', label: EVIDENCE_STATE_LABEL.unjudged, line: 'badge', glyph: '?', color: 'var(--st-cancelled)', decoration: 'transparent', soft: 'var(--st-cancelled-soft)', alert: false, phase: 4, - hint: '这句话带了引用，还没请模型判断引用支不支持它', + hint: '还没请模型判断证据支不支持这句话：探索运行点开再判，或者到了上限没判', }, none: { code: 'none', label: EVIDENCE_STATE_LABEL.none, line: 'dotted', glyph: '?', @@ -452,6 +457,8 @@ export function locatable(doc: EvidenceDocData, v: EvidenceViolation): EvidenceS export interface EvidenceTally { /** 数字总数：带引用的数字片段 + 裸数字（含列表序号、代码块标签里的） */ total: number + /** 四期：结论句按判定分的句数。文档没开裁判、也没有按需判过的句子时是 null（横幅不多说这一段） */ + claims?: ClaimTallyCounts | null /** 可疑实体（可能是编造的名字）、核对不了的名字：数字那半句之外另说 */ suspect?: number unverified?: number @@ -504,7 +511,7 @@ export function docTally(doc: EvidenceDocData): EvidenceTally { const suspect = num(st.unknown_entities) ?? all.filter((v) => v.code === 'unknown_entity').length const unverified = num(st.unverified_entities) ?? all.filter((v) => v.code === 'unverified_entity').length return { total, cited, none, other, hidden: hidden.length, structural, noSegment: hidden.length - structural, - suspect, unverified } + suspect, unverified, claims: claimCounts(doc) } } /** @@ -522,15 +529,16 @@ export function statsTally(stats: EvidenceDocData['stats'] | null | undefined): return { total, cited, none, other } } -/** 读屏摘要：计数、画不了线的两种各在哪、按键说明 */ +/** 读屏摘要：计数、结论句的判定、画不了线的两种各在哪、按键说明 */ export function tallySummary(t: EvidenceTally, keys: boolean): string { return [ evidenceTally(t.cited, t.total, t.other), + t.claims?.total ? `${claimTally(t.claims)}（模型判断，非确定）` : '', t.suspect ? `${formatNumber(t.suspect)} 个${EVIDENCE_STATE.suspect.label}` : '', t.unverified ? `${formatNumber(t.unverified)} 个名字${EVIDENCE_STATE.unverified.label}` : '', t.structural ? EVIDENCE_TEXT.structuralCount(t.structural) : '', t.noSegment ? EVIDENCE_TEXT.noSegmentCount(t.noSegment) : '', - keys ? EVIDENCE_TEXT.keysHint : '', + keys ? (t.claims?.unsupported || t.claims?.partial ? JUDGE_TEXT.keysHint : EVIDENCE_TEXT.keysHint) : '', ].filter(Boolean).join('。') } @@ -812,7 +820,8 @@ export function evidenceTrace( /** * 章上的两个数：「引用 N」是有出处的片段（数字、值、表名字段名、引文），「无证据 M」是没出处的 - * 数字、解析不了的引用、可疑实体，结论句策略为 require_citation 时再加上没挂依据的结论句。 + * 数字、解析不了的引用、可疑实体，结论句策略为 require_citation 或 judge 时再加上没挂依据的结论句 + * （judge 在挂依据这件事上和 require_citation 一样严，出口同样计入缺口）。 * 统计缺了数字那两项（别的版本的后端）返回 null，不画章 */ export function stampCounts(stats: EvidenceStats | null | undefined, claims?: string | null): @@ -821,10 +830,178 @@ export function stampCounts(stats: EvidenceStats | null | undefined, claims?: st if (!t) return null const cited = t.cited + (num(stats?.values) ?? 0) + (num(stats?.entities) ?? 0) + (num(stats?.quotes) ?? 0) const none = t.none + t.other + (num(stats?.unknown_entities) ?? 0) - + (claims === 'require_citation' ? num(stats?.uncited_claims) ?? 0 : 0) + + (claims === 'require_citation' || claims === 'judge' ? num(stats?.uncited_claims) ?? 0 : 0) return { cited, none } } +// ------------------------------------------------------------------------- +// 四期：结论句裁判 +// ------------------------------------------------------------------------- + +/** 判过了的四种（不含未裁判）：同后端 judge.JUDGED。判过的不再给「请模型判断这句」 */ +const JUDGED = new Set(['supported', 'partial', 'unsupported', 'not_a_claim']) +/** 触顶的四个上限：同后端 judge.LIMITS，这几种没判的写「已到上限」和怎么调 */ +export const JUDGE_LIMITS: readonly string[] = ['max_claims', 'max_cost_usd', 'daily_max_usd', 'timeout_s'] + +export const isJudged = (v: Pick | null | undefined): boolean => !!v && JUDGED.has(v.status) + +/** 没判的原因是不是上限，是的话返回是哪个 */ +export const limitOf = (v: Pick | null | undefined): string | null => + v?.status === 'unjudged' && v.reason && JUDGE_LIMITS.includes(v.reason) ? v.reason : null + +/** + * 判定在句末画成哪种徽标。「不是结论句」不挂：它不需要证据，挂个记号只添噪音（面板里照样写明模型的判断）； + * 认不出的判定也不挂——宁可不画，不能把不知道的东西画成某种结论 + */ +export function verdictState(v: Pick | null | undefined): EvidenceStateCode | null { + switch (v?.status) { + case 'supported': return 'supported' + case 'partial': return 'partial' + case 'unsupported': return 'unsupported' + case 'unjudged': return 'unjudged' + default: return null + } +} + +/** 一句的判定：封存之后按需追加的盖过文档里的（探索运行里文档记的是「未裁判 · 按需」） */ +export function unitVerdict( + unit: Pick, overlay?: Record | null, +): EvidenceVerdict | undefined { + return overlay?.[unit.id] ?? unit.verdict ?? undefined +} + +/** 这一句能不能请模型判断：结论句、不在表格和代码块里（同后端 candidates(units=…) 的条件） */ +export const judgeable = (unit: Pick, block?: Pick): boolean => + unit.kind === 'claim' && !['table', 'code', 'heading'].includes(block?.type ?? '') + +/** + * 结论句的计数（出具横幅、证据条）。文档开了裁判（有 judge 摘要）时按候选句的判定数，预筛放掉的、 + * 表格里的不算；没开裁判的文档只在有按需判过的句子时才数，没判的按有没有挂依据分成「未裁判」「无证据」。 + * 模型判为「不是结论句」的不算进总数。什么判定都没有返回 null：升级前的文档、没开裁判也没点过的，横幅照旧。 + * + * 「判过」认的是句子身上带着的判定，不只认 overlay：横幅拿到的是 withVerdicts 叠好的文档（判定已经写进 + * unit.verdict，没再单独给 overlay）。没开裁判的封存文档本身从不带判定，带着的只能是封存后追加的 + */ +export function claimCounts( + doc: Pick, overlay?: Record | null, +): ClaimTallyCounts | null { + const judged = !!doc.judge && typeof doc.judge === 'object' + const c: ClaimTallyCounts = { total: 0, supported: 0, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 } + let seen = judged + for (const block of doc.blocks ?? []) { + for (const unit of block.units ?? []) { + const v = unitVerdict(unit, overlay) + if (v) seen = true + if (v) { + if (v.status === 'supported' || v.status === 'partial' || v.status === 'unsupported' || v.status === 'unjudged') { + c[v.status] += 1 + c.total += 1 + } + continue + } + if (judged || !judgeable(unit, block)) continue + if (unit.cites?.length) c.unjudged += 1 + else c.uncited += 1 + c.total += 1 + } + } + return seen ? c : null +} + +/** 把封存之后追加的判定叠到文档上（只换有判定的那几句，别的块原样共用）。给横幅计数用，不写回封存的文档 */ +export function withVerdicts>( + doc: T, overlay: Record | null | undefined, +): T { + if (!overlay || !Object.keys(overlay).length) return doc + let touched = false + const blocks = (doc.blocks ?? []).map((b) => { + if (!(b.units ?? []).some((u) => overlay[u.id])) return b + touched = true + return { ...b, units: b.units.map((u) => (overlay[u.id] ? { ...u, verdict: overlay[u.id] } : u)) } + }) + return touched ? { ...doc, blocks } : doc +} + +/** 句子太长时截一截，给 aria-label 和面板标题用 */ +const clip = (text: string, n = 40) => (text.length > n ? `${text.slice(0, n - 1)}…` : text) + +/** + * 句末徽标的 aria-label：句子、判定都写全，封存后追加的另说。「结论句「增长主要来自新客首单…」， + * 模型判断：证据不支持」；到上限没判的说「未裁判：已到上限」 + */ +export function claimLabel(unit: Pick, v: EvidenceVerdict): string { + const state = verdictState(v) + const said = state ? EVIDENCE_STATE[state].label : JUDGE_TEXT.notClaim + const why = limitOf(v) ? `${said}：${JUDGE_TEXT.limit}` : said + return JUDGE_TEXT.badgeLabel(clip(unitText(unit).trim()), why, !!v.post_seal) +} + +/** 一条判定认成前端的形状：status 缺了（或者写在 verdict 里）的认 verdict，缺 judge 的补上答复里的模型 */ +function asVerdict(v: unknown, fallback: { model?: unknown; postSeal: boolean }): EvidenceVerdict | null { + if (!v || typeof v !== 'object') return null + const o = v as Record + const status = typeof o.status === 'string' ? o.status : typeof o.verdict === 'string' ? o.verdict : null + if (!status) return null + return { + ...(o as object), status, + rationale: typeof o.rationale === 'string' ? o.rationale : undefined, + judge: typeof o.judge === 'string' ? o.judge : typeof fallback.model === 'string' ? fallback.model : null, + post_seal: typeof o.post_seal === 'boolean' ? o.post_seal : fallback.postSeal, + used: Array.isArray(o.used) ? o.used.map(String) : undefined, + reason: typeof o.reason === 'string' ? o.reason : undefined, + } +} + +/** + * 按需裁判接口的答复 → {unit: 判定}。认 {verdicts: {u4: …}}（evidence.judged 的载荷）和 + * {verdicts: [{unit: 'u4', …}]}；外层没有 verdicts 时认 units。按需判的都是封存后追加的：缺 post_seal 的记 true + */ +export function judgedVerdicts(body: unknown): Record { + if (!body || typeof body !== 'object') return {} + const b = body as Record + const raw = b.verdicts ?? b.units + const fallback = { model: b.model, postSeal: typeof b.post_seal === 'boolean' ? b.post_seal : true } + const out: Record = {} + if (Array.isArray(raw)) { + for (const item of raw) { + const id = item && typeof item === 'object' ? (item as any).unit ?? (item as any).id : null + const v = asVerdict(item, fallback) + if (typeof id === 'string' && id && v) out[id] = v + } + } else if (raw && typeof raw === 'object') { + for (const [id, item] of Object.entries(raw)) { + const v = asVerdict(item, fallback) + if (v) out[id] = v + } + } + return out +} + +/** 证据图里某份报告封存之后追加的判定（reports[].post_seal_verdicts）；后端没给时是空的 */ +export function graphVerdicts(graph: EvidenceGraph | null | undefined, report: string | undefined): Record { + const hit = (graph?.reports ?? []).find((r) => !report || !r?.node_id || r.node_id === report) + return hit?.post_seal_verdicts && typeof hit.post_seal_verdicts === 'object' + ? judgedVerdicts({ verdicts: hit.post_seal_verdicts, post_seal: true }) : {} +} + +/** + * 这一句和「改写一次」（rewrite_once）的关系：改写稿采用了、这句是新写的（changed 里有它），给出交回 + * 改写的原句（units 里对应是 null 的那几句）；改写稿没采用、这句是交回过的，给出没采用的原因。其余 null。 + * 只按 changed / units 认编号——没按这两项映射过的编号在封存文档里指的是别的句子 + */ +export function rewriteOf(doc: Pick, uid: string): + { kind: 'changed'; sentences: string[] } | { kind: 'rejected'; reason: string } | null { + const r = doc.judge?.rewrite + if (!r || typeof r !== 'object') return null + const units = Array.isArray(r.units) ? r.units : [] + const sentences = Array.isArray(r.sentences) ? r.sentences.map(String) : [] + if (r.applied && Array.isArray(r.changed) && r.changed.includes(uid)) { + return { kind: 'changed', sentences: sentences.filter((_, i) => units[i] == null) } + } + if (!r.applied && typeof r.reason === 'string' && r.reason && units.includes(uid)) return { kind: 'rejected', reason: r.reason } + return null +} + // ------------------------------------------------------------------------- // 旧运行的猜测 // ------------------------------------------------------------------------- diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index da660422..784e296d 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -132,15 +132,15 @@ export const APPROVAL_POLICY_LABEL: Record<'dangerous' | 'always' | 'never', str * 可点击证据：片段状态的叫法。正文里片段的 aria-label、面板标题、图例都用这一份。 * 外观（线型、字形、颜色）在 lib/evidence.ts，和这里一一对应。 * - * 本期（数字层）实际只会出现「有出处」和「无证据」；其余几种是后续期的，先把叫法定下， - * 免得到时候各处各起一个名字。 + * 概率性的四种（四期的结论句裁判）挂在句末的徽标上：有依据、部分有依据、证据不支持都是模型的判断， + * 叫法里写明「模型判断」；未裁判不带「有引用」——没挂依据、只有方向词的结论句也会送裁判、也会没判。 */ export const EVIDENCE_STATE_LABEL = { deterministic: '有出处', supported: '模型判断：有依据', partial: '模型判断：部分有依据', unsupported: '模型判断：证据不支持', - unjudged: '有引用，未裁判', + unjudged: '未裁判', none: '无证据', connective: '连接性文字', candidate: '猜测的来源', @@ -313,6 +313,188 @@ export const EVIDENCE_TEXT = { guessCandidates: '可能来自', } as const +/** 结论句按裁判结论分成的几堆（lib/evidence 的 claimCounts 数出来的） */ +export interface ClaimTallyCounts { + /** 结论句总数：模型判为「不是结论句」的不算 */ + total: number + supported: number + partial: number + unsupported: number + /** 送了裁判、没判成（到上限、调用失败）或还没请模型判断的 */ + unjudged: number + /** 没有判定、也没挂依据的结论句 */ + uncited: number +} + +/** + * 「结论 4 句（支持 3 · 无证据 1）」：出具横幅、证据条、报告核对那一行共用，和数字那一段同一种说法—— + * 先说总数，括号里按状态分，为 0 的不写。没有结论句时返回空串 + */ +export function claimTally(c: ClaimTallyCounts): string { + if (!c.total) return '' + const parts = [ + ['支持', c.supported], ['部分支持', c.partial], ['不支持', c.unsupported], ['未裁判', c.unjudged], ['无证据', c.uncited], + ].filter(([, n]) => (n as number) > 0).map(([k, n]) => `${k} ${formatNumber(n as number)}`) + return `结论 ${formatNumber(c.total)} 句${parts.length ? `（${parts.join(' · ')}）` : ''}` +} + +/** + * 结论句裁判（四期）在证据面板、句末徽标里的说法。判断是模型给的：一律写明「模型判断」「非确定」， + * 封存之后按需追加的另写「封存后追加」 + */ +export const JUDGE_TEXT = { + section: '模型的解释', + claim: '结论句', + /** 读屏摘要末尾的操作说明：有证据不支持、部分支持的结论句时 n 也跳到它们的句末徽标 */ + keysHint: '用左右方向键逐个查看，上下键按句子走，n 跳到下一处无证据或证据不支持的句子，回车打开证据，Esc 关闭', + /** 面板里判断的徽标：谁判的、而且不是确定的 */ + badge: (model: string | null | undefined) => `模型判断 · ${model || '裁判模型'} · 非确定`, + postSeal: '封存后追加', + /** 判定里没记模型名时的叫法 */ + judgeModel: '裁判模型', + /** 句末徽标的悬停说明：到上限没判的 */ + limitNotJudged: '已到上限，这句没判（模型没有看过这句）', + /** 没判的句子（到上限、没跑成）记着的裁判模型：中性地说，不挂「模型判断」的徽标 */ + notJudgedBy: (model: string) => `裁判模型 ${model} 没有判这句`, + sealedDoc: '报告文档已封存 · 核对一致', + sealedWithDoc: '报告文档已封存 · 核对一致，这条判断随报告一起封存', + postSealHint: '运行封存之后按需追加的判断：不在封存范围内，封存核对照样一致。它是模型的解释，不是证据', + sealedHint: '正式运行里报告撰写节点当场判的，和报告一起封存。它是模型的解释，不是证据', + used: (aliases: string[]) => `裁判看过的证据：${aliases.join('、')}`, + cites: '挂的依据', + noCites: '这句没挂依据（只有方向词、因果词）：裁判看不到任何证据摘录', + notClaim: '模型判断：不是结论句', + ask: '请模型判断这句', + askAgain: '再请模型判断一次', + asking: '正在请模型判断…', + askHint: '探索运行按需裁判：点一次判这一句，花费计入每次点击和每日的上限', + notAsked: '还没请模型判断这句', + screened: '预筛认为这句不陈述数据事实（短句、只有过渡的话），没有送裁判', + noPrice: '裁判模型不在价格目录里，按令牌估不出金额：金额上限对它不起作用', + limit: '已到上限', + /** 触顶之后怎么调。按需裁判（每次点击）和正式运行（每份报告）调的地方不一样 */ + limitHow: { + click: { + max_cost_usd: '到「设置 → 偏好设置 → 证据裁判」调高「每次点击的金额上限」，或者设成不限，再点一次', + daily_max_usd: '到「设置 → 偏好设置 → 证据裁判」调高「每日金额上限」或设成不限；也可以等到明天（按本地日期重新计）', + max_claims: '到「设置 → 偏好设置 → 证据裁判」调高「每份报告最多判几句」（报告撰写节点里写了的以节点为准）', + timeout_s: '到「设置 → 偏好设置 → 证据裁判」调高「每份报告的时长上限」或设成不限，再点一次', + } as Record, + report: { + max_cost_usd: '到报告撰写节点的「结论句裁判」调高金额上限（没写就是设置里「证据裁判」的默认值），也可以设成不限，再发起一次运行', + daily_max_usd: '到「设置 → 偏好设置 → 证据裁判」调高「每日金额上限」或设成不限，再发起一次运行', + max_claims: '到报告撰写节点的「结论句裁判」调高最多判几句（或设置里的默认值），也可以设成不限，再发起一次运行', + timeout_s: '到报告撰写节点的「结论句裁判」调高时长上限（或设置里的默认值），也可以设成不限，再发起一次运行', + } as Record, + }, + failed: (why: string) => `没判成：${why}`, + oldBackend: '这个后端还不支持按需裁判', + notFound: '报告里没有这一句', + formalOnly: '正式运行的裁判在报告撰写节点里当场完成，不能按需追加', + rewritten: '这句是按裁判的意见改写过的（改写一次）', + rewriteFrom: '交回改写的原句', + rewriteRejected: (reason: string) => `这句交回写作者改写过一次，改写稿没有采用：${reason}`, + /** + * 横幅上结论句那一段之外另起的「没挂依据的结论句 N」：开了裁判的文档里，没挂依据的句子按判定数（或预筛 + * 放掉了不数），出具却照样按没挂依据计缺口——悬停时说清这两件事不冲突 + */ + uncitedBesides: '没写依据（[[see:]]）的结论句：出具照样计入缺口。模型判过的也算在里面——判定是模型的解释，代替不了依据', + /** 句末徽标的 aria-label：「结论句「增长主要来自新客首单」，模型判断：证据不支持，封存后追加」 */ + badgeLabel: (sentence: string, state: string, postSeal: boolean) => + `结论句「${sentence}」，${state}${postSeal ? '，封存后追加' : ''}`, +} as const + +/** + * 报告撰写节点的 judge 子配置（claims 为 judge 时生效）：检查器、发布前修复的预览（「结论句裁判 · 金额上限」） + * 共用这份叫法。键同后端 schema.JUDGE_KEYS + */ +export const JUDGE_FIELD_LABEL: Record = { + provider: '裁判模型 · 接入', + model: '裁判模型', + max_claims: '最多判几句', + max_cost_usd: '金额上限（美元）', + timeout_s: '时长上限（秒）', + rewrite_once: '证据不支持的句子交回改写一次', + on_unsupported: '证据不支持时', +} +/** judge.on_unsupported：正式运行里证据不支持的结论句怎么判档（探索运行只标注） */ +export const JUDGE_ON_UNSUPPORTED_LABEL: Record = { + degrade: '出具降档', + withhold: '不予出具', +} + +/** 设置页「证据裁判」一组（后端 engine/judge.py 的 JUDGE_DEFAULTS） */ +export const JUDGE_SETTING_TEXT = { + label: '证据裁判', + hint: '报告里的结论句（「增长主要来自新客首单」这类）由另一个模型按证据逐句判断：正式运行在报告撰写节点里当场判，探索运行点开哪句才判哪句。判断是模型给的，不是系统核对，界面上一律标「非确定」。', + provider: '裁判模型 · 接入', + model: '裁判模型 · 模型', + /** 接入留空、模型也留空：后端依次用 Copilot 的模型、默认接入（judge_model_spec）。只有这时才叫「跟随 Copilot」 */ + providerDefault: '跟随 Copilot 的模型', + /** 接入留空、模型填了：这一级写了就整组用这一级，后端按模型名找接入——不再跟随 Copilot */ + providerFromModel: '按模型名找接入', + modelDefault: '留空：用接入的默认模型', + modelFollow: '留空：跟随 Copilot 的模型', + differ: '建议和写报告的模型不同：同一个模型审自己写的，写错的地方它多半也看不出来', + limits: '上限（每一项都可以设成不限）', + reportClaims: '每份报告最多判几句', + reportCost: '每份报告的金额上限（美元）', + reportTimeout: '每份报告的时长上限（秒）', + clickCost: '每次点击的金额上限（美元）', + daily: '每日金额上限（美元）', + unlimited: '不限', + nodeWins: '报告撰写节点的「结论句裁判」里写了的上限以节点为准', + spend: (usd: string, calls: number) => `今天已花 $${usd}（${formatNumber(calls)} 次调用）`, + unpriced: (n: number) => `另有 ${formatNumber(n)} 次调用估不出金额（模型不在价格目录里），没算进去`, + spendNone: '今天还没有裁判调用', +} as const + +/** + * 只填了裁判模型、接入留空时实际用哪个接入（后端 judge_model_spec：哪一级写了就整组用哪一级，不跨级拼； + * 再由 resolve_provider 按模型名找接入）。scope：设置页（上一级是 Copilot 的模型）还是报告撰写节点（上一级是设置）。 + * found：按模型名找到的接入；fallback：认不出时后端去调的默认接入 + */ +export function judgeByModelText( + scope: 'settings' | 'node', model: string, found: { name: string; enabled: boolean } | null | undefined, + fallback: string | null | undefined, +): string { + const follow = scope === 'settings' ? '「跟随 Copilot」只在接入和模型都留空时生效' + : '接入和模型都没写时才跟随设置里的「证据裁判模型」' + if (found && !found.enabled) { + return `只填了模型：「${model}」属于已停用的接入「${found.name}」，裁判调用会失败、结论句都记为未裁判；启用它或选别的接入` + } + if (found) return `只填了模型：按模型名用接入「${found.name}」。${follow}` + return `只填了模型：没有哪个接入的模型清单里有「${model}」，会拿它去调默认接入${fallback ? `「${fallback}」` : ''}，` + + `调不通时结论句都记为未裁判；不是这家的模型请选上接入。${follow}` +} + +/** + * 某项上限设成不限时写明它还受什么约束——不要悄悄生效。别的上限也不限时照实少说几样，全不限时 + * 说到底只受调用次数（结论句的多少）约束。set：各项眼下是不是有上限 + */ +export function judgeUnlimitedText( + key: 'claims' | 'cost' | 'timeout' | 'click' | 'daily', + set: { claims: boolean; cost: boolean; timeout: boolean; click: boolean; daily: boolean }, +): string { + const list = (items: [boolean, string][]) => items.filter(([on]) => on).map(([, t]) => t) + switch (key) { + case 'claims': return '不设上限，句数只受报告里结论句多少约束' + case 'timeout': return '不设上限，时长只受模型接口自身的超时约束' + case 'cost': { + const by = list([[set.claims, '句数上限'], [set.timeout, '时长上限'], [set.daily, '每日上限']]) + return by.length ? `不设上限，费用只受${by.join('、')}约束` : '不设上限，费用只受报告里结论句的多少约束（句数、时长、每日也都不限）' + } + case 'click': { + const by = list([[set.claims, '句数上限'], [set.timeout, '时长上限'], [set.daily, '每日上限']]) + return by.length ? `不设上限，每次点击只受${by.join('、')}约束` : '不设上限，每次点击只受点开的句数约束（句数、时长、每日也都不限）' + } + case 'daily': { + const by = list([[set.cost, '每份报告的金额上限'], [set.click, '每次点击的金额上限']]) + return by.length ? `不设上限，每日费用只受${by.join('和')}约束` : '不设上限，每日费用只受模型调用次数约束（每份报告、每次点击也都不限）' + } + } +} + /** * 记录页「证据」页签：报告、常驻面板、审计表。审计表按状态分组，四组的说法和片段状态同一套 */ diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index a48b156b..e6cb3839 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState } from 'react' -import type { KeyboardEvent as ReactKeyboardEvent } from 'react' +import type { KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' import { Navigate, useParams } from 'react-router-dom' import { Check, Cpu, Download, KeyRound, Monitor, Moon, Plug, Plus, Settings as SettingsIcon, Sun, X, @@ -7,7 +7,7 @@ import { import clsx from 'clsx' import { api } from '../api/client' import type { ProviderDraft } from '../api/client' -import { useCatalog, useOnReconnect } from '../store/catalog' +import { modelOptions, providerOfModel, useCatalog, useOnReconnect } from '../store/catalog' import { confirmDialog, DeleteButton, EmptyState, ErrorState, Field, HealthPill, isComposing, Modal, PageHeader, promptDialog, SectionBar, Skeleton, Spinner, TabPanel, Tabs, toast, useRadioGroup, useTabRoute, @@ -22,7 +22,7 @@ import type { Provider } from '../types' import { normalizeTheme } from '../lib/theme' import { useLeaveGuard } from '../lib/leave' import { localActor, setLocalActor } from '../lib/actor' -import { AGENT_GUARD_TEXT, TOOL_GATE_TEXT } from '../lib/terms' +import { AGENT_GUARD_TEXT, JUDGE_SETTING_TEXT, TOOL_GATE_TEXT, judgeByModelText, judgeUnlimitedText } from '../lib/terms' import type { ThemePref } from '../lib/theme' // 提到模块级：tab 名同时是 URL 的最后一段，两处各写一份迟早对不上。 @@ -610,11 +610,17 @@ function PrefsTab() { const [draft, setDraft] = useState<{ actor: string; scope: string; collection: string; confirm: boolean; gateProvider: string; gateModel: string steps: string; budgetTokens: string; budgetUsd: string + judgeProvider: string; judgeModel: string + // 证据裁判的五项上限：null 是不限（存 null），字符串是正在填的数 + jClaims: string | null; jCost: string | null; jTimeout: string | null; jClick: string | null; jDaily: string | null } | null>(null) + const [spend, setSpend] = useState<{ usd?: number; calls?: number; unpriced_calls?: number } | null>(null) const [saving, setSaving] = useState(false) const [savedFlash, setSavedFlash] = useState(0) const [saveError, setSaveError] = useState(null) + // 今天裁判花了多少：只读，放在每日上限旁边。老后端没有这个接口就不显示 + const loadSpend = () => api.settings.judgeSpend().then(setSpend, () => setSpend(null)) const load = async () => { try { setValues(await api.settings.get()) @@ -623,6 +629,7 @@ function PrefsTab() { setLoadError(e) } api.memory.scopes().then(setScopes, () => {}) + void loadSpend() } useEffect(() => { void load() }, []) // 只在首次没拉到时重拉：重拉会覆盖正在编辑的表单 @@ -640,6 +647,14 @@ function PrefsTab() { steps: String(values?.run?.agent_max_steps ?? 100), budgetTokens: values?.run?.agent_budget_tokens == null ? '' : String(values.run.agent_budget_tokens), budgetUsd: values?.run?.agent_budget_usd == null ? '' : String(values.run.agent_budget_usd), + // 证据裁判（后端 engine/judge.py，GET 给的是引擎实际生效的样子：存坏了的显示默认，null 是不限） + judgeProvider: values?.judge?.provider ?? '', + judgeModel: values?.judge?.model ?? '', + jClaims: limitText(values?.judge, 'report_max_claims'), + jCost: limitText(values?.judge, 'report_max_cost_usd'), + jTimeout: limitText(values?.judge, 'report_timeout_s'), + jClick: limitText(values?.judge, 'click_max_cost_usd'), + jDaily: limitText(values?.judge, 'daily_max_usd'), } const cur = draft ?? saved const stepCap = Number(values?.limits?.max_agent_steps) || 100 @@ -657,6 +672,21 @@ function PrefsTab() { ? undefined : '填大于 0 的金额，或者留空表示不限', } const guardInvalid = Object.values(guardErrors).some(Boolean) + // 裁判的上限同样只校验改过的；null（不限）总是对的 + const judgeError = (key: 'jClaims' | 'jCost' | 'jTimeout' | 'jClick' | 'jDaily') => { + const v = cur[key] + if (v === saved[key] || v === null) return undefined + const n = Number(v.trim()) + if (key === 'jClaims') return /^\d+$/.test(v.trim()) && n >= 1 ? undefined : '填正整数，或者勾「不限」' + if (key === 'jTimeout') return v.trim() !== '' && Number.isFinite(n) && n > 0 ? undefined : '填大于 0 的秒数，或者勾「不限」' + return v.trim() !== '' && Number.isFinite(n) && n > 0 ? undefined : '填大于 0 的金额，或者勾「不限」' + } + const judgeErrors = { + jClaims: judgeError('jClaims'), jCost: judgeError('jCost'), jTimeout: judgeError('jTimeout'), + jClick: judgeError('jClick'), jDaily: judgeError('jDaily'), + } + const judgeInvalid = Object.values(judgeErrors).some(Boolean) + const invalid = guardInvalid || judgeInvalid const changed = (Object.keys(saved) as (keyof typeof saved)[]).filter((k) => cur[k] !== saved[k]) const dirty = values ? changed.length : 0 // 外壳唯一的 blocker 在地址（pathname）变化时问：切标签、导航、⌘K、⌥ 数字、后退都算 @@ -678,7 +708,7 @@ function PrefsTab() { * 有改动才 PUT——只改了署名时不必碰服务端，断线时也存得上 */ const save = async () => { - if (!values || !dirty || guardInvalid) return + if (!values || !dirty || invalid) return setSaving(true) setSaveError(null) const actor = cur.actor.trim() @@ -687,9 +717,20 @@ function PrefsTab() { setLocalActor(actor) savedActor.current = actor } - const runChanged = changed.some((k) => k !== 'actor') + const judgeChanged = changed.some((k) => JUDGE_DRAFT.has(k)) + const runChanged = changed.some((k) => k !== 'actor' && !JUDGE_DRAFT.has(k)) try { - if (runChanged) { + // 裁判一组整组写：后端对没给的键回落到默认值，只发改过的那一项会把别的项冲回默认 + const judge = judgeChanged ? { + provider: cur.judgeProvider || null, + model: cur.judgeModel.trim() || null, + report_max_claims: parseLimit(cur.jClaims), + report_max_cost_usd: parseLimit(cur.jCost), + report_timeout_s: parseLimit(cur.jTimeout), + click_max_cost_usd: parseLimit(cur.jClick), + daily_max_usd: parseLimit(cur.jDaily), + } : null + if (runChanged || judge) { const run = { ...(values.run ?? {}), default_memory_scope: cur.scope, @@ -701,8 +742,14 @@ function PrefsTab() { agent_budget_tokens: cur.budgetTokens.trim() === '' ? null : Number(cur.budgetTokens), agent_budget_usd: cur.budgetUsd.trim() === '' ? null : Number(cur.budgetUsd), } - const out = await api.settings.put({ run }) - setValues((v) => ({ ...(v ?? {}), run: out?.run ?? run })) + const out = await api.settings.put({ ...(runChanged ? { run } : {}), ...(judge ? { judge } : {}) }) + setValues((v) => ({ + ...(v ?? {}), + ...(runChanged ? { run: out?.run ?? run } : {}), + ...(judge ? { judge: out?.judge ?? judge } : {}), + })) + // 每日上限改了：旁边那句「今天已花」的分母跟着变，顺手再取一次 + if (judge) void loadSpend() } setDraft(null) setSavedFlash(Date.now()) @@ -724,6 +771,29 @@ function PrefsTab() { const gateProvider = cur.gateProvider ? providers.find((p) => p.name === cur.gateProvider) : defaultProvider const gateProviderMissing = !!cur.gateProvider && providers.length > 0 && !gateProvider const gateModels = (gateProvider?.models ?? []).map((m) => m.id) + // 裁判模型的接入：接入和模型都留空时后端依次用 Copilot 的模型、默认接入（judge_model_spec）。只填了模型时 + // 这一级写了就整组用这一级，按模型名找接入（resolve_provider）——不再跟随 Copilot，留空那一项不能还叫「跟随 Copilot」 + const judgeProvider = cur.judgeProvider ? providers.find((p) => p.name === cur.judgeProvider) : undefined + const judgeProviderMissing = !!cur.judgeProvider && providers.length > 0 && !judgeProvider + const judgeModelName = cur.judgeModel.trim() + const judgeByModel = !cur.judgeProvider && !!judgeModelName + const judgeOwner = judgeByModel ? providerOfModel(providers, judgeModelName) : undefined + const judgeByModelHint = judgeByModel && providers.length > 0 + ? judgeByModelText('settings', judgeModelName, judgeOwner && { name: judgeOwner.provider.name, enabled: judgeOwner.enabled }, + defaultProvider?.name) + : '' + // 接入留空时按模型名找接入：所有启用接入的模型都能选；选了接入只列这一家的 + const judgeModels = judgeProvider ? (judgeProvider.models ?? []).map((m) => m.id) + : [...new Set(modelOptions(providers).map((o) => o.value))] + const pickJudgeProvider = (name: string) => { + const next = name ? providers.find((p) => p.name === name) : undefined + const keep = !name || !cur.judgeModel || (next?.models ?? []).some((m) => m.id === cur.judgeModel) || next?.default_model === cur.judgeModel + edit({ judgeProvider: name, ...(keep ? {} : { judgeModel: '' }) }) + } + const limitSet = { + claims: cur.jClaims !== null, cost: cur.jCost !== null, timeout: cur.jTimeout !== null, + click: cur.jClick !== null, daily: cur.jDaily !== null, + } const pickGateProvider = (name: string) => { // 换了接入，原来填的模型不在新接入的清单里就清掉：拿 A 家的模型名去调 B 家必然失败 const next = name ? providers.find((p) => p.name === name) : defaultProvider @@ -864,6 +934,85 @@ function PrefsTab() {
+ + {values.judge && ( +
+ +
+
裁判模型
+

{JUDGE_SETTING_TEXT.differ}

+
+ {judgeByModelHint} + ) : undefined}> + {(p) => ( + + )} + + + {(p) => ( + <> + edit({ judgeModel: e.target.value })} /> + + {judgeModels.map((m) => + + )} + +
+
+
+
{JUDGE_SETTING_TEXT.limits}
+

{JUDGE_SETTING_TEXT.nodeWins}

+
+ edit({ jClaims: v })} /> + edit({ jCost: v })} /> + edit({ jTimeout: v })} /> + edit({ jClick: v })} /> + edit({ jDaily: v })} + extra={spend && ( +
+ {spend.calls ? JUDGE_SETTING_TEXT.spend(formatSpend(spend.usd), spend.calls) : JUDGE_SETTING_TEXT.spendNone} + {!!spend.unpriced_calls && ( + + {JUDGE_SETTING_TEXT.unpriced(spend.unpriced_calls)} + + )} +
+ )} /> +
+
+
+ )} )} @@ -884,6 +1033,9 @@ function PrefsTab() { actor: '署名', scope: '记忆作用域', collection: '知识库', confirm: '危险工具的默认审批策略', gateProvider: '门控模型的接入', gateModel: '门控模型', steps: '默认最大步数', budgetTokens: '令牌预算', budgetUsd: '金额预算', + judgeProvider: '裁判模型的接入', judgeModel: '裁判模型', + jClaims: '每份报告最多判几句', jCost: '每份报告的金额上限', jTimeout: '每份报告的时长上限', + jClick: '每次点击的金额上限', jDaily: '每日金额上限', })[k]).join('、')} @@ -893,8 +1045,9 @@ function PrefsTab() { - @@ -910,6 +1063,61 @@ function PrefsTab() { ) } +/** 裁判那一组在草稿里的键：它们改了才写 judge 一组 */ +const JUDGE_DRAFT = new Set(['judgeProvider', 'judgeModel', 'jClaims', 'jCost', 'jTimeout', 'jClick', 'jDaily']) + +/** 勾掉「不限」时填回的数：同后端 JUDGE_DEFAULTS */ +const JUDGE_FALLBACK = { + report_max_claims: '40', report_max_cost_usd: '0.05', report_timeout_s: '30', click_max_cost_usd: '0.01', daily_max_usd: '2', +} + +/** 设置里的一项上限 → 表单：null 是不限，没有这一项（老后端）当空串 */ +function limitText(group: Record | undefined, key: string): string | null { + const v = group?.[key] + return v === null ? null : v === undefined ? '' : String(v) +} + +/** 表单 → 存进去的值：不限存 null */ +const parseLimit = (v: string | null): number | null => (v === null ? null : Number(v.trim())) + +/** 今天花了多少美元：小额按分以下的精度写，不四舍五入成 0 */ +const formatSpend = (usd: number | undefined): string => + usd == null || !Number.isFinite(usd) ? '0' : usd >= 1 ? usd.toFixed(2) : String(Number(usd.toFixed(4))) + +/** + * 一项裁判上限：数字框加「不限」。勾了不限，框变灰，下面写明不设上限之后费用还受什么约束—— + * 不许悄悄生效；勾掉时填回之前的数（没有就填默认值） + */ +function LimitField({ id, dataKey, label, value, fallback, integer, error, unlimited, onChange, extra }: { + id: string; dataKey: string; label: string; value: string | null; fallback: string; integer?: boolean + error?: string; unlimited: string; onChange: (v: string | null) => void; extra?: ReactNode +}) { + const last = useRef(value ?? fallback) + if (value !== null) last.current = value || last.current + const off = value === null + return ( +
+ {unlimited} : undefined}> + {(p) => ( +
+ onChange(e.target.value)} /> + +
+ )} + + {extra} +
+ ) +} + /** * 主题：选中即生效、即保存。显示的是眼下实际应用的偏好（useTheme）——从导航、 * ⌘K 换了主题，这里跟着变，不会留着旧值。 diff --git a/frontend/src/pages/runs/EvidencePane.tsx b/frontend/src/pages/runs/EvidencePane.tsx index f05775c3..78cc956b 100644 --- a/frontend/src/pages/runs/EvidencePane.tsx +++ b/frontend/src/pages/runs/EvidencePane.tsx @@ -143,7 +143,7 @@ export function EvidencePane({ run, output, labelOf, refreshKey }: { {doc ? ( { handles.current.set(r.node_id!, h) }} doc={doc} artifact={r.doc_artifact} - runId={run.id} label={r.fields?.join('、') || r.node_id} tally + runId={run.id} runClass={run.run_class} label={r.fields?.join('、') || r.node_id} tally panel={wide ? 'dock' : 'auto'} dock={dock} onView={(o) => onView(r.node_id!, o)} /> ) : slot?.status === 'error' ? (

{T.docGone(labelOf(r.node_id) ?? r.node_id ?? '')}

diff --git a/frontend/src/run/AssistantStream.tsx b/frontend/src/run/AssistantStream.tsx index afe275d3..3fed5472 100644 --- a/frontend/src/run/AssistantStream.tsx +++ b/frontend/src/run/AssistantStream.tsx @@ -15,14 +15,14 @@ import { humanizeError } from '../lib/errors' import { explainRunError } from '../lib/explain' import { formatClock, formatDuration, formatNumber, NONE, shortId } from '../lib/format' import { statusLabel } from '../lib/status' -import { EVIDENCE_TEXT, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' +import { EVIDENCE_TEXT, JUDGE_TEXT, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' import { childrenByExec, compactSteps, parseQueryResult, progressOf, spread, teamVerdictOf, type Exec, type FixKind, type ResultTable as Table, type Step, type StepKind, type TeamMemberEx, type TeamRun, } from './decode' import { CODE_COLUMN, CopyChip, LEADING_ZERO, Markdown, type MarkSpec } from './Markdown' -import { EvidenceField, type EvidenceDocHandle } from './EvidenceDoc' +import { ClaimTally, EvidenceField, type EvidenceDocHandle } from './EvidenceDoc' import { EVIDENCE_STATE, docTally, evidenceFields, issuanceMarks, type EvidenceTally } from '../lib/evidence' import type { EvidenceDocData } from '../types' import { useRunClock } from './useRunClock' @@ -2027,7 +2027,7 @@ function Output({ output, runClass, broken, question, onFollowUp, runId }: { ) @@ -2262,6 +2262,12 @@ export function IssuanceBanner({ issuance, runClass, evidence }: { const uncitedClaims = Number(issuance?.claims?.uncited_claims ?? 0) || 0 const numbers = counts ? evidenceTally(counts.cited, counts.total, counts.other) : '' const numbersOk = !counts?.none && !counts?.other + // 四期：结论句有判定（开了裁判，或探索运行里按需判过）时多一段「结论 4 句（支持 3 · 无证据 1）」。 + // 没开裁判的文档里没挂依据的都数在这一段的「无证据」里，不再另起一句；开了裁判的文档里，没挂依据却送了 + // 裁判的句子按判定数、预筛放掉的不数——出具照样按没挂依据计缺口（require_citation、judge 都是），这时 + // 结论句那一段盖不全，照出具的数另起「没挂依据的结论句 N」，不能让花了档位的缺口从横幅上消失 + const claims = counts?.claims?.total ? counts.claims : null + const uncitedLine = uncitedClaims > (claims?.uncited ?? 0) return ( // data-issuance-banner：画布上成果节点的出具印章据此滚过来；tabIndex 让「去看出具」能把焦点带到这里 @@ -2281,9 +2287,10 @@ export function IssuanceBanner({ issuance, runClass, evidence }: { {meta.hint &&
{meta.hint}
} - {counts && (counts.total > 0 || counts.other > 0 || suspect > 0 || uncitedClaims > 0) && ( + {counts && (counts.total > 0 || counts.other > 0 || suspect > 0 || uncitedClaims > 0 || !!claims) && ( // 逐段证据的计数：和报告核对那一行、证据条同一种说法。可疑名字、没挂依据的结论句不是数字， - // 各自另起一句。有无证据的数字、可疑名字时才给「定位下一处」（结论句不是片段，定位不到） + // 各自另起一句。有无证据的数字、可疑名字、证据不支持（或部分支持）的结论句时才给「定位下一处」—— + // 结论句靠句末徽标定位；只是没挂依据的结论句没有徽标，定位不到
{numbers && ( {numbers} @@ -2293,12 +2300,14 @@ export function IssuanceBanner({ issuance, runClass, evidence }: { {numbers ? '· ' : ''}{EVIDENCE_STATE.suspect.glyph} {EVIDENCE_TEXT.suspectTag} {formatNumber(suspect)} )} - {uncitedClaims > 0 && ( - - {numbers || suspect > 0 ? '· ' : ''}{EVIDENCE_TEXT.uncitedClaimsTag} {formatNumber(uncitedClaims)} + {claims && 0} />} + {uncitedLine && ( + + {numbers || suspect > 0 || claims ? '· ' : ''}{EVIDENCE_TEXT.uncitedClaimsTag} {formatNumber(uncitedClaims)} )} - {(counts.none > 0 || counts.other > 0 || suspect > 0) && evidence?.onNext && ( + {(counts.none > 0 || counts.other > 0 || suspect > 0 || !!claims?.unsupported || !!claims?.partial) && evidence?.onNext && ( diff --git a/frontend/src/run/EvidenceDoc.tsx b/frontend/src/run/EvidenceDoc.tsx index dbe6d923..236747eb 100644 --- a/frontend/src/run/EvidenceDoc.tsx +++ b/frontend/src/run/EvidenceDoc.tsx @@ -8,13 +8,13 @@ import { ListChecks } from 'lucide-react' import clsx from 'clsx' import { rovingTarget } from '../components/ui' import { - EVIDENCE_STATE, docTally, segmentKind, segmentLabel, segmentState, tallySummary, unitText, type EvidenceStateCode, - type EvidenceTally as Tally, + EVIDENCE_STATE, claimCounts, claimLabel, docTally, segmentKind, segmentLabel, segmentState, tallySummary, unitText, + isJudged, limitOf, unitVerdict, verdictState, withVerdicts, type EvidenceStateCode, type EvidenceTally as Tally, } from '../lib/evidence' import { formatNumber } from '../lib/format' -import { EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, evidenceTally } from '../lib/terms' -import { useEvidence } from '../store/evidence' -import type { EvidenceBlock, EvidenceDocData, EvidenceSegment, EvidenceUnit } from '../types' +import { EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, JUDGE_TEXT, claimTally, evidenceTally, type ClaimTallyCounts } from '../lib/terms' +import { useEvidence, useVerdicts } from '../store/evidence' +import type { EvidenceBlock, EvidenceDocData, EvidenceSegment, EvidenceUnit, EvidenceVerdict } from '../types' import { EvidencePanel, type PanelMode, type PanelView } from './EvidencePanel' import { BlockView, Markdown, StyledSlice, inlineStyles, numericColumns, type InlineStyles } from './Markdown' @@ -26,10 +26,13 @@ import { BlockView, Markdown, StyledSlice, inlineStyles, numericColumns, type In * BlockView，和普通答案长得一模一样；有出处的数字画细实线，没有出处的画点状线、 * 句末挂「无证据」。 * - * 键盘：整份报告只占一个 Tab 位（roving tabindex）。←/→ 在有状态的片段间走，↑/↓ 按 + * 结论句的判定（四期）只在句末挂一枚小徽标（◆ ◇ ! ?），整句不画线；证据不支持的整句铺一层浅底。 + * 徽标也是按钮：点开是这一句的「模型的解释」。判定是模型给的，颜色一律不用确定性的绿。 + * + * 键盘：整份报告只占一个 Tab 位（roving tabindex）。←/→ 在有状态的片段、句末徽标间走，↑/↓ 按 * 句子走（表格里按列走到上一行 / 下一行，出了表格接着按句子走），n / N 跳到下一处 / - * 上一处无证据，回车打开证据面板，Esc 关掉并留在原片段；面板开着时方向键走到哪、面板 - * 跟到哪，Tab 进面板。 + * 上一处无证据、证据不支持的地方，回车打开证据面板，Esc 关掉并留在原片段；面板开着时方向键走到哪、 + * 面板跟到哪，Tab 进面板。 * * 长报告：事件都挂在容器上（片段只带 data-seg），块用 memo，外加 * content-visibility: auto；超过 TEXT_CAP 的部分先折叠，跳转到折叠里的片段时自动展开。 @@ -47,11 +50,19 @@ export interface EvidenceDocHandle { /** 和 AssistantStream 的长文本折叠同一个量级：先显示这么多字，后面的折起来 */ const TEXT_CAP = 3000 +/** 句末徽标在键盘顺序里的键：片段 id 是 s0…，不会撞 */ +const badgeKey = (unit: string) => `claim:${unit}` +const unitOfKey = (key: string) => (key.startsWith('claim:') ? key.slice(6) : null) + +interface Badge { unit: EvidenceUnit; block: number; verdict: EvidenceVerdict; state: EvidenceStateCode } + interface Model { - /** 可点的片段，按正文顺序 */ + /** 可点的片段和句末徽标，按正文顺序 */ order: string[] index: Map seg: Map + /** 句末徽标：键是 claim:，带着这一句眼下的判定 */ + badge: Map /** 每个有可点片段的句子里第一个可点片段，按正文顺序（↑/↓ 用） */ unitFirst: string[] unitOf: Map @@ -65,10 +76,11 @@ interface Model { grid: Map } -function buildModel(blocks: EvidenceBlock[]): Model { +function buildModel(blocks: EvidenceBlock[], overlay?: Record): Model { const order: string[] = [] const index = new Map() const seg = new Map() + const badge = new Map() const unitFirst: string[] = [] const unitOf = new Map() const alerts: string[] = [] @@ -96,12 +108,28 @@ function buildModel(blocks: EvidenceBlock[]): Model { if (!grid.has(key)) grid.set(key, s.id) } } + // 句末徽标排在这一句的片段后面：←/→ 走到句尾就是它，证据不支持的也进 n / N + const verdict = unitVerdict(unit, overlay) + const state = verdictState(verdict) + if (verdict && state) { + const key = badgeKey(unit.id) + badge.set(key, { unit, block: b, verdict, state }) + index.set(key, order.length) + order.push(key) + if (first) { unitFirst.push(key); first = false } + unitOf.set(key, unitFirst.length - 1) + if (EVIDENCE_STATE[state].alert) alerts.push(key) + } } chars.push(total) }) - return { order, index, seg, unitFirst, unitOf, alerts, chars, cell, grid } + return { order, index, seg, badge, unitFirst, unitOf, alerts, chars, cell, grid } } +/** 片段或徽标在第几块 */ +const blockOf = (model: Model, key: string | null | undefined): number => + (key ? model.seg.get(key)?.block ?? model.badge.get(key)?.block : undefined) ?? -1 + /** * ↑/↓ 该去哪。表格里按列走：下一行同一列（那一格空着就找这一行离它最近的一格）； * 走出表格的上下边，接着按句子走到表格前后那一句——不在表格里一格一格地横着挪 @@ -123,11 +151,11 @@ function verticalTarget(model: Model, id: string, down: boolean): string | null // 出了表格：往下找表格之后的第一句，往上找表格之前的最后一句 const u = model.unitOf.get(id) ?? 0 const list = down ? model.unitFirst.slice(u + 1) : model.unitFirst.slice(0, u).reverse() - const out = list.find((sid) => model.seg.get(sid)?.block !== at.block) + const out = list.find((sid) => blockOf(model, sid) !== at.block) if (out) return out // 表格就是整份报告的首尾：和句子一样首尾相接 const wrap = down ? model.unitFirst : [...model.unitFirst].reverse() - return wrap.find((sid) => model.seg.get(sid)?.block !== at.block) ?? null + return wrap.find((sid) => blockOf(model, sid) !== at.block) ?? null } const u = model.unitOf.get(id) ?? 0 return model.unitFirst[(u + (down ? 1 : -1) + n) % n] ?? null @@ -168,10 +196,24 @@ export const EvidenceDoc = forwardRef void /** 自己显示「N/N 数字有出处」那一条。上面已经有出具横幅（它会写这一行）时不用 */ tally?: boolean -}>(function EvidenceDoc({ doc, artifact, runId, dense = false, label, panel = 'auto', dock, onView, tally = false }, ref) { + /** + * 运行类别（formal / exploratory）：「请模型判断这句」只在探索运行里有。开了裁判的文档自己记着 + * （on_demand / inline），没开的看这里，不给就按运行 id 取一次 + */ + runClass?: string +}>(function EvidenceDoc({ doc, artifact, runId, dense = false, label, panel = 'auto', dock, onView, tally = false, runClass }, ref) { const blocks = useMemo(() => doc.blocks ?? [], [doc]) - const model = useMemo(() => buildModel(blocks), [blocks]) + const report = doc.node_id || undefined + // 封存之后按需追加的判定：盖过文档里的「未裁判 · 按需」。只换句末徽标，封存的文档一个字不改 + const overlay = useVerdicts(runId, report) + const model = useMemo(() => buildModel(blocks, overlay), [blocks, overlay]) const counts = useMemo(() => docTally(doc), [doc]) + const claims = useMemo(() => claimCounts(doc, overlay), [doc, overlay]) + // 探索运行的报告（开了裁判的按需文档，或调用方说了是探索运行）：证据图带着这次运行里之前按需判过的句子 + // （reports[].post_seal_verdicts），取一次，重新打开页面时徽标还在。别的报告不多发这个请求 + const loadGraph = useEvidence((s) => s.loadGraph) + const lateOnes = doc.judge?.mode === 'on_demand' || (runClass === 'exploratory' && doc.judge?.mode !== 'inline') + useEffect(() => { if (lateOnes && runId) void loadGraph(runId) }, [lateOnes, runId, loadGraph]) const rootRef = useRef(null) const instance = useId() const panelId = `${instance}-panel` @@ -183,11 +225,17 @@ export const EvidenceDoc = forwardRef(null) const mode = usePanelMode(panel, dense) - // 换了一份文档：焦点位回到第一个片段，面板关掉 + // 换了一份文档：焦点位回到第一个片段，面板关掉。只认文档本身——判定回来了（徽标变了）不算换文档， + // 面板正开着这一句的解释，不能因此收起 useEffect(() => { setCurrent(model.order[0] ?? null) setView(null) - }, [model]) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [blocks]) + // 拿着 Tab 位的徽标没了（判成「不是结论句」）：Tab 位交回第一个，整份报告不能一个 Tab 位都没有 + useEffect(() => { + if (current && !model.index.has(current)) setCurrent(model.order[0] ?? null) + }, [model, current]) // 同一时刻只开一个面板：另一份报告打开了自己的面板，这边的就收起 const owner = useEvidence((s) => s.owner) @@ -205,15 +253,18 @@ export const EvidenceDoc = forwardRef - rootRef.current?.querySelector(`[data-seg="${CSS.escape(id)}"]`) ?? null, []) + const segEl = useCallback((id: string) => { + const unit = unitOfKey(id) + const sel = unit ? `[data-ev-claim="${CSS.escape(unit)}"]` : `[data-seg="${CSS.escape(id)}"]` + return rootRef.current?.querySelector(sel) ?? null + }, []) - /** 把焦点挪到某个片段上。在折叠里的先展开，渲染出来再聚焦 */ + /** 把焦点挪到某个片段（或句末徽标）上。在折叠里的先展开，渲染出来再聚焦 */ const focusSeg = useCallback((id: string, opts?: { flash?: boolean }) => { - const hit = model.seg.get(id) - if (!hit) return + const block = blockOf(model, id) + if (block < 0) return setCurrent(id) - if (hit.block >= shownBlocks) { + if (block >= shownBlocks) { pendingFocus.current = id setExpanded(true) return @@ -233,6 +284,13 @@ export const EvidenceDoc = forwardRef { + const badge = model.badge.get(id) + if (badge) { + claim(instance) + setView({ kind: 'unit', id: badge.unit.id }) + setAnnounce(`${JUDGE_TEXT.section}：${claimLabel(badge.unit, badge.verdict)}`) + return + } const hit = model.seg.get(id) if (!hit) return claim(instance) @@ -247,7 +305,7 @@ export const EvidenceDoc = forwardRef { - const back = view?.kind === 'seg' ? view.id : current + const back = view?.kind === 'seg' ? view.id : view?.kind === 'unit' ? badgeKey(view.id) : current setView(null) setAnnounce('') if (refocus && back) requestAnimationFrame(() => segEl(back)?.focus()) @@ -265,16 +323,18 @@ export const EvidenceDoc = forwardRef { onView?.(open) }, [open, onView]) + const viewKey = view?.kind === 'seg' ? view.id : view?.kind === 'unit' ? badgeKey(view.id) : null + useImperativeHandle(ref, () => ({ open: (id, opts) => { - if (!model.seg.has(id)) return false + if (!model.seg.has(id) && !model.badge.has(id)) return false if (opts?.focus) focusSeg(id, { flash: true }) else setCurrent(id) openSeg(id) return true }, next: (dir = 1) => { - const target = step(model.alerts, view?.kind === 'seg' ? view.id : current, dir) + const target = step(model.alerts, viewKey ?? current, dir) if (target) { focusSeg(target, { flash: true }) openSeg(target) @@ -288,19 +348,20 @@ export const EvidenceDoc = forwardRef { - const el = (target as HTMLElement | null)?.closest?.('[data-seg]') - return el && rootRef.current?.contains(el) ? el.dataset.seg ?? null : null + const el = (target as HTMLElement | null)?.closest?.('[data-seg], [data-ev-claim]') + if (!el || !rootRef.current?.contains(el)) return null + return el.dataset.seg ?? (el.dataset.evClaim ? badgeKey(el.dataset.evClaim) : null) } const onClick = (e: ReactMouseEvent) => { const id = segOf(e.target) if (!id) return e.preventDefault() setCurrent(id) - if (view?.kind === 'seg' && view.id === id) close(false) + if (viewKey === id) close(false) else openSeg(id) } const onFocus = (e: ReactFocusEvent) => { @@ -341,9 +402,9 @@ export const EvidenceDoc = forwardRef 40 const panelNode = view && ( @@ -353,6 +414,7 @@ export const EvidenceDoc = forwardRef close()} onLocate={(id) => { focusSeg(id, { flash: true }); openSeg(id) }} @@ -361,14 +423,14 @@ export const EvidenceDoc = forwardRef 0) + const summary = tallySummary({ ...counts, claims }, model.order.length > 0) return (

{summary}

{announce}
- {tally && (counts.total > 0 || counts.other > 0 || !!counts.suspect) && ( - { + {tally && (counts.total > 0 || counts.other > 0 || !!counts.suspect || !!claims?.total) && ( + { const target = step(model.alerts, current, 1) if (target) { focusSeg(target, { flash: true }); openSeg(target) } else openViolations() }} onList={doc.violations?.length ? openViolations : undefined} /> @@ -388,7 +450,7 @@ export const EvidenceDoc = forwardRef + panelId={panelId} model={model} /> {mode === 'inline' && b === activeBlock && panelNode} ))} @@ -437,14 +499,18 @@ function usePanelMode(panel: 'auto' | PanelMode, dense: boolean): PanelMode { return wide ? 'side' : 'drawer' } -/** 证据条：没有出具横幅时由文档自己说「7/12 数字有出处 · 无证据 5 · 另有 1 处引用解析不了」 */ +/** + * 证据条：没有出具横幅时由文档自己说「7/12 数字有出处 · 无证据 5 · 另有 1 处引用解析不了」， + * 有结论句判定时接着说「结论 4 句（支持 3 · 不支持 1）」 + */ export function EvidenceTally({ counts, onNext, onList }: { - counts: Pick + counts: Pick onNext?: () => void onList?: () => void }) { const suspect = counts.suspect ?? 0 - const clean = !counts.none && !counts.other && !suspect + const claims = counts.claims + const clean = !counts.none && !counts.other && !suspect && !claims?.unsupported && !claims?.partial const numbers = evidenceTally(counts.cited, counts.total, counts.other) return (
@@ -459,6 +525,7 @@ export function EvidenceTally({ counts, onNext, onList }: { {numbers ? '· ' : ''}{EVIDENCE_STATE.suspect.glyph} {EVIDENCE_TEXT.suspectTag} {formatNumber(suspect)} )} + {claims && claims.total > 0 && 0} />} {!clean && onNext && ( ) }) + const key = badge ? badgeKey(unit.id) : '' + const meta = badge ? EVIDENCE_STATE[badge.state] : null + const open = !!badge && active === key return ( <> - {nodes} + {badge?.state === 'unsupported' + // 证据不支持：整句铺一层浅底（折行时每行各自带圆角），字色、字重都不动，也不画线 + ? {nodes} + : nodes} {tag && [...alerts].map((state) => ( ))} + {badge && meta && ( + + )} ) } @@ -631,8 +744,10 @@ function UnitView({ unit, doc, current, active, panelId, code, tag }: { */ export const EvidenceField = forwardRef void -}>(function EvidenceField({ artifact, text, runId, dense, label, tally, onDoc }, ref) { + runClass?: string +}>(function EvidenceField({ artifact, text, runId, dense, label, tally, onDoc, runClass }, ref) { const slot = useEvidence((s) => s.docs[artifact]) const loadDoc = useEvidence((s) => s.loadDoc) useEffect(() => { void loadDoc(artifact) }, [artifact, loadDoc]) @@ -640,9 +755,12 @@ export const EvidenceField = forwardRef { onDoc?.(usable) }, [usable, onDoc]) + const overlay = useVerdicts(runId, usable?.node_id || undefined) + const counted = useMemo(() => (usable ? withVerdicts(usable, overlay) : null), [usable, overlay]) + useEffect(() => { onDoc?.(counted) }, [counted, onDoc]) if (usable) { - return + return } const note = slot?.status === 'error' ? EVIDENCE_TEXT.docMissing : otherRun ? EVIDENCE_TEXT.docOtherRun : mismatch ? EVIDENCE_TEXT.docMismatch : '' diff --git a/frontend/src/run/EvidencePanel.tsx b/frontend/src/run/EvidencePanel.tsx index 109b3b28..14f87092 100644 --- a/frontend/src/run/EvidencePanel.tsx +++ b/frontend/src/run/EvidencePanel.tsx @@ -3,22 +3,25 @@ import { type ReactNode, } from 'react' import { createPortal } from 'react-dom' -import { ArrowLeft, CornerDownRight, ListChecks, ShieldCheck, X } from 'lucide-react' +import { Link } from 'react-router-dom' +import { ArrowLeft, CornerDownRight, ListChecks, Scale, ShieldCheck, X } from 'lucide-react' import clsx from 'clsx' -import { StatusBadge, isComposing } from '../components/ui' +import { ApiError } from '../api/client' +import { Spinner, StatusBadge, isComposing } from '../components/ui' import { EVIDENCE_KIND_STYLE, EVIDENCE_STATE, caliberSourceText, caliberUpgradeText, closestNames, docForeign, entitySources, - evidenceTrace, evidenceValue as valueText, graphDoc, inputSource, integrityFailures, locatable, locatorText, notableInput, - queryOf, queryWindow, quoteWhere, quoteWindow, reasonOf, sealVerdict, segName, segmentKind, segmentState, sourceOf, - type SealStatus, type SourceTone, + evidenceTrace, evidenceValue as valueText, graphDoc, inputSource, integrityFailures, isJudged, judgeable, limitOf, + locatable, locatorText, notableInput, queryOf, queryWindow, quoteWhere, quoteWindow, reasonOf, rewriteOf, sealVerdict, + segName, segmentKind, segmentState, sourceOf, unitText, unitVerdict, verdictState, type SealStatus, type SourceTone, } from '../lib/evidence' +import { humanizeError } from '../lib/errors' import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format' -import { EVIDENCE_TEXT } from '../lib/terms' +import { EVIDENCE_TEXT, JUDGE_TEXT } from '../lib/terms' import { useCatalog } from '../store/catalog' -import { segmentKey, useEvidence } from '../store/evidence' +import { askKey, segmentKey, useEvidence, useExplore, useVerdicts } from '../store/evidence' import type { - EvidenceDocData, EvidenceInput, EvidenceQuoteSource, EvidenceSeal, EvidenceSegment, EvidenceStep, EvidenceUnit, - EvidenceViolation, + EvidenceBlock, EvidenceDocData, EvidenceInput, EvidenceOnDemand, EvidenceQuoteSource, EvidenceSeal, EvidenceSegment, + EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation, } from '../types' import { ArtifactViewer, ResultTable } from './AssistantStream' import { useEvidenceHost } from './evidenceHost' @@ -46,21 +49,27 @@ import { CopyChip } from './Markdown' * 名字给出最接近的已知名字），逐字引文（原文所在的文档和片段，引文在原文里的位置高亮）。 * 画布右栏里点开片段时，把证据路径交给画布（evidenceHost），节点名点一下就对准那个节点。 * + * 四期多了「模型的解释」：只对结论句显示——裁判模型的判定、理由（斜体）、带「模型判断 · 模型名 · 非确定」 + * 的徽标，封存之后按需追加的另写「封存后追加」。探索运行里还没判过的句子给「请模型判断这句」；到了上限 + * 写明是哪个上限、到哪里调。点句末徽标打开的是整句（kind: 'unit'）：句子、模型的解释、挂的依据、封存。 + * * 四种摆法：side 从右侧弹出（宽屏），drawer 从底部抽出（窄屏），inline 在 360px 的 * 画布右栏里直接栏内展开，dock 放进页面给的一块常驻位置（记录页的「证据」页签）。 * 都不是模态的：开着面板照样能在正文里走。 */ export type PanelMode = 'side' | 'drawer' | 'inline' | 'dock' -export type PanelView = { kind: 'seg'; id: string } | { kind: 'violations' } +export type PanelView = { kind: 'seg'; id: string } | { kind: 'unit'; id: string } | { kind: 'violations' } -export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, onLocate, onViolations }: { +export function EvidencePanel({ id, mode, doc, artifact, runId, runClass, view, onClose, onLocate, onViolations }: { id: string mode: PanelMode doc: EvidenceDocData /** 正文这份文档的工件 id：和证据接口报的封存文档比对，不是同一份就不信接口给的链 */ artifact?: string runId?: string + /** 运行类别：没开裁判的文档靠它认探索运行（「请模型判断这句」只在探索运行里有） */ + runClass?: string view: PanelView onClose: () => void /** 跳到正文里的某个片段（违规清单里的「定位」） */ @@ -69,7 +78,11 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, o }) { const titleId = `${id}-title` const found = useMemo(() => (view.kind === 'seg' ? findSeg(doc, view.id) : null), [doc, view]) - const state = found ? segmentState(found.seg) : null + const claimAt = useMemo(() => (view.kind === 'unit' ? findUnit(doc, view.id) : null), [doc, view]) + const overlay = useVerdicts(runId, doc.node_id || undefined) + const claimVerdict = claimAt ? unitVerdict(claimAt.unit, overlay) : undefined + const claimState = verdictState(claimVerdict) + const state = found ? segmentState(found.seg) : claimState const meta = state ? EVIDENCE_STATE[state] : null const onKeyDown = (e: ReactKeyboardEvent) => { @@ -81,7 +94,8 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, o } const title = view.kind === 'violations' ? EVIDENCE_TEXT.violations - : found ? segName(found.seg) : EVIDENCE_TEXT.panelTitle + : found ? segName(found.seg) + : claimAt ? unitText(claimAt.unit).trim() || JUDGE_TEXT.claim : EVIDENCE_TEXT.panelTitle // 有出处的实体、引文：徽标写得更具体（「有出处 · 逐字引文」），颜色和字形照同一套 const kind = found && state === 'deterministic' ? segmentKind(found.seg) : null const badge = kind ? { ...meta!, label: EVIDENCE_KIND_STYLE[kind].label, glyph: EVIDENCE_KIND_STYLE[kind].glyph } : meta @@ -110,7 +124,10 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, o )} {badge && ( - + // 概率性的判定（结论句）用虚线框：和系统核对过的「有出处」连框线都不一样 + {badge.glyph && {badge.glyph}}{badge.label} )} @@ -129,24 +146,35 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, o {view.kind === 'violations' ? : found - ? - :

{NONE}

} + ? + : claimAt + ? + :

{NONE}

}
) } -function findSeg(doc: EvidenceDocData, id: string): { seg: EvidenceSegment; unit: EvidenceUnit } | null { +function findSeg(doc: EvidenceDocData, id: string): { seg: EvidenceSegment; unit: EvidenceUnit; block: EvidenceBlock } | null { for (const block of doc.blocks ?? []) { for (const unit of block.units ?? []) { const seg = (unit.segments ?? []).find((s) => s.id === id) - if (seg) return { seg, unit } + if (seg) return { seg, unit, block } } } return null } +function findUnit(doc: EvidenceDocData, id: string): { unit: EvidenceUnit; block: EvidenceBlock } | null { + for (const block of doc.blocks ?? []) { + const unit = (block.units ?? []).find((u) => u.id === id) + if (unit) return { unit, block } + } + return null +} + /** 面板里的一节：小标题 + 内容 */ function Part({ title, children, ...rest }: { title: string; children: ReactNode } & Record<`data-${string}`, string>) { return ( @@ -163,9 +191,10 @@ function flashOnce(el: HTMLElement) { setTimeout(() => { if (el.dataset.flash === 'focus') delete el.dataset.flash }, 1600) } -function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: { +function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block, onViolations }: { panelId: string - doc: EvidenceDocData; artifact?: string; runId?: string; seg: EvidenceSegment; unit: EvidenceUnit + doc: EvidenceDocData; artifact?: string; runId?: string; runClass?: string; seg: EvidenceSegment; unit: EvidenceUnit + block: EvidenceBlock onViolations: () => void }) { const report = doc.node_id || undefined @@ -191,6 +220,7 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: const inGraph = graphDoc(graphData, artifact) const docIssue: 'foreign' | 'tampered' | null = foreign || inGraph === 'foreign' ? 'foreign' : inGraph === 'tampered' ? 'tampered' : null + useLateVerdict(runId, report, docIssue ? undefined : detail?.unit, unit.id) const state = segmentState(seg) const cite = seg.cite @@ -355,6 +385,9 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }:

{sourceOf(seg, doc) || detail?.note}

)} + + {host.onNode && (trace.producers.length > 0 || trace.consumers.length > 0) && ( // 画布右栏：这段证据经过的节点，点一下在画布上选中并对准 @@ -386,6 +419,271 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: ) } +/** + * 片段接口答的这一句带着封存之后追加的判定（后端按最新的 evidence.judged 叠上的）：记进缓存，正文的句末 + * 徽标、横幅的计数跟着变。接口答的是另一份报告时调用方不传 unit + */ +function useLateVerdict(runId: string | undefined, report: string | undefined, + answered: { id?: string; verdict?: EvidenceVerdict } | undefined, unitId: string) { + const noteVerdicts = useEvidence((s) => s.noteVerdicts) + const late = answered?.verdict?.post_seal && (!answered.id || answered.id === unitId) ? answered.verdict : undefined + const sig = late ? JSON.stringify(late) : '' + useEffect(() => { + if (runId && late) noteVerdicts(runId, report, { [unitId]: late }) + // 按内容比：同一条判定不重记 + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [runId, report, unitId, sig, noteVerdicts]) +} + +/** + * 点句末徽标打开的整句：句子、模型的解释、挂的依据、封存。封存状态和叠上的判定借这句第一个片段的 + * 片段接口取（接口答的是另一份报告时一概不用） + */ +function ClaimBody({ doc, artifact, runId, runClass, unit, block }: { + doc: EvidenceDocData; artifact?: string; runId?: string; runClass?: string; unit: EvidenceUnit; block: EvidenceBlock +}) { + const report = doc.node_id || undefined + const first = (unit.segments ?? []).find((s) => s.kind !== 'structural') + const key = runId && first ? segmentKey(runId, first.id, report) : '' + const slot = useEvidence((s) => (key ? s.segments[key] : undefined)) + const loadSegment = useEvidence((s) => s.loadSegment) + useEffect(() => { + if (runId && first) void loadSegment(runId, first.id, report) + }, [runId, first, report, loadSegment]) + const answered = slot?.status === 'ok' ? slot.data : undefined + const foreign = first ? docForeign(answered, { artifact, node: report, seg: first }) : null + useLateVerdict(runId, report, foreign ? undefined : answered?.unit, unit.id) + const overlay = useVerdicts(runId, report) + const verdict = unitVerdict(unit, overlay) + const seal = foreign ? undefined : answered?.seal + const late = !!verdict?.post_seal + const sealed = sealVerdict(seal, { + foreign: !!foreign, pending: !!runId && !seal && slot?.status === 'loading', noRun: !runId, docOnly: true, + }) + // 一整句没有「这一段的证据」可言：封存核对通过时只说报告文档封存了，节点里当场判的判定跟着一起封存 + const sealLabel = sealed.status !== 'done' ? sealed.label + : verdict && !late && isJudged(verdict) ? JUDGE_TEXT.sealedWithDoc : JUDGE_TEXT.sealedDoc + const cites = (unit.cites ?? []).map((alias) => ({ alias, label: doc.catalog?.[alias]?.label as string | undefined })) + return ( + <> + {foreign && ( + + )} + + + + + + {cites.length ? ( +
    + {cites.map((c) => ( +
  • + {c.alias} + {c.label && {c.label}} +
    • + ))} +
+ ) :

{JUDGE_TEXT.noCites}

} + + + + {late && ( + // 封存后追加的判定：报告文档封存着，这条判断不在封存范围里——两件事分开说 +

+ + {JUDGE_TEXT.postSeal}：{JUDGE_TEXT.postSealHint} +

+ )} + + + ) +} + +/** + * 模型的解释：只对结论句。判定、理由（斜体）、「模型判断 · 模型名 · 非确定」的徽标，封存之后按需追加的 + * 另写「封存后追加」；到上限没判的写明哪个上限、到哪里调；改写过一次的写明原句。探索运行里还没判过的 + * （没有判定、未裁判）给「请模型判断这句」。什么都说不上的（正式运行、没开裁判的文档）不画这一节 + */ +function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: { + doc: EvidenceDocData; unit: EvidenceUnit; block: EvidenceBlock; runId?: string; runClass?: string + /** + * 片段接口答的「这句能不能请模型判断」：有它就照它（运行还没跑完封存、封存被改过这些前端认不出来）； + * 老后端没有时按运行类别和文档自己认 + */ + onDemand?: EvidenceOnDemand + /** 片段接口还在取：先不按运行类别另取一次 */ + pending?: boolean +}) { + const report = doc.node_id || undefined + const overlay = useVerdicts(runId, report) + const verdict = unitVerdict(unit, overlay) + const eligible = judgeable(unit, block) + const known = onDemand && typeof onDemand.available === 'boolean' ? onDemand : undefined + const explore = useExplore(doc, runId, runClass, eligible && !isJudged(verdict) && !known && !pending) + const ask = useEvidence((s) => (runId ? s.asks[askKey(runId, report, unit.id)] : undefined)) + const judge = useEvidence((s) => s.judge) + const state = verdictState(verdict) + const meta = state ? EVIDENCE_STATE[state] : null + const limit = limitOf(verdict) + const late = !!verdict?.post_seal + const rewrite = rewriteOf(doc, unit.id) + const judgeDoc = !!doc.judge + const canAsk = !!runId && !isJudged(verdict) && (known ? !!known.available : eligible && explore === true) + // 探索运行里暂时判不了（还没跑完封存、封存被改过、升级前的运行）：照接口的原话说为什么没有按钮 + const blocked = known && !known.available && BLOCKED.has(known.reason ?? '') ? known.message || '' : '' + // 这一次按需裁判的答复里没有这一句的判定：接口说了为什么（跳过了、触顶）就照说 + const res = ask?.status === 'ok' ? ask.data : undefined + const skipped = res?.skipped?.[unit.id] + const resLimit = !verdict || verdict.status !== 'unjudged' ? null + : (Array.isArray(res?.limits_hit) && res.limits_hit.find((l) => typeof l === 'string')) || null + const shownLimit = limit ?? resLimit + const judged = isJudged(verdict) + // 这一次答复的那句话（message）：判定之外接口还想说的，照原话写出来。最要紧的是「运行已经接着跑了，这次的 + // 判定没有记进运行记录」——判定照样交回来、也标着封存后追加，只看徽标会以为它记进去了。已经由别处说过的不重复： + // 跳过的那一行就是它、上限的那一框说的就是它、和判定里写的理由一字不差 + const resMessage = typeof res?.message === 'string' ? res.message.trim() : '' + const showMessage = !!resMessage && !(skipped && !judged) && !(shownLimit && resMessage.startsWith(JUDGE_TEXT.limit)) + && resMessage !== verdict?.rationale + // 只对结论句：表格单元格、代码、标题不是一句话，开了裁判的文档里也不画这一节 + const show = !!verdict || (judgeDoc && eligible) || canAsk || !!blocked || ask?.status === 'error' || showMessage + if (!show) return null + const asking = ask?.status === 'loading' + const how = shownLimit ? JUDGE_TEXT.limitHow[late || explore ? 'click' : 'report'][shownLimit] : '' + return ( + + {verdict && judged && ( + // 「模型判断 · 模型名 · 非确定」只挂在模型真判过的句子上：到上限、没跑成的未裁判也记着裁判模型， + // 但模型没看过这句，挂上这枚徽标就像它判过一样 +
+ + {JUDGE_TEXT.badge(verdict.judge)} + + {late && ( + + {JUDGE_TEXT.postSeal} + + )} +
+ )} + {verdict && ( +

+ {meta?.glyph && {meta.glyph}} + {meta ? meta.label : JUDGE_TEXT.notClaim} +

+ )} + {showMessage && ( +

+ {resMessage} +

+ )} + {verdict && !judged && verdict.judge && ( + // 没判的句子只中性地说一句裁判模型是谁（到上限时知道是哪个模型的价钱、哪个模型没跑成） +

{JUDGE_TEXT.notJudgedBy(verdict.judge)}

+ )} + {verdict?.rationale && !shownLimit && ( + // 理由是模型写的话：斜体，和系统给的说明分开 +

{verdict.rationale}

+ )} + {shownLimit && ( +
+

{JUDGE_TEXT.limit} + {limitWords(verdict?.rationale) ? `：${limitWords(verdict?.rationale)}` : ''}

+ {how && ( +

+ {how} + {how.startsWith('到「设置') && ( + <> 去设置 + )} +

+ )} +
+ )} + {verdict?.status === 'unjudged' && !shownLimit && verdict.reason !== 'on_demand' && verdict.rationale && ( +

{verdict.rationale}

+ )} + {!!verdict?.used?.length && judged && ( +

{JUDGE_TEXT.used(verdict.used)}

+ )} + {!verdict && judgeDoc && eligible && !canAsk && ( + // 开了裁判的文档里没有判定的结论句：预筛放掉的照实说；探索运行里暂时判不了的由上面 blocked 说 +

+ {doc.judge?.screened?.includes(unit.id) || explore !== false ? JUDGE_TEXT.screened : JUDGE_TEXT.formalOnly} +

+ )} + {rewrite?.kind === 'changed' && ( +
+

{JUDGE_TEXT.rewritten}

+ {rewrite.sentences.length > 0 && ( +

{JUDGE_TEXT.rewriteFrom}：{rewrite.sentences.map((t) => `「${t}」`).join('')}

+ )} +
+ )} + {rewrite?.kind === 'rejected' && ( +

+ {JUDGE_TEXT.rewriteRejected(rewrite.reason)} +

+ )} + {verdict && !late && judged && ( + // 封存后追加的那句说明放在「封存」一节（它说的是封存）；节点里当场判的在这里说一句随报告封存 +

{JUDGE_TEXT.sealedHint}

+ )} + {skipped && !judged && ( +

+ {res?.message || (skipped === 'not_a_claim' ? JUDGE_TEXT.notClaim : JUDGE_TEXT.notFound)} +

+ )} + {blocked && !canAsk && ( +

{blocked}

+ )} + {ask?.status === 'error' && ( +

+ {askError(ask.error)} +

+ )} + {canAsk && ( +
+ {!verdict || verdict.reason === 'on_demand' ? {JUDGE_TEXT.notAsked} : null} + + {JUDGE_TEXT.askHint} +
+ )} + + ) +} + +/** 「已到上限（这份报告的裁判金额上限 $0.05），这句没判」→「这份报告的裁判金额上限 $0.05，这句没判」 */ +function limitWords(rationale: string | undefined): string { + if (!rationale) return '' + const m = /^已到上限（(.+?)）[，,]?\s*(.*)$/.exec(rationale) + return m ? [m[1], m[2]].filter(Boolean).join('，') : rationale.replace(/^已到上限[：:]?/, '') +} + +/** 接口说这句暂时判不了、值得告诉人为什么的几种（正式运行、判过了、不是结论句不用说） */ +const BLOCKED = new Set(['unsealed', 'seal_broken', 'legacy']) + +/** + * 按需裁判没成：409 是这次运行不让判（正式运行、还没跑完封存、封存被改过），照后端的原话；老后端没有这个 + * 接口另有说法；其余按通用的错误说法 + */ +function askError(error: unknown): string { + if (error instanceof ApiError) { + if (error.status === 409) return error.message || JUDGE_TEXT.formalOnly + if ((error.status === 404 || error.status === 405) && !error.code) return JUDGE_TEXT.oldBackend + } + return JUDGE_TEXT.failed(humanizeError(error).title) +} + type InputLink = { index: number; alias: string; go: () => void } | undefined /** diff --git a/frontend/src/run/__tests__/evidence-judge.json b/frontend/src/run/__tests__/evidence-judge.json new file mode 100644 index 00000000..bde2fba5 --- /dev/null +++ b/frontend/src/run/__tests__/evidence-judge.json @@ -0,0 +1,4879 @@ +{ + "_comment": "可点击证据第四期（结论句裁判）的夹具：/tmp/agentlab-fix/E4-front/gen_fixture.py 生成。文档由 compose_doc 真跑，候选句由 judge.prepare 真跑，判定是剧本（模拟裁判模型），经 judge 的 summarize / apply_judgement 写回。只用通用名", + "model": "judge-model-b", + "formal": { + "run_id": "run-evidence-judge", + "run_class": "formal", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4", + "doc": { + "schema": "agentlab.report/1", + "run_id": "run-evidence-judge", + "node_id": "write", + "markdown": "## 本周概览\n\n本周（2026-W37）销售额 45,678.5，订单 1,234 单。\n\n增长主要来自新客首单，老客复购持平。\n\n退款 29 单，其中大多数是部分退款。\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。", + "source": "## 本周概览\n\n本周（[[i:week]]）销售额 [[v:Q1.r0.gmv]]，订单 [[v:Q1.r0.order_cnt]] 单。[[see:Q1]]\n\n增长主要来自新客首单，老客复购持平。[[see:Q1]]\n\n退款 [[v:Q2.r0.refund_cnt]] 单，其中大多数是部分退款。[[see:Q2]]\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。[[see:i:week]]", + "catalog": { + "Q1": { + "alias": "Q1", + "kind": "query", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {}, + "node_id": "fetch", + "exec": 1, + "artifact": "2ed2edc8ed904c049d96eecddc209e88ff3d0ccde4f06c0d6ee6d05f955bf4fa", + "via": "7a81331ed15bd9a9910bc90cc54ea6eb8b9dd21b8136464a342c19a49cf7a7c9", + "call_id": "toolu_01", + "tool": "db_query__shop", + "source": "shop", + "columns": [ + "gmv", + "order_cnt" + ], + "rows": 1, + "truncated": false, + "label": "db_query__shop · 1 行" + }, + "Q2": { + "alias": "Q2", + "kind": "query", + "eid": "ev:query:e483a806bd3a7ad9", + "locator": {}, + "node_id": "fetch", + "exec": 1, + "artifact": "79c1d7e581637df166e44a93e59b10f4eb0d94ba6f0fd5c76029a3b7c5e53853", + "via": "a61a900e68819eba5cc778d74e999759a3754a07ddcf9c5cf67b7856fb479f45", + "call_id": "toolu_02", + "tool": "db_query__shop", + "source": "shop", + "columns": [ + "refund_cnt", + "partial_cnt" + ], + "rows": 1, + "truncated": false, + "label": "db_query__shop · 1 行" + }, + "i:week": { + "alias": "i:week", + "kind": "input", + "eid": "ev:input:a398c5e60abfeedd", + "locator": { + "field": "week" + }, + "value": "2026-W37", + "rendered": "2026-W37", + "label": "week = 2026-W37" + } + }, + "blocks": [ + { + "type": "heading", + "level": 2, + "units": [ + { + "id": "u0", + "kind": "heading", + "span": [ + 3, + 7 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "## ", + "state": "neutral", + "span": [ + 0, + 3 + ], + "id": "s0" + }, + { + "kind": "text", + "text": "本周概览", + "state": "neutral", + "span": [ + 3, + 7 + ], + "id": "s1" + } + ] + } + ], + "id": "b0" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u1", + "kind": "claim", + "span": [ + 9, + 45 + ], + "cites": [ + "i:week", + "Q1" + ], + "see": [ + { + "ref": "Q1", + "role": "support", + "alias": "Q1", + "kind": "query", + "status": "resolved", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 7, + 9 + ], + "id": "s2" + }, + { + "kind": "text", + "text": "本周（", + "state": "neutral", + "span": [ + 9, + 12 + ], + "id": "s3" + }, + { + "kind": "value", + "text": "2026-W37", + "ref": "i:week", + "cite": { + "ref": "week", + "alias": "i:week", + "locator": { + "field": "week" + }, + "eid": "ev:input:a398c5e60abfeedd", + "kind": "input", + "role": "value", + "status": "resolved", + "value": "2026-W37", + "rendered": "2026-W37" + }, + "state": "deterministic", + "span": [ + 12, + 20 + ], + "id": "s4" + }, + { + "kind": "text", + "text": "）销售额 ", + "state": "neutral", + "span": [ + 20, + 25 + ], + "id": "s5" + }, + { + "kind": "number", + "text": "45,678.5", + "ref": "v:Q1.r0.gmv", + "cite": { + "ref": "Q1.r0.gmv", + "alias": "Q1", + "locator": { + "row": 0, + "column": "gmv" + }, + "eid": "ev:cell:dfba96352ac91db5", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 45678.5, + "rendered": "45,678.5" + }, + "state": "deterministic", + "span": [ + 25, + 33 + ], + "id": "s6" + }, + { + "kind": "text", + "text": "，订单 ", + "state": "neutral", + "span": [ + 33, + 37 + ], + "id": "s7" + }, + { + "kind": "number", + "text": "1,234", + "ref": "v:Q1.r0.order_cnt", + "cite": { + "ref": "Q1.r0.order_cnt", + "alias": "Q1", + "locator": { + "row": 0, + "column": "order_cnt" + }, + "eid": "ev:cell:a12c4cd776db6371", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 1234, + "rendered": "1,234" + }, + "state": "deterministic", + "span": [ + 37, + 42 + ], + "id": "s8" + }, + { + "kind": "text", + "text": " 单。", + "state": "neutral", + "span": [ + 42, + 45 + ], + "id": "s9" + } + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + } + ], + "id": "b1" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u2", + "kind": "claim", + "span": [ + 47, + 65 + ], + "cites": [ + "Q1" + ], + "see": [ + { + "ref": "Q1", + "role": "support", + "alias": "Q1", + "kind": "query", + "status": "resolved", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 45, + 47 + ], + "id": "s10" + }, + { + "kind": "text", + "text": "增长主要来自新客首单，老客复购持平。", + "state": "neutral", + "span": [ + 47, + 65 + ], + "id": "s11" + } + ], + "verdict": { + "status": "unsupported", + "rationale": "Q1 只有总销售额和订单数，看不出新客、老客各贡献多少", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1" + ] + } + } + ], + "id": "b2" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u3", + "kind": "claim", + "span": [ + 67, + 86 + ], + "cites": [ + "Q2" + ], + "see": [ + { + "ref": "Q2", + "role": "support", + "alias": "Q2", + "kind": "query", + "status": "resolved", + "eid": "ev:query:e483a806bd3a7ad9", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 65, + 67 + ], + "id": "s12" + }, + { + "kind": "text", + "text": "退款 ", + "state": "neutral", + "span": [ + 67, + 70 + ], + "id": "s13" + }, + { + "kind": "number", + "text": "29", + "ref": "v:Q2.r0.refund_cnt", + "cite": { + "ref": "Q2.r0.refund_cnt", + "alias": "Q2", + "locator": { + "row": 0, + "column": "refund_cnt" + }, + "eid": "ev:cell:e5d8f3b24016f3c9", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 29, + "rendered": "29" + }, + "state": "deterministic", + "span": [ + 70, + 72 + ], + "id": "s14" + }, + { + "kind": "text", + "text": " 单，其中大多数是部分退款。", + "state": "neutral", + "span": [ + 72, + 86 + ], + "id": "s15" + } + ], + "verdict": { + "status": "partial", + "rationale": "退款 29 单有依据；Q2 里部分退款 11 单，说不上「大多数」", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q2" + ] + } + } + ], + "id": "b3" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u4", + "kind": "claim", + "span": [ + 88, + 99 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 86, + 88 + ], + "id": "s16" + }, + { + "kind": "text", + "text": "华东贡献了最多的增量。", + "state": "neutral", + "span": [ + 88, + 99 + ], + "id": "s17" + } + ], + "verdict": { + "status": "unjudged", + "rationale": "已到上限（这份报告的裁判金额上限 $0.05），这句没判", + "judge": "judge-model-b", + "post_seal": false, + "reason": "max_cost_usd" + } + } + ], + "id": "b4" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u5", + "kind": "connective", + "span": [ + 101, + 109 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 99, + 101 + ], + "id": "s18" + }, + { + "kind": "text", + "text": "下面看退款情况。", + "state": "neutral", + "span": [ + 101, + 109 + ], + "id": "s19" + } + ] + } + ], + "id": "b5" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u6", + "kind": "claim", + "span": [ + 111, + 120 + ], + "cites": [ + "i:week" + ], + "see": [ + { + "ref": "i:week", + "role": "support", + "alias": "i:week", + "kind": "input", + "status": "resolved", + "eid": "ev:input:a398c5e60abfeedd", + "locator": { + "field": "week" + } + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 109, + 111 + ], + "id": "s20" + }, + { + "kind": "text", + "text": "周报按自然周统计。", + "state": "neutral", + "span": [ + 111, + 120 + ], + "id": "s21" + } + ], + "verdict": { + "status": "not_a_claim", + "rationale": "这句说的是统计口径，不陈述数据事实", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "i:week" + ] + } + } + ], + "id": "b6" + } + ], + "stats": { + "units": 7, + "segments": 22, + "claims": 5, + "connective": 1, + "headings": 1, + "numbers": 3, + "numbers_cited": 3, + "values": 1, + "uncited_numbers": 0, + "unresolved": 0, + "see": 4, + "uncited_claims": 1, + "violations": 0, + "supported": 1, + "partial": 1, + "unsupported": 1, + "not_a_claim": 1, + "unjudged": 1 + }, + "violations": [], + "judge": { + "mode": "inline", + "model": "judge-model-b", + "on_unsupported": "degrade", + "rewrite_once": false, + "limits": { + "max_claims": 40, + "max_cost_usd": 0.05, + "timeout_s": 30, + "daily_max_usd": 2.0 + }, + "candidates": 5, + "screened": [], + "counts": { + "supported": 1, + "partial": 1, + "unsupported": 1, + "not_a_claim": 1, + "unjudged": 1 + }, + "unjudged": { + "max_cost_usd": 1 + }, + "limits_hit": [ + "max_cost_usd" + ], + "complete": false, + "priced": true, + "cost_usd": 0.0412, + "calls": 1, + "reused": 0, + "duration_ms": 6120, + "gaps": [ + "有 1 句结论没裁判：已到上限（这份报告的裁判金额上限 $0.05）" + ], + "notes": [], + "same_model": false, + "rewrite": null + } + }, + "graph": { + "run_id": "run-evidence-judge", + "schema": "agentlab.evidence/1", + "mode": "cited", + "seal": { + "sealed": true, + "ok": true, + "manifest_seq": 64, + "legacy": false + }, + "reports": [ + { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4", + "doc_sealed": true, + "hash_ok": true, + "fields": [ + "answer" + ], + "stats": { + "units": 7, + "segments": 22, + "claims": 5, + "connective": 1, + "headings": 1, + "numbers": 3, + "numbers_cited": 3, + "values": 1, + "uncited_numbers": 0, + "unresolved": 0, + "see": 4, + "uncited_claims": 1, + "violations": 0, + "supported": 1, + "partial": 1, + "unsupported": 1, + "not_a_claim": 1, + "unjudged": 1 + } + } + ], + "evidence": [], + "edges": [] + }, + "segments": { + "s1": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s1", + "text": "本周概览", + "kind": "text", + "state": "neutral", + "span": [ + 3, + 7 + ], + "unit": "u0" + }, + "unit": { + "id": "u0", + "kind": "heading", + "text": "本周概览", + "span": [ + 3, + 7 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s3": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s3", + "text": "本周（", + "kind": "text", + "state": "neutral", + "span": [ + 9, + 12 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s4": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s4", + "text": "2026-W37", + "kind": "value", + "state": "deterministic", + "span": [ + 12, + 20 + ], + "unit": "u1", + "cite": { + "ref": "week", + "alias": "i:week", + "locator": { + "field": "week" + }, + "eid": "ev:input:a398c5e60abfeedd", + "kind": "input", + "role": "value", + "status": "resolved", + "value": "2026-W37", + "rendered": "2026-W37" + }, + "ref": "i:week" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s5": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s5", + "text": "）销售额 ", + "kind": "text", + "state": "neutral", + "span": [ + 20, + 25 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s6": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s6", + "text": "45,678.5", + "kind": "number", + "state": "deterministic", + "span": [ + 25, + 33 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.gmv", + "alias": "Q1", + "locator": { + "row": 0, + "column": "gmv" + }, + "eid": "ev:cell:dfba96352ac91db5", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 45678.5, + "rendered": "45,678.5" + }, + "ref": "v:Q1.r0.gmv" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s7": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s7", + "text": "，订单 ", + "kind": "text", + "state": "neutral", + "span": [ + 33, + 37 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s8": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s8", + "text": "1,234", + "kind": "number", + "state": "deterministic", + "span": [ + 37, + 42 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.order_cnt", + "alias": "Q1", + "locator": { + "row": 0, + "column": "order_cnt" + }, + "eid": "ev:cell:a12c4cd776db6371", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 1234, + "rendered": "1,234" + }, + "ref": "v:Q1.r0.order_cnt" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s9": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s9", + "text": " 单。", + "kind": "text", + "state": "neutral", + "span": [ + 42, + 45 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数，说的是同一周", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1", + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s11": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s11", + "text": "增长主要来自新客首单，老客复购持平。", + "kind": "text", + "state": "neutral", + "span": [ + 47, + 65 + ], + "unit": "u2" + }, + "unit": { + "id": "u2", + "kind": "claim", + "text": "增长主要来自新客首单，老客复购持平。", + "span": [ + 47, + 65 + ], + "verdict": { + "status": "unsupported", + "rationale": "Q1 只有总销售额和订单数，看不出新客、老客各贡献多少", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q1" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s13": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s13", + "text": "退款 ", + "kind": "text", + "state": "neutral", + "span": [ + 67, + 70 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "partial", + "rationale": "退款 29 单有依据；Q2 里部分退款 11 单，说不上「大多数」", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q2" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s14": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s14", + "text": "29", + "kind": "number", + "state": "deterministic", + "span": [ + 70, + 72 + ], + "unit": "u3", + "cite": { + "ref": "Q2.r0.refund_cnt", + "alias": "Q2", + "locator": { + "row": 0, + "column": "refund_cnt" + }, + "eid": "ev:cell:e5d8f3b24016f3c9", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 29, + "rendered": "29" + }, + "ref": "v:Q2.r0.refund_cnt" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "partial", + "rationale": "退款 29 单有依据；Q2 里部分退款 11 单，说不上「大多数」", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q2" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s15": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s15", + "text": " 单，其中大多数是部分退款。", + "kind": "text", + "state": "neutral", + "span": [ + 72, + 86 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "partial", + "rationale": "退款 29 单有依据；Q2 里部分退款 11 单，说不上「大多数」", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "Q2" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s17": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s17", + "text": "华东贡献了最多的增量。", + "kind": "text", + "state": "neutral", + "span": [ + 88, + 99 + ], + "unit": "u4" + }, + "unit": { + "id": "u4", + "kind": "claim", + "text": "华东贡献了最多的增量。", + "span": [ + 88, + 99 + ], + "verdict": { + "status": "unjudged", + "rationale": "已到上限（这份报告的裁判金额上限 $0.05），这句没判", + "judge": "judge-model-b", + "post_seal": false, + "reason": "max_cost_usd" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s19": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s19", + "text": "下面看退款情况。", + "kind": "text", + "state": "neutral", + "span": [ + 101, + 109 + ], + "unit": "u5" + }, + "unit": { + "id": "u5", + "kind": "connective", + "text": "下面看退款情况。", + "span": [ + 101, + 109 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s21": { + "report": { + "node_id": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4" + }, + "segment": { + "id": "s21", + "text": "周报按自然周统计。", + "kind": "text", + "state": "neutral", + "span": [ + 111, + 120 + ], + "unit": "u6" + }, + "unit": { + "id": "u6", + "kind": "claim", + "text": "周报按自然周统计。", + "span": [ + 111, + 120 + ], + "verdict": { + "status": "not_a_claim", + "rationale": "这句说的是统计口径，不陈述数据事实", + "judge": "judge-model-b", + "post_seal": false, + "used": [ + "i:week" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + } + }, + "units": { + "ok": "u1", + "bad": "u2", + "partial": "u3", + "limit": "u4", + "not": "u6" + }, + "text_seg": { + "ok": "s3", + "bad": "s11", + "partial": "s13", + "limit": "s17", + "not": "s21" + }, + "report_checked": { + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4", + "ok": true, + "stats": { + "units": 7, + "segments": 22, + "claims": 5, + "connective": 1, + "headings": 1, + "numbers": 3, + "numbers_cited": 3, + "values": 1, + "uncited_numbers": 0, + "unresolved": 0, + "see": 4, + "uncited_claims": 1, + "violations": 0, + "supported": 1, + "partial": 1, + "unsupported": 1, + "not_a_claim": 1, + "unjudged": 1 + }, + "violations": [], + "repairs": 0, + "claims": "judge", + "judge": { + "mode": "inline", + "model": "judge-model-b", + "on_unsupported": "degrade", + "rewrite_once": false, + "limits": { + "max_claims": 40, + "max_cost_usd": 0.05, + "timeout_s": 30, + "daily_max_usd": 2.0 + }, + "candidates": 5, + "screened": [], + "counts": { + "supported": 1, + "partial": 1, + "unsupported": 1, + "not_a_claim": 1, + "unjudged": 1 + }, + "unjudged": { + "max_cost_usd": 1 + }, + "limits_hit": [ + "max_cost_usd" + ], + "complete": false, + "priced": true, + "cost_usd": 0.0412, + "calls": 1, + "reused": 0, + "duration_ms": 6120, + "gaps": [ + "有 1 句结论没裁判：已到上限（这份报告的裁判金额上限 $0.05）" + ], + "notes": [], + "same_model": false, + "rewrite": null + } + }, + "output": { + "answer": "## 本周概览\n\n本周（2026-W37）销售额 45,678.5，订单 1,234 单。\n\n增长主要来自新客首单，老客复购持平。\n\n退款 29 单，其中大多数是部分退款。\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。", + "_evidence": { + "report_node": "write", + "doc_artifact": "66cab12caafe4ddd98548bcb2fa28ba385e820e2f9ee8728afa92b438bc43fa4", + "fields": [ + "answer" + ] + }, + "_issuance": { + "tier": "degraded", + "matched_numbers": 3, + "metrics_checked": 0, + "claims": { + "policy": "judge", + "uncited_claims": 1, + "uncited": [ + { + "unit": "u4", + "span": [ + 88, + 99 + ], + "text": "华东贡献了最多的增量。" + } + ], + "on_unsupported": "degrade", + "mode": "inline", + "counts": { + "supported": 1, + "partial": 1, + "unsupported": 1, + "not_a_claim": 1, + "unjudged": 1 + }, + "unsupported": [ + { + "unit": "u2", + "span": [ + 47, + 65 + ], + "text": "增长主要来自新客首单，老客复购持平。", + "rationale": "Q1 只有总销售额和订单数，看不出新客、老客各贡献多少" + } + ], + "partial": [ + { + "unit": "u3", + "span": [ + 67, + 86 + ], + "text": "退款 29 单，其中大多数是部分退款。", + "rationale": "退款 29 单有依据；Q2 里部分退款 11 单，说不上「大多数」" + } + ], + "unjudged": { + "max_cost_usd": 1 + }, + "limits_hit": [ + "max_cost_usd" + ], + "complete": false, + "model": "judge-model-b" + }, + "gaps": [ + "有 1 句结论没裁判：已到上限（这份报告的裁判金额上限 $0.05）" + ] + } + } + }, + "explore": { + "run_id": "run-evidence-judge-explore", + "run_class": "exploratory", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860", + "doc": { + "schema": "agentlab.report/1", + "run_id": "run-evidence-judge-explore", + "node_id": "write", + "markdown": "## 本周概览\n\n本周（2026-W37）销售额 45,678.5，订单 1,234 单。\n\n增长主要来自新客首单，老客复购持平。\n\n退款 29 单，其中大多数是部分退款。\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。", + "source": "## 本周概览\n\n本周（[[i:week]]）销售额 [[v:Q1.r0.gmv]]，订单 [[v:Q1.r0.order_cnt]] 单。[[see:Q1]]\n\n增长主要来自新客首单，老客复购持平。[[see:Q1]]\n\n退款 [[v:Q2.r0.refund_cnt]] 单，其中大多数是部分退款。[[see:Q2]]\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。[[see:i:week]]", + "catalog": { + "Q1": { + "alias": "Q1", + "kind": "query", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {}, + "node_id": "fetch", + "exec": 1, + "artifact": "2ed2edc8ed904c049d96eecddc209e88ff3d0ccde4f06c0d6ee6d05f955bf4fa", + "via": "7a81331ed15bd9a9910bc90cc54ea6eb8b9dd21b8136464a342c19a49cf7a7c9", + "call_id": "toolu_01", + "tool": "db_query__shop", + "source": "shop", + "columns": [ + "gmv", + "order_cnt" + ], + "rows": 1, + "truncated": false, + "label": "db_query__shop · 1 行" + }, + "Q2": { + "alias": "Q2", + "kind": "query", + "eid": "ev:query:e483a806bd3a7ad9", + "locator": {}, + "node_id": "fetch", + "exec": 1, + "artifact": "79c1d7e581637df166e44a93e59b10f4eb0d94ba6f0fd5c76029a3b7c5e53853", + "via": "a61a900e68819eba5cc778d74e999759a3754a07ddcf9c5cf67b7856fb479f45", + "call_id": "toolu_02", + "tool": "db_query__shop", + "source": "shop", + "columns": [ + "refund_cnt", + "partial_cnt" + ], + "rows": 1, + "truncated": false, + "label": "db_query__shop · 1 行" + }, + "i:week": { + "alias": "i:week", + "kind": "input", + "eid": "ev:input:a398c5e60abfeedd", + "locator": { + "field": "week" + }, + "value": "2026-W37", + "rendered": "2026-W37", + "label": "week = 2026-W37" + } + }, + "blocks": [ + { + "type": "heading", + "level": 2, + "units": [ + { + "id": "u0", + "kind": "heading", + "span": [ + 3, + 7 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "## ", + "state": "neutral", + "span": [ + 0, + 3 + ], + "id": "s0" + }, + { + "kind": "text", + "text": "本周概览", + "state": "neutral", + "span": [ + 3, + 7 + ], + "id": "s1" + } + ] + } + ], + "id": "b0" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u1", + "kind": "claim", + "span": [ + 9, + 45 + ], + "cites": [ + "i:week", + "Q1" + ], + "see": [ + { + "ref": "Q1", + "role": "support", + "alias": "Q1", + "kind": "query", + "status": "resolved", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 7, + 9 + ], + "id": "s2" + }, + { + "kind": "text", + "text": "本周（", + "state": "neutral", + "span": [ + 9, + 12 + ], + "id": "s3" + }, + { + "kind": "value", + "text": "2026-W37", + "ref": "i:week", + "cite": { + "ref": "week", + "alias": "i:week", + "locator": { + "field": "week" + }, + "eid": "ev:input:a398c5e60abfeedd", + "kind": "input", + "role": "value", + "status": "resolved", + "value": "2026-W37", + "rendered": "2026-W37" + }, + "state": "deterministic", + "span": [ + 12, + 20 + ], + "id": "s4" + }, + { + "kind": "text", + "text": "）销售额 ", + "state": "neutral", + "span": [ + 20, + 25 + ], + "id": "s5" + }, + { + "kind": "number", + "text": "45,678.5", + "ref": "v:Q1.r0.gmv", + "cite": { + "ref": "Q1.r0.gmv", + "alias": "Q1", + "locator": { + "row": 0, + "column": "gmv" + }, + "eid": "ev:cell:dfba96352ac91db5", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 45678.5, + "rendered": "45,678.5" + }, + "state": "deterministic", + "span": [ + 25, + 33 + ], + "id": "s6" + }, + { + "kind": "text", + "text": "，订单 ", + "state": "neutral", + "span": [ + 33, + 37 + ], + "id": "s7" + }, + { + "kind": "number", + "text": "1,234", + "ref": "v:Q1.r0.order_cnt", + "cite": { + "ref": "Q1.r0.order_cnt", + "alias": "Q1", + "locator": { + "row": 0, + "column": "order_cnt" + }, + "eid": "ev:cell:a12c4cd776db6371", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 1234, + "rendered": "1,234" + }, + "state": "deterministic", + "span": [ + 37, + 42 + ], + "id": "s8" + }, + { + "kind": "text", + "text": " 单。", + "state": "neutral", + "span": [ + 42, + 45 + ], + "id": "s9" + } + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + } + ], + "id": "b1" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u2", + "kind": "claim", + "span": [ + 47, + 65 + ], + "cites": [ + "Q1" + ], + "see": [ + { + "ref": "Q1", + "role": "support", + "alias": "Q1", + "kind": "query", + "status": "resolved", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 45, + 47 + ], + "id": "s10" + }, + { + "kind": "text", + "text": "增长主要来自新客首单，老客复购持平。", + "state": "neutral", + "span": [ + 47, + 65 + ], + "id": "s11" + } + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + } + ], + "id": "b2" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u3", + "kind": "claim", + "span": [ + 67, + 86 + ], + "cites": [ + "Q2" + ], + "see": [ + { + "ref": "Q2", + "role": "support", + "alias": "Q2", + "kind": "query", + "status": "resolved", + "eid": "ev:query:e483a806bd3a7ad9", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 65, + 67 + ], + "id": "s12" + }, + { + "kind": "text", + "text": "退款 ", + "state": "neutral", + "span": [ + 67, + 70 + ], + "id": "s13" + }, + { + "kind": "number", + "text": "29", + "ref": "v:Q2.r0.refund_cnt", + "cite": { + "ref": "Q2.r0.refund_cnt", + "alias": "Q2", + "locator": { + "row": 0, + "column": "refund_cnt" + }, + "eid": "ev:cell:e5d8f3b24016f3c9", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 29, + "rendered": "29" + }, + "state": "deterministic", + "span": [ + 70, + 72 + ], + "id": "s14" + }, + { + "kind": "text", + "text": " 单，其中大多数是部分退款。", + "state": "neutral", + "span": [ + 72, + 86 + ], + "id": "s15" + } + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + } + ], + "id": "b3" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u4", + "kind": "claim", + "span": [ + 88, + 99 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 86, + 88 + ], + "id": "s16" + }, + { + "kind": "text", + "text": "华东贡献了最多的增量。", + "state": "neutral", + "span": [ + 88, + 99 + ], + "id": "s17" + } + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + } + ], + "id": "b4" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u5", + "kind": "connective", + "span": [ + 101, + 109 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 99, + 101 + ], + "id": "s18" + }, + { + "kind": "text", + "text": "下面看退款情况。", + "state": "neutral", + "span": [ + 101, + 109 + ], + "id": "s19" + } + ] + } + ], + "id": "b5" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u6", + "kind": "claim", + "span": [ + 111, + 120 + ], + "cites": [ + "i:week" + ], + "see": [ + { + "ref": "i:week", + "role": "support", + "alias": "i:week", + "kind": "input", + "status": "resolved", + "eid": "ev:input:a398c5e60abfeedd", + "locator": { + "field": "week" + } + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 109, + 111 + ], + "id": "s20" + }, + { + "kind": "text", + "text": "周报按自然周统计。", + "state": "neutral", + "span": [ + 111, + 120 + ], + "id": "s21" + } + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + } + ], + "id": "b6" + } + ], + "stats": { + "units": 7, + "segments": 22, + "claims": 5, + "connective": 1, + "headings": 1, + "numbers": 3, + "numbers_cited": 3, + "values": 1, + "uncited_numbers": 0, + "unresolved": 0, + "see": 4, + "uncited_claims": 1, + "violations": 0, + "supported": 0, + "partial": 0, + "unsupported": 0, + "not_a_claim": 0, + "unjudged": 5 + }, + "violations": [], + "judge": { + "mode": "on_demand", + "model": null, + "on_unsupported": "degrade", + "rewrite_once": false, + "limits": null, + "candidates": 5, + "screened": [], + "counts": { + "supported": 0, + "partial": 0, + "unsupported": 0, + "not_a_claim": 0, + "unjudged": 5 + }, + "unjudged": { + "on_demand": 5 + }, + "limits_hit": [], + "complete": false, + "priced": null, + "cost_usd": 0.0, + "calls": 0, + "reused": 0, + "duration_ms": 0, + "gaps": [], + "notes": [], + "same_model": false, + "rewrite": null + } + }, + "graph": { + "run_id": "run-evidence-judge-explore", + "schema": "agentlab.evidence/1", + "mode": "cited", + "seal": { + "sealed": true, + "ok": true, + "manifest_seq": 64, + "legacy": false + }, + "reports": [ + { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860", + "doc_sealed": true, + "hash_ok": true, + "fields": [ + "answer" + ], + "stats": { + "units": 7, + "segments": 22, + "claims": 5, + "connective": 1, + "headings": 1, + "numbers": 3, + "numbers_cited": 3, + "values": 1, + "uncited_numbers": 0, + "unresolved": 0, + "see": 4, + "uncited_claims": 1, + "violations": 0, + "supported": 0, + "partial": 0, + "unsupported": 0, + "not_a_claim": 0, + "unjudged": 5 + } + } + ], + "evidence": [], + "edges": [] + }, + "segments": { + "s1": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s1", + "text": "本周概览", + "kind": "text", + "state": "neutral", + "span": [ + 3, + 7 + ], + "unit": "u0" + }, + "unit": { + "id": "u0", + "kind": "heading", + "text": "本周概览", + "span": [ + 3, + 7 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s3": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s3", + "text": "本周（", + "kind": "text", + "state": "neutral", + "span": [ + 9, + 12 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s4": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s4", + "text": "2026-W37", + "kind": "value", + "state": "deterministic", + "span": [ + 12, + 20 + ], + "unit": "u1", + "cite": { + "ref": "week", + "alias": "i:week", + "locator": { + "field": "week" + }, + "eid": "ev:input:a398c5e60abfeedd", + "kind": "input", + "role": "value", + "status": "resolved", + "value": "2026-W37", + "rendered": "2026-W37" + }, + "ref": "i:week" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s5": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s5", + "text": "）销售额 ", + "kind": "text", + "state": "neutral", + "span": [ + 20, + 25 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s6": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s6", + "text": "45,678.5", + "kind": "number", + "state": "deterministic", + "span": [ + 25, + 33 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.gmv", + "alias": "Q1", + "locator": { + "row": 0, + "column": "gmv" + }, + "eid": "ev:cell:dfba96352ac91db5", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 45678.5, + "rendered": "45,678.5" + }, + "ref": "v:Q1.r0.gmv" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s7": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s7", + "text": "，订单 ", + "kind": "text", + "state": "neutral", + "span": [ + 33, + 37 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s8": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s8", + "text": "1,234", + "kind": "number", + "state": "deterministic", + "span": [ + 37, + 42 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.order_cnt", + "alias": "Q1", + "locator": { + "row": 0, + "column": "order_cnt" + }, + "eid": "ev:cell:a12c4cd776db6371", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 1234, + "rendered": "1,234" + }, + "ref": "v:Q1.r0.order_cnt" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s9": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s9", + "text": " 单。", + "kind": "text", + "state": "neutral", + "span": [ + 42, + 45 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s11": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s11", + "text": "增长主要来自新客首单，老客复购持平。", + "kind": "text", + "state": "neutral", + "span": [ + 47, + 65 + ], + "unit": "u2" + }, + "unit": { + "id": "u2", + "kind": "claim", + "text": "增长主要来自新客首单，老客复购持平。", + "span": [ + 47, + 65 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s13": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s13", + "text": "退款 ", + "kind": "text", + "state": "neutral", + "span": [ + 67, + 70 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s14": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s14", + "text": "29", + "kind": "number", + "state": "deterministic", + "span": [ + 70, + 72 + ], + "unit": "u3", + "cite": { + "ref": "Q2.r0.refund_cnt", + "alias": "Q2", + "locator": { + "row": 0, + "column": "refund_cnt" + }, + "eid": "ev:cell:e5d8f3b24016f3c9", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 29, + "rendered": "29" + }, + "ref": "v:Q2.r0.refund_cnt" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s15": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s15", + "text": " 单，其中大多数是部分退款。", + "kind": "text", + "state": "neutral", + "span": [ + 72, + 86 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s17": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s17", + "text": "华东贡献了最多的增量。", + "kind": "text", + "state": "neutral", + "span": [ + 88, + 99 + ], + "unit": "u4" + }, + "unit": { + "id": "u4", + "kind": "claim", + "text": "华东贡献了最多的增量。", + "span": [ + 88, + 99 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s19": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s19", + "text": "下面看退款情况。", + "kind": "text", + "state": "neutral", + "span": [ + 101, + 109 + ], + "unit": "u5" + }, + "unit": { + "id": "u5", + "kind": "connective", + "text": "下面看退款情况。", + "span": [ + 101, + 109 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s21": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s21", + "text": "周报按自然周统计。", + "kind": "text", + "state": "neutral", + "span": [ + 111, + 120 + ], + "unit": "u6" + }, + "unit": { + "id": "u6", + "kind": "claim", + "text": "周报按自然周统计。", + "span": [ + 111, + 120 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + } + }, + "segments_after": { + "s1": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s1", + "text": "本周概览", + "kind": "text", + "state": "neutral", + "span": [ + 3, + 7 + ], + "unit": "u0" + }, + "unit": { + "id": "u0", + "kind": "heading", + "text": "本周概览", + "span": [ + 3, + 7 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s3": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s3", + "text": "本周（", + "kind": "text", + "state": "neutral", + "span": [ + 9, + 12 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s4": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s4", + "text": "2026-W37", + "kind": "value", + "state": "deterministic", + "span": [ + 12, + 20 + ], + "unit": "u1", + "cite": { + "ref": "week", + "alias": "i:week", + "locator": { + "field": "week" + }, + "eid": "ev:input:a398c5e60abfeedd", + "kind": "input", + "role": "value", + "status": "resolved", + "value": "2026-W37", + "rendered": "2026-W37" + }, + "ref": "i:week" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s5": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s5", + "text": "）销售额 ", + "kind": "text", + "state": "neutral", + "span": [ + 20, + 25 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s6": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s6", + "text": "45,678.5", + "kind": "number", + "state": "deterministic", + "span": [ + 25, + 33 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.gmv", + "alias": "Q1", + "locator": { + "row": 0, + "column": "gmv" + }, + "eid": "ev:cell:dfba96352ac91db5", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 45678.5, + "rendered": "45,678.5" + }, + "ref": "v:Q1.r0.gmv" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s7": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s7", + "text": "，订单 ", + "kind": "text", + "state": "neutral", + "span": [ + 33, + 37 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s8": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s8", + "text": "1,234", + "kind": "number", + "state": "deterministic", + "span": [ + 37, + 42 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.order_cnt", + "alias": "Q1", + "locator": { + "row": 0, + "column": "order_cnt" + }, + "eid": "ev:cell:a12c4cd776db6371", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 1234, + "rendered": "1,234" + }, + "ref": "v:Q1.r0.order_cnt" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s9": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s9", + "text": " 单。", + "kind": "text", + "state": "neutral", + "span": [ + 42, + 45 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s11": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s11", + "text": "增长主要来自新客首单，老客复购持平。", + "kind": "text", + "state": "neutral", + "span": [ + 47, + 65 + ], + "unit": "u2" + }, + "unit": { + "id": "u2", + "kind": "claim", + "text": "增长主要来自新客首单，老客复购持平。", + "span": [ + 47, + 65 + ], + "verdict": { + "status": "unsupported", + "rationale": "Q1 只有总销售额和订单数，看不出新客、老客各贡献多少", + "judge": "judge-model-b", + "post_seal": true, + "used": [ + "Q1" + ] + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s13": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s13", + "text": "退款 ", + "kind": "text", + "state": "neutral", + "span": [ + 67, + 70 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s14": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s14", + "text": "29", + "kind": "number", + "state": "deterministic", + "span": [ + 70, + 72 + ], + "unit": "u3", + "cite": { + "ref": "Q2.r0.refund_cnt", + "alias": "Q2", + "locator": { + "row": 0, + "column": "refund_cnt" + }, + "eid": "ev:cell:e5d8f3b24016f3c9", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 29, + "rendered": "29" + }, + "ref": "v:Q2.r0.refund_cnt" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s15": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s15", + "text": " 单，其中大多数是部分退款。", + "kind": "text", + "state": "neutral", + "span": [ + 72, + 86 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s17": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s17", + "text": "华东贡献了最多的增量。", + "kind": "text", + "state": "neutral", + "span": [ + 88, + 99 + ], + "unit": "u4" + }, + "unit": { + "id": "u4", + "kind": "claim", + "text": "华东贡献了最多的增量。", + "span": [ + 88, + 99 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s19": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s19", + "text": "下面看退款情况。", + "kind": "text", + "state": "neutral", + "span": [ + 101, + 109 + ], + "unit": "u5" + }, + "unit": { + "id": "u5", + "kind": "connective", + "text": "下面看退款情况。", + "span": [ + 101, + 109 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s21": { + "report": { + "node_id": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860" + }, + "segment": { + "id": "s21", + "text": "周报按自然周统计。", + "kind": "text", + "state": "neutral", + "span": [ + 111, + 120 + ], + "unit": "u6" + }, + "unit": { + "id": "u6", + "kind": "claim", + "text": "周报按自然周统计。", + "span": [ + 111, + 120 + ], + "verdict": { + "status": "unjudged", + "rationale": "探索运行按需裁判：点开这句、请模型判断时才判", + "judge": null, + "post_seal": false, + "reason": "on_demand" + } + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + } + }, + "units": { + "ok": "u1", + "bad": "u2", + "limit": "u4" + }, + "text_seg": { + "ok": "s3", + "bad": "s11", + "limit": "s17" + }, + "judge": { + "bad": { + "report": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860", + "verdicts": { + "u2": { + "status": "unsupported", + "rationale": "Q1 只有总销售额和订单数，看不出新客、老客各贡献多少", + "judge": "judge-model-b", + "post_seal": true, + "used": [ + "Q1" + ] + } + }, + "keys": { + "u2": "0000000000000000" + }, + "model": "judge-model-b", + "cost_usd": 0.0031, + "calls": 1, + "limits_hit": [], + "gaps": [], + "notes": [], + "skipped": {}, + "post_seal": true + }, + "ok": { + "report": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860", + "verdicts": { + "u1": { + "status": "supported", + "rationale": "Q1 第 1 行的 gmv 和 order_cnt 正是这两个数", + "judge": "judge-model-b", + "post_seal": true, + "used": [ + "Q1", + "i:week" + ] + } + }, + "keys": { + "u1": "0000000000000000" + }, + "model": "judge-model-b", + "cost_usd": 0.0031, + "calls": 1, + "limits_hit": [], + "gaps": [], + "notes": [], + "skipped": {}, + "post_seal": true + }, + "limit": { + "report": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860", + "verdicts": { + "u4": { + "status": "unjudged", + "rationale": "已到上限（这次点击的裁判金额上限 $0.01），这句没判", + "judge": "judge-model-b", + "post_seal": true, + "reason": "max_cost_usd" + } + }, + "keys": { + "u4": "0000000000000000" + }, + "model": "judge-model-b", + "cost_usd": 0.0, + "calls": 0, + "limits_hit": [ + "max_cost_usd" + ], + "gaps": [], + "notes": [], + "skipped": {}, + "post_seal": true + } + }, + "output": { + "answer": "## 本周概览\n\n本周（2026-W37）销售额 45,678.5，订单 1,234 单。\n\n增长主要来自新客首单，老客复购持平。\n\n退款 29 单，其中大多数是部分退款。\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。", + "_evidence": { + "report_node": "write", + "doc_artifact": "ea710b3978d0f2c98a71ea3ce0d672bb0717a889d70b4f89cd3207c7190f5860", + "fields": [ + "answer" + ] + }, + "_issuance": { + "tier": "degraded", + "matched_numbers": 3, + "metrics_checked": 0 + } + } + }, + "plain": { + "run_id": "run-evidence-judge-plain", + "run_class": "exploratory", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5", + "doc": { + "schema": "agentlab.report/1", + "run_id": "run-evidence-judge-plain", + "node_id": "write", + "markdown": "## 本周概览\n\n本周（2026-W37）销售额 45,678.5，订单 1,234 单。\n\n增长主要来自新客首单，老客复购持平。\n\n退款 29 单，其中大多数是部分退款。\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。", + "source": "## 本周概览\n\n本周（[[i:week]]）销售额 [[v:Q1.r0.gmv]]，订单 [[v:Q1.r0.order_cnt]] 单。[[see:Q1]]\n\n增长主要来自新客首单，老客复购持平。[[see:Q1]]\n\n退款 [[v:Q2.r0.refund_cnt]] 单，其中大多数是部分退款。[[see:Q2]]\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。[[see:i:week]]", + "catalog": { + "Q1": { + "alias": "Q1", + "kind": "query", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {}, + "node_id": "fetch", + "exec": 1, + "artifact": "2ed2edc8ed904c049d96eecddc209e88ff3d0ccde4f06c0d6ee6d05f955bf4fa", + "via": "7a81331ed15bd9a9910bc90cc54ea6eb8b9dd21b8136464a342c19a49cf7a7c9", + "call_id": "toolu_01", + "tool": "db_query__shop", + "source": "shop", + "columns": [ + "gmv", + "order_cnt" + ], + "rows": 1, + "truncated": false, + "label": "db_query__shop · 1 行" + }, + "Q2": { + "alias": "Q2", + "kind": "query", + "eid": "ev:query:e483a806bd3a7ad9", + "locator": {}, + "node_id": "fetch", + "exec": 1, + "artifact": "79c1d7e581637df166e44a93e59b10f4eb0d94ba6f0fd5c76029a3b7c5e53853", + "via": "a61a900e68819eba5cc778d74e999759a3754a07ddcf9c5cf67b7856fb479f45", + "call_id": "toolu_02", + "tool": "db_query__shop", + "source": "shop", + "columns": [ + "refund_cnt", + "partial_cnt" + ], + "rows": 1, + "truncated": false, + "label": "db_query__shop · 1 行" + }, + "i:week": { + "alias": "i:week", + "kind": "input", + "eid": "ev:input:a398c5e60abfeedd", + "locator": { + "field": "week" + }, + "value": "2026-W37", + "rendered": "2026-W37", + "label": "week = 2026-W37" + } + }, + "blocks": [ + { + "type": "heading", + "level": 2, + "units": [ + { + "id": "u0", + "kind": "heading", + "span": [ + 3, + 7 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "## ", + "state": "neutral", + "span": [ + 0, + 3 + ], + "id": "s0" + }, + { + "kind": "text", + "text": "本周概览", + "state": "neutral", + "span": [ + 3, + 7 + ], + "id": "s1" + } + ] + } + ], + "id": "b0" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u1", + "kind": "claim", + "span": [ + 9, + 45 + ], + "cites": [ + "i:week", + "Q1" + ], + "see": [ + { + "ref": "Q1", + "role": "support", + "alias": "Q1", + "kind": "query", + "status": "resolved", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 7, + 9 + ], + "id": "s2" + }, + { + "kind": "text", + "text": "本周（", + "state": "neutral", + "span": [ + 9, + 12 + ], + "id": "s3" + }, + { + "kind": "value", + "text": "2026-W37", + "ref": "i:week", + "cite": { + "ref": "week", + "alias": "i:week", + "locator": { + "field": "week" + }, + "eid": "ev:input:a398c5e60abfeedd", + "kind": "input", + "role": "value", + "status": "resolved", + "value": "2026-W37", + "rendered": "2026-W37" + }, + "state": "deterministic", + "span": [ + 12, + 20 + ], + "id": "s4" + }, + { + "kind": "text", + "text": "）销售额 ", + "state": "neutral", + "span": [ + 20, + 25 + ], + "id": "s5" + }, + { + "kind": "number", + "text": "45,678.5", + "ref": "v:Q1.r0.gmv", + "cite": { + "ref": "Q1.r0.gmv", + "alias": "Q1", + "locator": { + "row": 0, + "column": "gmv" + }, + "eid": "ev:cell:dfba96352ac91db5", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 45678.5, + "rendered": "45,678.5" + }, + "state": "deterministic", + "span": [ + 25, + 33 + ], + "id": "s6" + }, + { + "kind": "text", + "text": "，订单 ", + "state": "neutral", + "span": [ + 33, + 37 + ], + "id": "s7" + }, + { + "kind": "number", + "text": "1,234", + "ref": "v:Q1.r0.order_cnt", + "cite": { + "ref": "Q1.r0.order_cnt", + "alias": "Q1", + "locator": { + "row": 0, + "column": "order_cnt" + }, + "eid": "ev:cell:a12c4cd776db6371", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 1234, + "rendered": "1,234" + }, + "state": "deterministic", + "span": [ + 37, + 42 + ], + "id": "s8" + }, + { + "kind": "text", + "text": " 单。", + "state": "neutral", + "span": [ + 42, + 45 + ], + "id": "s9" + } + ] + } + ], + "id": "b1" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u2", + "kind": "claim", + "span": [ + 47, + 65 + ], + "cites": [ + "Q1" + ], + "see": [ + { + "ref": "Q1", + "role": "support", + "alias": "Q1", + "kind": "query", + "status": "resolved", + "eid": "ev:query:f0db81e2badfc3e2", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 45, + 47 + ], + "id": "s10" + }, + { + "kind": "text", + "text": "增长主要来自新客首单，老客复购持平。", + "state": "neutral", + "span": [ + 47, + 65 + ], + "id": "s11" + } + ] + } + ], + "id": "b2" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u3", + "kind": "claim", + "span": [ + 67, + 86 + ], + "cites": [ + "Q2" + ], + "see": [ + { + "ref": "Q2", + "role": "support", + "alias": "Q2", + "kind": "query", + "status": "resolved", + "eid": "ev:query:e483a806bd3a7ad9", + "locator": {} + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 65, + 67 + ], + "id": "s12" + }, + { + "kind": "text", + "text": "退款 ", + "state": "neutral", + "span": [ + 67, + 70 + ], + "id": "s13" + }, + { + "kind": "number", + "text": "29", + "ref": "v:Q2.r0.refund_cnt", + "cite": { + "ref": "Q2.r0.refund_cnt", + "alias": "Q2", + "locator": { + "row": 0, + "column": "refund_cnt" + }, + "eid": "ev:cell:e5d8f3b24016f3c9", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 29, + "rendered": "29" + }, + "state": "deterministic", + "span": [ + 70, + 72 + ], + "id": "s14" + }, + { + "kind": "text", + "text": " 单，其中大多数是部分退款。", + "state": "neutral", + "span": [ + 72, + 86 + ], + "id": "s15" + } + ] + } + ], + "id": "b3" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u4", + "kind": "claim", + "span": [ + 88, + 99 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 86, + 88 + ], + "id": "s16" + }, + { + "kind": "text", + "text": "华东贡献了最多的增量。", + "state": "neutral", + "span": [ + 88, + 99 + ], + "id": "s17" + } + ] + } + ], + "id": "b4" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u5", + "kind": "connective", + "span": [ + 101, + 109 + ], + "cites": [], + "see": [], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 99, + 101 + ], + "id": "s18" + }, + { + "kind": "text", + "text": "下面看退款情况。", + "state": "neutral", + "span": [ + 101, + 109 + ], + "id": "s19" + } + ] + } + ], + "id": "b5" + }, + { + "type": "paragraph", + "units": [ + { + "id": "u6", + "kind": "claim", + "span": [ + 111, + 120 + ], + "cites": [ + "i:week" + ], + "see": [ + { + "ref": "i:week", + "role": "support", + "alias": "i:week", + "kind": "input", + "status": "resolved", + "eid": "ev:input:a398c5e60abfeedd", + "locator": { + "field": "week" + } + } + ], + "segments": [ + { + "kind": "structural", + "text": "\n\n", + "state": "neutral", + "span": [ + 109, + 111 + ], + "id": "s20" + }, + { + "kind": "text", + "text": "周报按自然周统计。", + "state": "neutral", + "span": [ + 111, + 120 + ], + "id": "s21" + } + ] + } + ], + "id": "b6" + } + ], + "stats": { + "units": 7, + "segments": 22, + "claims": 5, + "connective": 1, + "headings": 1, + "numbers": 3, + "numbers_cited": 3, + "values": 1, + "uncited_numbers": 0, + "unresolved": 0, + "see": 4, + "uncited_claims": 1, + "violations": 0 + }, + "violations": [] + }, + "graph": { + "run_id": "run-evidence-judge-plain", + "schema": "agentlab.evidence/1", + "mode": "cited", + "seal": { + "sealed": true, + "ok": true, + "manifest_seq": 64, + "legacy": false + }, + "reports": [ + { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5", + "doc_sealed": true, + "hash_ok": true, + "fields": [ + "answer" + ], + "stats": { + "units": 7, + "segments": 22, + "claims": 5, + "connective": 1, + "headings": 1, + "numbers": 3, + "numbers_cited": 3, + "values": 1, + "uncited_numbers": 0, + "unresolved": 0, + "see": 4, + "uncited_claims": 1, + "violations": 0 + } + } + ], + "evidence": [], + "edges": [] + }, + "segments": { + "s1": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s1", + "text": "本周概览", + "kind": "text", + "state": "neutral", + "span": [ + 3, + 7 + ], + "unit": "u0" + }, + "unit": { + "id": "u0", + "kind": "heading", + "text": "本周概览", + "span": [ + 3, + 7 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s3": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s3", + "text": "本周（", + "kind": "text", + "state": "neutral", + "span": [ + 9, + 12 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s4": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s4", + "text": "2026-W37", + "kind": "value", + "state": "deterministic", + "span": [ + 12, + 20 + ], + "unit": "u1", + "cite": { + "ref": "week", + "alias": "i:week", + "locator": { + "field": "week" + }, + "eid": "ev:input:a398c5e60abfeedd", + "kind": "input", + "role": "value", + "status": "resolved", + "value": "2026-W37", + "rendered": "2026-W37" + }, + "ref": "i:week" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s5": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s5", + "text": "）销售额 ", + "kind": "text", + "state": "neutral", + "span": [ + 20, + 25 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s6": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s6", + "text": "45,678.5", + "kind": "number", + "state": "deterministic", + "span": [ + 25, + 33 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.gmv", + "alias": "Q1", + "locator": { + "row": 0, + "column": "gmv" + }, + "eid": "ev:cell:dfba96352ac91db5", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 45678.5, + "rendered": "45,678.5" + }, + "ref": "v:Q1.r0.gmv" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s7": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s7", + "text": "，订单 ", + "kind": "text", + "state": "neutral", + "span": [ + 33, + 37 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s8": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s8", + "text": "1,234", + "kind": "number", + "state": "deterministic", + "span": [ + 37, + 42 + ], + "unit": "u1", + "cite": { + "ref": "Q1.r0.order_cnt", + "alias": "Q1", + "locator": { + "row": 0, + "column": "order_cnt" + }, + "eid": "ev:cell:a12c4cd776db6371", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 1234, + "rendered": "1,234" + }, + "ref": "v:Q1.r0.order_cnt" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s9": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s9", + "text": " 单。", + "kind": "text", + "state": "neutral", + "span": [ + 42, + 45 + ], + "unit": "u1" + }, + "unit": { + "id": "u1", + "kind": "claim", + "text": "本周（2026-W37）销售额 45,678.5，订单 1,234 单。", + "span": [ + 9, + 45 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s11": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s11", + "text": "增长主要来自新客首单，老客复购持平。", + "kind": "text", + "state": "neutral", + "span": [ + 47, + 65 + ], + "unit": "u2" + }, + "unit": { + "id": "u2", + "kind": "claim", + "text": "增长主要来自新客首单，老客复购持平。", + "span": [ + 47, + 65 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s13": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s13", + "text": "退款 ", + "kind": "text", + "state": "neutral", + "span": [ + 67, + 70 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s14": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s14", + "text": "29", + "kind": "number", + "state": "deterministic", + "span": [ + 70, + 72 + ], + "unit": "u3", + "cite": { + "ref": "Q2.r0.refund_cnt", + "alias": "Q2", + "locator": { + "row": 0, + "column": "refund_cnt" + }, + "eid": "ev:cell:e5d8f3b24016f3c9", + "kind": "cell", + "role": "value", + "status": "resolved", + "value": 29, + "rendered": "29" + }, + "ref": "v:Q2.r0.refund_cnt" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s15": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s15", + "text": " 单，其中大多数是部分退款。", + "kind": "text", + "state": "neutral", + "span": [ + 72, + 86 + ], + "unit": "u3" + }, + "unit": { + "id": "u3", + "kind": "claim", + "text": "退款 29 单，其中大多数是部分退款。", + "span": [ + 67, + 86 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s17": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s17", + "text": "华东贡献了最多的增量。", + "kind": "text", + "state": "neutral", + "span": [ + 88, + 99 + ], + "unit": "u4" + }, + "unit": { + "id": "u4", + "kind": "claim", + "text": "华东贡献了最多的增量。", + "span": [ + 88, + 99 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s19": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s19", + "text": "下面看退款情况。", + "kind": "text", + "state": "neutral", + "span": [ + 101, + 109 + ], + "unit": "u5" + }, + "unit": { + "id": "u5", + "kind": "connective", + "text": "下面看退款情况。", + "span": [ + 101, + 109 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + }, + "s21": { + "report": { + "node_id": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5" + }, + "segment": { + "id": "s21", + "text": "周报按自然周统计。", + "kind": "text", + "state": "neutral", + "span": [ + 111, + 120 + ], + "unit": "u6" + }, + "unit": { + "id": "u6", + "kind": "claim", + "text": "周报按自然周统计。", + "span": [ + 111, + 120 + ] + }, + "chain": [], + "seal": { + "sealed": true, + "ok": true, + "covered": true + }, + "redacted": { + "columns": [] + } + } + }, + "units": { + "bad": "u2" + }, + "text_seg": { + "bad": "s11" + }, + "judge": { + "bad": { + "report": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5", + "verdicts": { + "u2": { + "status": "unsupported", + "rationale": "Q1 只有总销售额和订单数，看不出新客、老客各贡献多少", + "judge": "judge-model-b", + "post_seal": true, + "used": [ + "Q1" + ] + } + }, + "keys": { + "u2": "0000000000000000" + }, + "model": "judge-model-b", + "cost_usd": 0.0031, + "calls": 1, + "limits_hit": [], + "gaps": [], + "notes": [], + "skipped": {}, + "post_seal": true + } + }, + "output": { + "answer": "## 本周概览\n\n本周（2026-W37）销售额 45,678.5，订单 1,234 单。\n\n增长主要来自新客首单，老客复购持平。\n\n退款 29 单，其中大多数是部分退款。\n\n华东贡献了最多的增量。\n\n下面看退款情况。\n\n周报按自然周统计。", + "_evidence": { + "report_node": "write", + "doc_artifact": "a298926483566ca94190b676441c8e9ed914cb99d7ef1cbb2b40a10526e112d5", + "fields": [ + "answer" + ] + }, + "_issuance": { + "tier": "degraded", + "matched_numbers": 3, + "metrics_checked": 0, + "claims": { + "policy": "require_citation", + "uncited_claims": 1, + "uncited": [ + { + "unit": "u4", + "span": [ + 88, + 99 + ], + "text": "华东贡献了最多的增量。" + } + ] + } + } + } + } +} diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 703b72c8..acbb6887 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -1,7 +1,7 @@ import type { RunEvent, TeamMember, TeamRound, TeamRun, ToolChange } from '../types' import type { NodeState, RunPhase } from './trace' import { formatDuration, formatNumber } from '../lib/format' -import { TYPE_LABEL, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' +import { TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms' import { statsTally } from '../lib/evidence' // 泳道数据画布也要用（supervisor 节点要展开成协作矩阵），所以类型放在 @@ -646,6 +646,23 @@ function queryTitle(sql: string, source: string): string { /** agent 开了 cite_fields 时循环结束后那次结构化抽取 */ const EXTRACT_TITLE = '按出处抽取字段' +/** + * report.checked 带的裁判摘要 → 「结论 4 句（支持 1 · 不支持 1 · 未裁判 2）」，和出具横幅同一种说法。 + * 「不是结论句」不算；没有摘要（没开裁判、老后端）返回空串 + */ +function judgeTally(judge: unknown): string { + const c = judge && typeof judge === 'object' ? (judge as { counts?: Record }).counts : undefined + if (!c || typeof c !== 'object') return '' + const n = (k: string) => (typeof c[k] === 'number' && Number.isFinite(c[k] as number) ? c[k] as number : 0) + const counts = { supported: n('supported'), partial: n('partial'), unsupported: n('unsupported'), unjudged: n('unjudged'), + uncited: 0 } + return claimTally({ ...counts, total: counts.supported + counts.partial + counts.unsupported + counts.unjudged }) +} + +/** 报告撰写节点请裁判模型判断结论句（四期，claims: judge）：units 是这一批几句 */ +const judgeTitle = (units: unknown) => + `请裁判模型判断${typeof units === 'number' && units > 0 ? ` ${formatNumber(units)} 句` : ''}结论` +const JUDGE_FAILED_TITLE = '结论句裁判没跑成' /** 审批行末尾：批的时候点了「始终允许」，这个工具之后不再问人 */ const ALWAYS_NOTE = '，并设为「始终允许 · 门控把关」' @@ -731,6 +748,59 @@ function explainLog(code: string | undefined, message: string): Pick `· ${String(v?.message ?? v?.text ?? v?.code ?? '')}`).filter((x) => x.length > 2) const more = violations.length > listed.length ? [`…另有 ${violations.length - listed.length} 处`] : [] @@ -1701,6 +1781,21 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt break } + case 'evidence.judged': { + // 探索运行跑完、封存之后，有人点开结论句请模型判断：判定追加在封存之后，不改封存的报告 + const verdicts = d.verdicts && typeof d.verdicts === 'object' ? Object.values(d.verdicts as Record) : [] + const judged = verdicts.filter((v) => ['supported', 'partial', 'unsupported', 'not_a_claim'].includes(v?.status)).length + const hit: string[] = Array.isArray(d.limits_hit) ? d.limits_hit : [] + push({ + id: `ej-${seq}`, seq, kind: 'note', nodeId, status: 'done', code: 'evidence_judged', + level: hit.length || (Array.isArray(d.gaps) && d.gaps.length) ? 'warn' : 'info', + title: `封存后按需裁判了 ${formatNumber(judged)} 句结论${hit.length ? '，有的到了上限没判' : ''}`, + sub: '封存后追加 · 模型判断，非确定', + ...(d.model ? { detail: [`模型：${d.model}`, ...(Array.isArray(d.gaps) ? d.gaps.map(String) : [])].join('\n') } : {}), + }, nodeId) + break + } + case 'caliber.upgrade': out.push({ id: `cu-${seq}`, seq, kind: 'note', level: 'warn', status: 'done', diff --git a/frontend/src/store/catalog.ts b/frontend/src/store/catalog.ts index ce7ba302..85f23066 100644 --- a/frontend/src/store/catalog.ts +++ b/frontend/src/store/catalog.ts @@ -392,6 +392,21 @@ export function hasPendingApproval(approvals: Approval[], runId: string | null | return approvals.some((a) => a.run_id === runId && a.status === 'pending') } +/** + * 只写了模型、没选接入时，后端按模型名找接入（providers/factory.resolve_provider）：启用的接入里模型清单有它的， + * 再看默认模型是它的；只有停用的接入认得它时后端直接报错（不会悄悄换一家）。都认不出返回 undefined—— + * 后端这时用第一个启用的非 mock 接入去调 + */ +export function providerOfModel(providers: Provider[], model: string): + { provider: Provider; enabled: boolean } | undefined { + const owns = (p: Provider) => (p.models ?? []).some((m) => m?.id === model) + const enabled = providers.filter((p) => p.enabled) + const hit = enabled.find(owns) ?? enabled.find((p) => p.default_model === model) + if (hit) return { provider: hit, enabled: true } + const off = providers.find((p) => !p.enabled && (owns(p) || p.default_model === model)) + return off ? { provider: off, enabled: false } : undefined +} + /** 把所有 provider 的模型拉平成下拉选项。 */ export function modelOptions(providers: Provider[]): { value: string; label: string; group: string }[] { const out: { value: string; label: string; group: string }[] = [] diff --git a/frontend/src/store/evidence.ts b/frontend/src/store/evidence.ts index 3af05336..a4905371 100644 --- a/frontend/src/store/evidence.ts +++ b/frontend/src/store/evidence.ts @@ -1,6 +1,10 @@ +import { useEffect, useMemo } from 'react' import { create } from 'zustand' import { ApiError, api } from '../api/client' -import type { EvidenceDocData, EvidenceGraph, EvidenceSegmentDetail, EvidenceStats } from '../types' +import { graphVerdicts, judgedVerdicts } from '../lib/evidence' +import type { + EvidenceDocData, EvidenceGraph, EvidenceJudgeResult, EvidenceSegmentDetail, EvidenceStats, EvidenceVerdict, +} from '../types' /** * 可点击证据的缓存：报告文档、证据图、点开过的片段。 @@ -12,6 +16,10 @@ import type { EvidenceDocData, EvidenceGraph, EvidenceSegmentDetail, EvidenceSta * * 出错的不缓存成定论：下次点开再取一次（后端刚重启、接口还没部署到这个版本都会是 * 一时的）。正在取的不重复发。 + * + * 四期：封存之后按需追加的结论句判定按 `runId@报告` 记在 judged 里（按需裁判接口、片段接口叠上的 + * 判定都记进来），正文的句末徽标、横幅的结论句计数据此盖过文档里的「未裁判 · 按需」。封存的文档本身 + * 一个字不改。同一句正在判的不重复发；判过的（判定不是未裁判）不再给按钮，也就不会再发 */ export interface Slot { @@ -49,8 +57,23 @@ interface EvidenceState { * 不一定还是最近的。清掉以后，运行摘下来（「清除」、换一张图再回来）时 loadLatest 重新取 */ dropLatest: (workflowId: string) => void + /** 封存之后追加的判定：`runId@报告` → {unit: 判定} */ + judged: Record> + /** 按需裁判的请求：`runId@报告:unit` → 这一次的答复（触顶、失败说什么靠它） */ + asks: Record> + /** 运行类别（formal / exploratory）：没开裁判的文档靠它认探索运行，按需取一次 */ + runClasses: Record> + /** 记下几句封存后追加的判定（片段接口叠上的、证据图带来的） */ + noteVerdicts: (runId: string, report: string | undefined, verdicts: Record) => void + /** 请模型判断这几句（探索运行）。判定记进 judged；答复（含触顶、出错）记进 asks */ + judge: (runId: string, report: string | undefined, units: string[]) => Promise + loadRunClass: (runId: string) => Promise } +/** 封存后追加的判定按哪份报告记：一次运行里有好几份报告时 unit id 会撞（每份都从 u0 数起） */ +export const docKey = (runId: string, report?: string) => `${runId}@${report ?? ''}` +export const askKey = (runId: string, report: string | undefined, unit: string) => `${docKey(runId, report)}:${unit}` + /** * 片段的缓存键：`runId:segId`。一次运行里有好几份报告时片段 id 会撞（每份都从 s0 数起）， * 带上报告节点：`runId:segId@report` @@ -134,5 +157,73 @@ export const useEvidence = create((set, get) => { return { latest } }) }, + judged: {}, + asks: {}, + runClasses: {}, + noteVerdicts: (runId, report, verdicts) => { + const key = docKey(runId, report) + const cur = get().judged[key] ?? {} + // 内容一样就不换引用：正文按它重算徽标，片段接口每次答同样的判定不该让整份报告重渲染 + const fresh = Object.entries(verdicts).filter(([u, v]) => JSON.stringify(cur[u]) !== JSON.stringify(v)) + if (!fresh.length) return + set((s) => ({ judged: { ...s.judged, [key]: { ...(s.judged[key] ?? {}), ...Object.fromEntries(fresh) } } })) + }, + judge: async (runId, report, units) => { + const keys = units.map((u) => askKey(runId, report, u)) + if (!units.length || keys.some((k) => get().asks[k]?.status === 'loading')) return + const mark = (slot: Slot) => + set((s) => ({ asks: { ...s.asks, ...Object.fromEntries(keys.map((k) => [k, slot])) } })) + mark({ status: 'loading' }) + try { + const res = await api.evidence.judge(runId, { units, ...(report ? { report } : {}) }) + get().noteVerdicts(runId, report, judgedVerdicts(res)) + mark({ status: 'ok', data: res }) + } catch (error) { + mark({ status: 'error', error }) + } + }, + loadRunClass: (runId) => { + const cur = get().runClasses[runId] + if (cur && cur.status !== 'error') return Promise.resolve() + set((s) => ({ runClasses: { ...s.runClasses, [runId]: { status: 'loading' } } })) + return api.runs.get(runId).then( + (run) => set((s) => ({ runClasses: { ...s.runClasses, [runId]: { status: 'ok', data: run?.run_class ?? null } } })), + (error) => set((s) => ({ runClasses: { ...s.runClasses, [runId]: { status: 'error', error } } })), + ) + }, } }) + +/** + * 一份报告封存之后追加的判定：证据图带来的（reports[].judged），叠上这一页里按需判的、片段接口答的 + * （后者新）。没有运行、什么都没有时是 undefined + */ +export function useVerdicts(runId: string | undefined, report: string | undefined): + Record | undefined { + const own = useEvidence((s) => (runId ? s.judged[docKey(runId, report)] : undefined)) + const graph = useEvidence((s) => (runId ? s.graphs[runId]?.data : undefined)) + return useMemo(() => { + const fromGraph = graphVerdicts(graph, report) + if (!own && !Object.keys(fromGraph).length) return undefined + return { ...fromGraph, ...(own ?? {}) } + }, [own, graph, report]) +} + +/** + * 这份报告出自探索运行吗（按需裁判只在探索运行里有）。开了裁判的文档自己记着：on_demand 是探索运行、 + * inline 是正式运行；没开裁判的看调用方给的运行类别，再没有就按运行 id 取一次（enabled 时才取）。 + * 还不知道时是 null：按钮先不出，免得在正式运行里闪一下 + */ +export function useExplore(doc: Pick, runId: string | undefined, runClass: string | undefined, + enabled: boolean): boolean | null { + const mode = doc.judge?.mode + const known = mode === 'on_demand' ? true : mode === 'inline' ? false + : runClass ? runClass === 'exploratory' : undefined + const slot = useEvidence((s) => (runId ? s.runClasses[runId] : undefined)) + const loadRunClass = useEvidence((s) => s.loadRunClass) + const need = known === undefined && enabled && !!runId + useEffect(() => { if (need && runId) void loadRunClass(runId) }, [need, runId, loadRunClass]) + if (known !== undefined) return known + if (!runId) return false + return slot?.status === 'ok' ? slot.data === 'exploratory' : null +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index dff87883..19be2469 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -706,6 +706,95 @@ export interface EvidenceUnit { loc?: { row: number; col: number } /** 列表项的缩进层级 */ depth?: number + /** + * 结论句裁判的判定（四期）：只有送了裁判的候选句有。正式运行里节点当场判的进封存（post_seal false）； + * 探索运行里候选句先标 unjudged + reason on_demand，点开再判 + */ + verdict?: EvidenceVerdict +} + +/** + * 模型对一句结论的判定。是模型的判断，不是证据，不能被引用。 + * status 为 unjudged 时 reason 写明为什么没判：四个上限（max_claims / max_cost_usd / daily_max_usd / + * timeout_s，rationale 以「已到上限（…）」开头）、error、format、missing、on_demand（探索运行按需） + */ +export interface EvidenceVerdict { + status: 'supported' | 'partial' | 'unsupported' | 'not_a_claim' | 'unjudged' | string + /** 理由，最多 60 字 */ + rationale?: string + /** 裁判模型 id；探索运行还没判的是 null */ + judge?: string | null + /** 封存之后按需追加的（evidence.judged 事件） */ + post_seal?: boolean + /** 裁判看过、摘录里真有的证据编号 */ + used?: string[] + reason?: string + cost_usd?: number +} + +/** 文档（和 report.checked、节点产出）里的裁判摘要：键的顺序同后端 judge.SUMMARY_KEYS */ +export interface EvidenceJudgeSummary { + /** inline：正式运行节点里当场判；on_demand：探索运行按需 */ + mode?: 'inline' | 'on_demand' | string + model?: string | null + on_unsupported?: 'degrade' | 'withhold' | string + rewrite_once?: boolean + limits?: { max_claims?: number | null; max_cost_usd?: number | null; timeout_s?: number | null + daily_max_usd?: number | null } | null + candidates?: number + /** 预筛放掉、没送裁判的结论句 */ + screened?: string[] + counts?: Partial> + unjudged?: Record + limits_hit?: string[] + complete?: boolean + priced?: boolean | null + cost_usd?: number + calls?: number + reused?: number + duration_ms?: number + gaps?: string[] + notes?: string[] + same_model?: boolean + /** + * 不支持的句子交回写作者改写一次（rewrite_once）。units：交回的句子在封存文档里的编号（采用了改写稿时， + * 改掉的句子是 null）；sentences：交回的原句；changed：改写稿里新写的句子的编号（没采用时为空）； + * reason：没采用的原因 + */ + rewrite?: { units?: (string | null)[]; sentences?: string[]; applied?: boolean; reason?: string | null + changed?: string[] } | null +} + +/** + * POST /runs/{id}/evidence/judge 的答复：evidence.judged 事件的载荷（NOTES-A4 §8）。前端对缺字段宽容： + * verdicts 也认成 [{unit, …}] 的列表 + */ +export interface EvidenceJudgeResult { + /** 判的是哪份报告：{node_id, doc_artifact}（事件载荷里是节点 id 字符串） */ + report?: string | { node_id?: string; doc_artifact?: string } + doc_artifact?: string + verdicts?: Record | (EvidenceVerdict & { unit?: string })[] + model?: string | null + cost_usd?: number + calls?: number + limits_hit?: string[] + gaps?: string[] + notes?: string[] + skipped?: Record + post_seal?: boolean + /** 触顶、没跑成、点的不是结论句时的一句话；判成了是 null */ + message?: string | null + /** 触顶时怎么调，一个上限一句 */ + adjust?: string[] + [key: string]: unknown +} + +/** 这句能不能「请模型判断」：片段接口按运行的实际情况答（和按需裁判接口同一套条件） */ +export interface EvidenceOnDemand { + available?: boolean + /** formal / legacy / not_a_claim / judged / unsealed / seal_broken */ + reason?: string | null + message?: string | null } export interface EvidenceBlock { @@ -750,6 +839,12 @@ export interface EvidenceStats { quotes?: number unknown_entities?: number unverified_entities?: number + /** 四期：claims 为 judge 时追加，候选句按判定的句数 */ + supported?: number + partial?: number + unsupported?: number + not_a_claim?: number + unjudged?: number } /** report_doc 工件的内容 */ @@ -764,6 +859,8 @@ export interface EvidenceDocData { blocks: EvidenceBlock[] stats?: EvidenceStats violations?: EvidenceViolation[] + /** 四期：写作时结论句策略是 judge 才有 */ + judge?: EvidenceJudgeSummary } /** @@ -807,7 +904,14 @@ export interface EvidenceGraph { /** 文档取回时哈希复验：false 对不上（或不是这次运行这个节点写的），null 文件不在了 */ hash_ok?: boolean | null /** 写这份报告时的结论句策略（report.checked 里记的；老后端、升级前的运行没有） */ - claims?: string | null }[] + claims?: string | null + /** + * 四期：封存之后按需追加的判定（evidence.judged 叠在一起），{unit: 判定}。后端没给时前端只认 + * 点开时片段接口、按需裁判接口答的那几句 + */ + post_seal_verdicts?: Record | null + /** 封存的裁判摘要（文档的 judge） */ + judge?: EvidenceJudgeSummary | null }[] evidence?: { alias?: string; eid?: string; kind?: string; label?: string; node_id?: string; artifact?: string sealed?: boolean; cited_by?: string[]; report?: string }[] edges?: { from: string; to: string; rel: string }[] @@ -1042,7 +1146,11 @@ export interface EvidenceAudit { export interface EvidenceSegmentDetail { report?: { node_id?: string; doc_artifact?: string } segment?: Partial & { unit?: string } - unit?: { id?: string; kind?: string; text?: string; span?: [number, number]; cites?: string[] } + unit?: { id?: string; kind?: string; text?: string; span?: [number, number]; cites?: string[] + /** 这一句的判定：封存的文档里的，或者按最新的 evidence.judged 叠上的（post_seal） */ + verdict?: EvidenceVerdict + /** 这一句能不能请模型判断、不能的话为什么 */ + on_demand?: EvidenceOnDemand } block?: { id?: string; type?: string } chain?: EvidenceStep[] /** 片段为什么是现在这个状态，一句人话 */ diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 6736bd21..e37c9cdd 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -1313,6 +1313,103 @@ await section('agent 字段按出处核对（可点击证据第二期）：抽 && !odd.some((s) => /undefined|NaN/.test(`${s.title}${s.sub ?? ''}`)), odd.map((s) => s.title).join(' | ')) }) +await section('结论句裁判（可点击证据第四期）：裁判调用单独成行、几条警告说人话、封存后按需裁判有记录', async () => { + const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) + // 原话和后端同一个格式：engine/judge.py 的 run_request、engine/nodes/report.py 的改写一次。先核对后端源码还是这么说的 + const judgePy = readFileSync(`${root}backend/app/engine/judge.py`, 'utf8') + const reportPy = readFileSync(`${root}backend/app/engine/nodes/report.py`, 'utf8') + check('后端发裁判事件的格式还是夹具里这一种', judgePy.includes('purpose="judge"') && judgePy.includes('code="judge_limit"') + && judgePy.includes('结论句裁判已到上限（{labels}）：{n} 句没判，记为未裁判；已判的保留') + && judgePy.includes('"结论句裁判没跑完：" + "；".join(failed)') && judgePy.includes('code="judge_unpriced"') + && judgePy.includes('不在价格目录里，按令牌估不出金额：') && reportPy.includes('code="judge_same_model"') + && reportPy.includes('：等于自己审自己') && reportPy.includes('code="report_rewrite"') && reportPy.includes('句结论证据不支持（') + && reportPy.includes('已交回写作者只改这几句') && reportPy.includes('改写稿冒出 {len(fresh)} 处原稿没有的问题（') + && reportPy.includes('没有采用，') && reportPy.includes('保留原稿和原来的判定'), + '改了 judge.py / report.py 的说法就同步改 decode.ts 的 explainLog 和这里的夹具') + const LIMIT = '结论句裁判已到上限（这份报告的裁判金额上限 $0.05）：3 句没判，记为未裁判；已判的保留' + const FAILED = '结论句裁判没跑完：有 2 句结论没裁判：裁判调用失败（ConnectError: 连不上）' + const UNPRICED = '模型「jm-x」不在价格目录里，按令牌估不出金额：金额上限（每份报告、每次点击、每日）对它不起作用，费用只受句数、时长上限约束' + const SAME = '裁判模型和写作模型都是「wm-1」：等于自己审自己，模型写错的地方它多半也看不出来。到设置里选一个不同的「证据裁判模型」，或者在节点的 judge.model 里指定' + const REWRITE = '裁判认为 2 句结论证据不支持（「增长主要来自新客」「退款多数是部分退款」），已交回写作者只改这几句' + const REJECTED = '改写稿冒出 1 处原稿没有的问题（「12」），没有采用，保留原稿和原来的判定' + const events = [ + E(1, 'run.started', null, { total: 1 }), + E(2, 'node.started', 'write', { node_type: 'report', label: '写周报' }), + E(3, 'llm.start', 'write', { model: 'wm-1' }), + E(4, 'llm.end', 'write', { model: 'wm-1', duration_ms: 900 }), + E(5, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 4 }), + E(6, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 2100, input_tokens: 900, output_tokens: 300, units: 4 }), + E(7, 'log', 'write', { level: 'warn', code: 'judge_unpriced', message: UNPRICED }), + E(8, 'log', 'write', { level: 'warn', code: 'judge_same_model', message: SAME }), + E(9, 'log', 'write', { level: 'warn', code: 'judge_limit', limits: ['max_cost_usd'], unjudged: 3, message: LIMIT }), + E(10, 'log', 'write', { level: 'warn', code: 'judge_failed', message: FAILED }), + E(11, 'log', 'write', { level: 'warn', code: 'report_rewrite', message: REWRITE }), + E(12, 'log', 'write', { level: 'warn', code: 'report_rewrite_rejected', message: REJECTED }), + E(13, 'report.checked', 'write', { doc_artifact: 'doc-1', stats: { numbers: 3, numbers_cited: 3, uncited_numbers: 0, unresolved: 0 }, + violations: [], repairs: 0, claims: 'judge', + judge: { mode: 'inline', counts: { supported: 1, partial: 1, unsupported: 1, not_a_claim: 1, unjudged: 1 } } }), + E(14, 'node.finished', 'write', { duration_ms: 4000 }), + E(15, 'run.finished', null, { status: 'succeeded' }), + // 探索运行跑完之后有人点开结论句请模型判断：判定追加在封存之后 + E(16, 'evidence.judged', 'write', { report: 'write', doc_artifact: 'doc-1', units: ['u2'], model: 'jm-x', + verdicts: { u2: { status: 'unsupported', rationale: '看不出', judge: 'jm-x', post_seal: true } }, limits_hit: [], gaps: [], + post_seal: true }), + ] + const all = flatten(mod.decodeRun(events)) + check('新事件都有翻译', !all.some((s) => s.title === '一条还没翻译的记录'), all.filter((s) => s.title === '一条还没翻译的记录').map((s) => s.raw?.type).join(',')) + const llms = all.filter((s) => s.kind === 'llm') + check('写作照旧叫「思考并作答」，裁判那次单独起名、说判几句', llms.map((s) => s.title).join('|') === '思考并作答|请裁判模型判断 4 句结论', + llms.map((s) => s.title).join('|')) + check('……裁判那次跑完了、带耗时和模型', llms[1]?.status === 'done' && llms[1]?.meta === '2.1 s' && llms[1]?.code === 'judge' + && llms[1]?.detail === '模型：jm-x', `${llms[1]?.status} ${llms[1]?.meta} ${llms[1]?.code} ${llms[1]?.detail}`) + const by = (code) => all.find((s) => s.code === code) + check('judge_limit：几句没判、哪个上限、怎么调', by('judge_limit')?.title === '结论句裁判已到上限，3 句没判（记为未裁判，已判的保留）' + && by('judge_limit')?.sub === '这份报告的裁判金额上限 $0.05' && !!by('judge_limit')?.next?.includes('不限'), + `${by('judge_limit')?.title} / ${by('judge_limit')?.sub} / ${by('judge_limit')?.next}`) + check('judge_failed：没跑完、原因放副标题', by('judge_failed')?.title === '结论句裁判没跑完，没判的记为未裁判' + && by('judge_failed')?.sub === '有 2 句结论没裁判：裁判调用失败（ConnectError: 连不上）', `${by('judge_failed')?.title} / ${by('judge_failed')?.sub}`) + check('judge_unpriced：点名模型、金额上限不起作用', by('judge_unpriced')?.title === '裁判模型「jm-x」估不出金额，金额上限对它不起作用', + by('judge_unpriced')?.title) + check('judge_same_model：自己审自己、怎么改', by('judge_same_model')?.title === '裁判模型和写作模型都是「wm-1」：等于自己审自己' + && !!by('judge_same_model')?.next?.includes('证据裁判'), `${by('judge_same_model')?.title} / ${by('judge_same_model')?.next}`) + check('report_rewrite：几句不支持、交回改写，句子放副标题', by('report_rewrite')?.title === '裁判认为 2 句结论证据不支持，已交回写作者只改这几句' + && by('report_rewrite')?.sub === '「增长主要来自新客」「退款多数是部分退款」', `${by('report_rewrite')?.title} / ${by('report_rewrite')?.sub}`) + check('report_rewrite_rejected：冒出几处新问题、没有采用', by('report_rewrite_rejected')?.title === '改写稿冒出 1 处原稿没有的问题，没有采用' + && by('report_rewrite_rejected')?.sub === '「12」' && by('report_rewrite_rejected')?.next === '保留的是原稿和原来的判定', + `${by('report_rewrite_rejected')?.title} / ${by('report_rewrite_rejected')?.sub}`) + check('几条警告都是提醒（warn），原话留在展开区', ['judge_limit', 'judge_failed', 'judge_unpriced', 'judge_same_model', 'report_rewrite', + 'report_rewrite_rejected'].every((c) => by(c)?.level === 'warn' && by(c)?.detail)) + const rc = all.find((s) => s.code === 'report_checked') + check('报告核对那一行也数结论句，和出具横幅同一种说法（不是结论句的不算）', + rc?.title === '核对报告：3 个数字都有出处 · 结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）', rc?.title) + const ej = by('evidence_judged') + check('封存后按需裁判：单独一行，写明封存后追加、模型判断非确定', ej?.title === '封存后按需裁判了 1 句结论' + && ej?.sub === '封存后追加 · 模型判断，非确定' && ej?.level === 'info', `${ej?.title} / ${ej?.sub} / ${ej?.level}`) + + const failed = flatten(mod.decodeRun([ + E(1, 'node.started', 'write', { node_type: 'report' }), + E(2, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 2 }), + E(3, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 30000, units: 2, error: '已到上限（裁判时长上限 30 秒），这句没判' }), + E(4, 'node.finished', 'write', { duration_ms: 31000 }), + E(5, 'evidence.judged', 'write', { verdicts: { u4: { status: 'unjudged', reason: 'max_cost_usd' } }, limits_hit: ['max_cost_usd'] }), + ])) + const jx = failed.find((s) => s.kind === 'llm') + check('裁判调用没跑成：那一行说「没跑成」，是提醒不是失败，原因放展开区', jx?.title === '结论句裁判没跑成' && jx?.status === 'done' + && jx?.level === 'warn' && !!jx?.detail?.includes('裁判时长上限'), `${jx?.title} ${jx?.status} ${jx?.level}`) + check('……报告节点照样成功', failed.find((s) => s.kind === 'node')?.status === 'done') + const late = failed.find((s) => s.code === 'evidence_judged') + check('按需裁判触顶：写明有的到了上限没判（提醒）', late?.title === '封存后按需裁判了 0 句结论，有的到了上限没判' && late?.level === 'warn', + `${late?.title} ${late?.level}`) + const odd = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report' }), + E(2, 'log', 'w', { level: 'warn', code: 'judge_limit', message: '到上限了' }), + E(3, 'log', 'w', { level: 'warn', code: 'report_rewrite' }), + E(4, 'report.checked', 'w', { stats: { numbers: 1, numbers_cited: 1 }, judge: { counts: 'x' } })])) + check('别的说法（缺字段）：标题照样说人话，不出 undefined、不瞎填数', odd.find((s) => s.code === 'judge_limit')?.title === '结论句裁判已到上限（记为未裁判，已判的保留）' + && !odd.some((s) => /undefined|NaN/.test(`${s.title}${s.sub ?? ''}`)), odd.map((s) => s.title).join(' | ')) + check('……裁判摘要认不出时报告核对那一行只数数字', odd.find((s) => s.code === 'report_checked')?.title === '核对报告：1 个数字都有出处', + odd.find((s) => s.code === 'report_checked')?.title) +}) + await section('术语', async () => { // 没起名的节点退到类型名，类型名跟全站同一张表 const h = mod.decodeRun([{ seq: 1, type: 'node.started', node_id: 'h', ts: 1, data: { node_type: 'human', label: 'h' } }]) diff --git a/scripts/check-evidence.mjs b/scripts/check-evidence.mjs index a2e78cab..11c674a6 100644 --- a/scripts/check-evidence.mjs +++ b/scripts/check-evidence.mjs @@ -30,6 +30,10 @@ const fxq = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence- // 第三期：表名字段名（反引号里的、标记的、自动链接的）、可疑实体、核对不了的名字、逐字引文， // 以及没有契约的旧运行按数值猜的候选。文档和猜测都是后端真跑出来的（compose_doc / guess_sources） const fxe = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-entity.json`, 'utf8')) +// 第四期：结论句裁判。文档由 compose_doc 真跑，候选句由 judge.prepare 真跑，判定是剧本（模拟裁判模型）， +// 经 judge 的 summarize / apply_judgement 写回：正式运行（节点里判好的）、探索运行（按需）、没开裁判的探索运行 +const fxj = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-judge.json`, 'utf8')) +const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain] /** 夹具里某段文字是哪个片段：检查按文字认片段，重新生成夹具时编号变了也不用改 */ const segOf = (doc, text) => doc.blocks.flatMap((b) => b.units.flatMap((u) => u.segments)).find((s) => s.text === text)?.id @@ -52,13 +56,60 @@ async function section(key, name, fn) { const browser = await chromium.launch({ executablePath: CHROME }) -/** 伪造证据相关的 GET；片段请求记账（缓存：同一个片段点两次只取一次） */ -async function routed(page) { +/** + * 按需裁判的伪造答复：evidence.judged 的载荷形状（NOTES-A4 §8、E4-node 的 POST …/evidence/judge）。 + * 夹具里按句子备好了答复的用夹具的；别的句子给一条「有依据」。正式运行回 409 + */ +function judgeReply(set, body) { + if (set === fxj.formal) return { status: 409, json: { detail: '正式运行的裁判在节点内完成（claims: judge），判定随报告文档一起封存，不能再按需裁判', code: 'evidence_judge_formal' } } + const unit = body?.units?.[0] + const named = Object.entries(set.units).find(([, u]) => u === unit)?.[0] + if (named && set.judge?.[named]) return { json: set.judge[named] } + return { json: { report: { node_id: 'write', doc_artifact: set.doc_artifact }, verdicts: { [unit]: { status: 'supported', + rationale: '引用的证据支持这句话', judge: fxj.model, post_seal: true, used: [] } }, model: fxj.model, limits_hit: [], post_seal: true } } +} + +/** + * 伪造证据相关的 GET；片段请求记账（缓存：同一个片段点两次只取一次）。四期：按需裁判的 POST 也在这里认 + * （记进 hits 的 judge:<运行>:<句子>），别的非 GET 一律拦掉。judge 给了就用它答（改坏验证、触顶、出错）； + * segPatch 给了就改片段接口的答复（比如 on_demand 说运行还没封存） + */ +async function routed(page, { judge = null, segPatch = null } = {}) { const hits = [] + const judged = new Set() await page.route((u) => new URL(u).pathname.startsWith('/api/'), (r) => { const req = r.request() const url = new URL(req.url()) + const jset = JUDGE_SETS.find((x) => url.pathname.includes(`/runs/${x.run_id}/`) || url.pathname.endsWith(`/runs/${x.run_id}`)) + if (jset && req.method() === 'POST' && /\/evidence\/judge$/.test(url.pathname)) { + const body = req.postDataJSON() + hits.push(`judge:${jset.run_id}:${(body?.units ?? []).join(',')}:${body?.report ?? ''}`) + const reply = judge ? judge(jset, body) : judgeReply(jset, body) + if (reply.status == null || reply.status < 300) for (const u of body?.units ?? []) judged.add(`${jset.run_id}:${u}`) + return r.fulfill({ status: reply.status ?? 200, json: reply.json }) + } if (req.method() !== 'GET') return r.abort() + // 四期的报告工件：问数据页的答案（Output → EvidenceField）按工件 id 取文档 + const jart = JUDGE_SETS.find((x) => url.pathname === `/api/artifacts/${x.doc_artifact}`) + if (jart) return r.fulfill({ json: { id: jart.doc_artifact, content: jart.doc } }) + if (jset) { + const seg = url.pathname.match(/\/evidence\/segments\/([^/]+)$/) + if (seg) { + const id = decodeURIComponent(seg[1]) + hits.push(`j:${id}`) + // 判过之后片段接口按最新的 evidence.judged 叠上判定（封存后追加）：夹具里备着判过那一句的答复 + const after = jset.segments_after && judged.has(`${jset.run_id}:${jset.units.bad}`) ? jset.segments_after : jset.segments + let body = structuredClone(after[id]) + if (body && segPatch) body = segPatch(jset, body) + return body ? r.fulfill({ json: body }) + : r.fulfill({ status: 404, json: { detail: '报告里没有这个片段', code: 'evidence_segment_not_found' } }) + } + if (/\/evidence$/.test(url.pathname)) return r.fulfill({ json: jset.graph }) + if (url.pathname.endsWith(`/runs/${jset.run_id}`)) { + hits.push(`run:${jset.run_id}`) + return r.fulfill({ json: { id: jset.run_id, run_class: jset.run_class, status: 'succeeded' } }) + } + } const seg = url.pathname.match(/\/api\/runs\/([^/]+)\/evidence\/segments\/([^/]+)$/) // 按运行认夹具：第一期、第二期（查询链）、第三期（实体、引文）各有自己的片段 const set = url.pathname.includes(`/runs/${fxq.run_id}/`) ? fxq : url.pathname.includes(`/runs/${fxe.run_id}/`) ? fxe : fx @@ -83,14 +134,14 @@ async function routed(page) { return hits } -async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null } = {}) { +async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null, judge = null, segPatch = null } = {}) { const ctx = await browser.newContext({ viewport: { width: w, height: h }, ...(reduced ? { reducedMotion: 'reduce' } : {}) }) const page = await ctx.newPage() // 找不到元素时 8 秒就报，不等默认的 30 秒：改坏验证时一节里十几处都找不到 page.setDefaultTimeout(8000) const errors = [] page.on('pageerror', (e) => errors.push(e.message)) - const hits = await routed(page) + const hits = await routed(page, { judge, segPatch }) // 预览页直接 import 第二期夹具（vite 把 JSON 当模块给）：要一份改过的文档时把那个模块换掉 if (patchQuery) { await page.route((u) => new URL(u).pathname.endsWith('/run/__tests__/evidence-query.json'), (r) => @@ -1130,7 +1181,7 @@ await section('studio', '报告撰写节点：画布认得、检查器能配（R check('「指标来自」只列上游的口径卡（下游那张不列）', opts.length === 1 && opts[0].includes('周报口径'), opts.join(' | ')) const onViolation = await p.locator('[data-field="on_violation"] option').allInnerTexts() check('违规时的默认写明按运行类别', onViolation[0]?.includes('探索运行照常产出'), onViolation.join(' | ')) - // 三期：结论句策略 claims（off / require_citation，judge 是四期的，不出现在选项里）和表名字段名核对 entities + // 结论句策略 claims（off / require_citation / judge，四期放开了 judge）和表名字段名核对 entities // 表名字段名核对收在「高级选项」里（很少要改），先展开 await p.locator('button[aria-expanded]', { hasText: '高级选项' }).click().catch(() => {}) await p.waitForTimeout(100) @@ -1138,12 +1189,14 @@ await section('studio', '报告撰写节点：画布认得、检查器能配（R check('检查器有结论句策略、表名字段名核对（高级选项里）两项', fields.includes('claims') && all.includes('entities') && !fields.includes('entities'), all.join(',')) const claimVals = await p.locator('[data-field="claims"] option').evaluateAll((els) => els.map((e) => e.value)) - check('结论句策略只有 off / require_citation，没有 judge', claimVals.join(',') === 'off,require_citation', claimVals.join(',')) + check('结论句策略三档：off / require_citation / judge（四期放开了模型裁判）', claimVals.join(',') === 'off,require_citation,judge', + claimVals.join(',')) const claimsLabel = await p.locator('[data-field="claims"] label').first().innerText().catch(() => '') check('结论句策略有中文标签（发布前修复的预览也用它）', /结论句/.test(claimsLabel) && !/claims/.test(claimsLabel), claimsLabel) const claimsHelp = await p.locator('[data-field="claims"]').innerText() - check('……说明写明「结论句裁判在后续版本支持」、受管级别要显式选', claimsHelp.includes('结论句裁判在后续版本支持') - && claimsHelp.includes('受管'), claimsHelp.replace(/\s+/g, ' ').slice(0, 160)) + check('……说明写明模型逐句判断、受管级别要显式选并写金额上限，不再说「后续版本」', claimsHelp.includes('请模型逐句判断') + && claimsHelp.includes('受管') && claimsHelp.includes('金额上限') && !claimsHelp.includes('后续版本'), + claimsHelp.replace(/\s+/g, ' ').slice(0, 200)) check('没写 claims 时显示默认的 off', await p.locator('[data-field="claims"] select').inputValue() === 'off' || (await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.value)) === 'off') const entityVals = await p.locator('[data-field="entities"] option').evaluateAll((els) => els.map((e) => e.value)) @@ -1151,15 +1204,89 @@ await section('studio', '报告撰写节点：画布认得、检查器能配（R await p.locator('[data-field="claims"] select').selectOption('require_citation') const cfgOf = () => p.evaluate(() => window.__studio.getState().nodes.find((n) => n.id === 'write')?.data.config) check('选「要求挂依据」写进 config.claims', (await cfgOf())?.claims === 'require_citation') - // 手写或别处写进来的 judge：不能悄悄显示成 off，要照实写出来并说后续版本支持 + // 手写或别处写进来的不认识的值：不能悄悄显示成 off，要照实写出来 await p.evaluate(() => { const st = window.__studio.getState() const n = st.nodes.find((x) => x.id === 'write') - st.updateNode('write', { config: { ...n.data.config, claims: 'judge' } }) + st.updateNode('write', { config: { ...n.data.config, claims: 'sometimes' } }) }) await p.waitForTimeout(150) const shown = await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.textContent ?? '') - check('config 里写着 judge：下拉照实显示 judge 并说后续版本支持，不装成 off', shown.includes('judge') && shown.includes('后续版本'), shown) + check('config 里写着不认识的值：下拉照实显示，不装成 off', shown.includes('sometimes') && shown.includes('不认识'), shown) + + // 四期：选模型裁判，高级选项里出现结论句裁判的子配置（裁判模型、三项上限各自能设不限、改写一次、证据不支持时） + check('没选模型裁判时不出结论句裁判那一组', await p.locator('[data-field="judge"]').count() === 0) + await p.locator('[data-field="claims"] select').selectOption('judge') + check('选「请模型逐句判断」写进 config.claims = judge', (await cfgOf())?.claims === 'judge') + await p.waitForSelector('[data-field="judge"]', { timeout: 3000 }).catch(() => {}) + const jf = p.locator('[data-field="judge"]') + const jlabel = await jf.locator('label').first().innerText().catch(() => '') + check('高级选项里出现「结论句裁判」一组（中文标签）', await jf.count() === 1 && jlabel.includes('结论句裁判'), jlabel) + const keys = await jf.locator('[data-judge-key]').evaluateAll((els) => els.map((e) => e.getAttribute('data-judge-key'))) + check('……裁判模型、三项上限、改写一次、证据不支持时', keys.join(',') === 'model,max_claims,max_cost_usd,timeout_s,rewrite_once,on_unsupported', + keys.join(',')) + const labels = await jf.innerText() + // 细节只报缺了哪几个：整段文字里有沙箱的接入名（下拉的选项），日志常被贴进报告 + const missingLabels = ['裁判模型', '最多判几句', '金额上限（美元）', '时长上限（秒）', '交回改写一次', '证据不支持时'].filter((t) => !labels.includes(t)) + check('……每一项都有中文标签', !missingLabels.length, `缺：${missingLabels.join('、')}`) + check('……没写的上限说明跟着设置里的默认值', (await jf.locator('[data-judge-key="max_cost_usd"] [data-judge-note]').innerText()).includes('默认值')) + const cost = jf.locator('[data-judge-key="max_cost_usd"]') + await cost.locator('input[type="checkbox"]').click() + check('金额上限勾「不限」：写 null（不是删掉，也不是 0）', (await cfgOf())?.judge && (await cfgOf()).judge.max_cost_usd === null + && 'max_cost_usd' in (await cfgOf()).judge, JSON.stringify((await cfgOf())?.judge)) + const off = await cost.locator('[data-judge-note]').innerText().catch(() => '') + check('……写明「不设上限，费用只受……约束」，数字框变灰', off.includes('不设上限') && off.includes('约束') + && await cost.locator('input[type="number"]').isDisabled(), off) + await cost.locator('input[type="checkbox"]').click() + check('勾掉「不限」：回到没写（跟着设置里的默认值）', !((await cfgOf())?.judge && 'max_cost_usd' in (await cfgOf()).judge), + JSON.stringify((await cfgOf())?.judge)) + await cost.locator('input[type="number"]').fill('0.2') + await jf.locator('[data-judge-key="max_claims"] input[type="number"]').fill('20') + await jf.locator('[data-judge-key="timeout_s"] input[type="checkbox"]').click() + await jf.locator('[data-judge-key="rewrite_once"] input[type="checkbox"]').click() + await jf.locator('[data-judge-key="on_unsupported"] select').selectOption('withhold') + const j = (await cfgOf())?.judge ?? {} + check('写数就是数、时长不限写 null、改写一次写 true、不支持时写 withhold', + j.max_cost_usd === 0.2 && j.max_claims === 20 && j.timeout_s === null && j.rewrite_once === true && j.on_unsupported === 'withhold', + JSON.stringify(j)) + const provFirst = () => jf.locator('[data-judge-key="model"] select').evaluate((el) => el.options[0]?.textContent ?? '').catch(() => '') + check('接入和模型都没写：接入留空那一项叫「跟随设置」', await provFirst() === '跟随设置' && await jf.locator('[data-judge-by-model]').count() === 0) + await jf.locator('[data-judge-key="model"] input').fill('judge-model-b') + check('裁判模型写进 judge.model', (await cfgOf())?.judge?.model === 'judge-model-b') + // 节点上只写了模型：节点这一级整组生效（judge_model_spec 不跨级拼），按模型名找接入——已经不是「跟随设置」 + const pf = await provFirst() + check('只写了模型：接入留空那一项不再叫「跟随设置」，改叫「按模型名找接入」', pf === '按模型名找接入', pf) + const byModel = jf.locator('[data-judge-by-model]') + const byText = await byModel.innerText().catch(() => '') + // 说明里有沙箱的接入名：细节只报认没认出来，不打印原文 + check('……写明按模型名找接入、认不出就去调默认接入，两项都没写才跟随设置', await byModel.getAttribute('data-judge-by-model').catch(() => '') === 'default' + && byText.includes('只填了模型') && byText.includes('默认接入') && byText.includes('接入和模型都没写时才跟随设置'), + `data-judge-by-model=${await byModel.getAttribute('data-judge-by-model').catch(() => '（没有）')}`) + await jf.locator('[data-judge-key="model"] input').fill('') + check('……模型清空：又回到「跟随设置」，说明也收起', await provFirst() === '跟随设置' && await byModel.count() === 0 + && !('model' in ((await cfgOf())?.judge ?? {}))) + await jf.locator('[data-judge-key="model"] input').fill('judge-model-b') + const previews = await p.evaluate(async () => { + const url = performance.getEntriesByType('resource').map((e) => e.name) + .find((n) => { try { return new URL(n).pathname === '/src/canvas/issues.ts' } catch { return false } }) + const m = await import(url ?? '/src/canvas/issues.ts') + return [m.fixFieldLabel('judge.max_cost_usd', 'report'), m.fixValueText(null, 'judge.max_cost_usd'), + m.fixValueText(0.05, 'judge.max_cost_usd'), (m.fixValueLines({ max_cost_usd: null, model: 'x' }, 'judge') ?? []).join('；'), + m.fixValueText('withhold', 'judge.on_unsupported'), m.fixFieldLabel('defaults.model', 'report')] + }).catch((e) => [String(e)]) + check('发布前修复的预览：judge.max_cost_usd 写成「结论句裁判 · 金额上限（美元）」', previews[0] === '结论句裁判 · 金额上限（美元）', previews[0]) + check('……null 写「不限」（没写才是「（空）」）、金额带 $', previews[1] === '不限' && previews[2] === '$0.05', JSON.stringify(previews.slice(1, 3))) + check('……补预算按整份 judge 记时一键一行', previews[3] === '裁判模型：x；金额上限（美元）：不限', previews[3]) + check('……证据不支持时写选项文字', previews[4] === '不予出具', previews[4]) + check('……别处的 model 不借用裁判的叫法', previews[5] === '全图默认 · model', previews[5]) + if (SHOTS) { + await jf.scrollIntoViewIfNeeded() + for (const theme of ['dark', 'light']) { + await p.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await p.waitForTimeout(250) + await p.screenshot({ path: `${SHOTS}/evidence4-inspector-judge-${theme}.png` }) + } + } const fixLabel = await p.evaluate(async () => { // 按页面自己加载时的地址 import：改过的模块地址带 ?t=，直接写 /src/… 会拿到另一份实例 const url = performance.getEntriesByType('resource').map((e) => e.name) @@ -1586,6 +1713,401 @@ await section('guess', '旧运行按数值猜的候选：默认折叠，展开 } }) +// --------------------------------------------------------------------------- +// 第四期：结论句裁判 +// --------------------------------------------------------------------------- + +const J = '#evidence-judge' +const JX = '#evidence-judge-explore' +const JP = '#evidence-judge-plain' +const badge = (pg, sel, unit) => pg.locator(`${sel} [data-ev-claim="${unit}"]`) +/** 令牌算出来的颜色（借查询步骤那节的 tokenColor 探针），和徽标的计算色比 */ +const colorOf = (pg, token) => tokenColor(pg, 'color', `var(${token})`) +const activeKey = (pg) => pg.evaluate(() => document.activeElement?.getAttribute('data-seg') + ?? (document.activeElement?.getAttribute('data-ev-claim') ? `claim:${document.activeElement.getAttribute('data-ev-claim')}` : null) + ?? (document.activeElement?.closest('[data-evidence-panel]') ? 'panel' : document.activeElement?.tagName ?? null)) + +await section('judge', '四期：结论句的句末徽标——四通道、不用确定性的绿、整句不画线、n / N 也跳到证据不支持的句子', async () => { + const u = fxj.formal.units + const look = await page.evaluate(({ sel, units }) => Object.fromEntries(Object.entries(units).map(([k, id]) => { + const el = document.querySelector(`${sel} [data-ev-claim="${id}"]`) + if (!el) return [k, null] + const cs = getComputedStyle(el) + return [k, { verdict: el.getAttribute('data-ev-verdict'), glyph: el.textContent, color: cs.color, tag: el.tagName, + type: el.getAttribute('type'), label: el.getAttribute('aria-label'), deco: cs.textDecorationLine, + align: cs.verticalAlign }] + })), { sel: J, units: u }) + check('有判定的结论句句末都挂徽标（有依据、证据不支持、部分有依据、未裁判）', + look.ok?.verdict === 'supported' && look.bad?.verdict === 'unsupported' && look.partial?.verdict === 'partial' + && look.limit?.verdict === 'unjudged', JSON.stringify(Object.fromEntries(Object.entries(look).map(([k, v]) => [k, v?.verdict])))) + check('判成「不是结论句」的不挂徽标', look.not === null) + check('字形：◆ 有依据、◇ 部分、! 不支持、? 未裁判', [look.ok, look.partial, look.bad, look.limit].map((x) => x?.glyph).join('') === '◆◇!?', + [look.ok, look.partial, look.bad, look.limit].map((x) => x?.glyph).join('')) + const [accent, waiting, failed, dim, done] = await Promise.all(['--accent', '--st-waiting', '--st-failed', '--text-dim', '--st-done'] + .map((t) => colorOf(page, t))) + check('颜色令牌：有依据 --accent、部分 --st-waiting、不支持 --st-failed、未裁判 --text-dim', + look.ok?.color === accent && look.partial?.color === waiting && look.bad?.color === failed && look.limit?.color === dim, + JSON.stringify({ got: [look.ok?.color, look.partial?.color, look.bad?.color, look.limit?.color], want: [accent, waiting, failed, dim] })) + check('概率性的判定一个都不用确定性的绿', [look.ok, look.partial, look.bad, look.limit].every((x) => x && x.color !== done), done) + check('徽标是原生 button type=button、上标、不画线', [look.ok, look.bad].every((x) => x?.tag === 'BUTTON' && x.type === 'button' + && x.align === 'super' && x.deco === 'none'), JSON.stringify(look.ok)) + check('aria-label 写全：句子和判定', look.bad?.label === '结论句「增长主要来自新客首单，老客复购持平。」，模型判断：证据不支持', look.bad?.label) + check('……未裁判（到上限）的说「已到上限」', look.limit?.label?.endsWith('未裁判：已到上限'), look.limit?.label) + const titles = await page.evaluate(({ a, b, c }) => [document.querySelector(a)?.getAttribute('title') ?? '', + document.querySelector(b)?.getAttribute('title') ?? '', document.querySelector(c)?.getAttribute('title') ?? ''], + { a: `${J} [data-ev-claim="${u.bad}"]`, b: `${JX} [data-ev-claim="${fxj.explore.units.ok}"]`, c: `${J} [data-ev-claim="${u.limit}"]` }) + check('悬停说明：模型判过的写明谁判的、非确定', titles[0] === `模型判断：证据不支持（${fxj.model} · 非确定）`, titles[0]) + check('……还没有模型判过的（探索运行按需）不说谁判的，只说还没判', !titles[1].includes('非确定') && titles[1].startsWith('未裁判'), + titles[1]) + check('……到上限没判的（记着裁判模型）也不写成「（模型 · 非确定）」，说已到上限、这句没判', + titles[2].startsWith('未裁判：已到上限') && !titles[2].includes('非确定') && !titles[2].includes(fxj.model), titles[2]) + const shade = await page.evaluate(({ sel, bad, ok }) => { + const box = document.querySelector(`${sel} [data-ev-claim-shade="${bad}"]`) + const other = document.querySelector(`${sel} [data-ev-claim-shade="${ok}"]`) + const probe = document.createElement('span') + probe.style.background = 'var(--st-failed-soft)' + document.body.appendChild(probe) + const soft = getComputedStyle(probe).backgroundColor + probe.remove() + if (!box) return { found: false } + const texts = [...box.querySelectorAll('span')].map((el) => getComputedStyle(el).textDecorationLine) + return { found: true, bg: getComputedStyle(box).backgroundColor, soft, deco: getComputedStyle(box).textDecorationLine, + texts, other: !!other, text: box.textContent } + }, { sel: J, bad: u.bad, ok: u.ok }) + check('证据不支持的整句铺浅底（--st-failed-soft）', shade.found && shade.bg === shade.soft && shade.bg !== 'rgba(0, 0, 0, 0)', + JSON.stringify(shade)) + check('……只有不支持的那句铺底', shade.found && !shade.other) + check('整句的文字不画下划线（浅底里的字、别的结论句的字都没有线）', shade.found && shade.deco === 'none' + && shade.texts.every((d) => d === 'none'), JSON.stringify(shade.texts)) + const plainText = await page.evaluate(({ sel }) => { + const b = document.querySelector(`${sel} [data-ev-claim]`) + const sentence = b?.previousElementSibling + return sentence ? getComputedStyle(sentence).textDecorationLine : '' + }, { sel: J }) + check('……有依据的那句同样不画线（只有数字自己的线）', plainText === 'none' || plainText === '', plainText) + + // 键盘：徽标在 roving 顺序里，句尾就是它；n / N 跳到证据不支持、部分有依据的句子 + const stops = await page.locator(`${J} [data-seg][tabindex="0"], ${J} [data-ev-claim][tabindex="0"]`).count() + check('整份报告仍只有一个 Tab 位（徽标也在 roving 里）', stops === 1, String(stops)) + const firstNum = fxj.formal.doc.blocks.flatMap((b) => b.units).find((x) => x.id === u.ok).segments.filter((s) => s.kind === 'number').at(-1).id + await page.locator(`${J} [data-seg="${firstNum}"]`).focus() + await page.keyboard.press('ArrowRight') + check('→ 从句子最后一个片段走到句末徽标', await activeKey(page) === `claim:${u.ok}`, await activeKey(page)) + const jumps = [] + for (const key of ['n', 'n', 'N']) { + await page.keyboard.press(key) + jumps.push(await activeKey(page)) + } + check('n 跳到证据不支持、再到部分有依据的句子，N 回来', jumps.join(',') === `claim:${u.bad},claim:${u.partial},claim:${u.bad}`, jumps.join(',')) + check('只是走动不打开面板', await panel(page).count() === 0) + await page.keyboard.press('Enter') + await page.waitForSelector('[data-evidence-panel] [data-ev-judge]', { timeout: 4000 }) + check('回车打开这一句的「模型的解释」', (await panel(page).locator('[data-ev-judge]').getAttribute('data-ev-judge')) === 'unsupported') + check('徽标标着展开了面板', await badge(page, J, u.bad).getAttribute('aria-expanded') === 'true') + check('读屏从 live region 听到这一句的判定', (await page.locator(`${J} [aria-live="polite"]`).innerText()).includes('模型判断：证据不支持')) + await page.keyboard.press('Escape') + await page.waitForTimeout(150) + check('Esc 关掉，焦点回到徽标', await panel(page).count() === 0 && await activeKey(page) === `claim:${u.bad}`, await activeKey(page)) + const tally = await page.locator(`${J} [data-evidence-claims]`).innerText().catch(() => '') + check('证据条接着数结论句，和数字那一段同一种说法', tally.includes('结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）'), tally) + const summary = await page.locator(`${J} [data-evidence-summary]`).innerText() + check('读屏摘要也说结论句（写明是模型判断）', summary.includes('结论 4 句') && summary.includes('模型判断'), summary) + check('对照：一期的报告（没有判定）一个徽标都不挂', await page.locator(`${wide} [data-ev-claim], ${wide} [data-ev-claim-shade]`).count() === 0) +}) + +await section('j-panel', '四期：面板里「模型的解释」、「请模型判断这句」只在探索运行没判过时出现、触顶、封存后追加', async () => { + const r = await open('/ui-harness.html?evidence=1') + const pg = r.page + const p = pg.locator('[data-evidence-panel]') + const u = fxj.formal.units + await badge(pg, J, u.bad).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]') + const chip = await p.locator('[data-ev-judge-badge]').innerText().catch(() => '') + check('带「模型判断 · 模型名 · 非确定」徽标', chip.trim() === `模型判断 · ${fxj.model} · 非确定`, chip) + const why = await p.locator('[data-ev-rationale]').evaluate((el) => ({ text: el.textContent, style: getComputedStyle(el).fontStyle })) + .catch(() => null) + check('理由用斜体', why?.style === 'italic' && why.text.includes('看不出新客、老客'), JSON.stringify(why)) + const head = await p.locator('header [data-ev-badge]').evaluate((el) => ({ code: el.getAttribute('data-ev-badge'), + style: getComputedStyle(el).borderTopStyle, text: el.textContent })).catch(() => null) + check('面板标题的徽标：证据不支持、虚线框（概率性）', head?.code === 'unsupported' && head.style === 'dashed', JSON.stringify(head)) + check('正式运行节点里判的：不写「封存后追加」，写明随报告封存', await p.locator('[data-ev-post-seal]').count() === 0 + && (await p.locator('[data-ev-judge-sealed]').innerText().catch(() => '')).includes('和报告一起封存')) + check('正式运行：没有「请模型判断这句」', await p.locator('[data-ev-judge-ask]').count() === 0) + const cites = await p.locator('[data-ev-cites]').innerText().catch(() => '') + check('整句的面板列出挂的依据', cites.includes('Q1') && cites.includes('db_query__shop'), cites) + await badge(pg, J, u.limit).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]') + const lim = await p.locator('[data-ev-judge-limit]').innerText() + check('到上限没判：写明「已到上限」和是哪个上限', lim.includes('已到上限') && lim.includes('这份报告的裁判金额上限 $0.05'), lim) + check('……写明怎么调（正式运行：报告撰写节点的结论句裁判，可以设成不限）', lim.includes('报告撰写节点') && lim.includes('设成不限'), lim) + check('……正式运行也不给按钮', await p.locator('[data-ev-judge-ask]').count() === 0) + // 到上限没判的句子也记着裁判模型（judge 字段），但模型没看过它：不能画成「模型判断 · 模型名 · 非确定」 + await pg.waitForSelector('[data-evidence-panel] [data-ev-seal-state="done"]', { timeout: 3000 }).catch(() => {}) + check('……模型没看过这句：不挂「模型判断 · 模型名 · 非确定」', await p.locator('[data-ev-judge-badge]').count() === 0) + check('……也不说这条判断「和报告一起封存」', await p.locator('[data-ev-judge-sealed]').count() === 0) + const limSeal = await p.locator('[data-ev-seal]').innerText().catch(() => '') + check('……封存那一节只说报告文档封存了', limSeal.includes('报告文档已封存') && !limSeal.includes('这条判断随报告一起封存'), limSeal) + const limModel = await p.locator('[data-ev-judge-model]').innerText().catch(() => '') + check('……中性地说一句裁判模型是谁、没判这句', limModel.includes(fxj.model) && limModel.includes('没有判这句'), limModel) + // 点开结论句里的一个数字：片段面板里也有「模型的解释」（只对结论句） + const num = fxj.formal.doc.blocks.flatMap((b) => b.units).find((x) => x.id === u.ok).segments.find((s) => s.kind === 'number').id + await pg.locator(`${J} [data-seg="${num}"]`).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="supported"]') + check('结论句里的片段：面板里有这一句的模型的解释', (await p.locator('[data-ev-verdict]').innerText()).includes('模型判断：有依据')) + // 一期的报告：没有判定、没开裁判、不知道运行类别也取不到——不画这一节 + await pg.locator(`${wide} [data-seg="s8"]`).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-substituted]') + await pg.waitForTimeout(300) + check('对照：没有判定的老报告不画「模型的解释」', await p.locator('[data-ev-judge]').count() === 0) + await pg.keyboard.press('Escape') + + // 探索运行、开了裁判（按需）：还没判过的句子给按钮 + const x = fxj.explore.units + await badge(pg, JX, x.bad).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-ask]') + check('探索运行、还没判过：有「请模型判断这句」', (await p.locator('[data-ev-judge-ask]').innerText()).includes('请模型判断这句')) + check('……还没判过：没有模型徽标（还没有模型判过）', await p.locator('[data-ev-judge-badge]').count() === 0) + await p.locator('[data-ev-judge-ask]').click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]').catch(() => {}) + const sent = r.hits.filter((h) => h.startsWith(`judge:${fxj.explore.run_id}:`)) + check('请求体带着句子和报告节点', sent.length === 1 && sent[0].endsWith(`:${x.bad}:write`), sent.join(' | ')) + check('判完：写「封存后追加」', (await p.locator('[data-ev-post-seal]').innerText().catch(() => '')).trim() === '封存后追加') + check('……答复没带话（判成了）：不多写一行', await p.locator('[data-ev-judge-message]').count() === 0) + check('……封存那一节也写明封存后追加（报告文档仍是封存的）', (await p.locator('[data-ev-seal-late]').innerText().catch(() => '')).includes('封存后追加') + && (await p.locator('[data-ev-seal]').innerText()).includes('报告文档已封存')) + check('……判过了就不再给按钮', await p.locator('[data-ev-judge-ask]').count() === 0) + const after = await badge(pg, JX, x.bad).evaluate((el) => ({ v: el.getAttribute('data-ev-verdict'), late: el.hasAttribute('data-ev-post-seal') })) + check('正文的句末徽标跟着变：证据不支持、标着封存后追加', after.v === 'unsupported' && after.late, JSON.stringify(after)) + check('同一份报告在窄栏里的那份也跟着变（按运行和报告记）', + await badge(pg, '#evidence-judge-explore-narrow', x.bad).getAttribute('data-ev-verdict') === 'unsupported') + check('证据条的结论句计数跟着变', (await pg.locator(`${JX} [data-evidence-claims]`).innerText()).includes('不支持 1')) + await badge(pg, JX, x.bad).click() + await pg.waitForTimeout(100) + await badge(pg, JX, x.bad).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]') + check('同一句再点开：直接用已有的判定，不再请求', r.hits.filter((h) => h.startsWith(`judge:${fxj.explore.run_id}:`)).length === 1) + // 触顶：不报错，说明「已到上限」和怎么调（每次点击的上限），还能再点一次 + await badge(pg, JX, x.limit).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-ask]') + await p.locator('[data-ev-judge-ask]').click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]') + const top = await p.locator('[data-ev-judge-limit]').innerText() + check('按需裁判触顶：写「已到上限」和这次点击的上限', top.includes('已到上限') && top.includes('这次点击的裁判金额上限 $0.01'), top) + check('……写明怎么调：设置里「每次点击的金额上限」、可以设成不限', top.includes('每次点击的金额上限') && top.includes('设成不限') + && await p.locator('[data-ev-judge-settings]').count() === 1, top) + check('……不是报错', await p.locator('[data-ev-judge-error]').count() === 0) + check('……还能再请模型判断一次', (await p.locator('[data-ev-judge-ask]').innerText().catch(() => '')).includes('再请模型判断一次')) + if (SHOTS) { + for (const theme of ['dark', 'light']) { + await pg.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await badge(pg, JX, x.bad).click() + await pg.waitForTimeout(100) + if (await p.count() === 0) await badge(pg, JX, x.bad).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]') + await pg.waitForTimeout(250) + await pg.screenshot({ path: `${SHOTS}/evidence4-explore-judged-${theme}.png` }) + await badge(pg, J, u.limit).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]') + await pg.waitForTimeout(250) + await pg.screenshot({ path: `${SHOTS}/evidence4-formal-limit-${theme}.png` }) + await pg.keyboard.press('Escape') + } + await pg.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + } + + // 没开裁判的探索运行：文档里不记运行类别，面板按运行 id 取一次；结论句照样能请模型判断 + const pu = fxj.plain.doc.blocks.flatMap((b) => b.units).find((x2) => x2.id === 'u3') + const pnum = pu.segments.find((s) => s.kind === 'number').id + await pg.locator(`${JP} [data-seg="${pnum}"]`).click() + await pg.waitForSelector('[data-evidence-panel] [data-ev-judge-ask]').catch(() => {}) + check('没开裁判的报告：按运行 id 认出是探索运行，给按钮', r.hits.includes(`run:${fxj.plain.run_id}`) + && await p.locator('[data-ev-judge-ask]').count() === 1) + check('……判之前正文没有徽标', await pg.locator(`${JP} [data-ev-claim]`).count() === 0) + await p.locator('[data-ev-judge-ask]').click().catch(() => {}) + await pg.waitForSelector(`${JP} [data-ev-claim="u3"]`).catch(() => {}) + check('……判完正文挂上徽标（封存后追加）', await badge(pg, JP, 'u3').getAttribute('data-ev-verdict') === 'supported' + && await badge(pg, JP, 'u3').getAttribute('data-ev-post-seal') === '') + const ptally = await pg.locator(`${JP} [data-evidence-claims]`).innerText().catch(() => '') + check('……证据条多出结论句：挂了依据没判的算未裁判、没挂的算无证据', ptally.includes('结论 5 句（支持 1 · 未裁判 3 · 无证据 1）'), ptally) + check('没有运行时报错（裁判面板）', r.errors.length === 0, r.errors.join(' | ')) + await r.ctx.close() + + // 片段接口说暂时判不了（运行还没跑完封存）：不给按钮，照原话说为什么 + const un = await open('/ui-harness.html?evidence=1', { segPatch: (set, body) => (set === fxj.explore + ? { ...body, unit: { ...body.unit, on_demand: { available: false, reason: 'unsealed', message: '运行还没跑完封存，跑完再请模型判断' } } } : body) }) + const tnum = fxj.explore.doc.blocks.flatMap((b) => b.units).find((x2) => x2.id === x.ok).segments.find((s) => s.kind === 'number').id + await un.page.locator(`${JX} [data-seg="${tnum}"]`).click() + await un.page.waitForSelector('[data-evidence-panel] [data-ev-judge]') + await un.page.waitForTimeout(300) + check('接口说运行还没封存：不给按钮，照原话说', await un.page.locator('[data-ev-judge-ask]').count() === 0 + && (await un.page.locator('[data-ev-judge-blocked]').innerText()).includes('跑完再请模型判断')) + await un.ctx.close() + // 接口 409（正式运行、封存被改过）：不白屏，照后端原话说 + const e409 = await open('/ui-harness.html?evidence=1', { judge: () => ({ status: 409, json: { detail: '封存核对没通过：不再请模型判断', code: 'evidence_seal_broken' } }) }) + await badge(e409.page, JX, x.bad).click() + await e409.page.locator('[data-ev-judge-ask]').click() + await e409.page.waitForSelector('[data-evidence-panel] [data-ev-judge-error]') + check('按需裁判回 409：照后端原话说，不白屏', (await e409.page.locator('[data-ev-judge-error]').innerText()).includes('封存核对没通过')) + check('……徽标还是未裁判', await badge(e409.page, JX, x.bad).getAttribute('data-ev-verdict') === 'unjudged') + await e409.ctx.close() + // 答复里的那句话（message）：判定照样交回来，但运行已经接着跑了、没记进运行记录——得照原话写出来， + // 不能只挂一枚「封存后追加」让人以为记进去了。触顶的那句已经由上限那一框说了，不重复 + const lostText = '运行已经接着跑了，这次的判定没有记进运行记录（只在这里看得到，刷新就没了）；等它跑完封存，再请模型判断' + const lost = await open('/ui-harness.html?evidence=1', { judge: (set, body) => { + const unit = body?.units?.[0] + if (set === fxj.explore && unit === x.limit) { + return { json: { ...fxj.explore.judge.limit, message: '已到上限（这次点击的裁判金额上限 $0.01）：1 句没判，记为未裁判；已判的保留' } } + } + return { json: { report: { node_id: 'write', doc_artifact: set.doc_artifact }, model: fxj.model, limits_hit: [], post_seal: true, + message: lostText, event: null, calls: 1, + verdicts: { [unit]: { status: 'supported', rationale: '引用的证据支持这句话', judge: fxj.model, post_seal: true, used: [] } } } } + } }) + const lp = lost.page.locator('[data-evidence-panel]') + await badge(lost.page, JX, x.ok).click() + await lp.locator('[data-ev-judge-ask]').click().catch(() => {}) + await lost.page.waitForSelector('[data-evidence-panel] [data-ev-judge="supported"]').catch(() => {}) + const lostMsg = await lp.locator('[data-ev-judge-message]').innerText().catch(() => '') + check('答复带着「没有记进运行记录」：照原话写在判定下面', lostMsg.includes('没有记进运行记录') && lostMsg.includes('刷新就没了'), lostMsg || '（没写）') + check('……判定照样显示（有依据）', (await lp.locator('[data-ev-verdict]').innerText().catch(() => '')).includes('模型判断：有依据')) + await badge(lost.page, JX, x.limit).click() + await lp.locator('[data-ev-judge-ask]').click().catch(() => {}) + await lost.page.waitForSelector('[data-evidence-panel] [data-ev-judge-limit]').catch(() => {}) + check('触顶那句话已经由上限那一框说了：不再另写一遍', await lp.locator('[data-ev-judge-limit]').count() === 1 + && await lp.locator('[data-ev-judge-message]').count() === 0) + check('没有运行时报错（答复里的那句话）', lost.errors.length === 0, lost.errors.join(' | ')) + await lost.ctx.close() +}) + +await section('j-narrow', '四期在 360px：徽标和「模型的解释」栏内展开不横向滚动；减少动效', async () => { + const narrow = '#evidence-judge-narrow' + const overflow = () => page.evaluate((sel) => { + const el = document.querySelector(sel) + return { sw: el.scrollWidth, cw: el.clientWidth, w: el.getBoundingClientRect().width } + }, narrow) + const before = await overflow() + await badge(page, narrow, fxj.formal.units.bad).click() + await page.waitForSelector(`${narrow} [data-evidence-panel="inline"] [data-ev-judge]`) + const after = await overflow() + check('窄栏里徽标照样挂在句末', await page.locator(`${narrow} [data-ev-claim]`).count() === 4) + check('窄栏里没有横向滚动（面板打开前后）', Math.round(before.w) === 360 && before.sw <= before.cw && after.sw <= after.cw, + JSON.stringify({ before, after })) + if (SHOTS) { + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await page.waitForTimeout(200) + await page.locator(narrow).screenshot({ path: `${SHOTS}/evidence4-360-inline-${theme}.png` }) + } + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + } + await page.locator(`${narrow} [data-evidence-panel] button`, { hasText: '回到正文' }).click() + await page.waitForTimeout(150) + const small = await open('/ui-harness.html?evidence=1', { w: 360, h: 780 }) + const scroll = () => small.page.evaluate(() => ({ sw: document.documentElement.scrollWidth, vw: innerWidth })) + const a = await scroll() + await badge(small.page, J, fxj.formal.units.bad).click() + await small.page.waitForSelector('[data-evidence-panel] [data-ev-judge]') + const b = await scroll() + check('360px 宽的屏幕：整句的面板从底部抽出、整页没有横向滚动', + await small.page.locator('[data-evidence-panel]').getAttribute('data-evidence-panel') === 'drawer' && a.sw <= a.vw && b.sw <= b.vw, + JSON.stringify({ a, b })) + await small.ctx.close() + const r = await open('/ui-harness.html?evidence=1', { reduced: true }) + await badge(r.page, J, fxj.formal.units.bad).click() + await r.page.waitForSelector('[data-evidence-panel] [data-ev-judge]').catch(() => {}) + const still = await r.page.evaluate(() => { + const p = document.querySelector('[data-evidence-panel]') + return { n: p ? p.getAnimations({ subtree: true }) + .filter((a2) => !(a2 instanceof CSSTransition && Number(a2.effect?.getTiming().duration) <= 0.01)).length : -1, + name: p ? getComputedStyle(p).animationName : '' } + }) + check('减少动效时整句的面板没有任何动画', still.n === 0 && still.name === 'none', JSON.stringify(still)) + await r.ctx.close() +}) + +await section('j-banner', '四期：出具横幅多一段「结论 N 句（支持 a · 无证据 b）」', async () => { + const at = (key) => page.locator(`#issuance-claims [data-issuance-case="${key}"]`) + const judged = await at('judged').locator('[data-evidence-line]').innerText().catch(() => '') + check('结论句那一段：和数字那一段同一种说法', judged.includes('3 个数字都有出处') && judged.includes('结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）'), judged) + check('有证据不支持的结论句：给「定位下一处」', await at('judged').locator('[data-evidence-next]').count() === 1) + await page.evaluate(() => { window.__claimsNext = 0 }) + await at('judged').locator('[data-evidence-next]').click() + check('……点了就去找下一处', await page.evaluate(() => window.__claimsNext) === 1) + const cited = await at('cited').locator('[data-evidence-line]').innerText().catch(() => '') + check('方案的例子：结论 4 句（支持 3 · 无证据 1）', cited.includes('结论 4 句（支持 3 · 无证据 1）'), cited) + check('……没挂依据的算进「无证据」，不再另起「没挂依据的结论句」', await at('cited').locator('[data-evidence-line-claims]').count() === 0) + const all = at('all').locator('[data-evidence-claims]') + const [color, done] = [await all.evaluate((el) => getComputedStyle(el).color).catch(() => ''), await colorOf(page, '--st-done')] + check('全都支持：不给「定位下一处」，也不用确定性的绿', await at('all').locator('[data-evidence-next]').count() === 0 && !!color && color !== done, + `${color} / ${done}`) + check('对照：三期那一份（没有判定）照旧写「没挂依据的结论句」', await page.locator('#issuance-line [data-issuance-case="suspect"] [data-evidence-line-claims]').count() === 1) + // 开了裁判的文档：没挂依据的那句送了裁判、按判定数进了「支持 3」，出具照样按没挂依据记 1 句——不能被吞掉 + const ju = await at('judge-uncited').locator('[data-evidence-line]').innerText().catch(() => '') + check('开了裁判、出具还记着没挂依据的结论句：结论句那一段旁边另起「没挂依据的结论句 1」', + ju.includes('结论 3 句（支持 3）') && await at('judge-uncited').locator('[data-evidence-line-claims]').count() === 1 + && ju.includes('· 没挂依据的结论句 1'), ju) + const juTitle = await at('judge-uncited').locator('[data-evidence-line-claims]').getAttribute('title').catch(() => '') + check('……悬停说明：判定代替不了依据，出具照样计缺口', !!juTitle && juTitle.includes('代替不了依据') && juTitle.includes('缺口'), juTitle ?? '') + if (SHOTS) { + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await page.waitForTimeout(200) + await page.locator('#issuance-claims').screenshot({ path: `${SHOTS}/evidence4-banner-${theme}.png` }) + await page.locator(J).screenshot({ path: `${SHOTS}/evidence4-doc-${theme}.png` }) + } + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + } +}) + +await section('j-stream', '四期：问数据页、画布右栏的横幅（Output 这条路）——按需判过的结论句数进横幅，开了裁判的报告没挂依据的照样写出来', async () => { + // 按需裁判把「退款 29 单，其中大多数是部分退款」判成证据不支持（这句有数字，点得开） + const pu = fxj.plain.doc.blocks.flatMap((b) => b.units).find((x2) => x2.id === 'u3') + const pnum = pu.segments.find((x2) => x2.kind === 'number').id + const s = await open('/preview.html?syn=evidence-judge', { judge: (set, body) => (set === fxj.plain ? { json: { + report: { node_id: 'write', doc_artifact: set.doc_artifact }, model: fxj.model, limits_hit: [], post_seal: true, + verdicts: { [body.units[0]]: { status: 'unsupported', rationale: 'Q2 里部分退款 11 单，说不上「大多数」', judge: fxj.model, post_seal: true, used: ['Q2'] } }, + } } : judgeReply(set, body)) }) + const p = s.page + const turn = (id) => p.locator(`[data-turn="${id}"]`) + await p.waitForSelector('[data-turn="judge-plain"] [data-evidence-line]', { timeout: 6000 }).catch(() => {}) + await p.waitForSelector('[data-turn="judge-formal"] [data-evidence-line]', { timeout: 6000 }).catch(() => {}) + const plainLine = () => turn('judge-plain').locator('[data-evidence-line]').innerText().catch(() => '') + const before = await plainLine() + check('没开裁判、还没判过：横幅照旧写「没挂依据的结论句 1」，没有结论句那一段', + before.includes('没挂依据的结论句 1') && await turn('judge-plain').locator('[data-evidence-claims]').count() === 0, before) + // 点开那一句的数字，请模型判断（Output 没给运行类别：按运行 id 认出探索运行） + await turn('judge-plain').locator(`[data-seg="${pnum}"]`).click().catch(() => {}) + const ask = p.locator('[data-evidence-panel] [data-ev-judge-ask]') + await ask.waitFor({ timeout: 6000 }).catch(() => {}) + await ask.click().catch(() => {}) + await p.waitForSelector('[data-turn="judge-plain"] [data-ev-claim="u3"]', { timeout: 6000 }).catch(() => {}) + await p.waitForSelector('[data-turn="judge-plain"] [data-evidence-line] [data-evidence-claims]', { timeout: 3000 }).catch(() => {}) + check('……请模型判断那一下发出去了（按运行、报告、句子）', s.hits.includes(`judge:${fxj.plain.run_id}:u3:write`), + s.hits.filter((h) => h.startsWith('judge:')).join(' ')) + const claims = await turn('judge-plain').locator('[data-evidence-line] [data-evidence-claims]').innerText().catch(() => '') + check('按需判过一句：横幅多出结论句那一段（EvidenceField 叠好判定交出的文档，docTally 照样数）', + claims.includes('结论 5 句（不支持 1 · 未裁判 3 · 无证据 1）'), claims || '（没有这一段）') + const after = await plainLine() + check('……没挂依据的那句已经数在「无证据」里，不再另起一句', !!claims && await turn('judge-plain').locator('[data-evidence-line-claims]').count() === 0, after) + check('……有证据不支持的句子：横幅给「定位下一处」', !!claims && await turn('judge-plain').locator('[data-evidence-line] [data-evidence-next]').count() === 1, after) + await p.keyboard.press('Escape') + const formal = await turn('judge-formal').locator('[data-evidence-line]').innerText().catch(() => '') + check('开了裁判的正式运行：结论句那一段照节点里的判定数', formal.includes('结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）'), formal) + check('……没挂依据的那句（送了裁判、到上限没判）出具照样计缺口：另起「没挂依据的结论句 1」', + await turn('judge-formal').locator('[data-evidence-line-claims]').count() === 1 && formal.includes('没挂依据的结论句 1'), formal) + check('没有运行时报错（问数据页 · 结论句裁判）', s.errors.length === 0, s.errors.join(' | ')) + if (SHOTS) { + // 侧面板还开着会压住横幅的右半边：先关掉 + await p.locator('[data-evidence-panel] button[aria-label="关闭证据"]').click().catch(() => {}) + for (const theme of ['dark', 'light']) { + await p.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await p.waitForTimeout(200) + await turn('judge-plain').locator('[data-issuance-banner]').screenshot({ path: `${SHOTS}/evidence4-stream-plain-${theme}.png` }) + await turn('judge-formal').locator('[data-issuance-banner]').screenshot({ path: `${SHOTS}/evidence4-stream-formal-${theme}.png` }) + } + } + await s.ctx.close() +}) + if (SHOTS) { for (const theme of ['dark', 'light']) { await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) diff --git a/scripts/check-manage.mjs b/scripts/check-manage.mjs index db4425ee..21df9f17 100644 --- a/scripts/check-manage.mjs +++ b/scripts/check-manage.mjs @@ -609,6 +609,95 @@ await section('设置 · 智能体护栏', async () => { await t.close() }) +await section('设置 · 证据裁判', async () => { + // 结论句裁判的模型和各项上限（后端 engine/judge.py）：每一项都能设成不限（存 null），不限时写明还受什么约束。 + // 读写照运行默认值：GET 读、PUT 整组写。写请求在浏览器层拦下，不落库 + let stored = await get('/settings') + stored = { ...stored, judge: { provider: null, model: null, report_max_claims: 40, report_max_cost_usd: null, report_timeout_s: 30, + click_max_cost_usd: 0.01, daily_max_usd: 2 } } + let spendGets = 0 + const { page, sent, errors, close } = await open('/settings/prefs', { + handlers: [ + [/^GET \/settings$/, (route) => json(stored)(route)], + [/^PUT \/settings$/, (route, { body }) => { stored = { ...stored, ...body.values }; return json(stored)(route) }], + [/^GET \/settings\/judge\/spend$/, (route) => { spendGets += 1; return json({ date: '2026-09-29', usd: 0.0123, calls: 7, unpriced_calls: 2, daily_max_usd: 2 })(route) }], + ], + }) + const box = page.locator('[data-judge-settings]') + const all = (await box.innerText().catch(() => '')).replace(/\s+/g, ' ') + check('偏好里有「证据裁判」一组，写明判断是模型给的、不是系统核对', all.includes('证据裁判') && all.includes('非确定'), all.slice(0, 80)) + check('裁判模型：接入和模型，提示建议和写作模型不同', await page.locator('#pref-judge-provider').count() === 1 + && await page.locator('#pref-judge-model').count() === 1 && all.includes('建议和写报告的模型不同')) + check('……接入留空时跟随 Copilot 的模型', (await page.locator('#pref-judge-provider option').first().innerText()).includes('跟随 Copilot')) + const val = (id) => page.locator(`#${id}`).inputValue() + check('读回各项上限：40 句、30 秒、每次点击 $0.01、每日 $2', await val('pref-judge-claims') === '40' && await val('pref-judge-timeout') === '30' + && await val('pref-judge-click') === '0.01' && await val('pref-judge-daily') === '2') + const cost = page.locator('[data-judge-limit="report_max_cost_usd"]') + check('存着 null 的每份报告金额：勾着「不限」、框变灰', await cost.locator('[data-judge-unlimited-toggle]').isChecked() + && await page.locator('#pref-judge-cost').isDisabled()) + const costNote = await cost.locator('[data-judge-unlimited]').innerText().catch(() => '') + check('……写明「不设上限，费用只受……约束」', costNote === '不设上限，费用只受句数上限、时长上限、每日上限约束', costNote) + const spend = await page.locator('[data-judge-spend]').innerText().catch(() => '') + check('每日上限旁边写今天花了多少', spend.includes('今天已花 $0.0123（7 次调用）'), spend) + check('……估不出金额的调用另说', spend.includes('另有 2 次调用估不出金额') && await page.locator('[data-judge-unpriced]').count() === 1, spend) + + const daily = page.locator('[data-judge-limit="daily_max_usd"]') + await daily.locator('[data-judge-unlimited-toggle]').check() + const dailyNote = await daily.locator('[data-judge-unlimited]').innerText().catch(() => '') + check('每日上限设成不限：写明每日费用还受什么约束', dailyNote.startsWith('不设上限，每日费用只受') && dailyNote.includes('每次点击的金额上限'), dailyNote) + const costNote2 = await cost.locator('[data-judge-unlimited]').innerText().catch(() => '') + check('……每日也不限了：每份报告金额那句不再说「每日上限」', !costNote2.includes('每日') && costNote2.startsWith('不设上限'), costNote2) + await page.locator('#pref-judge-click').fill('0') + const save = page.getByRole('button', { name: /保存设置/ }) + check('每次点击填 0：标红、保存按钮不可用', await save.isDisabled() + && (await page.locator('[data-judge-limit="click_max_cost_usd"]').innerText()).includes('填大于 0 的金额，或者勾「不限」')) + await page.locator('#pref-judge-click').fill('0.02') + check('……接入留空时模型框的占位也写跟随 Copilot', await page.locator('#pref-judge-model').getAttribute('placeholder') === '留空：跟随 Copilot 的模型') + await page.locator('#pref-judge-model').fill('judge-model-b') + // 只填了模型：设置这一级整组生效（judge_model_spec 不跨级拼），后端按模型名找接入——已经不跟随 Copilot + const provFirst = await page.locator('#pref-judge-provider option').first().innerText().catch(() => '') + check('只填了模型：接入留空那一项不再叫「跟随 Copilot」，改叫「按模型名找接入」', provFirst === '按模型名找接入', provFirst) + const byModel = page.locator('[data-judge-by-model]') + const byText = await byModel.innerText().catch(() => '') + // 说明里有沙箱的接入名：细节只报认没认出来，不打印原文 + check('……写明按模型名找接入（认不出就去调默认接入），「跟随 Copilot」只在两项都留空时生效', + await byModel.getAttribute('data-judge-by-model').catch(() => '') === 'default' && byText.includes('只填了模型') + && byText.includes('默认接入') && byText.includes('「跟随 Copilot」只在接入和模型都留空时生效'), + `data-judge-by-model=${await byModel.getAttribute('data-judge-by-model').catch(() => '（没有）')}`) + const bar = (await page.locator('[data-prefs-bar]').innerText().catch(() => '')).replace(/\s+/g, ' ') + check('保存条写明改了哪几项', bar.includes('有 3 项未保存') && bar.includes('每日金额上限') && bar.includes('每次点击的金额上限') + && bar.includes('裁判模型'), bar) + const gets = spendGets + await save.click() + await page.waitForTimeout(500) + const put = sent.filter((s) => s.key === 'PUT /settings').at(-1)?.body?.values ?? {} + const j = put.judge ?? {} + check('保存：整组写 judge（七项都在），不限存 null', Object.keys(j).sort().join(',') + === 'click_max_cost_usd,daily_max_usd,model,provider,report_max_claims,report_max_cost_usd,report_timeout_s' + && j.daily_max_usd === null && j.report_max_cost_usd === null && j.click_max_cost_usd === 0.02 && j.report_max_claims === 40 + && j.report_timeout_s === 30 && j.model === 'judge-model-b' && j.provider === null, JSON.stringify(j)) + check('……只改了裁判：不碰运行默认值那一组', !('run' in put), Object.keys(put).join(',')) + check('……存完再取一次今天的花费（每日上限变了）', spendGets > gets, `${gets} → ${spendGets}`) + check('……存完显示「已保存」', (await page.locator('[data-prefs-bar]').innerText().catch(() => '')).includes('已保存')) + await daily.locator('[data-judge-unlimited-toggle]').uncheck() + check('勾掉「不限」：填回原来的数', await val('pref-judge-daily') === '2') + await daily.locator('[data-judge-unlimited-toggle]').check() + check('没有运行时报错', errors.length === 0, errors[0] ?? '') + if (SHOTS) { + await box.scrollIntoViewIfNeeded() + await page.waitForTimeout(200) + await box.screenshot({ path: `${SHOTS}/prefs-judge-${THEME}.png` }) + } + await close() + + // 老后端没有 judge 这一组：不显示，也不写 + const legacy = { ...stored } + delete legacy.judge + const old = await open('/settings/prefs', { handlers: [[/^GET \/settings$/, (route) => json(legacy)(route)]] }) + check('老后端没有裁判设置：这一组不出现', await old.page.locator('[data-judge-settings]').count() === 0) + await old.close() +}) + await section('数据 · 数据库', async () => { const dbs = sources.filter((s) => !/[\\/]uploads[\\/]tables[\\/]/.test(s.database ?? '')) const first = dbs.find((s) => s.table_count > 0) ?? dbs[0] @@ -1441,8 +1530,10 @@ await section('工具', async () => { check('……详情区写明原因和「绑了它的节点一定失败」', (await alert.innerText().catch(() => '')).includes('一定失败')) await shot(page, 'tool-broken-detail') await alert.getByRole('button', { name: '去改参数定义' }).click() - await page.waitForTimeout(700) const ed0 = page.getByRole('dialog', { name: '编辑工具「broken_demo」' }) + // 等编辑框真的出来再判：并行跑时固定等 700ms 不够，编辑框还没挂上就判成没打开（check-all 里偶发过） + await ed0.waitFor({ timeout: 5000 }).catch(() => {}) + await page.waitForTimeout(200) check('……「去改参数定义」直接打开它的编辑框（/tools/custom?edit=）', await ed0.count() === 1, page.url()) check('……读完 ?edit= 就从地址里摘掉', !page.url().includes('edit='), page.url()) const perr = await ed0.locator('[data-params-field]').innerText().catch(() => '') diff --git a/scripts/check-ui-kit.mjs b/scripts/check-ui-kit.mjs index 9803d29a..0baec97d 100644 --- a/scripts/check-ui-kit.mjs +++ b/scripts/check-ui-kit.mjs @@ -308,6 +308,8 @@ await section('lib/evidence：证据状态的四通道元数据、报告节点 unknown_entities: 2, uncited_claims: 1 }, 'require_citation')), JSON.stringify({ cited: 10, none: 5 })) eq('……结论句策略是 off 时不算没挂依据的结论句', ev.stampCounts({ numbers: 4, numbers_cited: 3, uncited_numbers: 1, unresolved: 0, uncited_claims: 3 }, 'off')?.none, 1) + eq('……judge 在挂依据上和 require_citation 一样严：没挂依据的结论句也算', ev.stampCounts({ numbers: 4, numbers_cited: 3, + uncited_numbers: 1, unresolved: 0, uncited_claims: 3 }, 'judge')?.none, 4) eq('可疑实体的提示：目录键、{name, table} 都写成去掉前缀的名字，最多 3 个', ev.closestNames(['c:orders.amount', { alias: 't:refunds' }, { name: 'region', table: 'orders', kind: 'column' }, 'c:x']).join(','), 'orders.amount,refunds,orders.region') @@ -409,6 +411,90 @@ await section('lib/evidence：证据状态的四通道元数据、报告节点 })) check(name, ok, detail) }) +await section('lib/evidence：结论句裁判（四期）——判定的徽标、计数、答复形状、改写一次', async () => { + for (const [name, ok, detail] of await page.evaluate(() => { + const { evidence: ev, terms: t } = window.__ui.lib + const fj = window.__ui.evidenceJudge + const out = [] + const eq = (name, got, want) => out.push([name, got === want, got === want ? '' : `得到 ${JSON.stringify(got)}，应为 ${JSON.stringify(want)}`]) + const ok = (name, cond, detail = '') => out.push([name, !!cond, cond ? '' : detail]) + const S = ev.EVIDENCE_STATE + ok('概率性四种都挂在句末徽标上（line=badge），字形 ◆ ◇ ! ?', ['supported', 'partial', 'unsupported', 'unjudged'] + .map((c) => `${S[c].line}/${S[c].glyph}`).join(' ') === 'badge/◆ badge/◇ badge/! badge/?', + ['supported', 'partial', 'unsupported', 'unjudged'].map((c) => `${S[c].line}/${S[c].glyph}`).join(' ')) + // 颜色令牌照方案 6.1：支持用强调色（--st-running 就是 --accent）、部分用 --st-waiting、不支持用 --st-failed、 + // 未裁判用暗字色（--st-cancelled 就是 --text-dim）。四种都不是确定性的绿 + eq('四种的颜色令牌', ['supported', 'partial', 'unsupported', 'unjudged'].map((c) => S[c].color).join(' '), + 'var(--st-running) var(--st-waiting) var(--st-failed) var(--st-cancelled)') + ok('概率性四种都不用确定性的绿（--st-done）', ['supported', 'partial', 'unsupported', 'unjudged'] + .every((c) => !S[c].color.includes('--st-done') && !S[c].soft.includes('--st-done'))) + ok('证据不支持、部分支持醒目（进 n / N），有依据、未裁判安静', S.unsupported.alert && S.partial.alert && !S.supported.alert && !S.unjudged.alert) + eq('未裁判的叫法不说「有引用」（没挂依据的结论句也会没判）', S.unjudged.label, '未裁判') + eq('判定 → 徽标：不是结论句不挂', [ev.verdictState({ status: 'supported' }), ev.verdictState({ status: 'not_a_claim' }), + ev.verdictState({ status: 'unjudged' }), ev.verdictState({ status: 'maybe' }), ev.verdictState(undefined)].join(','), + 'supported,,unjudged,,') + eq('封存后追加的判定盖过文档里的「未裁判 · 按需」', ev.unitVerdict({ id: 'u2', verdict: { status: 'unjudged', reason: 'on_demand' } }, + { u2: { status: 'unsupported', post_seal: true } }).status, 'unsupported') + eq('没有追加的用文档里的', ev.unitVerdict({ id: 'u2', verdict: { status: 'partial' } }, { u9: { status: 'supported' } }).status, 'partial') + eq('到上限没判的认得出是哪个上限', ev.limitOf({ status: 'unjudged', reason: 'daily_max_usd' }), 'daily_max_usd') + eq('按需、出错不是上限', `${ev.limitOf({ status: 'unjudged', reason: 'on_demand' })}|${ev.limitOf({ status: 'unjudged', reason: 'error' })}`, 'null|null') + ok('判过的四种（不含未裁判）', ev.isJudged({ status: 'not_a_claim' }) && ev.isJudged({ status: 'partial' }) && !ev.isJudged({ status: 'unjudged' })) + const k = (c) => (c ? `${c.total}:${c.supported}/${c.partial}/${c.unsupported}/${c.unjudged}/${c.uncited}` : 'null') + eq('正式运行（节点里判好的）：不是结论句不算，4 句', k(ev.claimCounts(fj.formal.doc)), '4:1/1/1/1/0') + eq('探索运行按需：5 句都未裁判', k(ev.claimCounts(fj.explore.doc)), '5:0/0/0/5/0') + eq('没开裁判、也没按需判过：不数（横幅照旧）', k(ev.claimCounts(fj.plain.doc)), 'null') + // 开了裁判的文档里预筛放掉的结论句（没有判定、列在 judge.screened 里）不算进结论句，也不算「无证据」 + eq('开了裁判：预筛放掉的结论句不数', k(ev.claimCounts({ judge: { mode: 'inline', screened: ['u9'] }, blocks: [{ type: 'paragraph', + units: [{ id: 'u9', kind: 'claim', cites: [], segments: [] }, { id: 'u1', kind: 'claim', cites: ['Q1'], segments: [], + verdict: { status: 'supported' } }] }] })), '1:1/0/0/0/0') + eq('没开裁判、按需判过一句：挂了依据的算未裁判，没挂的算无证据', k(ev.claimCounts(fj.plain.doc, + { [fj.plain.units.bad]: { status: 'unsupported', post_seal: true } })), '5:0/0/1/3/1') + eq('docTally 带上结论句计数', k(ev.docTally(fj.formal.doc).claims), '4:1/1/1/1/0') + // 问数据页、画布右栏的横幅走的是这条路：EvidenceField 把判定叠进文档（withVerdicts）交出去，Output 再 + // docTally——这时没有单独的 overlay，判定写在 unit.verdict 上，也得认得出「按需判过」 + eq('没开裁判、按需判过一句：叠好判定的文档交给 docTally 照样数（横幅的路径）', + k(ev.docTally(ev.withVerdicts(fj.plain.doc, { [fj.plain.units.bad]: { status: 'unsupported', post_seal: true } })).claims), + '5:0/0/1/3/1') + eq('叠上判定给横幅数：只换判过的那几句', k(ev.claimCounts(ev.withVerdicts(fj.explore.doc, + { [fj.explore.units.bad]: { status: 'unsupported', post_seal: true } }))), '5:0/0/1/4/0') + ok('叠上判定不改原文档', ev.withVerdicts(fj.explore.doc, { [fj.explore.units.bad]: { status: 'unsupported' } }) !== fj.explore.doc + && fj.explore.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fj.explore.units.bad).verdict.status === 'unjudged') + eq('计数的说法：和数字那一段同一种', t.claimTally({ total: 4, supported: 1, partial: 1, unsupported: 1, unjudged: 1, uncited: 0 }), + '结论 4 句（支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1）') + eq('……方案的例子', t.claimTally({ total: 4, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 1 }), '结论 4 句（支持 3 · 无证据 1）') + eq('……没有结论句给空串', t.claimTally({ total: 0, supported: 0, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }), '') + const unit = fj.formal.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fj.formal.units.bad) + eq('句末徽标的 aria-label：句子、判定写全', ev.claimLabel(unit, unit.verdict), '结论句「增长主要来自新客首单，老客复购持平。」，模型判断：证据不支持') + ok('……封存后追加的另说', ev.claimLabel(unit, { ...unit.verdict, post_seal: true }).endsWith('，封存后追加')) + ok('……到上限没判的说「已到上限」', ev.claimLabel(unit, { status: 'unjudged', reason: 'max_cost_usd' }).endsWith('未裁判：已到上限')) + // 按需裁判接口的答复：evidence.judged 的载荷（{verdicts: {u: …}}），也认列表；缺 post_seal 的记 true，缺 judge 的补模型 + const a = ev.judgedVerdicts({ verdicts: { u4: { status: 'supported', rationale: '对' } }, model: 'jm' }) + ok('答复 {verdicts: {u: …}}：补上模型、记封存后追加', a.u4?.status === 'supported' && a.u4.judge === 'jm' && a.u4.post_seal === true, + JSON.stringify(a)) + const b = ev.judgedVerdicts({ verdicts: [{ unit: 'u2', verdict: 'partial', judge: 'x', post_seal: false }, { unit: 'u3' }] }) + ok('答复是列表、判定写在 verdict 里：照认；没有判定的丢掉；接口给了 post_seal 就照它', b.u2?.status === 'partial' && b.u2.post_seal === false + && !('u3' in b), JSON.stringify(b)) + eq('认不出的答复给空表', Object.keys(ev.judgedVerdicts('oops')).length + Object.keys(ev.judgedVerdicts({ verdicts: 3 })).length, 0) + const g = ev.graphVerdicts({ reports: [{ node_id: 'w1', post_seal_verdicts: { u1: { status: 'supported' } } }, + { node_id: 'w2', post_seal_verdicts: { u1: { status: 'unsupported' } } }] }, 'w2') + ok('证据图里按报告取封存后追加的判定（reports[].post_seal_verdicts）', g.u1?.status === 'unsupported' && g.u1.post_seal === true, JSON.stringify(g)) + const applied = { judge: { rewrite: { applied: true, units: ['u1', null], sentences: ['原句甲', '原句乙'], changed: ['u5'], reason: null } } } + ok('改写一次采用了：changed 里的句子给出交回改写的原句（units 为 null 的那几句）', + JSON.stringify(ev.rewriteOf(applied, 'u5')) === JSON.stringify({ kind: 'changed', sentences: ['原句乙'] }), JSON.stringify(ev.rewriteOf(applied, 'u5'))) + eq('……没改的句子不标', ev.rewriteOf(applied, 'u1'), null) + const rejected = { judge: { rewrite: { applied: false, units: ['u2'], sentences: ['原句'], changed: [], reason: '改写稿冒出 1 处原稿没有的问题' } } } + eq('改写稿没采用：交回过的句子写明原因', ev.rewriteOf(rejected, 'u2')?.kind, 'rejected') + eq('……没交回的不标', ev.rewriteOf(rejected, 'u3'), null) + // 设置页：设成不限时写明还受什么约束；别的也不限时照实少说 + const all = { claims: true, cost: true, timeout: true, click: true, daily: true } + eq('不限的说法：每日', t.judgeUnlimitedText('daily', all), '不设上限，每日费用只受每份报告的金额上限和每次点击的金额上限约束') + eq('……每份报告的金额', t.judgeUnlimitedText('cost', all), '不设上限，费用只受句数上限、时长上限、每日上限约束') + ok('……每日、每份、每次都不限：说到底只受调用次数约束', t.judgeUnlimitedText('daily', { ...all, cost: false, click: false }).includes('调用次数')) + ok('……每日不限时每份报告的金额只剩句数、时长', !t.judgeUnlimitedText('cost', { ...all, daily: false }).includes('每日')) + return out + })) check(name, ok, detail) +}) + await section('lib/actor：本机署名', async () => { for (const [name, ok, detail] of await page.evaluate(() => { const a = window.__ui.lib.actor