mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 09:52:01 +00:00
可点击证据第四期:结论句由模型裁判;只引单元格的问数据图能发成受管
一、结论句裁判
- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定
二、成本上限
- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额
三、正式运行与探索运行
- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」
四、治理与自动修复
- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略
五、界面
- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题
六、门禁修正
- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
This commit is contained in:
+138
-27
@@ -23,8 +23,23 @@ from typing import Any, Callable
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.engine.governance import REPORT_POLICY, exit_text_fields, publish_issues
|
||||
from app.engine.schema import REPORT_CLAIMS, GraphNode, GraphSpec, NodeType, _ancestors, claims_problem
|
||||
from app.engine.governance import (
|
||||
JUDGE_UNLIMITED,
|
||||
REPORT_POLICY,
|
||||
REPORT_POLICY_ALSO,
|
||||
exit_text_fields,
|
||||
publish_issues,
|
||||
)
|
||||
from app.engine.schema import (
|
||||
CLAIMS_JUDGE,
|
||||
JUDGE_ON_UNSUPPORTED,
|
||||
REPORT_CLAIMS,
|
||||
GraphNode,
|
||||
GraphSpec,
|
||||
NodeType,
|
||||
_ancestors,
|
||||
claims_problem,
|
||||
)
|
||||
|
||||
#: 被选中的修复按这个顺序应用:先补契约骨架,再补契约里的各项,最后是节点上的开关。
|
||||
#: 前面的修复可能改变后面的候选(metrics_from 决定 required 从哪几张卡里挑,report_from 决定
|
||||
@@ -34,7 +49,7 @@ _ORDER = (
|
||||
"contract.report_from_missing", "contract.report_from_invalid", "governed.report_from_required",
|
||||
"governed.exit_text_source", "report.metrics_from_invalid",
|
||||
"contract.required_missing", "contract.strict_off", "contract.claims_ignored", "report.claims_invalid",
|
||||
"governed.report_policy",
|
||||
"governed.report_policy", "governed.judge_budget",
|
||||
"governed.caliber_agent_cite_fields", "governed.caliber_compute_input", "governed.agent_approval_never",
|
||||
"governed.default_approval_never", "governed.subgraph_unpinned", "governed.supervisor", "contract.not_object",
|
||||
"governed.text_bypass", "governed.caliber_agent_schema", "governed.caliber_model_input",
|
||||
@@ -210,7 +225,8 @@ def _plan_exit_fields(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None)
|
||||
|
||||
|
||||
def _plan_report_policy(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None:
|
||||
"""G3:报告撰写节点写明最严的三项。答案唯一,只往严里改;写了不支持的 claims 由 claims_invalid 管。"""
|
||||
"""G3:报告撰写节点写明最严的三项。答案唯一,只往严里改;写了不认识的 claims 由 claims_invalid 管。
|
||||
已经写了更严的写法(claims: judge)的不动——改回 require_citation 就是替人放宽了。"""
|
||||
if node is None:
|
||||
return None
|
||||
writes: dict[str, Any] = {}
|
||||
@@ -218,7 +234,9 @@ def _plan_report_policy(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None
|
||||
for key, want, _ in REPORT_POLICY:
|
||||
value = node.config.get(key)
|
||||
effective = ctx.spec.defaults.get(key) if value in (None, "") else value
|
||||
if effective != want and not (key == "claims" and claims_problem(effective)):
|
||||
if effective in (want, *REPORT_POLICY_ALSO.get(key, ())):
|
||||
continue
|
||||
if not (key == "claims" and claims_problem(effective)):
|
||||
writes[key], before[key] = want, copy.deepcopy(value)
|
||||
if not writes:
|
||||
return None
|
||||
@@ -230,18 +248,38 @@ def _plan_report_policy(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None
|
||||
|
||||
|
||||
def _plan_claims(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None:
|
||||
"""claims 写了不支持的值。受管级别只认 require_citation,答案唯一;已发布级别 off 和 require_citation
|
||||
都行,要人选——写的是 judge(还不支持的模型裁判)时建议 require_citation:它离裁判最近。"""
|
||||
"""claims 写了不认识的值。受管级别改成 require_citation:确定性的两档里只有它合规,答案唯一(judge 要花钱、
|
||||
要写预算,不替人开);已发布级别 off 和 require_citation 都行,要人选。"""
|
||||
if node is None:
|
||||
return None
|
||||
before = node.config.get("claims")
|
||||
if ctx.level == "governed":
|
||||
return _auto(f"把「{node.title}」的 claims 改成 require_citation:受管出具要求每句结论挂引用"
|
||||
+ ("(结论句裁判在后续版本支持)" if before == "judge" else ""), "claims", before, "require_citation")
|
||||
return _auto(f"把「{node.title}」的 claims 改成 require_citation:受管出具要求每句结论挂引用", "claims",
|
||||
before, "require_citation")
|
||||
hints = {"off": "结论句不参与判档,和以前一样", "require_citation": "没挂引用的结论句计入缺口,按出具档位降档"}
|
||||
options = [{"value": v, "label": v, "hint": hints[v]} for v in REPORT_CLAIMS]
|
||||
return _choice(f"选「{node.title}」的结论句策略(claims)", "claims", options,
|
||||
default="require_citation" if before == "judge" else None)
|
||||
return _choice(f"选「{node.title}」的结论句策略(claims)", "claims", options)
|
||||
|
||||
|
||||
def _plan_judge_budget(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None:
|
||||
"""G3:claims: judge 没写每份报告的裁判预算。写多少钱只有作者能定:候选是默认金额和「不限」,不替人选、
|
||||
不给建议。judge 原来写在图级 defaults 里的,写到节点上时带上 defaults 那一整份——节点写了 judge 就整个
|
||||
用节点的,只补一个 max_cost_usd 会把 defaults 里选的裁判模型、上限悄悄丢掉。"""
|
||||
from app.engine.judge import JUDGE_DEFAULTS
|
||||
|
||||
if node is None:
|
||||
return None
|
||||
judge = node.config.get("judge")
|
||||
if judge in (None, ""):
|
||||
judge = ctx.spec.defaults.get("judge")
|
||||
base = copy.deepcopy(judge) if isinstance(judge, dict) else {}
|
||||
amount = JUDGE_DEFAULTS["report_max_cost_usd"]
|
||||
options = [
|
||||
{"value": amount, "label": f"每份报告最多 ${amount:g}", "hint": "系统默认的每份报告上限;写进节点以后,改设置不影响这份报告"},
|
||||
{"value": None, "label": "不限(写 null)", "hint": JUDGE_UNLIMITED},
|
||||
]
|
||||
return _choice(f"给「{node.title}」的结论句裁判写每份报告的预算(judge.max_cost_usd)", "judge.max_cost_usd",
|
||||
options, _judge=base)
|
||||
|
||||
|
||||
def _plan_compute_input(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None:
|
||||
@@ -419,6 +457,7 @@ _PLANNERS: dict[str, Callable[[_Ctx, dict[str, Any], GraphNode | None], dict[str
|
||||
"governed.text_bypass": _plan_text_bypass,
|
||||
"governed.report_policy": _plan_report_policy,
|
||||
"report.claims_invalid": _plan_claims,
|
||||
"governed.judge_budget": _plan_judge_budget,
|
||||
"governed.caliber_compute_input": _plan_compute_input,
|
||||
"governed.caliber_agent_cite_fields": _plan_cite_fields,
|
||||
"governed.caliber_agent_schema": _plan_agent_schema,
|
||||
@@ -580,12 +619,25 @@ def _edit_exit_fields(graph: dict[str, Any], fix: dict[str, Any], value: Any) ->
|
||||
return out
|
||||
|
||||
|
||||
def _edit_judge_budget(graph: dict[str, Any], fix: dict[str, Any], value: Any) -> list[tuple[str, str, Any, Any]]:
|
||||
"""G3 的裁判预算:节点的 judge 写成(原来生效的那份 judge + 选中的 max_cost_usd)。选「不限」写显式的 null,
|
||||
和没写不是一回事。变更按整份 judge 记:前后都是 null 的 max_cost_usd 在预览里分不出写没写。"""
|
||||
node = _node(graph, fix["node_id"])
|
||||
if node is None:
|
||||
return []
|
||||
config = _config(node)
|
||||
after = {**copy.deepcopy(fix.get("_judge") or {}), "max_cost_usd": value}
|
||||
before = _write(config, "judge", after)
|
||||
return [(str(node["id"]), "judge", before, copy.deepcopy(after))]
|
||||
|
||||
|
||||
#: 每类问题怎么改图。大多是同一种「按节点字段写值」,改全图默认、改成果字段的各走一格;留着这张表是为了
|
||||
#: 某类修复要特殊处理时只换它自己的那一格(测试里也借它模拟一条出错的修复规则)
|
||||
_EDITORS: dict[str, Callable[[dict[str, Any], dict[str, Any], Any], list[tuple[str | None, str, Any, Any]]]] = {
|
||||
code: (_edit_defaults if code in _GRAPH_FIXES else _edit) for code in _PLANNERS
|
||||
}
|
||||
_EDITORS["governed.exit_text_source"] = _edit_exit_fields
|
||||
_EDITORS["governed.judge_budget"] = _edit_judge_budget
|
||||
|
||||
|
||||
def _handoff(fix: dict[str, Any], value: Any) -> bool:
|
||||
@@ -811,25 +863,34 @@ def _contract_loosened(name: str, prior: Any, contract: Any) -> list[str]:
|
||||
if contract.get("cells") is True and prior.get("cells") is not True:
|
||||
out.append(f"替你打开了「{name}」出具契约的 cells(允许报告直接引用查询单元格),这要你自己决定")
|
||||
was, now = _claims_rank(prior.get("claims")), _claims_rank(contract.get("claims"))
|
||||
if was is not None and now is not None and now < was:
|
||||
out.append(f"把「{name}」出具契约的 claims 往宽里改了(没挂依据的结论句{_CLAIMS_WORDS[was]} → "
|
||||
f"{_CLAIMS_WORDS[now]})")
|
||||
if was is not None and now is not None:
|
||||
looser = [f"{what}{words[a]} → {words[b]}" for (what, words), a, b in zip(_CLAIMS_AXES, was, now) if b < a]
|
||||
if looser:
|
||||
out.append(f"把「{name}」出具契约的 claims 往宽里改了({';'.join(looser)})")
|
||||
return out
|
||||
|
||||
|
||||
#: 契约 claims 的严格程度:不查 < 查了不算缺口(ignore)< 计入缺口降档(degrade)< 不予出具(withhold)
|
||||
_CLAIMS_WORDS = ("不检查", "只标出来", "计入缺口", "不予出具")
|
||||
#: 契约 claims 的严格程度分两样,各自从宽到严:没挂依据的结论句(不查 < 查了不算缺口 ignore < 计入缺口降档
|
||||
#: degrade < 不予出具 withhold),证据不支持的结论句(不裁判 < 裁判、降档 < 裁判、不予出具)。judge 两样都管:
|
||||
#: 挂依据的要求和 require_citation 一样,另外按裁判的判定判档
|
||||
_CLAIMS_AXES = (("没挂依据的结论句", ("不检查", "只标出来", "计入缺口", "不予出具")),
|
||||
("证据不支持的结论句", ("不裁判", "降档出具", "不予出具")))
|
||||
|
||||
|
||||
def _claims_rank(value: Any) -> int | None:
|
||||
"""契约里 claims 的严格程度,越大越严。judge 这类还不支持的写法不比较(validate 另报)。"""
|
||||
def _claims_rank(value: Any) -> tuple[int, int] | None:
|
||||
"""契约里 claims 的严格程度:(没挂依据的, 证据不支持的),各自越大越严,任何一样变小都算放宽。
|
||||
认不出的写法不比较(出口按缺口处理,不会因此放松)。"""
|
||||
name = value.get("policy") if isinstance(value, dict) else value
|
||||
if name in (None, "", "off"):
|
||||
return 0
|
||||
if name != "require_citation":
|
||||
return 0, 0
|
||||
if name not in ("require_citation", CLAIMS_JUDGE):
|
||||
return None
|
||||
on_uncited = value.get("on_uncited") if isinstance(value, dict) else None
|
||||
return {"ignore": 1, "withhold": 3}.get(on_uncited, 2)
|
||||
uncited = {"ignore": 1, "withhold": 3}.get(on_uncited, 2)
|
||||
if name != CLAIMS_JUDGE:
|
||||
return uncited, 0
|
||||
on_unsupported = value.get("on_unsupported") if isinstance(value, dict) else None
|
||||
return uncited, 2 if on_unsupported == "withhold" else 1
|
||||
|
||||
|
||||
def forbidden_changes(before: dict[str, Any], after: dict[str, Any],
|
||||
@@ -844,7 +905,8 @@ def forbidden_changes(before: dict[str, Any], after: dict[str, Any],
|
||||
- 审批策略往宽里改(每次调用都审批 > 仅危险工具需要审批 > 全部自动放行)、删掉写明的审批策略,
|
||||
把审批写成(或新加一个)「全部自动放行」;全图默认同理
|
||||
- 取消子工作流钉住的版本
|
||||
- 报告撰写节点的 numbers、on_violation、claims 往宽里改;关掉 agent 的 cite_fields
|
||||
- 报告撰写节点的 numbers、on_violation、claims 往宽里改(claims 改成认不出的写法也算),claims 一直是
|
||||
judge 时 judge.on_unsupported 从 withhold 退回去;关掉 agent 的 cite_fields
|
||||
- 替人把沙箱代码标成取数(evidence_role: source):它在取数还是在算只有作者知道。chosen 里的
|
||||
(节点 id, 字段) 是人在 choice 里亲手选的,不算
|
||||
- 改发布级别——级别不在图里,修复接口也从不写工作流的 status,所以只可能以一个不认识的
|
||||
@@ -928,12 +990,17 @@ def forbidden_changes(before: dict[str, Any], after: dict[str, Any],
|
||||
#: 报告撰写节点三项设置从严到宽的次序:没写时运行时的缺省值(on_violation 没写时探索运行按 flag)
|
||||
_REPORT_STRICT = {"numbers": ("strict", "strict"), "on_violation": ("fail", "flag"),
|
||||
"claims": ("require_citation", "off")}
|
||||
#: claims 从宽到严:off < require_citation < judge(在挂引用之外再请模型逐句裁判,不支持的按 on_unsupported
|
||||
#: 判档)。不在这里的写法不比较(validate 报 report.claims_invalid)
|
||||
_CLAIMS_ORDER = ("off", "require_citation", CLAIMS_JUDGE)
|
||||
|
||||
|
||||
def _report_loosened(name: str, prev: dict[str, Any], cfg: dict[str, Any], old_defaults: Any,
|
||||
new_defaults: Any) -> list[str]:
|
||||
"""报告撰写节点的三项设置有没有往宽里改:原来(按节点、再按全图默认)是最严的值,改完不是了。
|
||||
claims 改成 judge 不算放宽——那是还不支持的更严的写法,validate 会报。"""
|
||||
"""报告撰写节点的三项设置有没有往宽里改:numbers、on_violation 原来(按节点、再按全图默认)是最严的值,
|
||||
改完不是了;claims 按 off < require_citation < judge 排,往下走一格也算(require_citation → judge 是收紧),
|
||||
从 require_citation、judge 改成认不出的写法也算(原来就认不出的不比较)。claims 前后都是 judge 的,
|
||||
再看 judge.on_unsupported 有没有从 withhold 退回去。"""
|
||||
def effective(config: dict[str, Any], defaults: Any, key: str, fallback: str) -> Any:
|
||||
value = config.get(key)
|
||||
if value in (None, ""):
|
||||
@@ -943,12 +1010,56 @@ def _report_loosened(name: str, prev: dict[str, Any], cfg: dict[str, Any], old_d
|
||||
out = []
|
||||
for key, (strict, fallback) in _REPORT_STRICT.items():
|
||||
was, now = effective(prev, old_defaults, key, fallback), effective(cfg, new_defaults, key, fallback)
|
||||
if was == strict and now != strict and not (key == "claims" and now == "judge"):
|
||||
out.append(f"把「{name}」的 {key} 从 {strict} 放宽成了 {now}"
|
||||
+ ("(去掉了写明的值,运行时退回缺省)" if cfg.get(key) in (None, "") else ""))
|
||||
if key == "claims":
|
||||
if was not in _CLAIMS_ORDER or (was == "off" and now not in _CLAIMS_ORDER):
|
||||
continue
|
||||
if now not in _CLAIMS_ORDER:
|
||||
out.append(f"把「{name}」的 claims 从 {was} 改成了不认识的 {now}")
|
||||
continue
|
||||
if _CLAIMS_ORDER.index(now) >= _CLAIMS_ORDER.index(was):
|
||||
continue
|
||||
elif not (was == strict and now != strict):
|
||||
continue
|
||||
out.append(f"把「{name}」的 {key} 从 {was} 放宽成了 {now}"
|
||||
+ ("(去掉了写明的值,运行时退回缺省)" if cfg.get(key) in (None, "") else ""))
|
||||
if effective(prev, old_defaults, "claims", "off") == effective(cfg, new_defaults, "claims", "off") \
|
||||
== CLAIMS_JUDGE and (why := _judge_loosened(name, prev, cfg, old_defaults, new_defaults)):
|
||||
out.append(why)
|
||||
return out
|
||||
|
||||
|
||||
def _judge_loosened(name: str, prev: dict[str, Any], cfg: dict[str, Any], old_defaults: Any,
|
||||
new_defaults: Any) -> str | None:
|
||||
"""claims 一直是 judge 时,judge.on_unsupported 从 withhold(证据不支持的结论句不予出具)退回 degrade
|
||||
(只降档)。judge 和运行时(report.report_judge)同一个取法:节点上写了就整个用节点的,没写取图级
|
||||
defaults;on_unsupported 没写、写了认不出的都按 degrade。所以补预算时把 judge 整个换成
|
||||
{max_cost_usd: …}、节点另写一份不带 withhold 的 judge 盖掉全图默认,都算放宽。"""
|
||||
def block(config: dict[str, Any], defaults: Any) -> tuple[Any, bool]:
|
||||
own = config.get("judge")
|
||||
if own not in (None, ""):
|
||||
return own, False
|
||||
return (defaults if isinstance(defaults, dict) else {}).get("judge"), True
|
||||
|
||||
def written(judge: Any) -> Any:
|
||||
return judge.get("on_unsupported") if isinstance(judge, dict) else None
|
||||
|
||||
(old_block, old_inherited), (new_block, new_inherited) = block(prev, old_defaults), block(cfg, new_defaults)
|
||||
if written(old_block) != "withhold" or (raw := written(new_block)) == "withhold":
|
||||
return None
|
||||
was = "withhold" + ("(跟随全图默认)" if old_inherited else "")
|
||||
if raw in JUDGE_ON_UNSUPPORTED:
|
||||
now = f"{raw}" + ("(跟随全图默认)" if new_inherited else "")
|
||||
elif raw not in (None, ""):
|
||||
now = f"认不出的 {raw}(运行时按 degrade)"
|
||||
elif new_inherited:
|
||||
now = "degrade(跟随全图默认,那里没写 on_unsupported,运行时退回缺省)"
|
||||
elif old_inherited:
|
||||
now = "degrade(节点上另写了 judge,整个用节点的、不再跟随全图默认,没写 on_unsupported 就退回缺省)"
|
||||
else:
|
||||
now = "degrade(去掉了写明的值,运行时退回缺省)"
|
||||
return f"把「{name}」的 judge.on_unsupported 从 {was}{'' if old_inherited else ' '}放宽成了 {now}"
|
||||
|
||||
|
||||
def _type_word(node_type: Any, type_label: Callable[[Any], str]) -> str:
|
||||
try:
|
||||
return f"「{type_label(NodeType(node_type))}」"
|
||||
|
||||
@@ -8,12 +8,15 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.engine.schema import (
|
||||
CLAIMS_JUDGE,
|
||||
GraphNode,
|
||||
GraphSpec,
|
||||
NodeType,
|
||||
ValidationResult,
|
||||
_caliber_inputs,
|
||||
claims_problem,
|
||||
contract_needs_metrics,
|
||||
judge_problems,
|
||||
type_label,
|
||||
validate_graph,
|
||||
)
|
||||
@@ -158,7 +161,7 @@ def _lint_contract(
|
||||
if isinstance(sources, str):
|
||||
sources = [sources]
|
||||
|
||||
if not sources:
|
||||
if not sources and contract_needs_metrics(contract):
|
||||
flag("出具契约没有声明 metrics_from(指标来自哪个「口径卡」节点),叙述里的数字无从回指",
|
||||
code="contract.metrics_from_missing", node_id=node_id, hard=True, field="contract.metrics_from")
|
||||
for src in sources:
|
||||
@@ -177,7 +180,11 @@ def _lint_contract(
|
||||
"(叙述),数字回指校验不会执行", code="contract.report_from_missing", node_id=node_id, hard=True,
|
||||
field="contract.report_from")
|
||||
|
||||
if strict and not (contract.get("required") or []):
|
||||
# 引用模式下整张图没有口径卡(只引查询单元格的问数据图):没有指标可列,strict + cells 已经保证
|
||||
# 每个数点得开出处、对不上就不予出具(用户 2026-09-29 拍板)。有口径卡的照旧要列必需指标——
|
||||
# 按图里有没有口径卡判,不按契约写没写 metrics_from,免得不写 metrics_from 就绕过去
|
||||
cells_only = contract.get("report_from") not in (None, "") and not metric_nodes
|
||||
if strict and not (contract.get("required") or []) and not cells_only:
|
||||
flag(
|
||||
"受管模板的出具契约必须声明 required(必需指标),否则「不予出具」这一档永远触发不了",
|
||||
code="contract.required_missing", node_id=node_id, hard=True, field="contract.required",
|
||||
@@ -202,7 +209,7 @@ def _lint_contract(
|
||||
#
|
||||
# G1 带契约的出口核对报告撰写节点的文档,成果字段的文字只能来自报告撰写节点
|
||||
# G2 模型写的文字不绕过报告撰写节点直接流进出口(从图上追)
|
||||
# G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation
|
||||
# G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation(或 judge,带上预算)
|
||||
# G4 口径卡的输入不来自计算角色的沙箱代码
|
||||
# G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields(模型调用节点不能给口径卡供数)
|
||||
#
|
||||
@@ -216,6 +223,11 @@ REPORT_POLICY = (
|
||||
("on_violation", "fail", "重写之后还违规就让节点失败,不带着问题出具"),
|
||||
("claims", "require_citation", "没挂引用的结论句计入缺口,按出具档位降档"),
|
||||
)
|
||||
#: 除了 REPORT_POLICY 里写的那个值,还认的更严的写法。claims: judge 在挂引用之外再请另一个模型按证据逐句
|
||||
#: 裁判,不支持的按 on_unsupported 判档——比 require_citation 严,但要写预算(见 judge_budget_missing)
|
||||
REPORT_POLICY_ALSO: dict[str, tuple[str, ...]] = {"claims": (CLAIMS_JUDGE,)}
|
||||
#: 不限的写法:judge.max_cost_usd 写 null。说给人听的时候照 engine/judge 设置项的说法
|
||||
JUDGE_UNLIMITED = "不设上限,费用只受句数、时长上限和每日上限约束"
|
||||
|
||||
#: 产出本身就是证据、或者由系统确定地算出来的节点。成果字段取它们的值,不算「别的节点写的字」
|
||||
_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE)
|
||||
@@ -289,6 +301,21 @@ def _effective(spec: GraphSpec, node: GraphNode, key: str, default: Any = None)
|
||||
return spec.defaults.get(key, default) if value in (None, "") else value
|
||||
|
||||
|
||||
def judge_budget_missing(spec: GraphSpec, node: GraphNode) -> bool:
|
||||
"""claims 是 judge、生效的 judge 配置里却没有 max_cost_usd 这个键(G3)。
|
||||
|
||||
写 null 是显式的不限,算写了;没写的键运行时取设置里的默认——受管出具要作者自己定。judge 本身写坏了
|
||||
(不是对象、max_cost_usd 写成了别的东西)由 validate 的 report.judge_invalid 报,这里不重复。
|
||||
judge 和 claims 一样按运行时的规则取:节点上写了就整个用节点的,不和图级 defaults 拼。
|
||||
"""
|
||||
if _effective(spec, node, "claims") != CLAIMS_JUDGE:
|
||||
return False
|
||||
judge = _effective(spec, node, "judge")
|
||||
if any(field in ("judge", "judge.max_cost_usd") for field, _ in judge_problems(judge)):
|
||||
return False
|
||||
return not (isinstance(judge, dict) and "max_cost_usd" in judge)
|
||||
|
||||
|
||||
class _Graph:
|
||||
"""一张图上反复要用的查找:谁产出了某个值、最后一条消息是谁写的。"""
|
||||
|
||||
@@ -489,13 +516,20 @@ def _lint_evidence(spec: GraphSpec, flag: Callable[..., None], *, strict: bool)
|
||||
for key, want, why in REPORT_POLICY:
|
||||
value = _effective(spec, node, key)
|
||||
if key == "claims" and claims_problem(value):
|
||||
continue # 写了不支持的值(比如 judge),validate 的 report.claims_invalid 报
|
||||
if value != want:
|
||||
continue # 写了不认识的值,validate 的 report.claims_invalid 报
|
||||
also = REPORT_POLICY_ALSO.get(key, ())
|
||||
if value != want and value not in also:
|
||||
now = "现在没写" if value in (None, "") else f"现在是 {value}"
|
||||
loose.append((key, f"{key}: {want}({why};{now})"))
|
||||
more = "".join(f",或者更严的 {v}" for v in also)
|
||||
loose.append((key, f"{key}: {want}({why}{more};{now})"))
|
||||
if loose:
|
||||
flag(f"{_who(node)}要写明 {';'.join(text for _, text in loose)}。受管模板的报告撰写节点按最严的规则自查",
|
||||
code="governed.report_policy", node_id=node.id, hard=True, field=loose[0][0])
|
||||
if judge_budget_missing(spec, node):
|
||||
flag(f"{_who(node)}的结论句由模型裁判(claims: judge),却没有写每份报告的裁判预算:在 judge 里写 "
|
||||
"max_cost_usd——写一个金额(美元),或者显式写 null 表示不限(" + JUDGE_UNLIMITED + ")。"
|
||||
"受管出具不让裁判的花费悄悄按设置里的默认值走",
|
||||
code="governed.judge_budget", node_id=node.id, hard=True, field="judge.max_cost_usd")
|
||||
|
||||
# G4、G5
|
||||
feeders: dict[str, tuple[GraphNode, list[str]]] = {}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -237,6 +237,7 @@ def _apply_contract(
|
||||
strict=bool(contract.get("strict")),
|
||||
gaps=gaps,
|
||||
unresolved=cited.get("unresolved"),
|
||||
unsupported=cited.get("unsupported"),
|
||||
uncited_claims=cited.get("uncited"),
|
||||
claims_policy=cited.get("claims_policy"),
|
||||
)
|
||||
@@ -298,7 +299,7 @@ def _check_citations(
|
||||
reports: list[dict[str, Any]], *, cells: bool = True, governed: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""独立复核报告文档:{matched, unmatched, unresolved, gaps, doc_artifact, stats, claims_policy, uncited,
|
||||
claims, entities}。
|
||||
unsupported, claims, entities}。
|
||||
|
||||
不信报告节点自己的统计:文档按 id 从工件库取回(取回时复验哈希),目录按报告节点
|
||||
同一套参数从状态里重建(不用文档里存的那份),每个带引用的片段重新解析、重新渲染、
|
||||
@@ -313,7 +314,7 @@ def _check_citations(
|
||||
|
||||
out: dict[str, Any] = {"matched": [], "unmatched": [], "unresolved": [], "gaps": [],
|
||||
"doc_artifact": None, "stats": None, "claims_policy": None, "uncited": None,
|
||||
"claims": None, "entities": None}
|
||||
"unsupported": None, "claims": None, "entities": None}
|
||||
gaps = out["gaps"]
|
||||
spec = ctx.run.spec
|
||||
node = spec.node_map().get(report_from)
|
||||
@@ -403,7 +404,7 @@ def _check_citations(
|
||||
"span": [start, end], "eid": cite["eid"]})
|
||||
|
||||
_claims(out, contract, payload, evidence_on=ledger_enabled(ctx.run), governed=governed,
|
||||
uncited=checked.get("uncited") or [])
|
||||
uncited=checked.get("uncited") or [], doc=doc, title=node.title)
|
||||
|
||||
mine = next((r for r in reports if r["node"].id == report_from), None)
|
||||
edited = mine["edited"] if mine else []
|
||||
@@ -416,20 +417,29 @@ def _check_citations(
|
||||
|
||||
#: 没挂依据的结论句怎么处置,从松到严。认不出的写法按 degrade 算:写错一个词不能让缺口悄悄消失
|
||||
_ON_UNCITED = ("ignore", "degrade", "withhold")
|
||||
#: 证据不支持的结论句怎么处置(claims: judge),从松到严。认不出的写法同样按 degrade 算
|
||||
_ON_UNSUPPORTED = ("degrade", "withhold")
|
||||
#: 要求结论句挂依据的策略:judge 在这一点上和 require_citation 一样严,另外再按裁判的判定判档
|
||||
_CITING = ("require_citation", "judge")
|
||||
|
||||
|
||||
def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, Any], *, evidence_on: bool,
|
||||
governed: bool = False, uncited: list[dict[str, Any]]) -> None:
|
||||
"""结论句策略:没挂依据的结论句按 claims_policy 交给 decide_tier 判档。
|
||||
governed: bool = False, uncited: list[dict[str, Any]], doc: dict[str, Any] | None = None,
|
||||
title: str = "") -> None:
|
||||
"""结论句策略:没挂依据的、裁判判为证据不支持的结论句按 claims_policy 交给 decide_tier 判档。
|
||||
|
||||
两处可以要求挂依据:报告节点写作时记下的 claims(节点产出里的,写作提示按它要求过),和契约自己写的
|
||||
require_citation(对象形式可以另写 on_uncited)。两处都写了取更严的那个——契约只能收紧、不能放松:
|
||||
require_citation / judge(对象形式可以另写 on_uncited)。两处都写了取更严的那个——契约只能收紧、不能放松:
|
||||
报告节点要求了,契约写 ignore 或 off 也照旧计入缺口,写 withhold 就收紧成不予出具。受管级别的正式运行
|
||||
里 ignore 一律当 degrade:没有哪道门禁查契约里的 claims,不能让它把用户要的底线拉低。
|
||||
|
||||
报告节点写的是 judge 时,判定在文档里(按哈希取回的那份:判定是模型给的,出口复算不了,只能信封存的
|
||||
文档)。正式运行在节点里判过:证据不支持的按 on_unsupported 判档(报告节点和契约取更严的),没判完的
|
||||
(裁判没跑成、触顶)把裁判摘要里的缺口照抄进来——不能判完整出具,也不当成「不支持」。探索运行按需裁判,
|
||||
这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。
|
||||
|
||||
数的是这里重新核对出来的 uncited,不信文档里记的 cites。升级前写的报告(节点产出里没有 claims 这个键,
|
||||
包括跨着升级还没跑完的运行)一律不管,契约写了也照旧记「这一版还不支持」。judge(裁判模型)是后续版本
|
||||
的事,契约写了照实记缺口。
|
||||
包括跨着升级还没跑完的运行)一律不管,契约写了也照旧记「这一版还不支持」。
|
||||
"""
|
||||
written = payload.get("claims") if isinstance(payload, dict) else None
|
||||
# 报告节点升级后才在产出里记 claims(off 也记):没有这个键就是升级前写的报告
|
||||
@@ -437,23 +447,66 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
declared = contract.get("claims")
|
||||
name = declared.get("policy") if isinstance(declared, dict) else declared
|
||||
asks: list[str] = []
|
||||
if upgraded and written == "require_citation":
|
||||
if upgraded and written in _CITING:
|
||||
asks.append("degrade")
|
||||
if upgraded and name == "require_citation":
|
||||
if upgraded and name in _CITING:
|
||||
wanted = declared.get("on_uncited") if isinstance(declared, dict) else None
|
||||
asks.append(wanted if wanted in _ON_UNCITED else "degrade")
|
||||
elif name not in (None, "", "off"):
|
||||
out["gaps"].append("契约声明了结论句检查(claims),这一版还不支持,结论句没有核对")
|
||||
judged = upgraded and written == "judge"
|
||||
if upgraded and name == "judge" and not judged:
|
||||
out["gaps"].append(f"契约要求结论句由模型裁判(claims: judge),报告「{title}」没有开 claims: judge,"
|
||||
"结论句没有裁判")
|
||||
if not asks:
|
||||
return
|
||||
on_uncited = max(asks, key=_ON_UNCITED.index)
|
||||
if governed and on_uncited == "ignore":
|
||||
on_uncited = "degrade"
|
||||
policy: Any = "require_citation" if on_uncited == "degrade" else \
|
||||
{"policy": "require_citation", "on_uncited": on_uncited}
|
||||
out["claims_policy"], out["uncited"] = policy, uncited
|
||||
out["claims"] = {"policy": "require_citation", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED],
|
||||
**({"on_uncited": on_uncited} if on_uncited != "degrade" else {})}
|
||||
extra = {"on_uncited": on_uncited} if on_uncited != "degrade" else {}
|
||||
if not judged:
|
||||
policy: Any = "require_citation" if on_uncited == "degrade" else \
|
||||
{"policy": "require_citation", "on_uncited": on_uncited}
|
||||
out["claims_policy"], out["uncited"] = policy, uncited
|
||||
out["claims"] = {"policy": "require_citation", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED],
|
||||
**extra}
|
||||
return
|
||||
|
||||
from app.engine.evidence import iter_units
|
||||
|
||||
summary = (doc or {}).get("judge")
|
||||
if not isinstance(summary, dict):
|
||||
# 报告节点开了 judge、文档里却没有裁判摘要(不该发生):当作没判,不能判完整出具
|
||||
summary = {"mode": "inline", "counts": {}, "unjudged": {}, "limits_hit": [], "complete": False,
|
||||
"gaps": ["开了 claims: judge,文档里却没有裁判结果,结论句没有裁判"]}
|
||||
asked = [summary.get("on_unsupported")]
|
||||
if name == "judge" and isinstance(declared, dict):
|
||||
asked.append(declared.get("on_unsupported"))
|
||||
on_unsupported = max((a if a in _ON_UNSUPPORTED else "degrade" for a in asked), key=_ON_UNSUPPORTED.index)
|
||||
inline = summary.get("mode") == "inline"
|
||||
flagged: dict[str, list[dict[str, Any]]] = {"unsupported": [], "partial": []}
|
||||
if inline:
|
||||
# 探索运行按需裁判的判定不在文档里(在封存之后追加的 evidence.judged 事件里),这里只数正式运行的
|
||||
markdown = str((doc or {}).get("markdown") or "")
|
||||
for _, unit in iter_units(doc or {}):
|
||||
verdict = unit.get("verdict")
|
||||
status = verdict.get("status") if isinstance(verdict, dict) else None
|
||||
if status in flagged:
|
||||
span = unit.get("span") or [0, 0]
|
||||
flagged[status].append({"unit": unit.get("id"), "span": span, "text": markdown[span[0]:span[1]],
|
||||
"rationale": verdict.get("rationale") or ""})
|
||||
out["gaps"].extend(f"报告「{title}」{gap}" for gap in summary.get("gaps") or [])
|
||||
counts = {k: int((summary.get("counts") or {}).get(k) or 0)
|
||||
for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")}
|
||||
out["claims_policy"] = {"policy": "judge", "on_unsupported": on_unsupported, "on_uncited": on_uncited}
|
||||
out["uncited"], out["unsupported"] = uncited, flagged["unsupported"]
|
||||
out["claims"] = {
|
||||
"policy": "judge", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED], **extra,
|
||||
"on_unsupported": on_unsupported, "mode": "inline" if inline else "on_demand", "counts": counts,
|
||||
"unsupported": flagged["unsupported"][:_LISTED], "partial": flagged["partial"][:_LISTED],
|
||||
"unjudged": dict(summary.get("unjudged") or {}), "limits_hit": list(summary.get("limits_hit") or []),
|
||||
"complete": bool(summary.get("complete")), "model": summary.get("model"),
|
||||
}
|
||||
|
||||
|
||||
def _declare_citations(
|
||||
@@ -497,8 +550,10 @@ def _declare_citations(
|
||||
metrics_checked=len(metrics),
|
||||
matched_numbers=len(cited["matched"]),
|
||||
matched=cited["matched"][:50],
|
||||
# on_uncited 只在不是默认的 degrade 时才有(收紧成 withhold、探索运行的契约写了 ignore)
|
||||
**({"claims": {k: cited["claims"][k] for k in ("policy", "uncited_claims", "on_uncited")
|
||||
# on_uncited 只在不是默认的 degrade 时才有(收紧成 withhold、探索运行的契约写了 ignore);
|
||||
# judge 另带不支持时怎么判档、在哪判的和各判定的句数(名单只在出具声明里)
|
||||
**({"claims": {k: cited["claims"][k] for k in ("policy", "uncited_claims", "on_uncited", "on_unsupported",
|
||||
"mode", "counts")
|
||||
if k in cited["claims"]}} if cited.get("claims") else {}),
|
||||
# 计数取复核的统计(声明里的名单最多列 _LISTED 个)
|
||||
**({"entities": {"unknown": (cited["stats"] or {}).get("unknown_entities", len(entities["unknown"])),
|
||||
|
||||
@@ -15,20 +15,32 @@
|
||||
写了个哪里都找不到的名字)只标注:不重写、不失败,受管级别的正式出具按缺口降档;表结构快照
|
||||
不全时核对不了的名字(unverified_entity)只标注。entities: off 整层不管表名、字段名。
|
||||
结论句策略 claims:off(默认)不管;require_citation 在写作提示里要求结论句挂依据(自动链接的
|
||||
名字不算依据),没挂的计入缺口、出口降档;judge(模型裁判)是后续版本的事,现在写了直接报错。
|
||||
名字不算依据),没挂的计入缺口、出口降档;judge 另外请一个模型按证据逐句判断(engine/judge.py):
|
||||
|
||||
- 正式运行在节点里判:写完、核对完之后判,判定写进文档的 unit.verdict,随文档一起封存。判定只标注、
|
||||
不改写正文;裁判没跑成、触顶了照样产出,没判的记未裁判并留下缺口,出口按 on_unsupported 判档
|
||||
- 探索运行不在节点里花这笔钱:候选句标「未裁判 · 按需」,点开哪句再判哪句(证据接口)
|
||||
- rewrite_once(默认关):证据不支持的句子连同理由交回写作者只改这几句,再判一次,最多一轮;
|
||||
改写稿冒出原稿没有的违规(只标注的可疑实体也算)就不采用
|
||||
- 裁判调用包在 @task 里,排在节点所有已有调用之后:「接着跑」时直接取 checkpoint 里的判定,
|
||||
不重复调用、不重复计费。写作调用不进 checkpoint,接着跑时会重写一稿——所以判定带着输入指纹,
|
||||
对不上这一稿的不用
|
||||
|
||||
出口契约复核(io.py)用 report_catalog 按同一套参数重建目录、独立重算,不信这里的自查。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
|
||||
from langgraph.func import task
|
||||
|
||||
from app.core.errors import describe_exception
|
||||
from app.core.events import EventType
|
||||
from app.db.base import SessionLocal
|
||||
from app.engine import judge as judging
|
||||
from app.engine.context import NodeContext, NodeError
|
||||
from app.engine.evidence import (
|
||||
CELLS_REASON,
|
||||
@@ -43,7 +55,17 @@ from app.engine.evidence import (
|
||||
ledger_enabled,
|
||||
)
|
||||
from app.engine.nodes.llm import _invoke_streaming, _model_spec, _report_call
|
||||
from app.engine.schema import REPORT_CLAIMS, GraphNode, GraphSpec, NodeType, _ancestors, claims_problem
|
||||
from app.engine.schema import (
|
||||
CLAIMS_JUDGE,
|
||||
CLAIMS_VALUES,
|
||||
JUDGE_ON_UNSUPPORTED,
|
||||
GraphNode,
|
||||
GraphSpec,
|
||||
NodeType,
|
||||
_ancestors,
|
||||
claims_problem,
|
||||
judge_problems,
|
||||
)
|
||||
from app.engine.state import GraphState, message_text, thinking_text
|
||||
from app.engine.toolcalls import leaked_markup, markup_warning
|
||||
from app.providers.factory import ProviderNotConfigured, get_chat_model
|
||||
@@ -52,8 +74,8 @@ ON_VIOLATION = ("fail", "flag")
|
||||
NUMBERS = ("strict", "off")
|
||||
#: 实体层:link(默认)核对表名、字段名并自动链接;off 整层不管(目录里不收表和字段)
|
||||
ENTITIES = ("link", "off")
|
||||
#: 结论句策略,和画布校验同一份(schema.REPORT_CLAIMS);judge 是后续版本的事
|
||||
CLAIMS = REPORT_CLAIMS
|
||||
#: 结论句策略,和画布校验同一份(schema.CLAIMS_VALUES):off / require_citation / judge
|
||||
CLAIMS = CLAIMS_VALUES
|
||||
#: 只标注、不让写作者重写也不判失败的违规:可疑实体(受管的正式出具按缺口降档)、表结构快照不全时
|
||||
#: 核对不了的名字(出口只标注)
|
||||
SOFT = frozenset({"unknown_entity", "unverified_entity"})
|
||||
@@ -67,6 +89,8 @@ MARKUP_NUDGE = ("你上一条回复把工具调用写成了文字。报告撰写
|
||||
"证据目录里,直接用引用标记写报告正文。")
|
||||
MARKUP_ERROR = ("模型输出了工具调用的原始标记,而报告撰写节点不调用工具——数据已经在证据目录里。"
|
||||
"换一个模型再试")
|
||||
#: 接着跑时 checkpoint 里缓存的判定最多翻几条去找这一稿的(每条都是先前某一稿的,对不上就往后翻)
|
||||
_STALE_LIMIT = 16
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -120,14 +144,26 @@ def report_entities(spec: GraphSpec, node: GraphNode) -> str:
|
||||
|
||||
|
||||
def report_claims(spec: GraphSpec, node: GraphNode) -> str:
|
||||
"""报告节点的结论句策略:off / require_citation。出口判档按它(和节点执行时同一个取值规则)。
|
||||
"""报告节点的结论句策略:off / require_citation / judge。出口判档按它(和节点执行时同一个取值规则)。
|
||||
|
||||
写错的值(包括 judge)画布校验就拦了,执行时节点也会报错,这里只把没写当成 off。
|
||||
写错的值画布校验就拦了,执行时节点也会报错,这里只把没写当成 off。
|
||||
"""
|
||||
value = _setting(spec, node, "claims", "off")
|
||||
return value if isinstance(value, str) and value in CLAIMS else "off"
|
||||
|
||||
|
||||
def report_judge(spec: GraphSpec, node: GraphNode) -> dict[str, Any]:
|
||||
"""claims: judge 的子配置:节点上的 judge(没写取图级 defaults),写了的键原样保留——上限写 null 就是
|
||||
不限,没写的键运行时取设置里的默认(judge.report_budget)。rewrite_once、on_unsupported 补上缺省
|
||||
(false、degrade):出口判档读 on_unsupported。写错的值画布校验就拦了,执行时节点也会报错。
|
||||
"""
|
||||
raw = _setting(spec, node, "judge")
|
||||
raw = raw if isinstance(raw, dict) else {}
|
||||
return {**raw, "rewrite_once": raw.get("rewrite_once") is True,
|
||||
"on_unsupported": raw.get("on_unsupported") if raw.get("on_unsupported") in JUDGE_ON_UNSUPPORTED
|
||||
else "degrade"}
|
||||
|
||||
|
||||
def report_allowance(spec: GraphSpec, node: GraphNode) -> list[Any]:
|
||||
"""报告自查放行的字面量:出口契约 report_from 指向它时,契约的 allow_numbers。
|
||||
|
||||
@@ -209,13 +245,18 @@ class _RenderedStream:
|
||||
self._ctx.emit(EventType.LLM_TOKEN, delta=tail)
|
||||
|
||||
|
||||
async def _default_on_violation(run_id: str) -> str:
|
||||
"""探索运行默认标注、照常产出;正式运行默认失败——封存的发布版不能带着裸数字出具。"""
|
||||
async def _run_class(run_id: str) -> str | None:
|
||||
"""按运行记录上的 run_class 取:子图、审批之后续跑都是另起的执行上下文,只有运行记录靠得住。"""
|
||||
from app.db.models import Run
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
return "fail" if run is not None and run.run_class == "formal" else "flag"
|
||||
return run.run_class if run is not None else None
|
||||
|
||||
|
||||
async def _default_on_violation(run_id: str) -> str:
|
||||
"""探索运行默认标注、照常产出;正式运行默认失败——封存的发布版不能带着裸数字出具。"""
|
||||
return "fail" if await _run_class(run_id) == "formal" else "flag"
|
||||
|
||||
|
||||
def _options(ctx: NodeContext) -> tuple[str, int]:
|
||||
@@ -225,6 +266,8 @@ def _options(ctx: NodeContext) -> tuple[str, int]:
|
||||
# 画布校验也拦这两项;执行时再守一道,报的是同一句话,免得写错的值被悄悄当成默认
|
||||
if problem := claims_problem(ctx.cfg("claims")):
|
||||
raise NodeError(ctx.node.id, problem)
|
||||
if ctx.cfg("claims") == CLAIMS_JUDGE and (problems := judge_problems(ctx.cfg("judge"))):
|
||||
raise NodeError(ctx.node.id, ";".join(message for _, message in problems))
|
||||
entities = ctx.cfg("entities", "link") or "link"
|
||||
if entities not in ENTITIES:
|
||||
raise NodeError(ctx.node.id, f"entities 只能是 {' / '.join(ENTITIES)},写的是 {entities!r}")
|
||||
@@ -248,6 +291,29 @@ def _blocking(violations: list[dict[str, Any]], numbers: str) -> list[dict[str,
|
||||
and not (numbers == "off" and v.get("code") == "uncited_number")]
|
||||
|
||||
|
||||
def _violation_key(violation: dict[str, Any]) -> tuple[str, str]:
|
||||
"""认一处违规:(code, 点名的字)。点名的字按 text、ref、message 的顺序取第一个有的。"""
|
||||
named = violation.get("text") or violation.get("ref") or violation.get("message") or ""
|
||||
return str(violation.get("code") or ""), str(named)
|
||||
|
||||
|
||||
def _new_violations(before: list[dict[str, Any]], after: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""after 里 before 没有的违规,按 (code, 点名的字) 逐个抵消(同一个裸数字多写一处也算新的)。
|
||||
|
||||
全部违规都比,包括只标注、不让重写的那几类(SOFT):改写稿冒出一个编造的表名,受管的正式出具就会
|
||||
因此降档;去掉一处、又冒出另一处,总数没变,冒出来的也还是新的。
|
||||
"""
|
||||
left = Counter(_violation_key(v) for v in before)
|
||||
fresh: list[dict[str, Any]] = []
|
||||
for violation in after:
|
||||
key = _violation_key(violation)
|
||||
if left[key] > 0:
|
||||
left[key] -= 1
|
||||
else:
|
||||
fresh.append(violation)
|
||||
return fresh
|
||||
|
||||
|
||||
def _named(violations: list[dict[str, Any]], limit: int = 3) -> str:
|
||||
"""违规里点名的那几个字:「45678」「m:nope」。"""
|
||||
names = [str(v.get("text") or v.get("ref") or "") for v in violations]
|
||||
@@ -269,7 +335,10 @@ def _messages(ctx: NodeContext, state: GraphState, catalog: dict[str, Any], cell
|
||||
system = ctx.render_str(ctx.cfg("system", ""), state)
|
||||
instructions = ctx.render_str(ctx.cfg("instructions", ""), state).strip() \
|
||||
or "根据下面的证据写一份简洁的报告,先总后分。"
|
||||
rules = f"{MARKER_RULES}\n{CLAIMS_RULE}" if claims == "require_citation" else MARKER_RULES
|
||||
rules = MARKER_RULES
|
||||
if claims in ("require_citation", CLAIMS_JUDGE):
|
||||
# 裁判只看写作者挂的依据:要判,就更得每句都挂上
|
||||
rules = f"{rules}\n{CLAIMS_RULE}" + (f"\n{judging.JUDGE_RULE}" if claims == CLAIMS_JUDGE else "")
|
||||
return [
|
||||
SystemMessage(content="\n\n".join(p for p in (system, ROLE) if p)),
|
||||
HumanMessage(content=(
|
||||
@@ -362,11 +431,26 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
response, raw = await draft(messages)
|
||||
doc = compose(raw)
|
||||
|
||||
judgement: dict[str, Any] | None = None
|
||||
if claims == CLAIMS_JUDGE:
|
||||
judge_cfg = report_judge(spec, ctx.node)
|
||||
if await _run_class(ctx.run.run_id) != "formal":
|
||||
# 探索运行默认按需裁判:点开哪句才判哪句,节点里不花这笔钱
|
||||
judgement = judging.on_demand(doc, on_unsupported=judge_cfg["on_unsupported"],
|
||||
rewrite_once=judge_cfg["rewrite_once"])
|
||||
elif not (blocking and on_violation == "fail"):
|
||||
# 反正要判失败的报告不值得再花钱判(节点紧接着就报错)
|
||||
doc, response, blocking, judgement, usages = await _judge_inline(
|
||||
ctx, doc, catalog, judge_cfg, model_id=model_id, messages=[*messages, response],
|
||||
draft=draft, compose=compose, numbers=numbers, blocking=blocking)
|
||||
spent.extend(usages)
|
||||
|
||||
doc_artifact = await _store(doc, ctx)
|
||||
ctx.emit(EventType.REPORT_CHECKED, doc_artifact=doc_artifact, ok=not doc["violations"],
|
||||
stats=doc["stats"], violations=doc["violations"][:20], repairs=repairs,
|
||||
on_violation=on_violation, failed=bool(blocking) and on_violation == "fail",
|
||||
**({"claims": claims} if evidence_on else {}))
|
||||
**({"claims": claims} if evidence_on else {}),
|
||||
**({"judge": judgement} if judgement is not None else {}))
|
||||
if blocking and on_violation == "fail":
|
||||
head = ";".join(v["message"] for v in blocking[:5])
|
||||
more = f";…另有 {len(blocking) - 5} 处" if len(blocking) > 5 else ""
|
||||
@@ -388,6 +472,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
"model": model_id,
|
||||
"duration_ms": int((time.perf_counter() - started) * 1000),
|
||||
**({"claims": claims} if evidence_on else {}),
|
||||
**({"judge": judgement} if judgement is not None else {}),
|
||||
}
|
||||
updates: dict[str, Any] = {"nodes": {ctx.node.id: result}, "usage": usage}
|
||||
if ctx.cfg("emit_message", True):
|
||||
@@ -396,3 +481,131 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
if var_name:
|
||||
updates["vars"] = {var_name: text}
|
||||
return updates
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# claims: judge——正式运行在节点里裁判
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _rewrite_request(sentences: list[tuple[str, str]]) -> str:
|
||||
listed = "\n".join(f"{i}.「{text}」——{why or '证据不支持这句话'}" for i, (text, why) in enumerate(sentences, 1))
|
||||
return (
|
||||
"另一个模型按证据逐句核对了你的报告,下面这几句它认为证据不支持:\n"
|
||||
f"{listed}\n\n"
|
||||
"请只改这几句:补上证据目录里真有的依据 [[see:…]],或者把说法收窄到证据能支持的程度,或者删掉;"
|
||||
"其余部分一字不改。数字仍然只能用引用标记写。不要解释改了什么,只输出改好的整篇报告正文。"
|
||||
)
|
||||
|
||||
|
||||
def _merge(first: dict[str, Any], second: dict[str, Any], budget: judging.Budget) -> dict[str, Any]:
|
||||
"""改写前后两轮裁判合成一份:判定、句数、缺口按改写后这一稿,花费、调用、用时两轮相加。"""
|
||||
merged = dict(second)
|
||||
for key in ("cost_usd", "estimated_usd"):
|
||||
merged[key] = round(first[key] + second[key], 6)
|
||||
for key in ("calls", "duration_ms"):
|
||||
merged[key] = first[key] + second[key]
|
||||
merged["notes"] = list(dict.fromkeys([*first["notes"], *second["notes"]]))
|
||||
merged["budget"] = budget.as_dict()
|
||||
merged["model"] = second.get("model") or first.get("model")
|
||||
merged["priced"] = first.get("priced")
|
||||
return merged
|
||||
|
||||
|
||||
async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str, Any], judge_cfg: dict[str, Any], *,
|
||||
model_id: str, messages: list[BaseMessage], draft: Any, compose: Any, numbers: str,
|
||||
blocking: list[dict[str, Any]]
|
||||
) -> tuple[dict[str, Any], BaseMessage, list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]:
|
||||
"""写完、核对完之后请裁判判一遍,判定写进文档。返回 (文档, 写作者最后一稿, 要拦的违规, 裁判摘要, 用量)。
|
||||
|
||||
messages 是写作的对话、最后一条是写作者这一稿。rewrite_once 时把证据不支持的句子交回去只改这几句,
|
||||
再判一次(没改的句子直接用第一次的判定),最多一轮;改写稿冒出原稿没有的违规就不采用,保留原稿。
|
||||
|
||||
摘要里的 rewrite = {units, sentences, applied, reason, changed}:
|
||||
- sentences:交回去的句子(原稿里的原文)
|
||||
- units:这几句在封存文档里的编号。没采用时封存的是原稿,就是原稿的编号;采用了封存的是改写稿,
|
||||
编号按改写稿重排,按原文认:原句原样留在改写稿里的给新编号,改掉了的是 null
|
||||
- changed:采用时改写稿里新出来的句子(原稿里没有这段原文)在封存文档里的编号;没采用是 []
|
||||
"""
|
||||
from app.engine.replay import _in_graph
|
||||
|
||||
async with SessionLocal() as session:
|
||||
settings = await judging.judge_settings(session)
|
||||
judge_spec = await judging.judge_model_spec(session, judge_cfg, settings=settings)
|
||||
budget = judging.report_budget(settings, judge_cfg)
|
||||
masks = await judging.source_masks(catalog)
|
||||
|
||||
@task
|
||||
async def judge_step(request: judging.JudgeRequest, key: str, limits: judging.Budget, known: dict[str, Any],
|
||||
first: bool) -> dict[str, Any]:
|
||||
# 裁判进 checkpoint:「接着跑」时直接取这里的判定,不再调用、不再计费(每日计数也记在这里面)。
|
||||
# 事件都在这里面发:取缓存时不会再发一遍。第二轮 limits 是剩下的钱和时间,说给人看的仍是配置的上限
|
||||
outcome = await judging.run_request(request, spec=judge_spec, budget=limits, known=known, emit=ctx.emit,
|
||||
shown=budget)
|
||||
if first and outcome.get("model") and outcome["model"] == model_id:
|
||||
outcome["same_model"] = True
|
||||
ctx.emit(EventType.LOG, level="warn", code="judge_same_model",
|
||||
message=f"裁判模型和写作模型都是「{model_id}」:等于自己审自己,模型写错的地方它多半也看不出来。"
|
||||
"到设置里选一个不同的「证据裁判模型」,或者在节点的 judge.model 里指定")
|
||||
return {**outcome, "key": key}
|
||||
|
||||
async def judge_round(round_doc: dict[str, Any], limits: judging.Budget, known: dict[str, Any],
|
||||
first: bool) -> dict[str, Any]:
|
||||
request = judging.prepare(round_doc, catalog, masked=masks)
|
||||
key = judging.request_key(request, judge_spec, limits, known)
|
||||
if _in_graph():
|
||||
# 写作不进 checkpoint,接着跑时写作者会重写一稿:缓存里先前那一稿的判定对不上指纹,往后翻
|
||||
for _ in range(_STALE_LIMIT):
|
||||
outcome = await judge_step(request, key, limits, known, first)
|
||||
if outcome.get("key") == key:
|
||||
return outcome
|
||||
return await judging.run_request(request, spec=judge_spec, budget=limits, known=known, emit=ctx.emit,
|
||||
shown=budget)
|
||||
|
||||
outcome = await judge_round(doc, budget, {}, True)
|
||||
same_model = bool(outcome.get("same_model"))
|
||||
usages = [outcome["usage"]]
|
||||
response = messages[-1]
|
||||
rewrite: dict[str, Any] | None = None
|
||||
bad = [uid for uid, v in outcome["verdicts"].items() if v["status"] == "unsupported"]
|
||||
if judge_cfg["rewrite_once"] and bad:
|
||||
texts = judging.unit_texts(doc)
|
||||
sentences = [(texts.get(uid, ""), outcome["verdicts"][uid].get("rationale") or "") for uid in bad]
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_rewrite",
|
||||
message=f"裁判认为 {len(bad)} 句结论证据不支持({_named([{'text': t} for t, _ in sentences])}),"
|
||||
"已交回写作者只改这几句")
|
||||
again, raw = await draft([*messages, HumanMessage(content=_rewrite_request(sentences))])
|
||||
rewritten = compose(raw)
|
||||
worse = _blocking(rewritten["violations"], numbers)
|
||||
fresh = _new_violations(doc["violations"], rewritten["violations"])
|
||||
rewrite = {"units": bad, "sentences": [t for t, _ in sentences], "applied": False, "reason": None,
|
||||
"changed": []}
|
||||
if fresh:
|
||||
rewrite["reason"] = (f"改写稿冒出 {len(fresh)} 处原稿没有的问题({_named(fresh)}),没有采用,"
|
||||
"保留原稿和原来的判定")
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_rewrite_rejected", message=rewrite["reason"])
|
||||
else:
|
||||
known = {outcome["keys"][u]: v for u, v in outcome["verdicts"].items() if v["status"] in judging.JUDGED}
|
||||
# 第二轮用第一轮剩下的钱和时间:每份报告的上限管的是整份报告
|
||||
left = judging.Budget(
|
||||
max_claims=budget.max_claims,
|
||||
max_cost_usd=None if budget.max_cost_usd is None else max(0.0, budget.max_cost_usd - outcome["cost_usd"]),
|
||||
timeout_s=None if budget.timeout_s is None else max(0.0, budget.timeout_s - outcome["duration_ms"] / 1000),
|
||||
daily_max_usd=budget.daily_max_usd)
|
||||
second = await judge_round(rewritten, left, known, False)
|
||||
usages.append(second["usage"])
|
||||
outcome = _merge(outcome, second, budget)
|
||||
doc, response, blocking = rewritten, again, worse
|
||||
# 封存的是改写稿,编号按它重排:按原文认回去,改掉了的句子在改写稿里没有编号
|
||||
now = judging.unit_texts(rewritten)
|
||||
at: dict[str, str] = {}
|
||||
for uid, text in now.items():
|
||||
at.setdefault(text, uid)
|
||||
before = set(texts.values())
|
||||
rewrite.update(applied=True, units=[at.get(texts.get(uid, "")) for uid in bad],
|
||||
changed=[uid for uid, text in now.items() if text not in before])
|
||||
summary = judging.summarize(outcome, mode="inline", on_unsupported=judge_cfg["on_unsupported"],
|
||||
rewrite_once=judge_cfg["rewrite_once"], same_model=same_model, rewrite=rewrite)
|
||||
judging.apply_judgement(doc, outcome, summary)
|
||||
return doc, response, blocking, summary, usages
|
||||
|
||||
|
||||
@@ -95,6 +95,16 @@ WARN_CODES: dict[str, tuple[str, str]] = {
|
||||
# (记成空值,不兜底成 0)。答案能用,缺口要说清是哪一种
|
||||
"agent_field_mismatch": ("agent_field_mismatch", DEGRADED),
|
||||
"agent_field_unverified": ("agent_field_unverified", DEGRADED),
|
||||
# 结论句裁判(claims: judge):触顶、没跑成的句子记为未裁判,出具不能判完整——是缺口,要说明
|
||||
"judge_limit": ("judge_limit", DEGRADED),
|
||||
"judge_failed": ("judge_failed", DEGRADED),
|
||||
# 证据不支持的句子交回写作者改过一次(和 report_repair 一样只记缺口);改写稿冒出新问题没采用的,
|
||||
# 原稿里那几句仍是证据不支持
|
||||
"report_rewrite": ("report_rewrite", DEGRADED),
|
||||
"report_rewrite_rejected": ("report_rewrite_rejected", DEGRADED),
|
||||
# 估不出裁判的金额、裁判和写作是同一个模型:配置上的提醒,答案本身没受影响
|
||||
"judge_unpriced": ("judge_unpriced", BENIGN),
|
||||
"judge_same_model": ("judge_same_model", BENIGN),
|
||||
}
|
||||
|
||||
# 这些 kind 重跑一次大概率能好:原因说得清,且不是「问题本身没问明白」。
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from enum import StrEnum
|
||||
@@ -423,31 +424,101 @@ def _check_report_sources(node: GraphNode, spec: GraphSpec, result: ValidationRe
|
||||
|
||||
|
||||
#: 报告撰写节点的结论句策略(claims)。off:结论句不参与判档(默认,保持以前的行为);
|
||||
#: require_citation:没挂引用的结论句计入缺口、按出具档位降档。judge(模型裁判)是后续版本的
|
||||
#: require_citation:没挂引用的结论句计入缺口、按出具档位降档。这两档是确定性的(发布前自动修复
|
||||
#: 给人选的也只有这两档)
|
||||
REPORT_CLAIMS = ("off", "require_citation")
|
||||
CLAIMS_LATER = "结论句裁判在后续版本支持"
|
||||
#: judge:结论句由另一个模型按证据逐句判断(engine/judge.py),子配置写在节点的 judge 里
|
||||
CLAIMS_JUDGE = "judge"
|
||||
CLAIMS_VALUES = (*REPORT_CLAIMS, CLAIMS_JUDGE)
|
||||
#: judge 子配置能写的键。三个上限写 null 表示不限;没写的键运行时取设置里的默认(设置 → judge 一组)
|
||||
JUDGE_KEYS = ("provider", "model", "max_claims", "max_cost_usd", "timeout_s", "rewrite_once", "on_unsupported")
|
||||
#: 证据不支持时正式运行怎么判档:degrade 降档(默认)/ withhold 不予出具。探索运行只标注
|
||||
JUDGE_ON_UNSUPPORTED = ("degrade", "withhold")
|
||||
|
||||
|
||||
def claims_problem(value: Any) -> str | None:
|
||||
"""claims 的取值有什么问题,没问题返回 None。校验和报告撰写节点执行时报的是同一句话。"""
|
||||
if value in (None, "") or (isinstance(value, str) and value in REPORT_CLAIMS):
|
||||
if value in (None, "") or (isinstance(value, str) and value in CLAIMS_VALUES):
|
||||
return None
|
||||
if value == "judge":
|
||||
return f"claims 写的是 judge:{CLAIMS_LATER},现在只能写 off 或 require_citation"
|
||||
return f"claims 只能是 {' / '.join(REPORT_CLAIMS)},写的是 {value!r}"
|
||||
return f"claims 只能是 {' / '.join(CLAIMS_VALUES)},写的是 {value!r}"
|
||||
|
||||
|
||||
def _positive(value: Any, *, integer: bool = False) -> bool:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
|
||||
return False
|
||||
return not integer or int(value) == value
|
||||
|
||||
|
||||
def judge_problems(value: Any) -> list[tuple[str, str]]:
|
||||
"""claims: judge 的子配置有什么问题:[(字段, 人话)],没问题返回 []。校验和节点执行时报的是同一句话。
|
||||
|
||||
上限写 null 就是不限(和没写不一样:没写取设置里的默认)。键写错了也报:写成 max_cost 的预算会被
|
||||
悄悄当成没写、按默认值花钱。
|
||||
"""
|
||||
if value in (None, ""):
|
||||
return []
|
||||
if not isinstance(value, dict):
|
||||
return [("judge", f'judge 要写成一个对象,比如 {{"max_cost_usd": 0.05, "max_claims": 40}},写的是 {value!r}')]
|
||||
out: list[tuple[str, str]] = []
|
||||
for key, item in value.items():
|
||||
field = f"judge.{key}"
|
||||
if key not in JUDGE_KEYS:
|
||||
out.append((field, f"judge 里不认识「{key}」:能写的是 {'、'.join(JUDGE_KEYS)}"))
|
||||
elif key in ("provider", "model") and item is not None and not isinstance(item, str):
|
||||
what = "模型接入的名字" if key == "provider" else "模型 id"
|
||||
out.append((field, f"judge.{key} 要写{what}(字符串),不写或写 null 就用设置里的「证据裁判模型」,"
|
||||
f"写的是 {item!r}"))
|
||||
elif key == "max_claims" and item is not None and not _positive(item, integer=True):
|
||||
out.append((field, f"judge.max_claims 要写正整数(每份报告最多判几句),写 null 表示不限,写的是 {item!r}"))
|
||||
elif key == "max_cost_usd" and item is not None and not _positive(item):
|
||||
out.append((field, "judge.max_cost_usd 要写大于 0 的金额(美元,每份报告的裁判花费上限),"
|
||||
f"写 null 表示不限,写的是 {item!r}"))
|
||||
elif key == "timeout_s" and item is not None and not _positive(item):
|
||||
out.append((field, f"judge.timeout_s 要写大于 0 的秒数(每份报告的裁判时长上限),写 null 表示不限,"
|
||||
f"写的是 {item!r}"))
|
||||
elif key == "rewrite_once" and not isinstance(item, bool):
|
||||
out.append((field, f"judge.rewrite_once 只能是 true 或 false,写的是 {item!r}"))
|
||||
elif key == "on_unsupported" and item not in JUDGE_ON_UNSUPPORTED:
|
||||
out.append((field, f"judge.on_unsupported 只能是 {' / '.join(JUDGE_ON_UNSUPPORTED)},写的是 {item!r}"))
|
||||
return out
|
||||
|
||||
|
||||
def _check_report_claims(node: GraphNode, spec: GraphSpec, result: ValidationResult) -> None:
|
||||
"""claims 写错了(包括还不支持的 judge)挡住运行:执行时报告节点照样会报这一句,
|
||||
与其跑完取数、算完口径卡才死在写报告那一步,不如在画图时说破。
|
||||
"""claims 写错了挡住运行:执行时报告节点照样会报这一句,与其跑完取数、算完口径卡才死在写报告
|
||||
那一步,不如在画图时说破。claims 是 judge 时再查 judge 子配置的形状;裁判模型和写作模型写成
|
||||
同一个的给警告——等于自己审自己。
|
||||
|
||||
和运行时同一个取值规则:节点上没写就取图级 defaults(NodeContext.cfg)。
|
||||
"""
|
||||
value = node.config.get("claims")
|
||||
if value in (None, ""):
|
||||
value = spec.defaults.get("claims")
|
||||
def effective(key: str) -> Any:
|
||||
value = node.config.get(key)
|
||||
return spec.defaults.get(key) if value in (None, "") else value
|
||||
|
||||
value = effective("claims")
|
||||
if problem := claims_problem(value):
|
||||
result.add(problem, node_id=node.id, field="claims", code="report.claims_invalid")
|
||||
if value != CLAIMS_JUDGE:
|
||||
return
|
||||
judge = effective("judge")
|
||||
for field, message in judge_problems(judge):
|
||||
result.add(message, node_id=node.id, field=field, code="report.judge_invalid")
|
||||
judge_model = judge.get("model") if isinstance(judge, dict) else None
|
||||
if isinstance(judge_model, str) and judge_model and judge_model == effective("model"):
|
||||
result.add(f"裁判模型和写作模型都是「{judge_model}」:等于自己审自己,模型写错的地方它多半也看不出来。"
|
||||
"给 judge.model 换一个不同的模型", level="warning", node_id=node.id, field="judge.model",
|
||||
code="report.judge_same_model")
|
||||
|
||||
|
||||
def contract_needs_metrics(contract: dict[str, Any]) -> bool:
|
||||
"""出具契约要不要写 metrics_from(指标来自哪几张口径卡)。
|
||||
|
||||
旧写法按数值回指叙述,没有口径卡就无从回指,一定要写。引用模式(report_from)下,数字回指靠
|
||||
报告里的引用标记:只引查询单元格(cells)的问数据图根本没有口径卡,不该逼它写;契约里列了
|
||||
required / expected 指标的,指标总得有个来处,照样要写。运行时 io.py 也是这么判的。
|
||||
"""
|
||||
if contract.get("report_from") in (None, ""):
|
||||
return True
|
||||
return bool(contract.get("required") or contract.get("expected"))
|
||||
|
||||
|
||||
def _check_report_from(node: GraphNode, report_from: Any, spec: GraphSpec,
|
||||
@@ -542,7 +613,7 @@ def validate_graph(spec: GraphSpec) -> ValidationResult:
|
||||
# 报成 error,两档发布和运行前都挡住
|
||||
result.add("出具契约必须是一个 JSON 对象", node_id=node.id, field="contract",
|
||||
code="contract.not_object")
|
||||
elif contract and not contract.get("metrics_from"):
|
||||
elif contract and not contract.get("metrics_from") and contract_needs_metrics(contract):
|
||||
result.add("出具契约缺 metrics_from(指标来自哪个「口径卡」节点)",
|
||||
node_id=node.id, field="contract.metrics_from", code="contract.metrics_from_missing")
|
||||
if isinstance(contract, dict) and contract.get("report_from") not in (None, ""):
|
||||
|
||||
Reference in New Issue
Block a user