Files
AgentLab/backend/tests/test_report_claims.py
T
ThearchyHelios 9727c21d8d 可点击证据第四期:结论句由模型裁判;只引单元格的问数据图能发成受管
一、结论句裁判

- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
  判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
  规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
  和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
  改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定

二、成本上限

- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
  默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
  不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额

三、正式运行与探索运行

- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
  裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
  标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」

四、治理与自动修复

- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
  从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略

五、界面

- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
  「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
  报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题

六、门禁修正

- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
  strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
  以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
2026-09-29 11:10:53 +08:00

192 lines
8.2 KiB
Python

"""报告节点的结论句策略 claims:off(默认,和以前一样)/ require_citation(没挂依据的结论句计入缺口)。
这里测确定性的一半(模型裁判 judge 见 test_report_judge.py):
- require_citation 时写作提示明确要求每句结论挂 [[see:…]];没挂的照常产出、计数,
出口按出具档位降档(io.py),报告节点自己不为它失败、不为它重写
- report.checked 事件和节点产出都带上 claims,出口据此判档
- 写了不认识的值节点直接报错,不能悄悄当成 off
- 升级前发起的运行一个字都不变
"""
from __future__ import annotations
import asyncio
import pytest
from langchain_core.messages import AIMessage
from sqlalchemy import select
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent
from app.engine import runner as runner_mod
from app.engine.evidence import CLAIMS_RULE, build_catalog, compose_doc, uncited_claims
from app.engine.runner import run_manager
from app.engine.schema import CLAIMS_VALUES, claims_problem
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
#: 第一句挂了依据,第二句是没挂依据的结论(有方向词),第三句是连接性的话
TEXT = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客。下面看细节。"
def weekly(**report):
nodes = [
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
node("write", "report", instructions="写周报", **report),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
def script(monkeypatch, reply=TEXT) -> list:
from app.providers import mock_model
seen: list = []
def _decide(self, messages):
seen.append(list(messages))
return AIMessage(content=reply)
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
return seen
def lenient(monkeypatch):
"""装作画布校验没拦住(校验以后放宽、或者绕过校验发起的运行):节点执行时自己也要守住这一关,
报的是同一句话。"""
original = runner_mod.validate_graph
def permissive(spec):
report = original(spec)
report.issues = [i for i in report.issues if i.field != "claims"]
report.ok = not any(i.level == "error" for i in report.issues)
return report
monkeypatch.setattr(runner_mod, "validate_graph", permissive)
async def finish(graph) -> Run:
run = await run_manager.start(graph=graph, input_payload={})
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run.id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
return row
raise AssertionError(f"run {run.id} 没跑完:{row.status}")
async def events(run_id: str, etype: str) -> list[RunEvent]:
async with SessionLocal() as session:
q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype).order_by(RunEvent.seq)
return list((await session.execute(q)).scalars())
async def checked_and_output(run_id: str) -> tuple[dict, dict]:
[checked] = await events(run_id, "report.checked")
finished = [e for e in await events(run_id, "node.finished") if e.node_id == "write"][0]
return checked.data, artifact_store.load(finished.data["artifact"])
def prompt_of(seen: list) -> str:
return "\n".join(str(m.content) for m in seen[0])
async def test_default_is_off_and_changes_nothing(monkeypatch):
seen = script(monkeypatch)
row = await finish(weekly())
assert row.status == "succeeded", row.error
checked, output = await checked_and_output(row.id)
assert checked["claims"] == "off" and output["claims"] == "off"
assert CLAIMS_RULE not in prompt_of(seen)
assert checked["stats"]["uncited_claims"] == 1
async def test_require_citation_asks_for_support_but_does_not_block(monkeypatch):
seen = script(monkeypatch)
row = await finish(weekly(claims="require_citation", on_violation="fail"))
assert row.status == "succeeded", row.error
assert len(seen) == 1, "没挂依据的结论句不触发重写"
assert CLAIMS_RULE in prompt_of(seen)
checked, output = await checked_and_output(row.id)
assert checked["claims"] == "require_citation" and output["claims"] == "require_citation"
assert checked["ok"] is True and checked["violations"] == []
assert checked["stats"]["uncited_claims"] == output["stats"]["uncited_claims"] == 1
doc = artifact_store.load(checked["doc_artifact"])
[flagged] = uncited_claims(doc)
assert flagged["text"] == "增长主要来自新客。" and doc["markdown"][slice(*flagged["span"])] == flagged["text"]
async def test_a_bad_value_is_refused_before_any_model_call(monkeypatch):
seen = script(monkeypatch)
with pytest.raises(ValueError) as info: # 画布校验先拦
await run_manager.start(graph=weekly(claims="sometimes"), input_payload={})
assert claims_problem("sometimes") in str(info.value)
lenient(monkeypatch)
row = await finish(weekly(claims="sometimes"))
assert row.status == "failed" and claims_problem("sometimes") in (row.error or ""), row.error
assert seen == [], "不支持的策略不该先花一次模型调用"
@pytest.mark.parametrize("value", ["sometimes", 1, ["require_citation"]])
async def test_other_values_are_refused(monkeypatch, value):
script(monkeypatch)
lenient(monkeypatch)
row = await finish(weekly(claims=value))
assert row.status == "failed", row.error
assert claims_problem(value) in row.error and "require_citation" in row.error
async def test_runs_from_before_the_upgrade_are_untouched(monkeypatch):
async def no_snapshot(session):
return None
monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot)
seen = script(monkeypatch)
row = await finish(weekly())
assert row.status == "succeeded", row.error
checked, output = await checked_and_output(row.id)
assert "claims" not in checked and "claims" not in output
assert CLAIMS_RULE not in prompt_of(seen)
# --------------------------------------------------------------------------
# 出口和界面共用的纯函数
# --------------------------------------------------------------------------
def test_the_supported_policies_match_the_validator():
from app.engine.nodes.report import CLAIMS, report_claims
from app.engine.schema import GraphSpec
assert CLAIMS == CLAIMS_VALUES == ("off", "require_citation", "judge")
spec = GraphSpec.model_validate(weekly(claims="require_citation"))
assert report_claims(spec, spec.node_map()["write"]) == "require_citation"
spec = GraphSpec.model_validate(weekly())
assert report_claims(spec, spec.node_map()["write"]) == "off"
spec.defaults["claims"] = "require_citation" # 图级 defaults 同样生效,和执行时一个规则
assert report_claims(spec, spec.node_map()["write"]) == "require_citation"
def test_uncited_claims_lists_claim_units_without_support():
catalog = build_catalog(nodes={"caliber": {"kind": "metric_set", "caliber": "c", "caliber_version": "v1",
"artifact": "f" * 64,
"metrics": [{"id": "gmv", "name": "销售额", "value": 1, "unit": ""}]}})
doc = compose_doc("销售额 [[m:gmv]]。环比下降。[[see:m:gmv]]主要来自东区。连接的话。", catalog)
assert [u["text"] for u in uncited_claims(doc)] == ["主要来自东区。"]
assert all(set(u) == {"unit", "span", "text"} for u in uncited_claims(doc))