Files
AgentLab/backend/tests/test_review.py
T
ThearchyHelios e4f2693918 可点击证据第一期:报告里的每个数字都点得开出处
「引用即渲染」:写报告的模型只写引用标记([[m:指标id]]、[[i:输入字段]]、[[see:…]]),数值由系统按口径卡
填进去,每个数字都挂着一条能复算的证据链。本期是数字层闭环;实体、结论句的概率性证据放在后续几期。

内核(engine/evidence.py,口径卡、出具契约、表达式等)
- 口径卡记录每个指标的来源:原式、代入式(每个输入写成「值 ← 路径」)、复算结果、口径与版本
- 出具契约的数值匹配改为按位置标注;同一个值有几个出处时记为「出处不唯一」并列出候选,不再随便挑一个
- 证据 id 由内容派生,事后对不上就是被改过;零值、结构性数字(列表序号、代码块标签)单独处理
- 口径卡新增 on_missing:缺输入时默认仍是整个节点失败,可以改成记为空值、交给契约判档

报告撰写节点(report)与引用模式的出具契约
- 流式生成时 token 里就是真实数字,不会漏出原始标记;裸数字、不存在的引用当场判违规,交回去重写
  (max_repairs,默认 1 次);仍不过时按 on_violation 失败(正式运行默认)或把违规标在报告里(探索运行默认)
- report.checked 事件记下核对结果,落在封存范围内;fail 模式下仍不过时带 failed,运行记录里画成失败
- 出具契约写了 report_from 就按引用核对:取回文档时复验哈希,独立重算一遍,全部带引用才判完整出具
- 成果字段逐字等于报告正文时标注 _evidence;复核只能加说明,不能改写带证据的答案(前后端各拦一道)

证据接口
- GET /api/runs/{id}/evidence 返回证据图;GET …/evidence/segments/{id} 返回一个片段的证据链
- 只认封存范围内的事件能追到的工件,不读 artifacts 表;封存被改过时不把任何一项标成已封存
- 没有报告的旧运行降级为 none 或 legacy_contract

界面
- EvidenceDoc 按块、句、片段渲染报告,片段是和正文一样的按钮;整份报告只占一个 Tab 位,
  ←/→ 在片段间移动、↑/↓ 按句、n/N 跳到下一处「无证据」、Enter 打开面板、Esc 关闭并把焦点还回去
- EvidencePanel:宽屏侧边、窄屏底部抽屉、360px 画布右栏里栏内展开;显示指标步骤、代入式、复算结果、
  封存状态。正文不是封存的那份报告时明确说出来,不拿别的报告的封存画绿
- 出具横幅多一行「N/N 数字有出处 · 无证据 M」和「定位下一处」;没有 _evidence 的旧回答渲染一字不差
- 画布认得报告撰写节点;Copilot 手册补上口径卡、报告节点、出具契约的写法;新增模板「⑨ 可追溯周报」

验证(沙箱):tsc 干净,生产构建通过,pytest 1601 passed,check-all 16 项检查全绿(新增 check-evidence)。
2026-09-28 11:47:23 +08:00

404 lines
16 KiB
Python

"""跑完之后的复核层。
核心不变量有两条,其余都是围着它们转的:
1. 干净的运行**一次模型都不调**。复核是补救手段,不是每次问数的固定开销。
2. 复核模型看不到原始数据,所以它写出来的任何新数字都是现编的,必须机械拦住。
"""
from __future__ import annotations
import pytest
from httpx import ASGITransport, AsyncClient
from app.core.events import EventType
from app.engine import review as rv
from app.main import app
@pytest.fixture
async def client():
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as c:
yield c
def ev(etype, **data):
return {"type": str(etype), "node_id": data.pop("node_id", ""), "data": data}
def warn(message, code=""):
return ev(EventType.LOG, level="warn", message=message, code=code)
OK_OUTPUT = {"结果": "管理员有 6 个。"}
# --------------------------------------------------------------------------
# scan:规则层
# --------------------------------------------------------------------------
def test_tool_error_is_broken():
signals = rv.scan([ev(EventType.TOOL_ERROR, tool="db_query__x", error="no such table")],
OK_OUTPUT)
assert [s.kind for s in signals] == ["tool_error"]
assert signals[0].severity == rv.BROKEN
assert "no such table" in signals[0].detail
def test_tool_error_then_success_is_only_degraded():
"""自愈接住了就不该报"答案不可信"。
参数自动纠正那条路每修一次就发一条 tool.error,不区分的话每次自愈
都会触发一次复核,还告诉用户答案不可信——而它明明跑通了。
"""
signals = rv.scan([
ev(EventType.TOOL_ERROR, tool="db_query__x", error="unexpected keyword 'query'"),
ev(EventType.TOOL_END, tool="db_query__x"),
], OK_OUTPUT)
assert [s.kind for s in signals] == ["tool_args_fixed"]
assert signals[0].severity == rv.DEGRADED
def test_retrieve_degraded_and_empty():
degraded = rv.scan([ev(EventType.RETRIEVE_END, count=3, degraded=True)], OK_OUTPUT)
assert degraded[0].kind == "retrieve_degraded"
assert degraded[0].severity == rv.DEGRADED
empty = rv.scan([ev(EventType.RETRIEVE_END, count=0, query="商家账户")], OK_OUTPUT)
assert empty[0].kind == "retrieve_empty"
assert empty[0].severity == rv.BROKEN
assert "商家账户" in empty[0].detail
def test_specific_retrieve_reason_wins_over_the_generic_one():
"""检索降级会发两条:knowledge.py 先发具体原因的 warn,再发 retrieve.end。
两边都收就是同一件事报两遍,用户读到一条含糊的加一条具体的,还得自己
判断是不是两个问题。
"""
signals = rv.scan([
warn("default 里有 32 条向量由「local-hashing」建立,和当前的对不上", "retrieve_degraded"),
ev(EventType.RETRIEVE_END, count=5, degraded=True),
], OK_OUTPUT)
assert len(signals) == 1
assert "32 条向量" in signals[0].detail
def test_node_and_run_failures_are_broken():
signals = rv.scan([
ev(EventType.NODE_FAILED, node_id="agent1", error="boom"),
ev(EventType.RUN_FAILED, error="超时"),
], OK_OUTPUT)
assert {s.kind for s in signals} == {"node_failed", "run_failed"}
assert all(s.severity == rv.BROKEN for s in signals)
def test_warn_classified_by_code_not_by_message():
"""分类挂在 code 上。消息是给人读的,随时会改。"""
assert rv.scan([warn("agent 用满了 8 步还没给出结论。", "step_limit")], OK_OUTPUT)[0] \
.severity == rv.BROKEN
assert rv.scan([warn("第 1 次校验失败:不是合法 JSON", "validate_retry")], OK_OUTPUT)[0] \
.severity == rv.DEGRADED
def test_benign_warns_do_not_trigger_review():
"""流式回退、隔离降级对答案内容零影响。
为它们叫一次模型,等于每次网络抖一下就多花一次调用,还要用户读一句
与结论无关的告警。轨迹里有记录就够了。
"""
events = [
warn("流式失败,回退非流式:connection reset", "stream_fallback"),
warn("节点要求 strict 隔离,但 microVM 未就绪,实际用了 seatbelt", "isolation_fallback"),
]
assert rv.scan(events, OK_OUTPUT) == []
def test_warn_without_code_defaults_to_degraded():
"""库里的老事件没有 code。宁可多说一句,也好过把真问题当噪音吞掉。"""
signals = rv.scan([warn("某个没带 code 的告警")], OK_OUTPUT)
assert [s.kind for s in signals] == ["warn"]
assert signals[0].severity == rv.DEGRADED
def test_empty_and_placeholder_output_are_broken():
assert rv.scan([], None)[0].kind == "empty_output"
assert rv.scan([], {"结果": ""})[0].kind == "empty_output"
# run dd9927e6 真的把这句内部管道文案当答案交出去过
leaked = {"结果": "本轮只执行了第一个工具。看到它的结果之后,再决定这一个还要不要调。"}
assert rv.scan([], leaked)[0].kind == "placeholder_output"
def test_duplicate_signals_collapse():
"""一次运行里同一条告警发五遍,用户只需要看到一次。"""
events = [warn("第 1 次校验失败:不是合法 JSON", "validate_retry")] * 5
assert len(rv.scan(events, OK_OUTPUT)) == 1
def test_scan_accepts_orm_like_rows():
"""事件既可能是 dict 也可能是 ORM 行,两种都要吃得下。"""
class Row:
type = str(EventType.TOOL_ERROR)
node_id = "n1"
data = {"tool": "t", "error": "e"}
assert rv.scan([Row()], OK_OUTPUT)[0].kind == "tool_error"
def test_worst_and_should_retry():
broken = rv.Signal("step_limit", "步数用满", rv.BROKEN)
degraded = rv.Signal("retrieve_degraded", "检索降级", rv.DEGRADED)
assert rv.worst([degraded, broken]) == rv.BROKEN
assert rv.worst([degraded]) == rv.DEGRADED
assert rv.worst([]) == ""
assert rv.should_retry([broken]) is True
# 检索什么都没查到,重跑一遍还是查不到——那不是重跑能解决的
assert rv.should_retry([rv.Signal("retrieve_empty", "没命中", rv.BROKEN)]) is False
assert rv.should_retry([degraded]) is False
# --------------------------------------------------------------------------
# check_numbers:不得新增数字
# --------------------------------------------------------------------------
def test_check_numbers_allows_reordering():
original = "管理员有 6 个,商家有 5 个,占比 45.5%。"
revised = "占比 45.5%。其中商家 5 个、管理员 6 个。"
assert rv.check_numbers(original, revised) == []
def test_check_numbers_catches_invented():
invented = rv.check_numbers("管理员有 6 个。", "管理员有 6 个,商家有 5 个。")
assert invented == ["5"]
def test_check_numbers_ignores_prose_without_numbers():
assert rv.check_numbers("管理员有 6 个。", "这次没能查到有效数据,建议重试。") == []
# --------------------------------------------------------------------------
# review:调模型那一段
# --------------------------------------------------------------------------
class FakeModel:
"""最小的 with_structured_output 替身。calls 记录被调了几次。"""
def __init__(self, reply):
self.reply = reply
self.calls = 0
def with_structured_output(self, schema):
return self
async def ainvoke(self, messages):
self.calls += 1
if isinstance(self.reply, Exception):
raise self.reply
return self.reply
SIGNALS = [rv.Signal("step_limit", "agent 用满了 8 步还没给出结论", rv.BROKEN)]
@pytest.mark.asyncio
async def test_review_annotates_without_rewriting():
model = FakeModel({"note": "这次没跑完就收尾了。", "answer": "", "retry": True})
result = await rv.review(model, question="有几个管理员", answer="管理员有 6 个。",
signals=SIGNALS)
assert result.verdict == "annotated"
assert result.answer is None # 沿用原答案
assert result.note == "这次没跑完就收尾了。"
assert result.retry is True
assert result.severity == rv.BROKEN
@pytest.mark.asyncio
async def test_review_adopts_clean_rewrite():
model = FakeModel({
"note": "步数用满了。", "answer": "管理员有 6 个(这个数可能不完整)。", "retry": False,
})
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
assert result.verdict == "rewritten"
assert result.answer == "管理员有 6 个(这个数可能不完整)。"
# 规则层管资格、模型有否决权:step_limit 本来够格重跑,模型说不必就不跑
assert result.retry is False
@pytest.mark.asyncio
async def test_model_cannot_force_an_unhelpful_retry():
"""检索什么都没查到,重跑一遍还是查不到——模型说要重跑也不能听。
否则就是白烧一次建图加一次跑图,而用户等的时间翻倍。
"""
model = FakeModel({"note": "知识库里没有这个。", "answer": "", "retry": True})
result = await rv.review(
model, question="q", answer="没查到。",
signals=[rv.Signal("retrieve_empty", "知识库里没有检索到相关内容", rv.BROKEN)],
)
assert result.severity == rv.BROKEN
assert result.retry is False
@pytest.mark.asyncio
async def test_review_reverts_rewrite_with_invented_numbers():
"""复核手上只有原答案,它写出来的新数字只能是凭空来的。"""
model = FakeModel({
"note": "步数用满了。", "answer": "管理员有 6 个,商家有 5 个。", "retry": False,
})
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
assert result.verdict == "reverted"
assert result.answer is None # 那份正文一个字都不能用
assert "5" in result.note # 但要说清是为什么退的
assert "步数用满了。" in result.note # 说明本身还是要给
@pytest.mark.asyncio
async def test_review_survives_model_failure():
"""复核只是加一层说明,它自己挂了不能把已经到手的答案也带走。"""
model = FakeModel(RuntimeError("502"))
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
assert result.verdict == "annotated"
assert result.answer is None
assert result.note == SIGNALS[0].detail # 退回规则层自己的措辞
assert result.retry is True
@pytest.mark.asyncio
async def test_review_survives_garbage_reply():
model = FakeModel("这不是 JSON,也不是对象")
result = await rv.review(model, question="q", answer="管理员有 6 个。", signals=SIGNALS)
assert result.verdict == "annotated"
assert result.note
# --------------------------------------------------------------------------
# 接口
# --------------------------------------------------------------------------
async def _make_run(client, output, events):
"""直接落一条运行和它的事件,省掉真跑一遍图。"""
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent
async with SessionLocal() as session:
run = Run(workflow_id=None, status="succeeded", output=output, graph={})
session.add(run)
await session.flush()
for i, e in enumerate(events):
session.add(RunEvent(
run_id=run.id, seq=i + 1, type=e["type"],
node_id=e["node_id"] or None, ts=0.0, data=e["data"],
))
await session.commit()
return run.id
@pytest.mark.asyncio
async def test_clean_run_never_touches_the_model(client, monkeypatch):
"""干净运行零成本——这条是整层设计的前提,要用「模型根本没被碰过」来断言。"""
async def explode(*a, **k):
raise AssertionError("干净的运行不该调模型")
monkeypatch.setattr("app.providers.factory.get_chat_model", explode)
monkeypatch.setattr("app.api.copilot.get_chat_model", explode)
run_id = await _make_run(client, OK_OUTPUT, [
ev(EventType.TOOL_START, tool="t"), ev(EventType.TOOL_END, tool="t"),
])
resp = await client.post("/api/copilot/review", json={"run_id": run_id, "question": "q"})
assert resp.status_code == 200
body = resp.json()
assert body["verdict"] == "ok"
assert body["signals"] == []
assert body["answer"] is None
@pytest.mark.asyncio
async def test_review_endpoint_reports_signals(client, monkeypatch):
"""没配模型时也得把规则层扫到的东西说出来,而不是静默放行。"""
from app.providers.factory import ProviderNotConfigured
async def unconfigured(*a, **k):
raise ProviderNotConfigured("没有可用的 provider")
monkeypatch.setattr("app.api.copilot.get_chat_model", unconfigured)
run_id = await _make_run(client, OK_OUTPUT, [
ev(EventType.TOOL_ERROR, tool="db_query__x", error="no such table: users"),
])
body = (await client.post(
"/api/copilot/review", json={"run_id": run_id, "question": "q"}
)).json()
assert body["verdict"] == "annotated"
assert body["severity"] == rv.BROKEN
assert [s["kind"] for s in body["signals"]] == ["tool_error"]
assert "no such table" in body["note"]
assert body["retry"] is True
@pytest.mark.asyncio
async def test_review_endpoint_404(client):
resp = await client.post("/api/copilot/review", json={"run_id": "没这个", "question": "q"})
assert resp.status_code == 404
# --------------------------------------------------------------------------
# 带逐段证据的答案:只加说明,不改写
# --------------------------------------------------------------------------
class RecordingModel(FakeModel):
def __init__(self, reply):
super().__init__(reply)
self.seen: list = []
async def ainvoke(self, messages):
self.seen.append(messages)
return await super().ainvoke(messages)
@pytest.mark.asyncio
async def test_review_never_rewrites_an_answer_that_carries_evidence():
"""报告节点产出的答案,每个数字都按位置对应着证据文档。复核把它整段改写,
文档和答案就对不上了——界面上点开的出处指向一段已经不存在的文字。"""
model = RecordingModel({"note": "报告被要求重写过一次。", "answer": "改写后的周报,销售额 45,678.5元。",
"retry": False})
result = await rv.review(model, question="q", answer="周报:销售额 45,678.5元。", signals=SIGNALS,
locked=True)
assert result.verdict == "annotated"
assert result.answer is None and result.original is None
assert result.note == "报告被要求重写过一次。"
prompt = "\n".join(str(text) for _, text in model.seen[0])
assert "不能改写" in prompt
def test_report_repair_is_a_degraded_signal():
signals = rv.scan([warn("报告里有 1 处没通过核对(「45678」),已要求写作者重写(第 1 次)",
"report_repair")], OK_OUTPUT)
assert [(s.kind, s.severity) for s in signals] == [("report_repair", rv.DEGRADED)]
@pytest.mark.asyncio
async def test_review_endpoint_keeps_answers_with_evidence(client, monkeypatch):
model = FakeModel({"note": "报告被要求重写过一次。", "answer": "整段改写过的周报。", "retry": False})
async def fake_model(*a, **k):
return model, "fake"
monkeypatch.setattr("app.api.copilot.get_chat_model", fake_model)
output = {"周报": "周报:销售额 45,678.5元。",
"_evidence": {"report_node": "write", "doc_artifact": "a" * 64, "fields": ["周报"]}}
run_id = await _make_run(client, output, [warn("报告里有 1 处没通过核对,已要求写作者重写", "report_repair")])
body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "q"})).json()
assert model.calls == 1
assert body["verdict"] == "annotated" and body["answer"] is None
assert body["note"] == "报告被要求重写过一次。"