Files
AgentLab/backend/tests/test_evidence_quotes.py
T
ThearchyHelios d071bf119f 可点击证据第三期:表名字段名和引文也点得开、可疑实体与结论句、受管门禁 G1–G5、证据审计导出
一、表结构、实体与引文

- 数据源工具查库时顺带存一份表结构快照(schema_snapshot),和查询快照一起进台账、进封存;
  台账里的查询记下 SQL 用到的表
- 报告里的表名、字段名可以点:反引号里的名字一律核对,正文里的裸名字只在明显是标识符时才自动链接,
  常见英文单词不链接;点开看到它出现在哪次查询、类型、所属的表,快照被截断时写明
- 找不到的名字标成可疑实体:在快照、SQL、结果列里都没有的记「可能是编造的名字」并给出最接近的 3 个;
  快照被截断、核对不了的记「核对不了」,不冤枉成编造
- 检索引文 [[q:K1|原文]]:原文在检索快照的命中片段里逐字出现(空白归一化后)才算有出处,点开高亮原文位置
- 自动链接的名字不算句子的引用,只有写作者写的引用才算

二、结论句与出具

- 报告节点新增 claims:off(默认,保持原样)或 require_citation(没有引用的结论句计入缺口)
- 受管级别的正式运行里,可疑实体和没有引用的结论句按「有缺口」降档,不拦截;探索运行和已发布级别只标注
- 契约的 claims 和报告节点的设置取更严的那个;受管正式运行里写 ignore 也按降档处理,并给出警告
- 出具横幅除了数字,也写出可疑实体和没有引用的结论句

三、受管门禁 G1–G5(只看图和配置)

- G1 带契约的出口,成果字段的文字只能来自报告撰写节点
- G2 模型写的文字不绕过报告撰写节点直接流进出口
- G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation
- G4 口径卡的输入不来自计算角色的沙箱代码
- G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields
- 受管级别是错误、挡住发布;已发布级别只给警告;已经发布的版本照常运行,下次发布时按新规则检查
- 每条规则都接进发布前自动修复(一键修复 / 给选项 / 交给 Copilot);禁止的改动补上关 cite_fields、
  删 output_schema、放宽契约 claims;Copilot 兜底不许自己把代码节点标成 source、不许放宽报告节点
- 发布弹窗里选「交给 Copilot」时明确写「正在交给 Copilot」

四、旧运行与审计

- 没有契约的旧答案:数字和已封存的查询单元格、口径卡值按数值比对,每个数最多 3 个候选,
  默认折叠,写明「猜测的来源,不能当证据」
- 有契约的旧运行:候选只从已封存的口径卡里找,人为插进工件表的行不当证据
- 证据审计:GET /api/runs/{id}/evidence/audit 按「无证据 / 可疑实体 / 有出处 / 旧运行猜测」分组,
  可导出 JSON 和 CSV(带封存状态);记录页和画布上的报告节点显示「有出处 N · 无证据 M」

五、SQL 守卫修正(安全)

- 去注释改成单遍扫描,认得字符串和各方言的引号(单引号转义、MySQL 反斜杠与 #、PostgreSQL 的 $$ 与嵌套注释、
  Oracle 的 q'[…]'、SQLite 的方括号);以前字符串里的 -- 会被当成注释,SELECT '--'; DROP … 这类写法能骗过多语句拦截
- 看不到服务器设置时每种读法都扫一遍,读法之间切分不一致就拒绝,任何一种读法是写操作就按写操作处理
- 执行查询和工具危险判定都按数据源的方言检查

六、其他

- 修掉检查脚本里画布检查的一处竞态,并行估时按实测更新
2026-09-29 05:26:34 +08:00

268 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""引文:[[q:K1|原话]] 必须在那次检索命中的片段里逐字出现。
知识库的结论以前只能写成 [[see:K1]]:读者知道依据是哪次检索,看不到是哪一句。引文把
原话写进报告,系统拿检索快照(取回时复验哈希)逐字核对:
- 空白归一化后逐字出现才算有出处;改一个字就是没有出处(state none,issue unresolved_ref)
- 正文里显示引文本身;目录里根本没有这次检索时显示占位,和别的解析不了的引用一样
- eid 带上命中片段的定位:第几条命中、原文里的起止
- 出口复核重新取快照、重新比对,不信文档里记的结论
"""
from __future__ import annotations
import asyncio
import random
import pytest
from langchain_core.messages import AIMessage
from sqlalchemy import select
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent
from app.engine.evidence import (
LATER_REASON,
StreamRenderer,
build_catalog,
catalog_prompt,
compose_doc,
iter_segments,
make_eid,
render_markers,
verify_doc,
)
from app.engine.runner import run_manager
K_ART = "c" * 64
HITS = [
{"chunk_id": "ch-1", "document_id": "doc-1", "title": "售后月报", "ordinal": 3,
"content": "九月退款主要集中在\n东区,原因是物流延误。退款率 2.35%,高于上月。"},
{"chunk_id": "ch-2", "document_id": "doc-2", "title": "Ops notes", "ordinal": 0,
"content": "Net revenue\nrose after the promotion ended. 物流延误已经解决。"},
]
SNAPSHOT = {"query": "退款", "collection": "kb", "hits": HITS}
def loader_of(content):
def load(artifact):
return content if artifact == K_ART else None
return load
LOAD = loader_of(SNAPSHOT)
@pytest.fixture
def catalog():
return build_catalog(nodes={}, ledger=[{"kind": "retrieval", "node_id": "kb", "exec": 1, "artifact": K_ART,
"source": "kb", "rows": len(HITS)}])
def quotes(doc):
return [s for s in iter_segments(doc) if s["kind"] == "quote"]
def test_verbatim_quote_resolves_with_its_location(catalog):
doc = compose_doc("售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]", catalog, loader=LOAD)
[seg] = quotes(doc)
assert doc["markdown"] == "售后说「退款主要集中在东区」。"
assert seg["text"] == "退款主要集中在东区" and seg["state"] == "deterministic"
cite = seg["cite"]
assert cite["kind"] == "quote" and cite["role"] == "quote" and cite["alias"] == "K1"
loc = cite["locator"]
assert loc["hit"] == 0 and HITS[0]["content"][loc["start"]:loc["end"]] == "退款主要集中在\n东区"
assert cite["eid"] == make_eid("quote", K_ART, loc)
assert cite["source"] == {"artifact": K_ART, "document": "doc-1", "chunk": "ch-1", "title": "售后月报",
"ordinal": 3}
assert doc["violations"] == [] and doc["stats"]["quotes"] == 1
assert doc["blocks"][0]["units"][0]["cites"] == ["K1"]
def test_one_changed_character_is_not_a_quote(catalog):
doc = compose_doc("售后说「[[q:K1|退款主要集中在西区]]」。", catalog, loader=LOAD)
[seg] = quotes(doc)
# 正文照样显示引文本身,只是标成没有出处
assert seg["text"] == "退款主要集中在西区" and seg["state"] == "none" and seg["issue"] == "unresolved_ref"
assert "逐字" in seg["cite"]["reason"]
assert [v["code"] for v in doc["violations"]] == ["unresolved_ref"]
assert doc["stats"]["quotes"] == 0
@pytest.mark.parametrize("quote", [
"Net revenue rose after the promotion ended.",
"Net revenue rose",
"退款率 2.35%,高于上月",
"退款主要集中在 东区",
])
def test_whitespace_is_normalised(catalog, quote):
doc = compose_doc(f"原话:[[q:K1|{quote}]]", catalog, loader=LOAD)
[seg] = quotes(doc)
assert seg["state"] == "deterministic", seg["cite"].get("reason")
loc = seg["cite"]["locator"]
assert " ".join(HITS[loc["hit"]]["content"][loc["start"]:loc["end"]].split()).replace(" ", "") \
== quote.replace(" ", "")
@pytest.mark.parametrize("quote", ["Netrevenue rose", "net revenue rose", "退款主要集中在东区。九月"])
def test_normalising_is_not_fuzzy(catalog, quote):
doc = compose_doc(f"原话:[[q:K1|{quote}]]", catalog, loader=LOAD)
assert quotes(doc)[0]["state"] == "none"
def test_numbers_inside_a_verified_quote_are_not_bare(catalog):
doc = compose_doc("[[q:K1|退款率 2.35%,高于上月]]", catalog, loader=LOAD)
assert doc["violations"] == [] and doc["stats"]["uncited_numbers"] == 0
@pytest.mark.parametrize("text, words", [
("[[q:K9|退款主要集中在东区]]", "目录里没有 K9"),
("[[q:K1|]]", "引文是空的"),
("[[q:K1|东区]]", "太短"),
])
def test_quotes_that_cannot_be_checked(catalog, text, words):
doc = compose_doc(text, catalog, loader=LOAD)
[seg] = quotes(doc)
assert seg["state"] == "none" and words in seg["cite"]["reason"], seg["cite"]["reason"]
def test_missing_retrieval_shows_a_placeholder(catalog):
doc = compose_doc("[[q:K9|退款主要集中在东区]]。", catalog, loader=LOAD)
assert doc["markdown"] == "⟦?q:K9|退款主要集中在东区⟧。"
def test_tampered_snapshot_is_not_evidence(catalog):
def tampered(artifact):
raise ValueError("hash mismatch")
doc = compose_doc("[[q:K1|退款主要集中在东区]]", catalog, loader=tampered)
[seg] = quotes(doc)
assert seg["state"] == "none" and "哈希" in seg["cite"]["reason"]
def test_exit_check_rereads_the_snapshot(catalog):
doc = compose_doc("[[q:K1|物流延误已经解决]]。", catalog, loader=LOAD)
assert doc["violations"] == [] and quotes(doc)[0]["cite"]["locator"]["hit"] == 1
changed = {**SNAPSHOT, "hits": [HITS[0], {**HITS[1], "content": "物流延误仍在持续。"}]}
checked = verify_doc(doc, catalog, loader=loader_of(changed))
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"]
assert verify_doc(doc, catalog, loader=LOAD)["ok"] is True
def phase_two(doc):
"""把文档改回二期组装的样子:那时 q / t / c 一律判为解析不了,正文是占位,原因是 LATER_REASON。
占位的字和现在「目录里没有 K9」时一样,只有记下的原因不同。"""
for seg in iter_segments(doc):
if seg.get("issue") == "unresolved_ref" and seg["ref"][:2] in ("q:", "t:", "c:"):
seg["cite"]["reason"] = LATER_REASON
return doc
def test_quotes_from_before_the_upgrade_are_rechecked_the_old_way(catalog):
"""二期的报告文档(升级前组装,停在报告和出口之间)升级后被复核:原话今天查得到也不能算
「渲染对不上」——那会被出口当成完整性问题记缺口。按当时的规矩:占位,解析不了。"""
doc = phase_two(compose_doc("售后说[[q:K1|退款主要集中在东区]]。", {}, loader=LOAD))
[seg] = quotes(doc)
assert seg["text"] == "⟦?q:K1|退款主要集中在东区⟧"
checked = verify_doc(doc, catalog, loader=LOAD)
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"]
assert LATER_REASON in checked["violations"][0]["message"]
# 只认占位:正文被改成了引文本身、又自称是二期的,照样按现在的规矩查
seg["text"] = "退款主要集中在东区"
doc["markdown"] = "售后说退款主要集中在东区。"
tail = [x for x in iter_segments(doc)][-1]
seg["span"] = [3, 3 + len(seg["text"])]
tail["span"] = [seg["span"][1], seg["span"][1] + len(tail["text"])]
[unit] = [u for b in doc["blocks"] for u in b["units"]]
unit["span"] = [0, tail["span"][1]]
assert [v["code"] for v in verify_doc(doc, catalog, loader=LOAD)["violations"]] == ["state_mismatch"]
def test_entity_markers_from_before_the_upgrade_are_rechecked_the_old_way():
ledger = [{"kind": "query", "node_id": "f", "exec": 1, "artifact": "a" * 64, "tool": "db_query__w", "source": "w",
"columns": ["gmv"], "rows": 1, "truncated": False, "schema_artifact": "5" * 64, "tables": ["orders"]},
{"kind": "schema", "node_id": "f", "exec": 1, "artifact": "5" * 64, "source": "w",
"tables": {"orders": ["id", "gmv"]}}]
doc = phase_two(compose_doc("看 [[t:orders]] 和 [[c:orders.gmv]]。", {}))
assert [s["text"] for s in iter_segments(doc) if s.get("ref")] == ["⟦?t:orders⟧", "⟦?c:orders.gmv⟧"]
checked = verify_doc(doc, build_catalog(nodes={}, ledger=ledger))
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref", "unresolved_ref"]
def test_streaming_equals_full_render(catalog):
text = "开头 [[q:K1|退款主要集中在东区]],又说 [[q:K1|退款主要集中在西区]];[[q:K9|无此检索]] 完。[[see:K1]]"
whole = render_markers(text, catalog, loader=LOAD)
assert whole == "开头 退款主要集中在东区,又说 退款主要集中在西区;⟦?q:K9|无此检索⟧ 完。"
rng = random.Random(11)
for _ in range(200):
renderer = StreamRenderer(catalog, loader=LOAD)
out, i = [], 0
while i < len(text):
step = rng.randint(1, 7)
out.append(renderer.feed(text[i:i + step]))
i += step
out.append(renderer.flush())
assert "".join(out) == whole
def test_catalog_prompt_shows_hits_to_quote_from(catalog):
prompt = catalog_prompt(catalog, loader=LOAD)
assert "[[q:K1|" in prompt and "逐字" in prompt and "[[see:K1]]" in prompt
assert "售后月报" in prompt and "退款主要集中在" in prompt and "Ops notes" in prompt
# --------------------------------------------------------------------------
# 跑一次:检索 → 报告撰写,引错了原话会被要求重写
# --------------------------------------------------------------------------
@pytest.fixture
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
async def test_report_node_checks_quotes_against_the_retrieval(engine_up, monkeypatch):
from app.memory import kb
from app.providers import mock_model
async def fake_search(session, **kw):
return [dict(HITS[0], score=0.9)]
monkeypatch.setattr(kb, "search", fake_search)
replies = ["售后说「[[q:K1|退款主要集中在西区]]」。[[see:K1]]", "售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]"]
seen: list = []
def decide(self, messages):
seen.append(list(messages))
return AIMessage(content=replies[min(len(seen), len(replies)) - 1])
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
nodes = [{"id": nid, "type": t, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": cfg}} for nid, t, cfg in [
("start", "input", {}), ("docs", "retrieve", {"query": "退款", "collection": "kb"}),
("write", "report", {"instructions": "引用售后的原话"}),
("out", "output", {"fields": [{"name": "r", "value": "{{ nodes.write.text }}"}]})]]
graph = {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
run = await run_manager.start(graph=graph, input_payload={})
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run.id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
break
assert row.status == "succeeded", row.error
assert row.output["r"] == "售后说「退款主要集中在东区」。"
first = "\n".join(str(m.content) for m in seen[0])
assert "[[q:K1|" in first and "退款主要集中在" in first and "售后月报" in first
assert len(seen) == 2 and "找不到这句原话" in str(seen[1][-1].content)
async with SessionLocal() as session:
[checked] = (await session.execute(select(RunEvent).where(
RunEvent.run_id == run.id, RunEvent.type == "report.checked"))).scalars().all()
assert checked.data["ok"] is True and checked.data["repairs"] == 1 and checked.data["stats"]["quotes"] == 1
doc = artifact_store.load(checked.data["doc_artifact"])
[seg] = quotes(doc)
retrieval = doc["catalog"]["K1"]["artifact"]
assert seg["cite"]["eid"] == make_eid("quote", retrieval, seg["cite"]["locator"])
assert seg["cite"]["source"]["chunk"] == "ch-1"