mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 08:31:59 +00:00
d071bf119f
一、表结构、实体与引文
- 数据源工具查库时顺带存一份表结构快照(schema_snapshot),和查询快照一起进台账、进封存;
台账里的查询记下 SQL 用到的表
- 报告里的表名、字段名可以点:反引号里的名字一律核对,正文里的裸名字只在明显是标识符时才自动链接,
常见英文单词不链接;点开看到它出现在哪次查询、类型、所属的表,快照被截断时写明
- 找不到的名字标成可疑实体:在快照、SQL、结果列里都没有的记「可能是编造的名字」并给出最接近的 3 个;
快照被截断、核对不了的记「核对不了」,不冤枉成编造
- 检索引文 [[q:K1|原文]]:原文在检索快照的命中片段里逐字出现(空白归一化后)才算有出处,点开高亮原文位置
- 自动链接的名字不算句子的引用,只有写作者写的引用才算
二、结论句与出具
- 报告节点新增 claims:off(默认,保持原样)或 require_citation(没有引用的结论句计入缺口)
- 受管级别的正式运行里,可疑实体和没有引用的结论句按「有缺口」降档,不拦截;探索运行和已发布级别只标注
- 契约的 claims 和报告节点的设置取更严的那个;受管正式运行里写 ignore 也按降档处理,并给出警告
- 出具横幅除了数字,也写出可疑实体和没有引用的结论句
三、受管门禁 G1–G5(只看图和配置)
- G1 带契约的出口,成果字段的文字只能来自报告撰写节点
- G2 模型写的文字不绕过报告撰写节点直接流进出口
- G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation
- G4 口径卡的输入不来自计算角色的沙箱代码
- G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields
- 受管级别是错误、挡住发布;已发布级别只给警告;已经发布的版本照常运行,下次发布时按新规则检查
- 每条规则都接进发布前自动修复(一键修复 / 给选项 / 交给 Copilot);禁止的改动补上关 cite_fields、
删 output_schema、放宽契约 claims;Copilot 兜底不许自己把代码节点标成 source、不许放宽报告节点
- 发布弹窗里选「交给 Copilot」时明确写「正在交给 Copilot」
四、旧运行与审计
- 没有契约的旧答案:数字和已封存的查询单元格、口径卡值按数值比对,每个数最多 3 个候选,
默认折叠,写明「猜测的来源,不能当证据」
- 有契约的旧运行:候选只从已封存的口径卡里找,人为插进工件表的行不当证据
- 证据审计:GET /api/runs/{id}/evidence/audit 按「无证据 / 可疑实体 / 有出处 / 旧运行猜测」分组,
可导出 JSON 和 CSV(带封存状态);记录页和画布上的报告节点显示「有出处 N · 无证据 M」
五、SQL 守卫修正(安全)
- 去注释改成单遍扫描,认得字符串和各方言的引号(单引号转义、MySQL 反斜杠与 #、PostgreSQL 的 $$ 与嵌套注释、
Oracle 的 q'[…]'、SQLite 的方括号);以前字符串里的 -- 会被当成注释,SELECT '--'; DROP … 这类写法能骗过多语句拦截
- 看不到服务器设置时每种读法都扫一遍,读法之间切分不一致就拒绝,任何一种读法是写操作就按写操作处理
- 执行查询和工具危险判定都按数据源的方言检查
六、其他
- 修掉检查脚本里画布检查的一处竞态,并行估时按实测更新
268 lines
12 KiB
Python
268 lines
12 KiB
Python
"""引文:[[q:K1|原话]] 必须在那次检索命中的片段里逐字出现。
|
||
|
||
知识库的结论以前只能写成 [[see:K1]]:读者知道依据是哪次检索,看不到是哪一句。引文把
|
||
原话写进报告,系统拿检索快照(取回时复验哈希)逐字核对:
|
||
|
||
- 空白归一化后逐字出现才算有出处;改一个字就是没有出处(state none,issue unresolved_ref)
|
||
- 正文里显示引文本身;目录里根本没有这次检索时显示占位,和别的解析不了的引用一样
|
||
- eid 带上命中片段的定位:第几条命中、原文里的起止
|
||
- 出口复核重新取快照、重新比对,不信文档里记的结论
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import random
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import select
|
||
|
||
from app.core import artifact_store
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run, RunEvent
|
||
from app.engine.evidence import (
|
||
LATER_REASON,
|
||
StreamRenderer,
|
||
build_catalog,
|
||
catalog_prompt,
|
||
compose_doc,
|
||
iter_segments,
|
||
make_eid,
|
||
render_markers,
|
||
verify_doc,
|
||
)
|
||
from app.engine.runner import run_manager
|
||
|
||
K_ART = "c" * 64
|
||
HITS = [
|
||
{"chunk_id": "ch-1", "document_id": "doc-1", "title": "售后月报", "ordinal": 3,
|
||
"content": "九月退款主要集中在\n东区,原因是物流延误。退款率 2.35%,高于上月。"},
|
||
{"chunk_id": "ch-2", "document_id": "doc-2", "title": "Ops notes", "ordinal": 0,
|
||
"content": "Net revenue\nrose after the promotion ended. 物流延误已经解决。"},
|
||
]
|
||
SNAPSHOT = {"query": "退款", "collection": "kb", "hits": HITS}
|
||
|
||
|
||
def loader_of(content):
|
||
def load(artifact):
|
||
return content if artifact == K_ART else None
|
||
return load
|
||
|
||
|
||
LOAD = loader_of(SNAPSHOT)
|
||
|
||
|
||
@pytest.fixture
|
||
def catalog():
|
||
return build_catalog(nodes={}, ledger=[{"kind": "retrieval", "node_id": "kb", "exec": 1, "artifact": K_ART,
|
||
"source": "kb", "rows": len(HITS)}])
|
||
|
||
|
||
def quotes(doc):
|
||
return [s for s in iter_segments(doc) if s["kind"] == "quote"]
|
||
|
||
|
||
def test_verbatim_quote_resolves_with_its_location(catalog):
|
||
doc = compose_doc("售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]", catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
assert doc["markdown"] == "售后说「退款主要集中在东区」。"
|
||
assert seg["text"] == "退款主要集中在东区" and seg["state"] == "deterministic"
|
||
cite = seg["cite"]
|
||
assert cite["kind"] == "quote" and cite["role"] == "quote" and cite["alias"] == "K1"
|
||
loc = cite["locator"]
|
||
assert loc["hit"] == 0 and HITS[0]["content"][loc["start"]:loc["end"]] == "退款主要集中在\n东区"
|
||
assert cite["eid"] == make_eid("quote", K_ART, loc)
|
||
assert cite["source"] == {"artifact": K_ART, "document": "doc-1", "chunk": "ch-1", "title": "售后月报",
|
||
"ordinal": 3}
|
||
assert doc["violations"] == [] and doc["stats"]["quotes"] == 1
|
||
assert doc["blocks"][0]["units"][0]["cites"] == ["K1"]
|
||
|
||
|
||
def test_one_changed_character_is_not_a_quote(catalog):
|
||
doc = compose_doc("售后说「[[q:K1|退款主要集中在西区]]」。", catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
# 正文照样显示引文本身,只是标成没有出处
|
||
assert seg["text"] == "退款主要集中在西区" and seg["state"] == "none" and seg["issue"] == "unresolved_ref"
|
||
assert "逐字" in seg["cite"]["reason"]
|
||
assert [v["code"] for v in doc["violations"]] == ["unresolved_ref"]
|
||
assert doc["stats"]["quotes"] == 0
|
||
|
||
|
||
@pytest.mark.parametrize("quote", [
|
||
"Net revenue rose after the promotion ended.",
|
||
"Net revenue rose",
|
||
"退款率 2.35%,高于上月",
|
||
"退款主要集中在 东区",
|
||
])
|
||
def test_whitespace_is_normalised(catalog, quote):
|
||
doc = compose_doc(f"原话:[[q:K1|{quote}]]", catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
assert seg["state"] == "deterministic", seg["cite"].get("reason")
|
||
loc = seg["cite"]["locator"]
|
||
assert " ".join(HITS[loc["hit"]]["content"][loc["start"]:loc["end"]].split()).replace(" ", "") \
|
||
== quote.replace(" ", "")
|
||
|
||
|
||
@pytest.mark.parametrize("quote", ["Netrevenue rose", "net revenue rose", "退款主要集中在东区。九月"])
|
||
def test_normalising_is_not_fuzzy(catalog, quote):
|
||
doc = compose_doc(f"原话:[[q:K1|{quote}]]", catalog, loader=LOAD)
|
||
assert quotes(doc)[0]["state"] == "none"
|
||
|
||
|
||
def test_numbers_inside_a_verified_quote_are_not_bare(catalog):
|
||
doc = compose_doc("[[q:K1|退款率 2.35%,高于上月]]", catalog, loader=LOAD)
|
||
assert doc["violations"] == [] and doc["stats"]["uncited_numbers"] == 0
|
||
|
||
|
||
@pytest.mark.parametrize("text, words", [
|
||
("[[q:K9|退款主要集中在东区]]", "目录里没有 K9"),
|
||
("[[q:K1|]]", "引文是空的"),
|
||
("[[q:K1|东区]]", "太短"),
|
||
])
|
||
def test_quotes_that_cannot_be_checked(catalog, text, words):
|
||
doc = compose_doc(text, catalog, loader=LOAD)
|
||
[seg] = quotes(doc)
|
||
assert seg["state"] == "none" and words in seg["cite"]["reason"], seg["cite"]["reason"]
|
||
|
||
|
||
def test_missing_retrieval_shows_a_placeholder(catalog):
|
||
doc = compose_doc("[[q:K9|退款主要集中在东区]]。", catalog, loader=LOAD)
|
||
assert doc["markdown"] == "⟦?q:K9|退款主要集中在东区⟧。"
|
||
|
||
|
||
def test_tampered_snapshot_is_not_evidence(catalog):
|
||
def tampered(artifact):
|
||
raise ValueError("hash mismatch")
|
||
|
||
doc = compose_doc("[[q:K1|退款主要集中在东区]]", catalog, loader=tampered)
|
||
[seg] = quotes(doc)
|
||
assert seg["state"] == "none" and "哈希" in seg["cite"]["reason"]
|
||
|
||
|
||
def test_exit_check_rereads_the_snapshot(catalog):
|
||
doc = compose_doc("[[q:K1|物流延误已经解决]]。", catalog, loader=LOAD)
|
||
assert doc["violations"] == [] and quotes(doc)[0]["cite"]["locator"]["hit"] == 1
|
||
changed = {**SNAPSHOT, "hits": [HITS[0], {**HITS[1], "content": "物流延误仍在持续。"}]}
|
||
checked = verify_doc(doc, catalog, loader=loader_of(changed))
|
||
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"]
|
||
assert verify_doc(doc, catalog, loader=LOAD)["ok"] is True
|
||
|
||
|
||
def phase_two(doc):
|
||
"""把文档改回二期组装的样子:那时 q / t / c 一律判为解析不了,正文是占位,原因是 LATER_REASON。
|
||
占位的字和现在「目录里没有 K9」时一样,只有记下的原因不同。"""
|
||
for seg in iter_segments(doc):
|
||
if seg.get("issue") == "unresolved_ref" and seg["ref"][:2] in ("q:", "t:", "c:"):
|
||
seg["cite"]["reason"] = LATER_REASON
|
||
return doc
|
||
|
||
|
||
def test_quotes_from_before_the_upgrade_are_rechecked_the_old_way(catalog):
|
||
"""二期的报告文档(升级前组装,停在报告和出口之间)升级后被复核:原话今天查得到也不能算
|
||
「渲染对不上」——那会被出口当成完整性问题记缺口。按当时的规矩:占位,解析不了。"""
|
||
doc = phase_two(compose_doc("售后说[[q:K1|退款主要集中在东区]]。", {}, loader=LOAD))
|
||
[seg] = quotes(doc)
|
||
assert seg["text"] == "⟦?q:K1|退款主要集中在东区⟧"
|
||
checked = verify_doc(doc, catalog, loader=LOAD)
|
||
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"]
|
||
assert LATER_REASON in checked["violations"][0]["message"]
|
||
# 只认占位:正文被改成了引文本身、又自称是二期的,照样按现在的规矩查
|
||
seg["text"] = "退款主要集中在东区"
|
||
doc["markdown"] = "售后说退款主要集中在东区。"
|
||
tail = [x for x in iter_segments(doc)][-1]
|
||
seg["span"] = [3, 3 + len(seg["text"])]
|
||
tail["span"] = [seg["span"][1], seg["span"][1] + len(tail["text"])]
|
||
[unit] = [u for b in doc["blocks"] for u in b["units"]]
|
||
unit["span"] = [0, tail["span"][1]]
|
||
assert [v["code"] for v in verify_doc(doc, catalog, loader=LOAD)["violations"]] == ["state_mismatch"]
|
||
|
||
|
||
def test_entity_markers_from_before_the_upgrade_are_rechecked_the_old_way():
|
||
ledger = [{"kind": "query", "node_id": "f", "exec": 1, "artifact": "a" * 64, "tool": "db_query__w", "source": "w",
|
||
"columns": ["gmv"], "rows": 1, "truncated": False, "schema_artifact": "5" * 64, "tables": ["orders"]},
|
||
{"kind": "schema", "node_id": "f", "exec": 1, "artifact": "5" * 64, "source": "w",
|
||
"tables": {"orders": ["id", "gmv"]}}]
|
||
doc = phase_two(compose_doc("看 [[t:orders]] 和 [[c:orders.gmv]]。", {}))
|
||
assert [s["text"] for s in iter_segments(doc) if s.get("ref")] == ["⟦?t:orders⟧", "⟦?c:orders.gmv⟧"]
|
||
checked = verify_doc(doc, build_catalog(nodes={}, ledger=ledger))
|
||
assert [v["code"] for v in checked["violations"]] == ["unresolved_ref", "unresolved_ref"]
|
||
|
||
|
||
def test_streaming_equals_full_render(catalog):
|
||
text = "开头 [[q:K1|退款主要集中在东区]],又说 [[q:K1|退款主要集中在西区]];[[q:K9|无此检索]] 完。[[see:K1]]"
|
||
whole = render_markers(text, catalog, loader=LOAD)
|
||
assert whole == "开头 退款主要集中在东区,又说 退款主要集中在西区;⟦?q:K9|无此检索⟧ 完。"
|
||
rng = random.Random(11)
|
||
for _ in range(200):
|
||
renderer = StreamRenderer(catalog, loader=LOAD)
|
||
out, i = [], 0
|
||
while i < len(text):
|
||
step = rng.randint(1, 7)
|
||
out.append(renderer.feed(text[i:i + step]))
|
||
i += step
|
||
out.append(renderer.flush())
|
||
assert "".join(out) == whole
|
||
|
||
|
||
def test_catalog_prompt_shows_hits_to_quote_from(catalog):
|
||
prompt = catalog_prompt(catalog, loader=LOAD)
|
||
assert "[[q:K1|" in prompt and "逐字" in prompt and "[[see:K1]]" in prompt
|
||
assert "售后月报" in prompt and "退款主要集中在" in prompt and "Ops notes" in prompt
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 跑一次:检索 → 报告撰写,引错了原话会被要求重写
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
async def test_report_node_checks_quotes_against_the_retrieval(engine_up, monkeypatch):
|
||
from app.memory import kb
|
||
from app.providers import mock_model
|
||
|
||
async def fake_search(session, **kw):
|
||
return [dict(HITS[0], score=0.9)]
|
||
|
||
monkeypatch.setattr(kb, "search", fake_search)
|
||
replies = ["售后说「[[q:K1|退款主要集中在西区]]」。[[see:K1]]", "售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]"]
|
||
seen: list = []
|
||
|
||
def decide(self, messages):
|
||
seen.append(list(messages))
|
||
return AIMessage(content=replies[min(len(seen), len(replies)) - 1])
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||
nodes = [{"id": nid, "type": t, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": cfg}} for nid, t, cfg in [
|
||
("start", "input", {}), ("docs", "retrieve", {"query": "退款", "collection": "kb"}),
|
||
("write", "report", {"instructions": "引用售后的原话"}),
|
||
("out", "output", {"fields": [{"name": "r", "value": "{{ nodes.write.text }}"}]})]]
|
||
graph = {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||
run = await run_manager.start(graph=graph, input_payload={})
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run.id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
break
|
||
assert row.status == "succeeded", row.error
|
||
assert row.output["r"] == "售后说「退款主要集中在东区」。"
|
||
|
||
first = "\n".join(str(m.content) for m in seen[0])
|
||
assert "[[q:K1|" in first and "退款主要集中在" in first and "售后月报" in first
|
||
assert len(seen) == 2 and "找不到这句原话" in str(seen[1][-1].content)
|
||
async with SessionLocal() as session:
|
||
[checked] = (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run.id, RunEvent.type == "report.checked"))).scalars().all()
|
||
assert checked.data["ok"] is True and checked.data["repairs"] == 1 and checked.data["stats"]["quotes"] == 1
|
||
doc = artifact_store.load(checked.data["doc_artifact"])
|
||
[seg] = quotes(doc)
|
||
retrieval = doc["catalog"]["K1"]["artifact"]
|
||
assert seg["cite"]["eid"] == make_eid("quote", retrieval, seg["cite"]["locator"])
|
||
assert seg["cite"]["source"]["chunk"] == "ch-1"
|