Files
AgentLab/backend/tests/test_autofix_assist.py
T
ThearchyHelios d071bf119f 可点击证据第三期:表名字段名和引文也点得开、可疑实体与结论句、受管门禁 G1–G5、证据审计导出
一、表结构、实体与引文

- 数据源工具查库时顺带存一份表结构快照(schema_snapshot),和查询快照一起进台账、进封存;
  台账里的查询记下 SQL 用到的表
- 报告里的表名、字段名可以点:反引号里的名字一律核对,正文里的裸名字只在明显是标识符时才自动链接,
  常见英文单词不链接;点开看到它出现在哪次查询、类型、所属的表,快照被截断时写明
- 找不到的名字标成可疑实体:在快照、SQL、结果列里都没有的记「可能是编造的名字」并给出最接近的 3 个;
  快照被截断、核对不了的记「核对不了」,不冤枉成编造
- 检索引文 [[q:K1|原文]]:原文在检索快照的命中片段里逐字出现(空白归一化后)才算有出处,点开高亮原文位置
- 自动链接的名字不算句子的引用,只有写作者写的引用才算

二、结论句与出具

- 报告节点新增 claims:off(默认,保持原样)或 require_citation(没有引用的结论句计入缺口)
- 受管级别的正式运行里,可疑实体和没有引用的结论句按「有缺口」降档,不拦截;探索运行和已发布级别只标注
- 契约的 claims 和报告节点的设置取更严的那个;受管正式运行里写 ignore 也按降档处理,并给出警告
- 出具横幅除了数字,也写出可疑实体和没有引用的结论句

三、受管门禁 G1–G5(只看图和配置)

- G1 带契约的出口,成果字段的文字只能来自报告撰写节点
- G2 模型写的文字不绕过报告撰写节点直接流进出口
- G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation
- G4 口径卡的输入不来自计算角色的沙箱代码
- G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields
- 受管级别是错误、挡住发布;已发布级别只给警告;已经发布的版本照常运行,下次发布时按新规则检查
- 每条规则都接进发布前自动修复(一键修复 / 给选项 / 交给 Copilot);禁止的改动补上关 cite_fields、
  删 output_schema、放宽契约 claims;Copilot 兜底不许自己把代码节点标成 source、不许放宽报告节点
- 发布弹窗里选「交给 Copilot」时明确写「正在交给 Copilot」

四、旧运行与审计

- 没有契约的旧答案:数字和已封存的查询单元格、口径卡值按数值比对,每个数最多 3 个候选,
  默认折叠,写明「猜测的来源,不能当证据」
- 有契约的旧运行:候选只从已封存的口径卡里找,人为插进工件表的行不当证据
- 证据审计:GET /api/runs/{id}/evidence/audit 按「无证据 / 可疑实体 / 有出处 / 旧运行猜测」分组,
  可导出 JSON 和 CSV(带封存状态);记录页和画布上的报告节点显示「有出处 N · 无证据 M」

五、SQL 守卫修正(安全)

- 去注释改成单遍扫描,认得字符串和各方言的引号(单引号转义、MySQL 反斜杠与 #、PostgreSQL 的 $$ 与嵌套注释、
  Oracle 的 q'[…]'、SQLite 的方括号);以前字符串里的 -- 会被当成注释,SELECT '--'; DROP … 这类写法能骗过多语句拦截
- 看不到服务器设置时每种读法都扫一遍,读法之间切分不一致就拒绝,任何一种读法是写操作就按写操作处理
- 执行查询和工具危险判定都按数据源的方言检查

六、其他

- 修掉检查脚本里画布检查的一处竞态,并行估时按实测更新
2026-09-29 05:26:34 +08:00

281 lines
14 KiB
Python

"""自动修复的 Copilot 兜底(autofix 的 assist: true)。
确定性修复修不了的 error 交给 Copilot:模型给剧本,看采纳规则——
- 正常的修复(error 变少、没有新 error、不降低要求)被采纳,只是预览,不落库
- 试图删契约的被拒
- 让错误变多的被拒
- 要人拿主意的放进 questions,不硬改
"""
from __future__ import annotations
import json
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select
from app.db.base import SessionLocal
from app.db.models import Workflow, WorkflowVersion
from app.main import app
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, label=None, **config):
return {"id": nid, "type": ntype, "position": {"x": 40, "y": 80}, "data": {"label": label or nid, "config": config}}
CONTRACT = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}
def two_exits() -> dict:
"""受管模板没有带契约的出口,而且出口有两个:该给哪个写契约说不准,确定性修复只能交给 Copilot。"""
nodes = [
node("start", "input", "周期"),
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}),
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
node("write", "report", "报告撰写", instructions="写周报",
numbers="strict", on_violation="fail", claims="require_citation"),
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
node("raw", "output", "原始数", fields=[{"name": "数", "value": "{{ nodes.fetch }}"}]),
]
edges = [{"id": f"e{i}", "source": a, "target": b} for i, (a, b) in enumerate(
[("start", "fetch"), ("fetch", "card"), ("card", "write"), ("write", "done"), ("fetch", "raw")])]
return {"nodes": nodes, "edges": edges, "viewport": {"x": 3, "y": 4, "zoom": 1.2}}
class Scripted:
"""按剧本吐操作流的模型;记下每次收到的请求。"""
def __init__(self, ops: list[dict]) -> None:
self.lines = [json.dumps(o, ensure_ascii=False) for o in ops]
self.calls: list[list] = []
async def astream(self, messages):
from langchain_core.messages import AIMessageChunk
self.calls.append(messages)
for line in self.lines:
yield AIMessageChunk(content=line + "\n")
def script(monkeypatch, ops: list[dict]) -> Scripted:
import app.api.copilot as copilot
model = Scripted(ops)
async def _model(*_a, **_k):
return model, "mock-fast"
monkeypatch.setattr(copilot, "get_chat_model", _model)
return model
async def create(client, graph: dict) -> str:
return (await client.post("/api/workflows", json={"name": "周报-兜底", "graph": graph})).json()["id"]
async def rows(wf: str) -> list:
async with SessionLocal() as session:
w = await session.get(Workflow, wf)
vs = (await session.execute(select(WorkflowVersion).where(WorkflowVersion.workflow_id == wf))).scalars()
return [{c.name: getattr(w, c.name) for c in Workflow.__table__.columns},
[{c.name: getattr(v, c.name) for c in WorkflowVersion.__table__.columns} for v in vs]]
async def assist(client, wf: str, **extra) -> dict:
r = await client.post(f"/api/workflows/{wf}/autofix", json={"level": "governed", "apply": [], "assist": True,
**extra})
assert r.status_code == 200, r.text
return r.json()
def done(ops: list[dict]) -> list[dict]:
return [{"op": "plan", "summary": "给出具节点补契约"}, *ops, {"op": "done", "explanation": "补上了出具契约"}]
async def test_a_sound_fix_is_adopted_as_a_preview(client, monkeypatch):
model = script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]))
wf = await create(client, two_exits())
before = await rows(wf)
out = await assist(client, wf)
assert out["assist"]["ok"] is True and out["assist"]["summary"] == "补上了出具契约"
assert out["applied"] == ["assist"] and out["ok"] is True, out
fixed = out["graph"]
assert next(n for n in fixed["nodes"] if n["id"] == "done")["data"]["config"]["contract"] == CONTRACT
assert fixed["viewport"] == two_exits()["viewport"] and [e["id"] for e in fixed["edges"]] == [
e["id"] for e in two_exits()["edges"]]
assert out["ops"] == [{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]
assert [(c["fix_id"], c["node_id"], c["field"]) for c in out["changes"]] == [("assist", "done", "contract")]
assert await rows(wf) == before # 只是预览
# 交给模型的请求里有剩下的问题和不许降低要求、不许替人选的约定
human = next(text for role, text in model.calls[0] if role == "human")
assert "受管模板至少要有一个「成果 / 出具」节点声明出具契约" in human
assert "不许降低要求" in human and "不替人选" in human and "question" in human
async def test_deleting_the_contract_is_refused(client, monkeypatch):
graph = two_exits()
graph["nodes"][4]["data"]["config"]["contract"] = {**CONTRACT, "report_from": "fetch"}
graph["nodes"][3] = node("write", "llm", "写周报", prompt="写周报") # 没有报告撰写节点:只能交给 Copilot
script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": None}}]))
wf = await create(client, graph)
out = await assist(client, wf)
assert out["assist"]["ok"] is False and "assist" not in out["applied"]
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
assert "契约" in rejected["reason"] and "降低" in rejected["reason"]
assert out["graph"] == graph and out["ok"] is False
async def test_a_fix_that_makes_errors_grow_is_refused(client, monkeypatch):
script(monkeypatch, done([
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
{"op": "update_node", "id": "card", "config": {"metrics": [{"id": "gmv"}]}}, # 口径卡的表达式被写丢了
]))
wf = await create(client, two_exits())
out = await assist(client, wf)
assert out["assist"]["ok"] is False
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
assert "新的问题" in rejected["reason"] and "expression" in rejected["reason"]
assert out["graph"] == two_exits()
async def test_removing_a_node_is_refused_even_if_it_helps(client, monkeypatch):
script(monkeypatch, done([
{"op": "remove_node", "id": "raw"},
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
]))
wf = await create(client, two_exits())
out = await assist(client, wf)
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
assert "删" in rejected["reason"] and out["graph"] == two_exits()
async def test_questions_are_passed_on_without_changes(client, monkeypatch):
script(monkeypatch, [
{"op": "plan", "summary": "要先问清楚"},
{"op": "question", "node_id": "done", "text": "两个出口里,哪个是要出具的?「出具」还是「原始数」?"},
{"op": "question", "node_id": "card", "text": "required 要包括哪些指标?候选:gmv"},
{"op": "question", "text": " "},
{"op": "done", "explanation": "等你决定出具的是哪个出口"},
])
wf = await create(client, two_exits())
out = await assist(client, wf)
# 一句一条的文字,前面标上是哪个节点的事(句子里已经点了名的不重复标)
assert out["assist"] == {"ok": False, "summary": "等你决定出具的是哪个出口", "questions": [
"两个出口里,哪个是要出具的?「出具」还是「原始数」?", "「周报口径卡」:required 要包括哪些指标?候选:gmv"]}
assert out["rejected"] == [] and out["graph"] == two_exits()
async def test_deterministic_fixes_run_first_and_copilot_sees_what_is_left(client, monkeypatch):
graph = two_exits()
graph["nodes"].insert(2, node("ask", "agent", "查数员", tools=["db_query__shop"], approval="never",
prompt="用 db_query__shop 查"))
graph["edges"] += [{"source": "start", "target": "ask"}, {"source": "ask", "target": "card"}]
model = script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]))
wf = await create(client, graph)
out = await assist(client, wf, apply=["governed.agent_approval_never:ask"])
assert out["applied"] == ["governed.agent_approval_never:ask", "assist"] and out["ok"] is True
human = next(text for role, text in model.calls[0] if role == "human")
assert "的审批策略是「全部自动放行」" not in human and '"approval": "dangerous"' in human
async def test_nothing_left_means_no_model_call(client, monkeypatch):
model = script(monkeypatch, [])
graph = two_exits()
graph["nodes"][4]["data"]["config"]["contract"] = CONTRACT
wf = await create(client, graph)
out = await assist(client, wf)
assert out["assist"]["ok"] is True and model.calls == []
async def test_an_unconfigured_model_is_said_plainly(client, monkeypatch):
import app.api.copilot as copilot
from app.providers.factory import ProviderNotConfigured
async def _nothing(*_a, **_k):
raise ProviderNotConfigured("没有可用的 provider")
monkeypatch.setattr(copilot, "get_chat_model", _nothing)
wf = await create(client, two_exits())
out = await assist(client, wf)
assert out["assist"]["ok"] is False and "设置" in out["assist"]["summary"]
assert out["rejected"] == []
async def test_blocking_issues_only_add_the_gate_when_asked():
from app.api.copilot import _blocking_issues
graph = two_exits()
nodes = {n["id"]: n for n in graph["nodes"]}
assert _blocking_issues(nodes, graph["edges"]) == []
gated = _blocking_issues(nodes, graph["edges"], level="governed")
assert [i["code"] for i in gated] == ["governed.no_contract"]
assert _blocking_issues(nodes, graph["edges"], level="published") == []
def guarded() -> dict:
"""受管图里只剩一处 error:协作团队节点。团队后面还有一个「人工把关」节点。"""
nodes = [
node("start", "input", "周期"),
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}),
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
node("team", "supervisor", "复核团队", goal="复核", agents=[{"name": "checker", "tools": []}]),
node("gate", "human", "人工把关", prompt="确认数字"),
node("write", "report", "报告撰写", instructions="写周报",
numbers="strict", on_violation="fail", claims="require_citation"),
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract=CONTRACT),
]
edges = [{"id": f"e{i}", "source": a, "target": b} for i, (a, b) in enumerate(
[("start", "fetch"), ("fetch", "card"), ("card", "team"), ("team", "gate"), ("gate", "write"),
("write", "done")])]
return {"nodes": nodes, "edges": edges}
@pytest.mark.parametrize("ops, word", [
# 同一个 id 换了类型:团队换成模型调用,顺手把人工把关换成整形——等于删了重建
([{"op": "add_node", "node": {"id": "team", "type": "llm", "label": "复核团队", "config": {"prompt": "复核"}}},
{"op": "add_node", "node": {"id": "gate", "type": "transform", "label": "人工把关",
"config": {"mode": "expression", "expression": "vars"}}}], "人工把关"),
# 类型没变,也是整个覆盖掉旧节点(配置、位置全丢)
([{"op": "add_node", "node": {"id": "gate", "type": "human", "label": "人工把关", "config": {"prompt": "随便"}}},
{"op": "add_node", "node": {"id": "team", "type": "llm", "label": "复核团队", "config": {"prompt": "复核"}}}],
"覆盖"),
])
async def test_overwriting_an_existing_node_with_add_node_is_refused(client, monkeypatch, ops, word):
graph = guarded()
script(monkeypatch, done(ops))
wf = await create(client, graph)
before = await rows(wf)
out = await assist(client, wf)
assert out["assist"]["ok"] is False and "assist" not in out["applied"] and out["ok"] is False
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
assert "降低" in rejected["reason"] and word in rejected["reason"], rejected
assert out["graph"] == graph and out["changes"] == [] and out["ops"] == []
assert [(n["id"], n["type"]) for n in out["graph"]["nodes"]] == [(n["id"], n["type"]) for n in graph["nodes"]]
assert await rows(wf) == before
async def test_renaming_a_node_while_fixing_another_error_is_adopted(client, monkeypatch):
"""改了名的节点上没修好的 error 文案跟着变了,但还是同一处问题:不算冒出新问题。"""
graph = guarded()
graph["nodes"][6]["data"]["config"].pop("contract") # 两处 error:协作团队、没有带契约的出口
script(monkeypatch, done([
{"op": "update_node", "id": "team", "label": "复核团队(待拆成固定步骤)"},
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
]))
wf = await create(client, graph)
out = await assist(client, wf)
assert out["applied"] == ["assist"] and out["assist"]["ok"] is True, out["rejected"]
assert [(i["code"], i["node_id"]) for i in out["remaining"] if i["level"] == "error"] == [
("governed.supervisor", "team")]
assert "复核团队(待拆成固定步骤)" in out["remaining"][0]["message"] or any(
"复核团队(待拆成固定步骤)" in i["message"] for i in out["remaining"])
assert {(c["node_id"], c["field"]) for c in out["changes"]} == {("team", "label"), ("done", "contract")}