mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 01:01:58 +00:00
d071bf119f
一、表结构、实体与引文
- 数据源工具查库时顺带存一份表结构快照(schema_snapshot),和查询快照一起进台账、进封存;
台账里的查询记下 SQL 用到的表
- 报告里的表名、字段名可以点:反引号里的名字一律核对,正文里的裸名字只在明显是标识符时才自动链接,
常见英文单词不链接;点开看到它出现在哪次查询、类型、所属的表,快照被截断时写明
- 找不到的名字标成可疑实体:在快照、SQL、结果列里都没有的记「可能是编造的名字」并给出最接近的 3 个;
快照被截断、核对不了的记「核对不了」,不冤枉成编造
- 检索引文 [[q:K1|原文]]:原文在检索快照的命中片段里逐字出现(空白归一化后)才算有出处,点开高亮原文位置
- 自动链接的名字不算句子的引用,只有写作者写的引用才算
二、结论句与出具
- 报告节点新增 claims:off(默认,保持原样)或 require_citation(没有引用的结论句计入缺口)
- 受管级别的正式运行里,可疑实体和没有引用的结论句按「有缺口」降档,不拦截;探索运行和已发布级别只标注
- 契约的 claims 和报告节点的设置取更严的那个;受管正式运行里写 ignore 也按降档处理,并给出警告
- 出具横幅除了数字,也写出可疑实体和没有引用的结论句
三、受管门禁 G1–G5(只看图和配置)
- G1 带契约的出口,成果字段的文字只能来自报告撰写节点
- G2 模型写的文字不绕过报告撰写节点直接流进出口
- G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation
- G4 口径卡的输入不来自计算角色的沙箱代码
- G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields
- 受管级别是错误、挡住发布;已发布级别只给警告;已经发布的版本照常运行,下次发布时按新规则检查
- 每条规则都接进发布前自动修复(一键修复 / 给选项 / 交给 Copilot);禁止的改动补上关 cite_fields、
删 output_schema、放宽契约 claims;Copilot 兜底不许自己把代码节点标成 source、不许放宽报告节点
- 发布弹窗里选「交给 Copilot」时明确写「正在交给 Copilot」
四、旧运行与审计
- 没有契约的旧答案:数字和已封存的查询单元格、口径卡值按数值比对,每个数最多 3 个候选,
默认折叠,写明「猜测的来源,不能当证据」
- 有契约的旧运行:候选只从已封存的口径卡里找,人为插进工件表的行不当证据
- 证据审计:GET /api/runs/{id}/evidence/audit 按「无证据 / 可疑实体 / 有出处 / 旧运行猜测」分组,
可导出 JSON 和 CSV(带封存状态);记录页和画布上的报告节点显示「有出处 N · 无证据 M」
五、SQL 守卫修正(安全)
- 去注释改成单遍扫描,认得字符串和各方言的引号(单引号转义、MySQL 反斜杠与 #、PostgreSQL 的 $$ 与嵌套注释、
Oracle 的 q'[…]'、SQLite 的方括号);以前字符串里的 -- 会被当成注释,SELECT '--'; DROP … 这类写法能骗过多语句拦截
- 看不到服务器设置时每种读法都扫一遍,读法之间切分不一致就拒绝,任何一种读法是写操作就按写操作处理
- 执行查询和工具危险判定都按数据源的方言检查
六、其他
- 修掉检查脚本里画布检查的一处竞态,并行估时按实测更新
281 lines
14 KiB
Python
281 lines
14 KiB
Python
"""自动修复的 Copilot 兜底(autofix 的 assist: true)。
|
|
|
|
确定性修复修不了的 error 交给 Copilot:模型给剧本,看采纳规则——
|
|
- 正常的修复(error 变少、没有新 error、不降低要求)被采纳,只是预览,不落库
|
|
- 试图删契约的被拒
|
|
- 让错误变多的被拒
|
|
- 要人拿主意的放进 questions,不硬改
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Workflow, WorkflowVersion
|
|
from app.main import app
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def node(nid, ntype, label=None, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 40, "y": 80}, "data": {"label": label or nid, "config": config}}
|
|
|
|
|
|
CONTRACT = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}
|
|
|
|
|
|
def two_exits() -> dict:
|
|
"""受管模板没有带契约的出口,而且出口有两个:该给哪个写契约说不准,确定性修复只能交给 Copilot。"""
|
|
nodes = [
|
|
node("start", "input", "周期"),
|
|
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}),
|
|
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
|
|
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
|
node("write", "report", "报告撰写", instructions="写周报",
|
|
numbers="strict", on_violation="fail", claims="require_citation"),
|
|
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
|
node("raw", "output", "原始数", fields=[{"name": "数", "value": "{{ nodes.fetch }}"}]),
|
|
]
|
|
edges = [{"id": f"e{i}", "source": a, "target": b} for i, (a, b) in enumerate(
|
|
[("start", "fetch"), ("fetch", "card"), ("card", "write"), ("write", "done"), ("fetch", "raw")])]
|
|
return {"nodes": nodes, "edges": edges, "viewport": {"x": 3, "y": 4, "zoom": 1.2}}
|
|
|
|
|
|
class Scripted:
|
|
"""按剧本吐操作流的模型;记下每次收到的请求。"""
|
|
|
|
def __init__(self, ops: list[dict]) -> None:
|
|
self.lines = [json.dumps(o, ensure_ascii=False) for o in ops]
|
|
self.calls: list[list] = []
|
|
|
|
async def astream(self, messages):
|
|
from langchain_core.messages import AIMessageChunk
|
|
|
|
self.calls.append(messages)
|
|
for line in self.lines:
|
|
yield AIMessageChunk(content=line + "\n")
|
|
|
|
|
|
def script(monkeypatch, ops: list[dict]) -> Scripted:
|
|
import app.api.copilot as copilot
|
|
|
|
model = Scripted(ops)
|
|
|
|
async def _model(*_a, **_k):
|
|
return model, "mock-fast"
|
|
|
|
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
|
return model
|
|
|
|
|
|
async def create(client, graph: dict) -> str:
|
|
return (await client.post("/api/workflows", json={"name": "周报-兜底", "graph": graph})).json()["id"]
|
|
|
|
|
|
async def rows(wf: str) -> list:
|
|
async with SessionLocal() as session:
|
|
w = await session.get(Workflow, wf)
|
|
vs = (await session.execute(select(WorkflowVersion).where(WorkflowVersion.workflow_id == wf))).scalars()
|
|
return [{c.name: getattr(w, c.name) for c in Workflow.__table__.columns},
|
|
[{c.name: getattr(v, c.name) for c in WorkflowVersion.__table__.columns} for v in vs]]
|
|
|
|
|
|
async def assist(client, wf: str, **extra) -> dict:
|
|
r = await client.post(f"/api/workflows/{wf}/autofix", json={"level": "governed", "apply": [], "assist": True,
|
|
**extra})
|
|
assert r.status_code == 200, r.text
|
|
return r.json()
|
|
|
|
|
|
def done(ops: list[dict]) -> list[dict]:
|
|
return [{"op": "plan", "summary": "给出具节点补契约"}, *ops, {"op": "done", "explanation": "补上了出具契约"}]
|
|
|
|
|
|
async def test_a_sound_fix_is_adopted_as_a_preview(client, monkeypatch):
|
|
model = script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]))
|
|
wf = await create(client, two_exits())
|
|
before = await rows(wf)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is True and out["assist"]["summary"] == "补上了出具契约"
|
|
assert out["applied"] == ["assist"] and out["ok"] is True, out
|
|
fixed = out["graph"]
|
|
assert next(n for n in fixed["nodes"] if n["id"] == "done")["data"]["config"]["contract"] == CONTRACT
|
|
assert fixed["viewport"] == two_exits()["viewport"] and [e["id"] for e in fixed["edges"]] == [
|
|
e["id"] for e in two_exits()["edges"]]
|
|
assert out["ops"] == [{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]
|
|
assert [(c["fix_id"], c["node_id"], c["field"]) for c in out["changes"]] == [("assist", "done", "contract")]
|
|
assert await rows(wf) == before # 只是预览
|
|
# 交给模型的请求里有剩下的问题和不许降低要求、不许替人选的约定
|
|
human = next(text for role, text in model.calls[0] if role == "human")
|
|
assert "受管模板至少要有一个「成果 / 出具」节点声明出具契约" in human
|
|
assert "不许降低要求" in human and "不替人选" in human and "question" in human
|
|
|
|
|
|
async def test_deleting_the_contract_is_refused(client, monkeypatch):
|
|
graph = two_exits()
|
|
graph["nodes"][4]["data"]["config"]["contract"] = {**CONTRACT, "report_from": "fetch"}
|
|
graph["nodes"][3] = node("write", "llm", "写周报", prompt="写周报") # 没有报告撰写节点:只能交给 Copilot
|
|
script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": None}}]))
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False and "assist" not in out["applied"]
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "契约" in rejected["reason"] and "降低" in rejected["reason"]
|
|
assert out["graph"] == graph and out["ok"] is False
|
|
|
|
|
|
async def test_a_fix_that_makes_errors_grow_is_refused(client, monkeypatch):
|
|
script(monkeypatch, done([
|
|
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
|
|
{"op": "update_node", "id": "card", "config": {"metrics": [{"id": "gmv"}]}}, # 口径卡的表达式被写丢了
|
|
]))
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "新的问题" in rejected["reason"] and "expression" in rejected["reason"]
|
|
assert out["graph"] == two_exits()
|
|
|
|
|
|
async def test_removing_a_node_is_refused_even_if_it_helps(client, monkeypatch):
|
|
script(monkeypatch, done([
|
|
{"op": "remove_node", "id": "raw"},
|
|
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
|
|
]))
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "删" in rejected["reason"] and out["graph"] == two_exits()
|
|
|
|
|
|
async def test_questions_are_passed_on_without_changes(client, monkeypatch):
|
|
script(monkeypatch, [
|
|
{"op": "plan", "summary": "要先问清楚"},
|
|
{"op": "question", "node_id": "done", "text": "两个出口里,哪个是要出具的?「出具」还是「原始数」?"},
|
|
{"op": "question", "node_id": "card", "text": "required 要包括哪些指标?候选:gmv"},
|
|
{"op": "question", "text": " "},
|
|
{"op": "done", "explanation": "等你决定出具的是哪个出口"},
|
|
])
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
# 一句一条的文字,前面标上是哪个节点的事(句子里已经点了名的不重复标)
|
|
assert out["assist"] == {"ok": False, "summary": "等你决定出具的是哪个出口", "questions": [
|
|
"两个出口里,哪个是要出具的?「出具」还是「原始数」?", "「周报口径卡」:required 要包括哪些指标?候选:gmv"]}
|
|
assert out["rejected"] == [] and out["graph"] == two_exits()
|
|
|
|
|
|
async def test_deterministic_fixes_run_first_and_copilot_sees_what_is_left(client, monkeypatch):
|
|
graph = two_exits()
|
|
graph["nodes"].insert(2, node("ask", "agent", "查数员", tools=["db_query__shop"], approval="never",
|
|
prompt="用 db_query__shop 查"))
|
|
graph["edges"] += [{"source": "start", "target": "ask"}, {"source": "ask", "target": "card"}]
|
|
model = script(monkeypatch, done([{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}}]))
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf, apply=["governed.agent_approval_never:ask"])
|
|
assert out["applied"] == ["governed.agent_approval_never:ask", "assist"] and out["ok"] is True
|
|
human = next(text for role, text in model.calls[0] if role == "human")
|
|
assert "的审批策略是「全部自动放行」" not in human and '"approval": "dangerous"' in human
|
|
|
|
|
|
async def test_nothing_left_means_no_model_call(client, monkeypatch):
|
|
model = script(monkeypatch, [])
|
|
graph = two_exits()
|
|
graph["nodes"][4]["data"]["config"]["contract"] = CONTRACT
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is True and model.calls == []
|
|
|
|
|
|
async def test_an_unconfigured_model_is_said_plainly(client, monkeypatch):
|
|
import app.api.copilot as copilot
|
|
from app.providers.factory import ProviderNotConfigured
|
|
|
|
async def _nothing(*_a, **_k):
|
|
raise ProviderNotConfigured("没有可用的 provider")
|
|
|
|
monkeypatch.setattr(copilot, "get_chat_model", _nothing)
|
|
wf = await create(client, two_exits())
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False and "设置" in out["assist"]["summary"]
|
|
assert out["rejected"] == []
|
|
|
|
|
|
async def test_blocking_issues_only_add_the_gate_when_asked():
|
|
from app.api.copilot import _blocking_issues
|
|
|
|
graph = two_exits()
|
|
nodes = {n["id"]: n for n in graph["nodes"]}
|
|
assert _blocking_issues(nodes, graph["edges"]) == []
|
|
gated = _blocking_issues(nodes, graph["edges"], level="governed")
|
|
assert [i["code"] for i in gated] == ["governed.no_contract"]
|
|
assert _blocking_issues(nodes, graph["edges"], level="published") == []
|
|
|
|
|
|
def guarded() -> dict:
|
|
"""受管图里只剩一处 error:协作团队节点。团队后面还有一个「人工把关」节点。"""
|
|
nodes = [
|
|
node("start", "input", "周期"),
|
|
node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}),
|
|
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
|
|
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
|
node("team", "supervisor", "复核团队", goal="复核", agents=[{"name": "checker", "tools": []}]),
|
|
node("gate", "human", "人工把关", prompt="确认数字"),
|
|
node("write", "report", "报告撰写", instructions="写周报",
|
|
numbers="strict", on_violation="fail", claims="require_citation"),
|
|
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract=CONTRACT),
|
|
]
|
|
edges = [{"id": f"e{i}", "source": a, "target": b} for i, (a, b) in enumerate(
|
|
[("start", "fetch"), ("fetch", "card"), ("card", "team"), ("team", "gate"), ("gate", "write"),
|
|
("write", "done")])]
|
|
return {"nodes": nodes, "edges": edges}
|
|
|
|
|
|
@pytest.mark.parametrize("ops, word", [
|
|
# 同一个 id 换了类型:团队换成模型调用,顺手把人工把关换成整形——等于删了重建
|
|
([{"op": "add_node", "node": {"id": "team", "type": "llm", "label": "复核团队", "config": {"prompt": "复核"}}},
|
|
{"op": "add_node", "node": {"id": "gate", "type": "transform", "label": "人工把关",
|
|
"config": {"mode": "expression", "expression": "vars"}}}], "人工把关"),
|
|
# 类型没变,也是整个覆盖掉旧节点(配置、位置全丢)
|
|
([{"op": "add_node", "node": {"id": "gate", "type": "human", "label": "人工把关", "config": {"prompt": "随便"}}},
|
|
{"op": "add_node", "node": {"id": "team", "type": "llm", "label": "复核团队", "config": {"prompt": "复核"}}}],
|
|
"覆盖"),
|
|
])
|
|
async def test_overwriting_an_existing_node_with_add_node_is_refused(client, monkeypatch, ops, word):
|
|
graph = guarded()
|
|
script(monkeypatch, done(ops))
|
|
wf = await create(client, graph)
|
|
before = await rows(wf)
|
|
out = await assist(client, wf)
|
|
assert out["assist"]["ok"] is False and "assist" not in out["applied"] and out["ok"] is False
|
|
[rejected] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
|
assert "降低" in rejected["reason"] and word in rejected["reason"], rejected
|
|
assert out["graph"] == graph and out["changes"] == [] and out["ops"] == []
|
|
assert [(n["id"], n["type"]) for n in out["graph"]["nodes"]] == [(n["id"], n["type"]) for n in graph["nodes"]]
|
|
assert await rows(wf) == before
|
|
|
|
|
|
async def test_renaming_a_node_while_fixing_another_error_is_adopted(client, monkeypatch):
|
|
"""改了名的节点上没修好的 error 文案跟着变了,但还是同一处问题:不算冒出新问题。"""
|
|
graph = guarded()
|
|
graph["nodes"][6]["data"]["config"].pop("contract") # 两处 error:协作团队、没有带契约的出口
|
|
script(monkeypatch, done([
|
|
{"op": "update_node", "id": "team", "label": "复核团队(待拆成固定步骤)"},
|
|
{"op": "update_node", "id": "done", "config": {"contract": CONTRACT}},
|
|
]))
|
|
wf = await create(client, graph)
|
|
out = await assist(client, wf)
|
|
assert out["applied"] == ["assist"] and out["assist"]["ok"] is True, out["rejected"]
|
|
assert [(i["code"], i["node_id"]) for i in out["remaining"] if i["level"] == "error"] == [
|
|
("governed.supervisor", "team")]
|
|
assert "复核团队(待拆成固定步骤)" in out["remaining"][0]["message"] or any(
|
|
"复核团队(待拆成固定步骤)" in i["message"] for i in out["remaining"])
|
|
assert {(c["node_id"], c["field"]) for c in out["changes"]} == {("team", "label"), ("done", "contract")}
|