mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 10:01:58 +00:00
ThearchyHelios
d071bf119f
一、表结构、实体与引文
- 数据源工具查库时顺带存一份表结构快照（schema_snapshot），和查询快照一起进台账、进封存；
台账里的查询记下 SQL 用到的表
- 报告里的表名、字段名可以点：反引号里的名字一律核对，正文里的裸名字只在明显是标识符时才自动链接，
常见英文单词不链接；点开看到它出现在哪次查询、类型、所属的表，快照被截断时写明
- 找不到的名字标成可疑实体：在快照、SQL、结果列里都没有的记「可能是编造的名字」并给出最接近的 3 个；
快照被截断、核对不了的记「核对不了」，不冤枉成编造
- 检索引文 [[q:K1|原文]]：原文在检索快照的命中片段里逐字出现（空白归一化后）才算有出处，点开高亮原文位置
- 自动链接的名字不算句子的引用，只有写作者写的引用才算
二、结论句与出具
- 报告节点新增 claims：off（默认，保持原样）或 require_citation（没有引用的结论句计入缺口）
- 受管级别的正式运行里，可疑实体和没有引用的结论句按「有缺口」降档，不拦截；探索运行和已发布级别只标注
- 契约的 claims 和报告节点的设置取更严的那个；受管正式运行里写 ignore 也按降档处理，并给出警告
- 出具横幅除了数字，也写出可疑实体和没有引用的结论句
三、受管门禁 G1–G5（只看图和配置）
- G1 带契约的出口，成果字段的文字只能来自报告撰写节点
- G2 模型写的文字不绕过报告撰写节点直接流进出口
- G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation
- G4 口径卡的输入不来自计算角色的沙箱代码
- G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields
- 受管级别是错误、挡住发布；已发布级别只给警告；已经发布的版本照常运行，下次发布时按新规则检查
- 每条规则都接进发布前自动修复（一键修复 / 给选项 / 交给 Copilot）；禁止的改动补上关 cite_fields、
删 output_schema、放宽契约 claims；Copilot 兜底不许自己把代码节点标成 source、不许放宽报告节点
- 发布弹窗里选「交给 Copilot」时明确写「正在交给 Copilot」
四、旧运行与审计
- 没有契约的旧答案：数字和已封存的查询单元格、口径卡值按数值比对，每个数最多 3 个候选，
默认折叠，写明「猜测的来源，不能当证据」
- 有契约的旧运行：候选只从已封存的口径卡里找，人为插进工件表的行不当证据
- 证据审计：GET /api/runs/{id}/evidence/audit 按「无证据 / 可疑实体 / 有出处 / 旧运行猜测」分组，
可导出 JSON 和 CSV（带封存状态）；记录页和画布上的报告节点显示「有出处 N · 无证据 M」
五、SQL 守卫修正（安全）
- 去注释改成单遍扫描，认得字符串和各方言的引号（单引号转义、MySQL 反斜杠与 #、PostgreSQL 的 $$ 与嵌套注释、
Oracle 的 q'[…]'、SQLite 的方括号）；以前字符串里的 -- 会被当成注释，SELECT '--'; DROP … 这类写法能骗过多语句拦截
- 看不到服务器设置时每种读法都扫一遍，读法之间切分不一致就拒绝，任何一种读法是写操作就按写操作处理
- 执行查询和工具危险判定都按数据源的方言检查
六、其他
- 修掉检查脚本里画布检查的一处竞态，并行估时按实测更新
263 lines
11 KiB
Python
263 lines
11 KiB
Python
"""「危险工具默认需要人工确认」是工具审批策略的全局默认，而不是一个摆设。
|
||
|
||
以前审批只看节点上的 approval（缺省 dangerous），设置页的开关勾不勾都一样——
|
||
取消勾选以为关掉了审批，勾上以为全局开了，两种理解都不对。
|
||
|
||
规则：节点自己配了就用节点的；没配时取全局设置（勾上 = dangerous，取消 = never）。
|
||
正式运行不吃"取消"：它跑的是封存的发布版，受管门禁要求危险工具至少人工确认，
|
||
一个全局开关不能把它悄悄降掉。
|
||
|
||
另外守一件连带的事：记忆域和知识库以前只在第一次 _drive 时传进去，审批恢复、
|
||
接着跑时回落成 "default"——同一次运行前后两段查的不是同一个库。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import sqlite3
|
||
import uuid
|
||
|
||
import pytest
|
||
from sqlalchemy import select
|
||
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import DataSource, Run, RunEvent, Setting, Workflow
|
||
from app.engine import compiler
|
||
from app.engine.runner import run_manager
|
||
from app.engine.schema import NodeType
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def confirm_setting():
|
||
"""把设置里的 run 组改成给定值；测试结束删掉，别影响别的测试。"""
|
||
async def put(value: bool) -> None:
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Setting, "run")
|
||
if row:
|
||
row.value = {**row.value, "confirm_dangerous_tools": value}
|
||
else:
|
||
session.add(Setting(key="run", value={"confirm_dangerous_tools": value}))
|
||
await session.commit()
|
||
|
||
yield put
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Setting, "run")
|
||
if row:
|
||
await session.delete(row)
|
||
await session.commit()
|
||
|
||
|
||
@pytest.fixture
|
||
async def writable(tmp_path):
|
||
path = tmp_path / "w.db"
|
||
c = sqlite3.connect(path)
|
||
c.execute("CREATE TABLE t (x INTEGER)")
|
||
c.executemany("INSERT INTO t VALUES (?)", [(i,) for i in range(5)])
|
||
c.commit()
|
||
c.close()
|
||
name = f"w{uuid.uuid4().hex[:6]}"
|
||
async with SessionLocal() as session:
|
||
session.add(DataSource(name=name, kind="sqlite", database=str(path), readonly=False,
|
||
description="可写测试库", options={}, schema_cache={}, enabled=True))
|
||
await session.commit()
|
||
|
||
def count() -> int:
|
||
conn = sqlite3.connect(path)
|
||
try:
|
||
return conn.execute("SELECT COUNT(*) FROM t").fetchone()[0]
|
||
finally:
|
||
conn.close()
|
||
|
||
return f"db_query__{name}", count
|
||
|
||
|
||
def node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
||
"data": {"label": nid, "config": config}}
|
||
|
||
|
||
def chain(*nodes):
|
||
return {"nodes": list(nodes),
|
||
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||
|
||
|
||
def _delete_graph(tool: str, **approval):
|
||
return chain(
|
||
node("start", "input", fields=[{"name": "question"}]),
|
||
node("q", "tool", tool=tool, args={"sql": "DELETE FROM t WHERE x = 0"}, **approval),
|
||
node("out", "output", fields=[{"name": "结果", "value": "{{ nodes.q }}"}]),
|
||
)
|
||
|
||
|
||
async def _settle(run_id: str) -> Run:
|
||
for _ in range(400):
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row.status in ("interrupted", "succeeded", "failed", "cancelled"):
|
||
return row
|
||
await asyncio.sleep(0.03)
|
||
raise AssertionError("等超时了")
|
||
|
||
|
||
async def test_unchecked_setting_lets_unconfigured_nodes_run(writable, confirm_setting):
|
||
tool, count = writable
|
||
await confirm_setting(False)
|
||
run = await run_manager.start(graph=_delete_graph(tool), input_payload={"question": "q"})
|
||
row = await _settle(run.id)
|
||
assert row.status == "succeeded", f"全局关了审批，节点没配，却还是停下来了：{row.status}"
|
||
assert count() == 4
|
||
|
||
|
||
async def test_checked_setting_keeps_asking(writable, confirm_setting):
|
||
tool, count = writable
|
||
await confirm_setting(True)
|
||
run = await run_manager.start(graph=_delete_graph(tool), input_payload={"question": "q"})
|
||
row = await _settle(run.id)
|
||
assert row.status == "interrupted"
|
||
assert count() == 5
|
||
|
||
|
||
async def test_what_the_node_says_wins_over_the_setting(writable, confirm_setting):
|
||
tool, count = writable
|
||
await confirm_setting(False)
|
||
run = await run_manager.start(
|
||
graph=_delete_graph(tool, approval="dangerous"), input_payload={"question": "q"})
|
||
row = await _settle(run.id)
|
||
assert row.status == "interrupted", "节点明说了要审批，全局开关不该盖过它"
|
||
assert count() == 5
|
||
|
||
|
||
async def test_governed_formal_runs_do_not_take_the_unchecked_setting(writable, confirm_setting):
|
||
tool, count = writable
|
||
await confirm_setting(False)
|
||
graph = _delete_graph(tool)
|
||
async with SessionLocal() as session:
|
||
wf = Workflow(name=f"受管-{uuid.uuid4().hex[:4]}", graph=graph, status="governed",
|
||
published_version=1)
|
||
session.add(wf)
|
||
await session.commit()
|
||
run = await run_manager.start(graph=graph, input_payload={"question": "q"},
|
||
workflow_id=wf.id, run_class="formal", version=1)
|
||
row = await _settle(run.id)
|
||
assert row.status == "interrupted", "受管门禁要独立生效"
|
||
assert count() == 5
|
||
|
||
|
||
def _governable(tool: str):
|
||
"""过得了受管门禁的最小图：危险工具节点不配 approval，后面接口径卡和出具契约。"""
|
||
return chain(
|
||
node("start", "input", fields=[{"name": "question"}]),
|
||
node("q", "tool", tool=tool, args={"sql": "DELETE FROM t WHERE x = 0"}),
|
||
node("caliber", "metrics", caliber="删除口径", caliber_version="v1", assign_to="m",
|
||
metrics=[{"id": "done", "name": "已执行", "unit": "次", "expression": "1"}]),
|
||
# 受管契约要核对报告撰写节点的文档（门禁 G1），叙述模式发不成受管
|
||
node("write", "report", instructions="写执行结果",
|
||
numbers="strict", on_violation="fail", claims="require_citation"),
|
||
node("out", "output", fields=[{"name": "结果", "value": "{{ nodes.q }}"}],
|
||
contract={"metrics_from": ["caliber"], "report_from": "write", "required": ["done"], "strict": True}),
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize("level", ["governed", "published"])
|
||
async def test_formal_runs_keep_asking_after_the_canvas_is_edited(
|
||
writable, confirm_setting, level
|
||
):
|
||
"""发布之后在画布上再存一次，status 就退回 draft，published_version 仍指着发布的那一版。
|
||
|
||
以前下限看的是 status == "governed"：受管模板只要被编辑过，它的正式运行就跟着
|
||
全局开关降到「全部自动放行」，DELETE 不经人手直接执行。现在正式运行一律不吃
|
||
"取消"——跑的是封存的发布版，它的审批行为不该取决于一个随时能改的偏好。
|
||
"""
|
||
from httpx import ASGITransport, AsyncClient
|
||
|
||
from app.main import app
|
||
|
||
tool, count = writable
|
||
await confirm_setting(False)
|
||
graph = _governable(tool)
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
wf = (await c.post("/api/workflows", json={"name": f"受管-{uuid.uuid4().hex[:4]}",
|
||
"graph": graph})).json()
|
||
published = (await c.post(f"/api/workflows/{wf['id']}/publish",
|
||
json={"level": level})).json()
|
||
assert published["ok"], published
|
||
edited = {**graph, "nodes": [*graph["nodes"][:-1],
|
||
{**graph["nodes"][-1], "data": {
|
||
**graph["nodes"][-1]["data"], "label": "改过的出口"}}]}
|
||
after = (await c.patch(f"/api/workflows/{wf['id']}", json={"graph": edited})).json()
|
||
assert after["status"] == "draft" and after["published_version"] == 1
|
||
r = await c.post("/api/runs", json={"workflow_id": wf["id"], "run_class": "formal",
|
||
"input": {"question": "q"}})
|
||
assert r.status_code == 201, r.text
|
||
row = await _settle(r.json()["id"])
|
||
assert row.status == "interrupted", f"编辑过画布后，正式运行跳过了审批：{row.status}"
|
||
assert row.approval_default == "dangerous"
|
||
assert count() == 5
|
||
|
||
|
||
async def test_exploratory_runs_of_a_governed_workflow_follow_the_setting(
|
||
writable, confirm_setting
|
||
):
|
||
"""下限只管正式运行。探索运行照旧听全局开关，不然这个开关又成了摆设。"""
|
||
tool, count = writable
|
||
await confirm_setting(False)
|
||
graph = _delete_graph(tool)
|
||
async with SessionLocal() as session:
|
||
wf = Workflow(name=f"受管-{uuid.uuid4().hex[:4]}", graph=graph, status="governed",
|
||
published_version=1)
|
||
session.add(wf)
|
||
await session.commit()
|
||
run = await run_manager.start(graph=graph, input_payload={"question": "q"}, workflow_id=wf.id)
|
||
row = await _settle(run.id)
|
||
assert row.status == "succeeded", row.status
|
||
assert count() == 4
|
||
|
||
|
||
async def test_effective_values_are_sealed_into_the_event_stream(confirm_setting):
|
||
await confirm_setting(False)
|
||
graph = chain(node("start", "input"), node("out", "output"))
|
||
run = await run_manager.start(graph=graph, input_payload={}, memory_scope="ops",
|
||
collection="manuals")
|
||
await _settle(run.id)
|
||
async with SessionLocal() as session:
|
||
started = (await session.execute(
|
||
select(RunEvent).where(RunEvent.run_id == run.id, RunEvent.type == "run.started")
|
||
)).scalars().first()
|
||
assert started.data["memory_scope"] == "ops"
|
||
assert started.data["collection"] == "manuals"
|
||
assert started.data["approval_default"] == "never"
|
||
|
||
|
||
async def test_resume_keeps_the_scope_the_run_started_with(monkeypatch):
|
||
seen: list[tuple[str, str]] = []
|
||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||
|
||
async def spy(state, ctx):
|
||
seen.append((ctx.run.memory_scope, ctx.run.collection))
|
||
return await original(state, ctx)
|
||
|
||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, spy)
|
||
graph = chain(
|
||
node("start", "input"),
|
||
node("before", "transform", mode="template", template="前"),
|
||
node("gate", "human", mode="approve", title="看一眼"),
|
||
node("after", "transform", mode="template", template="后"),
|
||
node("out", "output"),
|
||
)
|
||
run = await run_manager.start(graph=graph, input_payload={}, memory_scope="ops",
|
||
collection="manuals")
|
||
assert (await _settle(run.id)).status == "interrupted"
|
||
await run_manager.resume(run.id, {"approved": True})
|
||
for _ in range(200):
|
||
if len(seen) == 2:
|
||
break
|
||
await asyncio.sleep(0.03)
|
||
assert seen == [("ops", "manuals"), ("ops", "manuals")], "恢复之后换了个库在查"
|