Files
AgentLab/backend/tests/test_guardrails.py
T
ThearchyHelios 050bea4b72 可点击证据第五期:老工作流一键升级为可追溯结构;受管问数据图的 cells 一键选;pytest 并行
一、一键升级(只给预览,不自动保存)

- engine/upgrade.py:确定性改写,纯函数、重复执行结果不变
  - R1 契约的叙述指向模型调用节点 → 换成报告撰写节点(system 和 prompt 合成写作要求,模型配置照搬),契约改成 report_from
  - R2 没有契约、但模型写的文字进了出口,而且上游有取数 → 同样换成报告撰写
  - R3 input → 智能体 → 出口(问数据的典型图)→ 中间插入报告撰写节点,出口字段接报告的正文;
    只改直接相连的出口,经过别的节点读它的出口原样保留并写明原因
  - R4 配了 output_schema 的智能体 → 打开 cite_fields
  - R5 代码节点的产出被口径卡读 → 不改,只建议标成取数,等人确认
  - 一个叙述里混了几个写作者的,不猜,整步不做并写明原因
- POST /api/copilot/upgrade-evidence:返回改动、说明和改完的检查结果;assist 时再交给 Copilot 把纯算术的代码节点
  改写成口径卡表达式,不是纯算术的保留并给警告
- 升级和发布前修复走同一道「不许降低要求」的检查,只多两个由升级点名的例外:模型调用换成报告撰写、R3 改接的那条连线;
  改完不能更糟
- 校验对符合 R1–R4 的旧结构给一条建议「可以升级为可追溯结构」,不算提示、不挡发布
- Copilot 自查:报告撰写节点上游没有任何能引用数字的来源时打回

二、界面

- 问题面板的快速修复「升级为可追溯结构(预览改动)」:逐项写「节点 · 字段:原值 → 新值」,节点类型的变化写成
  「模型调用 → 报告撰写」,新插的节点和改接的连线另外写明;「应用」后走现有的保存,全图默认不丢
- 画布改过、存失败、画布被锁都有明确状态;保存失败后手动存成功会清掉「没存上」的提示,
  画布和预览对不上时「重试保存」不再盲存;校验接口失败时升级块照样在
- 记录页里没有逐段证据的运行给横幅「这次的报告没有逐段证据 → 升级这张图」,跳到编排页并打开升级预览
- 修掉一处旧缺陷:应用修复时只换配置、不换节点类型,模型调用换成报告撰写后存下去的还是模型调用

三、受管门禁:cells 一键选

- 图里没有口径卡、契约又没写 cells: true 的受管图,以前能过门禁、正式运行却每次失败;现在门禁拦下,
  发布弹窗给两个选项:写 cells: true,或交给 Copilot 改成口径卡指标(人亲手选的不算替人打开,Copilot 自己打开照样拦)
- 「修完要更好」的复核不再把改完才冒出来、只能由人拍板的这一处算作改坏:Copilot 补上契约、把 cells 留给人问,
  这一步会被采用,不再被判「冒出新的问题」作废

四、测试

- 开发依赖加上 pytest-xdist(锁文件只多了 pytest-xdist 和 execnet):全量后端测试 -n auto 约 16 秒,原来串行约 2 分钟
- 修掉并行下暴露的一处测试竞态:运行先落终态、再另一个事务封存,测试改成等封存完成再核对
2026-09-29 13:27:27 +08:00

186 lines
7.4 KiB
Python

"""三道"配了、显示了、却没人执行"的护栏。
- max_run_seconds:配置里有,没有任何代码执行它。一个挂住的模型或工具能把运行
和它占着的并发名额一直拖着。
- 模型调用超时:ModelSpec.timeout 从没被赋过值,所有调用都走 SDK 默认的 10 分钟
外加重试。
- manifest 哈希:只写不验,而且是在 run.finished 落库之前算的——清单里恰恰少了
那条宣布"跑完了、成果是什么"的事件。
显示成生效却没生效,比明说没有更危险:它让人以为有东西在兜底。
"""
from __future__ import annotations
import asyncio
import time
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select, update
from app.core.artifact_store import manifest_hash
from app.core.config import settings
from app.core.crypto import encrypt
from app.db.base import SessionLocal
from app.db.models import Provider, Run, RunEvent
from app.engine import compiler
from app.engine.runner import run_manager, verify_manifest
from app.engine.schema import NodeType
from app.main import app
from app.providers.factory import ModelSpec, build_chat_model
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
"data": {"label": nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes),
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
SIMPLE = chain(
node("start", "input", fields=[{"name": "question"}]),
node("t", "transform", mode="template", template="好", assign_to="x"),
node("out", "output", fields=[{"name": "结果", "value": "{{ vars.x }}"}]),
)
async def _run(graph, *, timeout=20.0) -> Run:
run = await run_manager.start(graph=graph, input_payload={"question": "随便"})
deadline = asyncio.get_event_loop().time() + timeout
while asyncio.get_event_loop().time() < deadline:
async with SessionLocal() as session:
row = await session.get(Run, run.id)
# 终态先落库、清单随后另一个事务封上(runner._finalize):只等状态的话,负载一高就会
# 读到「跑完了但还没封」的那一瞬。中断态不封存
if row and (row.status == "interrupted"
or row.status in ("succeeded", "failed", "cancelled") and row.manifest_seq is not None):
return row
await asyncio.sleep(0.05)
raise AssertionError("运行没有按时结束")
# --------------------------------------------------------------------------
# max_run_seconds
# --------------------------------------------------------------------------
async def test_a_run_past_its_time_limit_is_stopped(monkeypatch):
async def stuck(state, ctx):
await asyncio.sleep(30) # 一个不响应的模型或工具
return {}
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, stuck)
monkeypatch.setattr(settings, "max_run_seconds", 1)
started = time.perf_counter()
run = await _run(SIMPLE)
assert run.status == "failed"
assert "1 秒的上限" in (run.error or "")
assert time.perf_counter() - started < 10, "上限没起作用,等到了节点自己结束"
# --------------------------------------------------------------------------
# 模型调用超时
# --------------------------------------------------------------------------
def test_models_get_a_timeout_even_when_the_node_sets_none():
provider = Provider(name="t", kind="anthropic", api_key=encrypt("sk-test"), models=[],
default_model="claude-sonnet-5", enabled=True, extra={})
model = build_chat_model(provider, ModelSpec())
assert model.default_request_timeout == settings.model_timeout_seconds
# 节点显式配了的照旧优先
assert build_chat_model(provider, ModelSpec(timeout=42)).default_request_timeout == 42
# --------------------------------------------------------------------------
# manifest
# --------------------------------------------------------------------------
async def _events(run_id):
async with SessionLocal() as session:
return list((await session.execute(
select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
)).scalars())
async def test_the_seal_covers_run_finished_and_verifies():
run = await _run(SIMPLE)
assert run.status == "succeeded", run.error
events = await _events(run.id)
finished = next(e for e in events if e.type == "run.finished")
assert run.manifest_seq == finished.seq, "清单没有封到 run.finished"
report = await verify_manifest(run.id)
assert report["sealed"] and report["ok"], report
async def test_tampering_with_the_output_is_caught():
"""改成果——以前 run.finished 不在清单里,这一改清单照样对得上。"""
run = await _run(SIMPLE)
async with SessionLocal() as session:
await session.execute(
update(RunEvent)
.where(RunEvent.run_id == run.id, RunEvent.type == "run.finished")
.values(data={"output": {"结果": "被改过的结论"}})
)
await session.commit()
report = await verify_manifest(run.id)
assert report["ok"] is False
async def test_events_appended_after_the_seal_are_not_tampering():
"""封存之后追加的事件(复核批注之类)不在范围内,不该让核对失败。"""
run = await _run(SIMPLE)
async with SessionLocal() as session:
row = await session.get(Run, run.id)
session.add(RunEvent(run_id=run.id, seq=row.manifest_seq + 5, type="log", ts=time.time(),
node_id=None, data={"message": "事后追加的批注"}))
await session.commit()
assert (await verify_manifest(run.id))["ok"] is True
async def test_runs_sealed_before_manifest_seq_existed_verify_by_the_old_cut():
"""老运行的哈希是在 run.finished 落库之前算的,没有 manifest_seq——按当时的口径核。"""
run = await _run(SIMPLE)
rows = [(e.seq, e.type, e.node_id, e.data) for e in await _events(run.id)
if e.type != "run.finished"]
async with SessionLocal() as session:
await session.execute(update(Run).where(Run.id == run.id)
.values(manifest_hash=manifest_hash(rows), manifest_seq=None))
await session.commit()
report = await verify_manifest(run.id)
assert report["ok"] is True and report["legacy"] is True
async def test_a_run_waiting_for_a_human_is_not_sealed_yet():
graph = chain(
node("start", "input", fields=[{"name": "question"}]),
node("h", "human", mode="approve", title="批吗"),
node("out", "output", fields=[]),
)
run = await _run(graph)
assert run.status == "interrupted"
report = await verify_manifest(run.id)
assert report["sealed"] is False and report["ok"] is None
async def test_verify_through_the_api():
run = await _run(SIMPLE)
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
ok = await client.get(f"/api/runs/{run.id}/verify")
assert ok.status_code == 200 and ok.json()["ok"] is True
assert (await client.get("/api/runs/没有这个/verify")).status_code == 404