Files
AgentLab/backend/tests/test_guardrails.py
T
ThearchyHelios 5928cc052e 三道只配不管的护栏开始干活:运行时限、模型超时、清单核对
显示成生效却没生效,比明说没有更危险——它让人以为有东西在兜底。

1. max_run_seconds 只存在于配置和设置接口里,没有任何代码执行它:一个挂住的
   模型或工具能把运行和它占着的并发名额一直拖着。现在 runner 用 asyncio.timeout
   强制执行,按"一段执行"计(等人工审批时 _drive 已结束,不算);超时判 failed、
   说清是哪道上限,可以接着跑。别处抛的 TimeoutError 不冒充这道上限(看
   Timeout.expired())。

2. 模型调用从来没设过超时:ModelSpec.timeout 从没被赋值,那条传参分支一次都没
   执行过,所有调用走 SDK 默认的 10 分钟外加重试。新增 model_timeout_seconds
   (默认 300 秒)作为缺省,节点显式配的仍优先;已确认 ChatAnthropic / ChatOpenAI
   都接这个参数。

3. manifest 哈希只写不验,而且是在 run.finished 落库之前算的——清单里恰恰少了
   宣布"跑完了、成果是什么"的那条,改成果清单照样对得上。现在终态事件落库之后
   再封存,并记下封到哪一条(manifest_seq,新列);新增 verify_manifest 与
   GET /api/runs/{id}/verify,运行详情页加了核对按钮。封存之后追加的事件不算篡改。
   老运行没有 manifest_seq,按当时的口径还原范围——开发库里 300 条带哈希的历史
   运行全部核对通过(成功 / 失败 / 取消各类都有),没有一条误报。

设置页「运行环境」补上单次执行时限和模型调用超时两项,README 加了对应的常见问题。

顺带修了一个被 _seal 放大的老问题:_drive 的收尾被打断时(关停取消、库连接出错)
漏掉 _tasks.pop,留下的任务让下一次 shutdown 的 gather 在别的事件循环里抛
"future belongs to a different loop"——测试里表现为每八九轮一次、一出就是几十个
teardown error。pop 移进内层 finally,shutdown 只等未结束的任务并清空。修后全量
连跑 12 轮 12 轮全绿。

test_guardrails.py 8 例:超时中止(挂住的节点 1 秒后被停下)、缺省模型超时、封存覆盖
run.finished、改成果被戳穿、事后追加不误报、老运行按旧口径核、中断态未封存、接口
404。414 → 422 passed。
2026-09-25 15:59:32 +08:00

183 lines
7.1 KiB
Python

"""三道"配了、显示了、却没人执行"的护栏。
- max_run_seconds:配置里有,没有任何代码执行它。一个挂住的模型或工具能把运行
和它占着的并发名额一直拖着。
- 模型调用超时:ModelSpec.timeout 从没被赋过值,所有调用都走 SDK 默认的 10 分钟
外加重试。
- manifest 哈希:只写不验,而且是在 run.finished 落库之前算的——清单里恰恰少了
那条宣布"跑完了、成果是什么"的事件。
显示成生效却没生效,比明说没有更危险:它让人以为有东西在兜底。
"""
from __future__ import annotations
import asyncio
import time
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select, update
from app.core.artifact_store import manifest_hash
from app.core.config import settings
from app.core.crypto import encrypt
from app.db.base import SessionLocal
from app.db.models import Provider, Run, RunEvent
from app.engine import compiler
from app.engine.runner import run_manager, verify_manifest
from app.engine.schema import NodeType
from app.main import app
from app.providers.factory import ModelSpec, build_chat_model
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
"data": {"label": nid, "config": config}}
def chain(*nodes):
return {"nodes": list(nodes),
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
SIMPLE = chain(
node("start", "input", fields=[{"name": "question"}]),
node("t", "transform", mode="template", template="好", assign_to="x"),
node("out", "output", fields=[{"name": "结果", "value": "{{ vars.x }}"}]),
)
async def _run(graph, *, timeout=20.0) -> Run:
run = await run_manager.start(graph=graph, input_payload={"question": "随便"})
deadline = asyncio.get_event_loop().time() + timeout
while asyncio.get_event_loop().time() < deadline:
async with SessionLocal() as session:
row = await session.get(Run, run.id)
if row and row.status in ("succeeded", "failed", "cancelled", "interrupted"):
return row
await asyncio.sleep(0.05)
raise AssertionError("运行没有按时结束")
# --------------------------------------------------------------------------
# max_run_seconds
# --------------------------------------------------------------------------
async def test_a_run_past_its_time_limit_is_stopped(monkeypatch):
async def stuck(state, ctx):
await asyncio.sleep(30) # 一个不响应的模型或工具
return {}
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, stuck)
monkeypatch.setattr(settings, "max_run_seconds", 1)
started = time.perf_counter()
run = await _run(SIMPLE)
assert run.status == "failed"
assert "1 秒的上限" in (run.error or "")
assert time.perf_counter() - started < 10, "上限没起作用,等到了节点自己结束"
# --------------------------------------------------------------------------
# 模型调用超时
# --------------------------------------------------------------------------
def test_models_get_a_timeout_even_when_the_node_sets_none():
provider = Provider(name="t", kind="anthropic", api_key=encrypt("sk-test"), models=[],
default_model="claude-sonnet-5", enabled=True, extra={})
model = build_chat_model(provider, ModelSpec())
assert model.default_request_timeout == settings.model_timeout_seconds
# 节点显式配了的照旧优先
assert build_chat_model(provider, ModelSpec(timeout=42)).default_request_timeout == 42
# --------------------------------------------------------------------------
# manifest
# --------------------------------------------------------------------------
async def _events(run_id):
async with SessionLocal() as session:
return list((await session.execute(
select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
)).scalars())
async def test_the_seal_covers_run_finished_and_verifies():
run = await _run(SIMPLE)
assert run.status == "succeeded", run.error
events = await _events(run.id)
finished = next(e for e in events if e.type == "run.finished")
assert run.manifest_seq == finished.seq, "清单没有封到 run.finished"
report = await verify_manifest(run.id)
assert report["sealed"] and report["ok"], report
async def test_tampering_with_the_output_is_caught():
"""改成果——以前 run.finished 不在清单里,这一改清单照样对得上。"""
run = await _run(SIMPLE)
async with SessionLocal() as session:
await session.execute(
update(RunEvent)
.where(RunEvent.run_id == run.id, RunEvent.type == "run.finished")
.values(data={"output": {"结果": "被改过的结论"}})
)
await session.commit()
report = await verify_manifest(run.id)
assert report["ok"] is False
async def test_events_appended_after_the_seal_are_not_tampering():
"""封存之后追加的事件(复核批注之类)不在范围内,不该让核对失败。"""
run = await _run(SIMPLE)
async with SessionLocal() as session:
row = await session.get(Run, run.id)
session.add(RunEvent(run_id=run.id, seq=row.manifest_seq + 5, type="log", ts=time.time(),
node_id=None, data={"message": "事后追加的批注"}))
await session.commit()
assert (await verify_manifest(run.id))["ok"] is True
async def test_runs_sealed_before_manifest_seq_existed_verify_by_the_old_cut():
"""老运行的哈希是在 run.finished 落库之前算的,没有 manifest_seq——按当时的口径核。"""
run = await _run(SIMPLE)
rows = [(e.seq, e.type, e.node_id, e.data) for e in await _events(run.id)
if e.type != "run.finished"]
async with SessionLocal() as session:
await session.execute(update(Run).where(Run.id == run.id)
.values(manifest_hash=manifest_hash(rows), manifest_seq=None))
await session.commit()
report = await verify_manifest(run.id)
assert report["ok"] is True and report["legacy"] is True
async def test_a_run_waiting_for_a_human_is_not_sealed_yet():
graph = chain(
node("start", "input", fields=[{"name": "question"}]),
node("h", "human", mode="approve", title="批吗"),
node("out", "output", fields=[]),
)
run = await _run(graph)
assert run.status == "interrupted"
report = await verify_manifest(run.id)
assert report["sealed"] is False and report["ok"] is None
async def test_verify_through_the_api():
run = await _run(SIMPLE)
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
ok = await client.get(f"/api/runs/{run.id}/verify")
assert ok.status_code == 200 and ok.json()["ok"] is True
assert (await client.get("/api/runs/没有这个/verify")).status_code == 404