mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 05:22:01 +00:00
5928cc052e
显示成生效却没生效,比明说没有更危险——它让人以为有东西在兜底。
1. max_run_seconds 只存在于配置和设置接口里,没有任何代码执行它:一个挂住的
模型或工具能把运行和它占着的并发名额一直拖着。现在 runner 用 asyncio.timeout
强制执行,按"一段执行"计(等人工审批时 _drive 已结束,不算);超时判 failed、
说清是哪道上限,可以接着跑。别处抛的 TimeoutError 不冒充这道上限(看
Timeout.expired())。
2. 模型调用从来没设过超时:ModelSpec.timeout 从没被赋值,那条传参分支一次都没
执行过,所有调用走 SDK 默认的 10 分钟外加重试。新增 model_timeout_seconds
(默认 300 秒)作为缺省,节点显式配的仍优先;已确认 ChatAnthropic / ChatOpenAI
都接这个参数。
3. manifest 哈希只写不验,而且是在 run.finished 落库之前算的——清单里恰恰少了
宣布"跑完了、成果是什么"的那条,改成果清单照样对得上。现在终态事件落库之后
再封存,并记下封到哪一条(manifest_seq,新列);新增 verify_manifest 与
GET /api/runs/{id}/verify,运行详情页加了核对按钮。封存之后追加的事件不算篡改。
老运行没有 manifest_seq,按当时的口径还原范围——开发库里 300 条带哈希的历史
运行全部核对通过(成功 / 失败 / 取消各类都有),没有一条误报。
设置页「运行环境」补上单次执行时限和模型调用超时两项,README 加了对应的常见问题。
顺带修了一个被 _seal 放大的老问题:_drive 的收尾被打断时(关停取消、库连接出错)
漏掉 _tasks.pop,留下的任务让下一次 shutdown 的 gather 在别的事件循环里抛
"future belongs to a different loop"——测试里表现为每八九轮一次、一出就是几十个
teardown error。pop 移进内层 finally,shutdown 只等未结束的任务并清空。修后全量
连跑 12 轮 12 轮全绿。
test_guardrails.py 8 例:超时中止(挂住的节点 1 秒后被停下)、缺省模型超时、封存覆盖
run.finished、改成果被戳穿、事后追加不误报、老运行按旧口径核、中断态未封存、接口
404。414 → 422 passed。
183 lines
7.1 KiB
Python
183 lines
7.1 KiB
Python
"""三道"配了、显示了、却没人执行"的护栏。
|
|
|
|
- max_run_seconds:配置里有,没有任何代码执行它。一个挂住的模型或工具能把运行
|
|
和它占着的并发名额一直拖着。
|
|
- 模型调用超时:ModelSpec.timeout 从没被赋过值,所有调用都走 SDK 默认的 10 分钟
|
|
外加重试。
|
|
- manifest 哈希:只写不验,而且是在 run.finished 落库之前算的——清单里恰恰少了
|
|
那条宣布"跑完了、成果是什么"的事件。
|
|
|
|
显示成生效却没生效,比明说没有更危险:它让人以为有东西在兜底。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import select, update
|
|
|
|
from app.core.artifact_store import manifest_hash
|
|
from app.core.config import settings
|
|
from app.core.crypto import encrypt
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Provider, Run, RunEvent
|
|
from app.engine import compiler
|
|
from app.engine.runner import run_manager, verify_manifest
|
|
from app.engine.schema import NodeType
|
|
from app.main import app
|
|
from app.providers.factory import ModelSpec, build_chat_model
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
|
"data": {"label": nid, "config": config}}
|
|
|
|
|
|
def chain(*nodes):
|
|
return {"nodes": list(nodes),
|
|
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
SIMPLE = chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("t", "transform", mode="template", template="好", assign_to="x"),
|
|
node("out", "output", fields=[{"name": "结果", "value": "{{ vars.x }}"}]),
|
|
)
|
|
|
|
|
|
async def _run(graph, *, timeout=20.0) -> Run:
|
|
run = await run_manager.start(graph=graph, input_payload={"question": "随便"})
|
|
deadline = asyncio.get_event_loop().time() + timeout
|
|
while asyncio.get_event_loop().time() < deadline:
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row and row.status in ("succeeded", "failed", "cancelled", "interrupted"):
|
|
return row
|
|
await asyncio.sleep(0.05)
|
|
raise AssertionError("运行没有按时结束")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# max_run_seconds
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_a_run_past_its_time_limit_is_stopped(monkeypatch):
|
|
async def stuck(state, ctx):
|
|
await asyncio.sleep(30) # 一个不响应的模型或工具
|
|
return {}
|
|
|
|
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, stuck)
|
|
monkeypatch.setattr(settings, "max_run_seconds", 1)
|
|
|
|
started = time.perf_counter()
|
|
run = await _run(SIMPLE)
|
|
assert run.status == "failed"
|
|
assert "1 秒的上限" in (run.error or "")
|
|
assert time.perf_counter() - started < 10, "上限没起作用,等到了节点自己结束"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 模型调用超时
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_models_get_a_timeout_even_when_the_node_sets_none():
|
|
provider = Provider(name="t", kind="anthropic", api_key=encrypt("sk-test"), models=[],
|
|
default_model="claude-sonnet-5", enabled=True, extra={})
|
|
model = build_chat_model(provider, ModelSpec())
|
|
assert model.default_request_timeout == settings.model_timeout_seconds
|
|
# 节点显式配了的照旧优先
|
|
assert build_chat_model(provider, ModelSpec(timeout=42)).default_request_timeout == 42
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# manifest
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def _events(run_id):
|
|
async with SessionLocal() as session:
|
|
return list((await session.execute(
|
|
select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
|
|
)).scalars())
|
|
|
|
|
|
async def test_the_seal_covers_run_finished_and_verifies():
|
|
run = await _run(SIMPLE)
|
|
assert run.status == "succeeded", run.error
|
|
events = await _events(run.id)
|
|
finished = next(e for e in events if e.type == "run.finished")
|
|
assert run.manifest_seq == finished.seq, "清单没有封到 run.finished"
|
|
|
|
report = await verify_manifest(run.id)
|
|
assert report["sealed"] and report["ok"], report
|
|
|
|
|
|
async def test_tampering_with_the_output_is_caught():
|
|
"""改成果——以前 run.finished 不在清单里,这一改清单照样对得上。"""
|
|
run = await _run(SIMPLE)
|
|
async with SessionLocal() as session:
|
|
await session.execute(
|
|
update(RunEvent)
|
|
.where(RunEvent.run_id == run.id, RunEvent.type == "run.finished")
|
|
.values(data={"output": {"结果": "被改过的结论"}})
|
|
)
|
|
await session.commit()
|
|
report = await verify_manifest(run.id)
|
|
assert report["ok"] is False
|
|
|
|
|
|
async def test_events_appended_after_the_seal_are_not_tampering():
|
|
"""封存之后追加的事件(复核批注之类)不在范围内,不该让核对失败。"""
|
|
run = await _run(SIMPLE)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
session.add(RunEvent(run_id=run.id, seq=row.manifest_seq + 5, type="log", ts=time.time(),
|
|
node_id=None, data={"message": "事后追加的批注"}))
|
|
await session.commit()
|
|
assert (await verify_manifest(run.id))["ok"] is True
|
|
|
|
|
|
async def test_runs_sealed_before_manifest_seq_existed_verify_by_the_old_cut():
|
|
"""老运行的哈希是在 run.finished 落库之前算的,没有 manifest_seq——按当时的口径核。"""
|
|
run = await _run(SIMPLE)
|
|
rows = [(e.seq, e.type, e.node_id, e.data) for e in await _events(run.id)
|
|
if e.type != "run.finished"]
|
|
async with SessionLocal() as session:
|
|
await session.execute(update(Run).where(Run.id == run.id)
|
|
.values(manifest_hash=manifest_hash(rows), manifest_seq=None))
|
|
await session.commit()
|
|
report = await verify_manifest(run.id)
|
|
assert report["ok"] is True and report["legacy"] is True
|
|
|
|
|
|
async def test_a_run_waiting_for_a_human_is_not_sealed_yet():
|
|
graph = chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("h", "human", mode="approve", title="批吗"),
|
|
node("out", "output", fields=[]),
|
|
)
|
|
run = await _run(graph)
|
|
assert run.status == "interrupted"
|
|
report = await verify_manifest(run.id)
|
|
assert report["sealed"] is False and report["ok"] is None
|
|
|
|
|
|
async def test_verify_through_the_api():
|
|
run = await _run(SIMPLE)
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
|
ok = await client.get(f"/api/runs/{run.id}/verify")
|
|
assert ok.status_code == 200 and ok.json()["ok"] is True
|
|
assert (await client.get("/api/runs/没有这个/verify")).status_code == 404
|