mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:51:57 +00:00
三道只配不管的护栏开始干活：运行时限、模型超时、清单核对
显示成生效却没生效，比明说没有更危险——它让人以为有东西在兜底。
1. max_run_seconds 只存在于配置和设置接口里，没有任何代码执行它：一个挂住的
模型或工具能把运行和它占着的并发名额一直拖着。现在 runner 用 asyncio.timeout
强制执行，按"一段执行"计（等人工审批时 _drive 已结束，不算）；超时判 failed、
说清是哪道上限，可以接着跑。别处抛的 TimeoutError 不冒充这道上限（看
Timeout.expired()）。
2. 模型调用从来没设过超时：ModelSpec.timeout 从没被赋值，那条传参分支一次都没
执行过，所有调用走 SDK 默认的 10 分钟外加重试。新增 model_timeout_seconds
（默认 300 秒）作为缺省，节点显式配的仍优先；已确认 ChatAnthropic / ChatOpenAI
都接这个参数。
3. manifest 哈希只写不验，而且是在 run.finished 落库之前算的——清单里恰恰少了
宣布"跑完了、成果是什么"的那条，改成果清单照样对得上。现在终态事件落库之后
再封存，并记下封到哪一条（manifest_seq，新列）；新增 verify_manifest 与
GET /api/runs/{id}/verify，运行详情页加了核对按钮。封存之后追加的事件不算篡改。
老运行没有 manifest_seq，按当时的口径还原范围——开发库里 300 条带哈希的历史
运行全部核对通过（成功 / 失败 / 取消各类都有），没有一条误报。
设置页「运行环境」补上单次执行时限和模型调用超时两项，README 加了对应的常见问题。
顺带修了一个被 _seal 放大的老问题：_drive 的收尾被打断时（关停取消、库连接出错）
漏掉 _tasks.pop，留下的任务让下一次 shutdown 的 gather 在别的事件循环里抛
"future belongs to a different loop"——测试里表现为每八九轮一次、一出就是几十个
teardown error。pop 移进内层 finally，shutdown 只等未结束的任务并清空。修后全量
连跑 12 轮 12 轮全绿。
test_guardrails.py 8 例：超时中止（挂住的节点 1 秒后被停下）、缺省模型超时、封存覆盖
run.finished、改成果被戳穿、事后追加不误报、老运行按旧口径核、中断态未封存、接口
404。414 → 422 passed。
This commit is contained in:
ThearchyHelios
@@ -131,7 +131,8 @@ flowchart LR
|
||||
只是与上周不可比」。
|
||||
|
||||
每次运行的完整证据存于内容寻址的工件库（`data/artifacts/`，sha256 即地址，取回时复验哈希）；
|
||||
运行终态的事件清单哈希记录在 Run 上，事后修改流水将无法对齐。
|
||||
运行终态的事件清单哈希记录在 Run 上（封存到 `run.finished` 为止），事后修改流水将无法对齐。
|
||||
运行详情页一键核对，接口是 `GET /api/runs/{id}/verify`；封存之后追加的事件不在核对范围内。
|
||||
|
||||
---
|
||||
|
||||
@@ -416,6 +417,10 @@ agent 侧同样会收到实情，并被告知可直接查询 `information_schema
|
||||
|
||||
**端口冲突。** 前端默认 5273、后端 8000，通过 `AGENTLAB_WEB_PORT` / `AGENTLAB_PORT` 修改。
|
||||
|
||||
**运行被中止，提示超过时限。** 单次执行（两次人工介入之间的那一段）默认最多 600 秒，
|
||||
单次模型调用默认 300 秒没有输出即超时。多半是模型或工具没有响应；确实需要更久，调大
|
||||
`AGENTLAB_MAX_RUN_SECONDS` / `AGENTLAB_MODEL_TIMEOUT_SECONDS`。设置页「运行环境」里能看到当前值。
|
||||
|
||||
---
|
||||
|
||||
## 开发与验证
|
||||
|
||||
@@ -302,6 +302,17 @@ async def get_state(run_id: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{run_id}/verify")
|
||||
async def verify_run(run_id: str) -> dict[str, Any]:
|
||||
"""核对事件流和封存时的清单哈希是否一致：事后被改过、删过、插过都会对不上。"""
|
||||
from app.engine.runner import verify_manifest
|
||||
|
||||
try:
|
||||
return await verify_manifest(run_id)
|
||||
except KeyError as e:
|
||||
raise HTTPException(404, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/{run_id}/history")
|
||||
async def get_history(run_id: str, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""checkpoint 历史 —— 可以看到每一步之后的状态快照，用来做时间旅行式调试。"""
|
||||
|
||||
@@ -258,5 +258,7 @@ async def system_info() -> dict[str, Any]:
|
||||
"max_graph_steps": app_settings.max_graph_steps,
|
||||
"max_agent_steps": app_settings.max_agent_steps,
|
||||
"max_concurrent_runs": app_settings.max_concurrent_runs,
|
||||
"max_run_seconds": app_settings.max_run_seconds,
|
||||
"model_timeout_seconds": app_settings.model_timeout_seconds,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -63,7 +63,12 @@ class Settings(BaseSettings):
|
||||
microvm_boot_timeout: int = 300
|
||||
|
||||
# --- 执行引擎护栏 ---
|
||||
# 单次执行（两次人工介入之间的那一段）的墙钟上限，runner 里强制执行
|
||||
max_run_seconds: int = 600
|
||||
# 单次模型调用的超时。流式调用时是"两段输出之间最长沉默多久"，非流式是整次
|
||||
# 请求。不设的话走 SDK 默认的 10 分钟、外加重试——一个挂住的服务商能把一个
|
||||
# 节点拖上半小时
|
||||
model_timeout_seconds: int = 300
|
||||
max_graph_steps: int = 200
|
||||
# 节点上配多少步都过不去这个硬顶。默认值仍是节点上的 12（nodeDefs.ts），
|
||||
# 这里只决定"想调大的人最多能调到哪"——实测 87% 的 agent 节点 ≤10 步就收口，
|
||||
|
||||
@@ -107,6 +107,7 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [
|
||||
("runs", "version", "INTEGER"),
|
||||
("runs", "version_hash", "VARCHAR(64)"),
|
||||
("runs", "manifest_hash", "VARCHAR(64)"),
|
||||
("runs", "manifest_seq", "INTEGER"),
|
||||
("runs", "started_by", "VARCHAR(100)"),
|
||||
("workflows", "status", "TEXT DEFAULT 'draft'"),
|
||||
("workflows", "published_version", "INTEGER"),
|
||||
|
||||
@@ -176,6 +176,9 @@ class Run(Base, TimestampMixin):
|
||||
version_hash: Mapped[str | None] = mapped_column(String(64), default=None)
|
||||
# 结束时对全部事件计算的清单哈希，事后改动事件流会被它戳穿
|
||||
manifest_hash: Mapped[str | None] = mapped_column(String(64), default=None)
|
||||
# 清单封到了哪一条事件（含）。之后追加的事件（复核批注之类）不在封存范围内；
|
||||
# 为空的是记录这一列之前封存的老运行，核对时按当时的口径算
|
||||
manifest_seq: Mapped[int | None] = mapped_column(Integer, default=None)
|
||||
started_by: Mapped[str | None] = mapped_column(String(100), default=None)
|
||||
|
||||
events: Mapped[list["RunEvent"]] = relationship(
|
||||
|
||||
+103
-22
@@ -61,9 +61,13 @@ class RunManager:
|
||||
await session.commit()
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
for task in list(self._tasks.values()):
|
||||
# 只等还没结束的：结束了的没什么可等，而且它可能属于另一个事件循环，
|
||||
# gather 会直接抛 "future belongs to a different loop"
|
||||
pending = [t for t in self._tasks.values() if not t.done()]
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
await asyncio.gather(*self._tasks.values(), return_exceptions=True)
|
||||
await asyncio.gather(*pending, return_exceptions=True)
|
||||
self._tasks.clear()
|
||||
if self._cm is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self._cm.__aexit__(None, None, None)
|
||||
@@ -418,6 +422,7 @@ class RunManager:
|
||||
error: str | None = None
|
||||
final_state: dict[str, Any] = {}
|
||||
|
||||
deadline: asyncio.Timeout | None = None
|
||||
async with self._semaphore:
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
@@ -449,14 +454,19 @@ class RunManager:
|
||||
}
|
||||
|
||||
interrupted = False
|
||||
async for mode, chunk in app.astream(
|
||||
payload, config=config, stream_mode=["custom", "updates"]
|
||||
):
|
||||
if mode == "custom":
|
||||
await self._handle_custom(run_id, chunk)
|
||||
elif mode == "updates":
|
||||
if await self._handle_updates(run_id, chunk):
|
||||
interrupted = True
|
||||
# 单次执行的墙钟上限。以前 max_run_seconds 只存在于配置里，没有任何
|
||||
# 代码执行它——一个挂住的模型或工具能把运行和它占着的并发名额一直拖着。
|
||||
# 按"这一段执行"计：等人工审批时 _drive 已经结束，那段时间不算。
|
||||
deadline = asyncio.timeout(settings.max_run_seconds)
|
||||
async with deadline:
|
||||
async for mode, chunk in app.astream(
|
||||
payload, config=config, stream_mode=["custom", "updates"]
|
||||
):
|
||||
if mode == "custom":
|
||||
await self._handle_custom(run_id, chunk)
|
||||
elif mode == "updates":
|
||||
if await self._handle_updates(run_id, chunk):
|
||||
interrupted = True
|
||||
|
||||
snapshot = await app.aget_state(config)
|
||||
final_state = dict(snapshot.values or {})
|
||||
@@ -470,6 +480,17 @@ class RunManager:
|
||||
error = "用户取消"
|
||||
await self._emit(run_id, EventType.RUN_CANCELLED, data={})
|
||||
raise
|
||||
except TimeoutError as e:
|
||||
status = "failed"
|
||||
if deadline is not None and deadline.expired():
|
||||
error = (
|
||||
f"这次执行超过了 {settings.max_run_seconds} 秒的上限，已中止。"
|
||||
"跑得慢的多半是模型或工具没有响应；确实需要更久，调大 AGENTLAB_MAX_RUN_SECONDS"
|
||||
)
|
||||
else: # 别处抛的超时，不是这道上限——按普通失败说
|
||||
error = f"{type(e).__name__}: {e}"
|
||||
logger.exception("run %s 失败", run_id)
|
||||
await self._emit(run_id, EventType.RUN_FAILED, data={"error": error})
|
||||
except Exception as e: # noqa: BLE001
|
||||
status = "failed"
|
||||
# NodeError 的 message 本来就是写给人看的（"人工驳回：…"），
|
||||
@@ -479,8 +500,13 @@ class RunManager:
|
||||
await self._emit(run_id, EventType.RUN_FAILED, data={"error": error})
|
||||
finally:
|
||||
elapsed = int((time.perf_counter() - started) * 1000)
|
||||
await self._finalize(run_id, status, error, final_state, elapsed)
|
||||
self._tasks.pop(run_id, None)
|
||||
try:
|
||||
await self._finalize(run_id, status, error, final_state, elapsed)
|
||||
finally:
|
||||
# 收尾被打断（关停时的取消、库连接出错）也得把自己摘掉。以前写在
|
||||
# _finalize 后面，一被打断就漏掉，留下的任务让下一次 shutdown 的
|
||||
# gather 在别的事件循环里炸开
|
||||
self._tasks.pop(run_id, None)
|
||||
# 运行到终态就把沙箱会话收掉。thread_id 每个 run 都是新的，
|
||||
# 不收的话每跑一次带代码节点的图就多一台常驻 microVM（几百 MB），
|
||||
# 而且没有任何自动回收路径会碰它——闲置回收只在"下次有人执行
|
||||
@@ -566,16 +592,6 @@ class RunManager:
|
||||
run.usage = usage
|
||||
if status != "interrupted":
|
||||
run.finished_at = datetime.now(timezone.utc)
|
||||
# 终态时对全部已落库事件计算清单哈希：事后任何对事件流的
|
||||
# 增删改都会与这个值对不上。中断态不算——事件还会继续追加。
|
||||
from app.core.artifact_store import manifest_hash as _manifest
|
||||
|
||||
rows = await session.execute(
|
||||
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
|
||||
.where(RunEvent.run_id == run_id)
|
||||
.order_by(RunEvent.seq)
|
||||
)
|
||||
run.manifest_hash = _manifest([tuple(r) for r in rows])
|
||||
await session.commit()
|
||||
|
||||
if status == "succeeded":
|
||||
@@ -584,9 +600,74 @@ class RunManager:
|
||||
EventType.RUN_FINISHED,
|
||||
data={"output": _safe(output), "usage": usage, "duration_ms": elapsed_ms},
|
||||
)
|
||||
if status != "interrupted":
|
||||
# 终态事件落库之后再封存。以前在 run.finished 之前算，清单里恰恰少了
|
||||
# 那条宣布"跑完了、成果是什么"的事件——改它的成果，清单照样对得上。
|
||||
# 中断态不封：事件还会继续追加
|
||||
await self._seal(run_id)
|
||||
if status in ("succeeded", "failed", "cancelled"):
|
||||
await bus.close(run_id)
|
||||
|
||||
async def _seal(self, run_id: str) -> None:
|
||||
"""对到此为止的全部事件算清单哈希，记下封到了哪一条。"""
|
||||
from app.core.artifact_store import manifest_hash as _manifest
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
if run is None:
|
||||
return
|
||||
rows = [tuple(r) for r in await session.execute(
|
||||
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
|
||||
.where(RunEvent.run_id == run_id)
|
||||
.order_by(RunEvent.seq)
|
||||
)]
|
||||
run.manifest_hash = _manifest(rows)
|
||||
run.manifest_seq = rows[-1][0] if rows else 0
|
||||
await session.commit()
|
||||
|
||||
|
||||
_TERMINAL = (EventType.RUN_FINISHED, EventType.RUN_FAILED, EventType.RUN_CANCELLED)
|
||||
|
||||
|
||||
async def verify_manifest(run_id: str) -> dict[str, Any]:
|
||||
"""重算封存范围内的清单哈希，和封存时记下的对一下。
|
||||
|
||||
以前只写不验：README 说"事后修改流水将无法对齐"，可仓库里没有一行代码去对。
|
||||
封存范围之后追加的事件不参与核对。没有 manifest_seq 的是加这一列之前封存的
|
||||
老运行——当时的哈希是在 run.finished 落库之前算的，按那时的口径还原范围。
|
||||
"""
|
||||
from app.core.artifact_store import manifest_hash as _manifest
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
if run is None:
|
||||
raise KeyError(f"找不到运行 {run_id}")
|
||||
if not run.manifest_hash:
|
||||
return {"sealed": False, "ok": None,
|
||||
"message": "这次运行还没有封存（没跑完，或者停在人工介入）"}
|
||||
rows = [tuple(r) for r in await session.execute(
|
||||
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
|
||||
.where(RunEvent.run_id == run_id)
|
||||
.order_by(RunEvent.seq)
|
||||
)]
|
||||
expected, sealed_at = run.manifest_hash, run.manifest_seq
|
||||
|
||||
legacy = sealed_at is None
|
||||
if legacy:
|
||||
last = next((r for r in reversed(rows) if r[1] in _TERMINAL), None)
|
||||
if last is None:
|
||||
sealed_at = rows[-1][0] if rows else 0
|
||||
elif last[1] == EventType.RUN_FINISHED:
|
||||
sealed_at = last[0] - 1 # 老口径：run.finished 本身不在清单里
|
||||
else:
|
||||
sealed_at = last[0] # run.failed / run.cancelled 在封存前就落库了
|
||||
covered = [r for r in rows if r[0] <= sealed_at]
|
||||
ok = _manifest(covered) == expected
|
||||
return {
|
||||
"sealed": True, "ok": ok, "events": len(covered), "sealed_at": sealed_at, "legacy": legacy,
|
||||
"message": "事件流与封存时一致" if ok else "事件流和封存时对不上：封存之后有事件被改过、删过或插过",
|
||||
}
|
||||
|
||||
|
||||
def _safe(value: Any, limit: int = 4000) -> Any:
|
||||
"""把任意对象裁剪成能安全放进事件负载的形状。"""
|
||||
|
||||
@@ -8,6 +8,7 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.crypto import decrypt
|
||||
from app.db.models import Provider
|
||||
from app.providers import catalog
|
||||
@@ -106,8 +107,9 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel:
|
||||
kwargs: dict[str, Any] = {"model": model}
|
||||
if spec.max_tokens:
|
||||
kwargs["max_tokens"] = spec.max_tokens
|
||||
if spec.timeout:
|
||||
kwargs["timeout"] = spec.timeout
|
||||
# 以前只有节点显式配了才传，而没有任何地方给 spec.timeout 赋过值——所有调用
|
||||
# 都走 SDK 默认的 10 分钟
|
||||
kwargs["timeout"] = spec.timeout or settings.model_timeout_seconds
|
||||
|
||||
# 采样参数：能力允许才传
|
||||
if spec.temperature is not None and catalog.supports_sampling(model):
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""三道"配了、显示了、却没人执行"的护栏。
|
||||
|
||||
- max_run_seconds：配置里有，没有任何代码执行它。一个挂住的模型或工具能把运行
|
||||
和它占着的并发名额一直拖着。
|
||||
- 模型调用超时：ModelSpec.timeout 从没被赋过值，所有调用都走 SDK 默认的 10 分钟
|
||||
外加重试。
|
||||
- manifest 哈希：只写不验，而且是在 run.finished 落库之前算的——清单里恰恰少了
|
||||
那条宣布"跑完了、成果是什么"的事件。
|
||||
|
||||
显示成生效却没生效，比明说没有更危险：它让人以为有东西在兜底。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.core.artifact_store import manifest_hash
|
||||
from app.core.config import settings
|
||||
from app.core.crypto import encrypt
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Provider, Run, RunEvent
|
||||
from app.engine import compiler
|
||||
from app.engine.runner import run_manager, verify_manifest
|
||||
from app.engine.schema import NodeType
|
||||
from app.main import app
|
||||
from app.providers.factory import ModelSpec, build_chat_model
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
||||
"data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def chain(*nodes):
|
||||
return {"nodes": list(nodes),
|
||||
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||||
|
||||
|
||||
SIMPLE = chain(
|
||||
node("start", "input", fields=[{"name": "question"}]),
|
||||
node("t", "transform", mode="template", template="好", assign_to="x"),
|
||||
node("out", "output", fields=[{"name": "结果", "value": "{{ vars.x }}"}]),
|
||||
)
|
||||
|
||||
|
||||
async def _run(graph, *, timeout=20.0) -> Run:
|
||||
run = await run_manager.start(graph=graph, input_payload={"question": "随便"})
|
||||
deadline = asyncio.get_event_loop().time() + timeout
|
||||
while asyncio.get_event_loop().time() < deadline:
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
if row and row.status in ("succeeded", "failed", "cancelled", "interrupted"):
|
||||
return row
|
||||
await asyncio.sleep(0.05)
|
||||
raise AssertionError("运行没有按时结束")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# max_run_seconds
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_run_past_its_time_limit_is_stopped(monkeypatch):
|
||||
async def stuck(state, ctx):
|
||||
await asyncio.sleep(30) # 一个不响应的模型或工具
|
||||
return {}
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, stuck)
|
||||
monkeypatch.setattr(settings, "max_run_seconds", 1)
|
||||
|
||||
started = time.perf_counter()
|
||||
run = await _run(SIMPLE)
|
||||
assert run.status == "failed"
|
||||
assert "1 秒的上限" in (run.error or "")
|
||||
assert time.perf_counter() - started < 10, "上限没起作用，等到了节点自己结束"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 模型调用超时
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_models_get_a_timeout_even_when_the_node_sets_none():
|
||||
provider = Provider(name="t", kind="anthropic", api_key=encrypt("sk-test"), models=[],
|
||||
default_model="claude-sonnet-5", enabled=True, extra={})
|
||||
model = build_chat_model(provider, ModelSpec())
|
||||
assert model.default_request_timeout == settings.model_timeout_seconds
|
||||
# 节点显式配了的照旧优先
|
||||
assert build_chat_model(provider, ModelSpec(timeout=42)).default_request_timeout == 42
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# manifest
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _events(run_id):
|
||||
async with SessionLocal() as session:
|
||||
return list((await session.execute(
|
||||
select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
|
||||
)).scalars())
|
||||
|
||||
|
||||
async def test_the_seal_covers_run_finished_and_verifies():
|
||||
run = await _run(SIMPLE)
|
||||
assert run.status == "succeeded", run.error
|
||||
events = await _events(run.id)
|
||||
finished = next(e for e in events if e.type == "run.finished")
|
||||
assert run.manifest_seq == finished.seq, "清单没有封到 run.finished"
|
||||
|
||||
report = await verify_manifest(run.id)
|
||||
assert report["sealed"] and report["ok"], report
|
||||
|
||||
|
||||
async def test_tampering_with_the_output_is_caught():
|
||||
"""改成果——以前 run.finished 不在清单里，这一改清单照样对得上。"""
|
||||
run = await _run(SIMPLE)
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
update(RunEvent)
|
||||
.where(RunEvent.run_id == run.id, RunEvent.type == "run.finished")
|
||||
.values(data={"output": {"结果": "被改过的结论"}})
|
||||
)
|
||||
await session.commit()
|
||||
report = await verify_manifest(run.id)
|
||||
assert report["ok"] is False
|
||||
|
||||
|
||||
async def test_events_appended_after_the_seal_are_not_tampering():
|
||||
"""封存之后追加的事件（复核批注之类）不在范围内，不该让核对失败。"""
|
||||
run = await _run(SIMPLE)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
session.add(RunEvent(run_id=run.id, seq=row.manifest_seq + 5, type="log", ts=time.time(),
|
||||
node_id=None, data={"message": "事后追加的批注"}))
|
||||
await session.commit()
|
||||
assert (await verify_manifest(run.id))["ok"] is True
|
||||
|
||||
|
||||
async def test_runs_sealed_before_manifest_seq_existed_verify_by_the_old_cut():
|
||||
"""老运行的哈希是在 run.finished 落库之前算的，没有 manifest_seq——按当时的口径核。"""
|
||||
run = await _run(SIMPLE)
|
||||
rows = [(e.seq, e.type, e.node_id, e.data) for e in await _events(run.id)
|
||||
if e.type != "run.finished"]
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(update(Run).where(Run.id == run.id)
|
||||
.values(manifest_hash=manifest_hash(rows), manifest_seq=None))
|
||||
await session.commit()
|
||||
report = await verify_manifest(run.id)
|
||||
assert report["ok"] is True and report["legacy"] is True
|
||||
|
||||
|
||||
async def test_a_run_waiting_for_a_human_is_not_sealed_yet():
|
||||
graph = chain(
|
||||
node("start", "input", fields=[{"name": "question"}]),
|
||||
node("h", "human", mode="approve", title="批吗"),
|
||||
node("out", "output", fields=[]),
|
||||
)
|
||||
run = await _run(graph)
|
||||
assert run.status == "interrupted"
|
||||
report = await verify_manifest(run.id)
|
||||
assert report["sealed"] is False and report["ok"] is None
|
||||
|
||||
|
||||
async def test_verify_through_the_api():
|
||||
run = await _run(SIMPLE)
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
ok = await client.get(f"/api/runs/{run.id}/verify")
|
||||
assert ok.status_code == 200 and ok.json()["ok"] is True
|
||||
assert (await client.get("/api/runs/没有这个/verify")).status_code == 404
|
||||
@@ -129,6 +129,9 @@ export const api = {
|
||||
events: (id: string, after = 0) => get<RunEvent[]>(`/runs/${id}/events?after=${after}`),
|
||||
state: (id: string) => get<any>(`/runs/${id}/state`),
|
||||
history: (id: string) => get<any[]>(`/runs/${id}/history`),
|
||||
/** 核对事件流和封存时的清单哈希：事后被改过、删过、插过都会对不上 */
|
||||
verify: (id: string) => get<{ sealed: boolean; ok: boolean | null; message: string }>(
|
||||
`/runs/${id}/verify`),
|
||||
remove: (id: string) => del(`/runs/${id}`),
|
||||
},
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import { useNavigate, useParams } from 'react-router-dom'
|
||||
import { Code2, GitFork, History, RefreshCw, Trash2 } from 'lucide-react'
|
||||
import { Code2, GitFork, History, RefreshCw, ShieldCheck, Trash2 } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { api } from '../api/client'
|
||||
import { Empty, Spinner, StatusDot, useToast } from '../components/ui'
|
||||
@@ -157,6 +157,20 @@ export function RunsPage() {
|
||||
</div>
|
||||
</div>
|
||||
<StatusDot status={selected.status} />
|
||||
{selected.manifest_hash && (
|
||||
<button
|
||||
className="btn btn-sm btn-ghost"
|
||||
title="核对事件流和封存时的清单是否一致"
|
||||
onClick={async () => {
|
||||
try {
|
||||
const res = await api.runs.verify(selected.id)
|
||||
toast(res.message, res.ok ? 'ok' : 'error')
|
||||
} catch (e: any) { toast(e.message ?? '核对失败', 'error') }
|
||||
}}
|
||||
>
|
||||
<ShieldCheck size={11} />
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
className={clsx('btn btn-sm btn-ghost', raw && 'text-[var(--accent)]')}
|
||||
title={raw ? '回到可读视图' : `看原始事件（${events.length} 条）`}
|
||||
|
||||
@@ -471,6 +471,8 @@ function SystemTab() {
|
||||
<Row label="图最大步数" value={info.limits?.max_graph_steps} />
|
||||
<Row label="Agent 最大步数" value={info.limits?.max_agent_steps} />
|
||||
<Row label="最大并发运行" value={info.limits?.max_concurrent_runs} />
|
||||
<Row label="单次执行时限" value={info.limits?.max_run_seconds && `${info.limits.max_run_seconds} 秒`} />
|
||||
<Row label="模型调用超时" value={info.limits?.model_timeout_seconds && `${info.limits.model_timeout_seconds} 秒`} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user