mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 10:51:58 +00:00
ThearchyHelios
5928cc052e
显示成生效却没生效，比明说没有更危险——它让人以为有东西在兜底。
1. max_run_seconds 只存在于配置和设置接口里，没有任何代码执行它：一个挂住的
模型或工具能把运行和它占着的并发名额一直拖着。现在 runner 用 asyncio.timeout
强制执行，按"一段执行"计（等人工审批时 _drive 已结束，不算）；超时判 failed、
说清是哪道上限，可以接着跑。别处抛的 TimeoutError 不冒充这道上限（看
Timeout.expired()）。
2. 模型调用从来没设过超时：ModelSpec.timeout 从没被赋值，那条传参分支一次都没
执行过，所有调用走 SDK 默认的 10 分钟外加重试。新增 model_timeout_seconds
（默认 300 秒）作为缺省，节点显式配的仍优先；已确认 ChatAnthropic / ChatOpenAI
都接这个参数。
3. manifest 哈希只写不验，而且是在 run.finished 落库之前算的——清单里恰恰少了
宣布"跑完了、成果是什么"的那条，改成果清单照样对得上。现在终态事件落库之后
再封存，并记下封到哪一条（manifest_seq，新列）；新增 verify_manifest 与
GET /api/runs/{id}/verify，运行详情页加了核对按钮。封存之后追加的事件不算篡改。
老运行没有 manifest_seq，按当时的口径还原范围——开发库里 300 条带哈希的历史
运行全部核对通过（成功 / 失败 / 取消各类都有），没有一条误报。
设置页「运行环境」补上单次执行时限和模型调用超时两项，README 加了对应的常见问题。
顺带修了一个被 _seal 放大的老问题：_drive 的收尾被打断时（关停取消、库连接出错）
漏掉 _tasks.pop，留下的任务让下一次 shutdown 的 gather 在别的事件循环里抛
"future belongs to a different loop"——测试里表现为每八九轮一次、一出就是几十个
teardown error。pop 移进内层 finally，shutdown 只等未结束的任务并清空。修后全量
连跑 12 轮 12 轮全绿。
test_guardrails.py 8 例：超时中止（挂住的节点 1 秒后被停下）、缺省模型超时、封存覆盖
run.finished、改成果被戳穿、事后追加不误报、老运行按旧口径核、中断态未封存、接口
404。414 → 422 passed。
518 lines
18 KiB
Python
518 lines
18 KiB
Python
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import contextlib
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, Depends, Header, HTTPException, Query, WebSocket, WebSocketDisconnect
|
||
from pydantic import BaseModel, Field
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from sqlalchemy import select as _select
|
||
|
||
from app.core.bus import bus
|
||
from app.core import artifact_store
|
||
from app.db.base import SessionLocal, get_session
|
||
from app.db.models import Approval, Artifact, Run, RunEvent, Workflow, WorkflowVersion
|
||
from app.engine.governance import unresolved_caliber_upgrades
|
||
from app.engine.runner import run_manager
|
||
from app.engine.schema import GraphSpec
|
||
|
||
router = APIRouter(prefix="/api/runs", tags=["runs"])
|
||
|
||
|
||
class StartRunIn(BaseModel):
|
||
workflow_id: str | None = None
|
||
graph: dict[str, Any] | None = None # 直接跑一张未保存的图（画布上的"试运行"）
|
||
input: dict[str, Any] = Field(default_factory=dict)
|
||
memory_scope: str = "default"
|
||
collection: str = "default"
|
||
# formal：从已发布的不可变版本发起，可复现可出具；exploratory：随便玩
|
||
run_class: str = "exploratory"
|
||
version: int | None = None # formal 时可指定版本，默认用已发布版本
|
||
|
||
|
||
class RunOut(BaseModel):
|
||
id: str
|
||
workflow_id: str | None
|
||
workflow_name: str
|
||
status: str
|
||
input: dict[str, Any]
|
||
output: dict[str, Any]
|
||
error: str | None
|
||
usage: dict[str, Any]
|
||
run_class: str | None = "exploratory"
|
||
version: int | None = None
|
||
version_hash: str | None = None
|
||
manifest_hash: str | None = None
|
||
started_by: str | None = None
|
||
created_at: Any = None
|
||
started_at: Any = None
|
||
finished_at: Any = None
|
||
|
||
model_config = {"from_attributes": True}
|
||
|
||
|
||
@router.post("", response_model=RunOut, status_code=201)
|
||
async def start_run(
|
||
payload: StartRunIn,
|
||
session: AsyncSession = Depends(get_session),
|
||
x_actor: str | None = Header(default=None),
|
||
) -> Run:
|
||
actor = (x_actor or "").strip() or None
|
||
name = "临时图"
|
||
version: int | None = None
|
||
version_hash: str | None = None
|
||
|
||
if payload.run_class == "formal":
|
||
# 正式运行的全部语义就这一条规则：必须引用一个不可变的已发布版本。
|
||
# 传裸 graph、或工作流还没发布过，都进不了 formal。
|
||
if not payload.workflow_id:
|
||
raise HTTPException(400, "正式运行必须指定 workflow_id")
|
||
if payload.graph is not None:
|
||
raise HTTPException(400, "正式运行不接受临时 graph——先保存并发布版本")
|
||
workflow = await session.get(Workflow, payload.workflow_id)
|
||
if not workflow:
|
||
raise HTTPException(404, "工作流不存在")
|
||
# 只看 published_version：status 反映的是当前画布（改过图就退回 draft），
|
||
# 而 formal 跑的是已发布的那一版，两者本来就可以不一致
|
||
if not workflow.published_version:
|
||
raise HTTPException(409, f"「{workflow.name}」还没有发布版本，先在编排页发布")
|
||
version = payload.version or workflow.published_version
|
||
# "不可变"和"过过闸"是两件事。快照确实不可变，但 PATCH 保存出来的
|
||
# 草稿版本同样有快照——只认 version 存不存在的话，工作流只要发布过
|
||
# 任意一个版本，之后所有草稿都能以 formal 身份启动，发布期的
|
||
# validate_graph + lint_for_publish 全部绕过。
|
||
if version != workflow.published_version:
|
||
raise HTTPException(
|
||
409,
|
||
f"v{version} 不是当前发布版本（v{workflow.published_version}）。"
|
||
"正式运行只能引用发布过的版本——要跑这一版就先发布它。",
|
||
)
|
||
snapshot = (
|
||
await session.execute(
|
||
_select(WorkflowVersion).where(
|
||
WorkflowVersion.workflow_id == workflow.id,
|
||
WorkflowVersion.version == version,
|
||
)
|
||
)
|
||
).scalar_one_or_none()
|
||
if not snapshot:
|
||
raise HTTPException(404, f"版本 v{version} 不存在")
|
||
graph = snapshot.graph
|
||
version_hash = snapshot.graph_hash or artifact_store.graph_hash(graph)
|
||
name = workflow.name
|
||
|
||
# 口径升版是被迫处置的事件：钉住的方法卡有新版本而未声明策略，拒绝启动
|
||
spec = GraphSpec.model_validate(graph)
|
||
errors, upgrade_events = await unresolved_caliber_upgrades(session, spec)
|
||
if errors:
|
||
raise HTTPException(409, "；".join(errors))
|
||
else:
|
||
graph = payload.graph
|
||
upgrade_events = []
|
||
if payload.workflow_id:
|
||
workflow = await session.get(Workflow, payload.workflow_id)
|
||
if not workflow:
|
||
raise HTTPException(404, "工作流不存在")
|
||
graph = graph or workflow.graph
|
||
name = workflow.name
|
||
if not graph:
|
||
raise HTTPException(400, "需要提供 workflow_id 或 graph")
|
||
|
||
try:
|
||
run = await run_manager.start(
|
||
graph=graph,
|
||
input_payload=payload.input,
|
||
workflow_id=payload.workflow_id,
|
||
workflow_name=name,
|
||
memory_scope=payload.memory_scope,
|
||
collection=payload.collection,
|
||
run_class="formal" if payload.run_class == "formal" else "exploratory",
|
||
version=version,
|
||
version_hash=version_hash,
|
||
started_by=actor,
|
||
)
|
||
except ValueError as e:
|
||
raise HTTPException(400, str(e)) from e
|
||
|
||
# 已声明策略的升版记入事件流，出具物上能看到"这期换口径了、怎么处置的"
|
||
for ev in upgrade_events:
|
||
payload_ev = dict(ev)
|
||
node = payload_ev.pop("node_id", None)
|
||
await run_manager.note(run.id, "caliber.upgrade", node_id=node, **payload_ev)
|
||
return run
|
||
|
||
|
||
@router.get("", response_model=list[RunOut])
|
||
async def list_runs(
|
||
workflow_id: str | None = None,
|
||
status: str | None = None,
|
||
limit: int = Query(default=50, le=200),
|
||
session: AsyncSession = Depends(get_session),
|
||
) -> list[Run]:
|
||
stmt = select(Run).order_by(Run.created_at.desc()).limit(limit)
|
||
if workflow_id:
|
||
stmt = stmt.where(Run.workflow_id == workflow_id)
|
||
if status:
|
||
stmt = stmt.where(Run.status == status)
|
||
return list((await session.execute(stmt)).scalars())
|
||
|
||
|
||
@router.get("/{run_id}", response_model=RunOut)
|
||
async def get_run(run_id: str, session: AsyncSession = Depends(get_session)) -> Run:
|
||
run = await session.get(Run, run_id)
|
||
if not run:
|
||
raise HTTPException(404, "运行记录不存在")
|
||
return run
|
||
|
||
|
||
@router.get("/{run_id}/events")
|
||
async def get_events(
|
||
run_id: str,
|
||
after: int = 0,
|
||
session: AsyncSession = Depends(get_session),
|
||
) -> list[dict[str, Any]]:
|
||
"""拉历史事件。刷新页面后重建时间线用这个，再接 WebSocket 续上实时流。"""
|
||
rows = await session.execute(
|
||
select(RunEvent)
|
||
.where(RunEvent.run_id == run_id, RunEvent.seq > after)
|
||
.order_by(RunEvent.seq)
|
||
)
|
||
return [
|
||
{
|
||
"id": e.id,
|
||
"run_id": e.run_id,
|
||
"seq": e.seq,
|
||
"type": e.type,
|
||
"node_id": e.node_id,
|
||
"ts": e.ts,
|
||
"data": e.data,
|
||
}
|
||
for e in rows.scalars()
|
||
]
|
||
|
||
|
||
@router.get("/{run_id}/artifacts")
|
||
async def list_artifacts(
|
||
run_id: str, session: AsyncSession = Depends(get_session)
|
||
) -> list[dict[str, Any]]:
|
||
"""这次运行产出的全部工件引用。审阅出具物时从这里下钻到完整证据。"""
|
||
rows = await session.execute(
|
||
select(Artifact).where(Artifact.run_id == run_id).order_by(Artifact.created_at)
|
||
)
|
||
return [
|
||
{
|
||
"id": a.id,
|
||
"kind": a.kind,
|
||
"node_id": a.node_id,
|
||
"size": a.size,
|
||
"meta": a.meta,
|
||
"created_at": a.created_at.isoformat() if a.created_at else None,
|
||
}
|
||
for a in rows.scalars()
|
||
]
|
||
|
||
|
||
@router.post("/{run_id}/cancel")
|
||
async def cancel_run(run_id: str) -> dict[str, Any]:
|
||
stopped = await run_manager.cancel(run_id)
|
||
if not stopped:
|
||
raise HTTPException(409, "该运行当前不在执行中")
|
||
return {"ok": True}
|
||
|
||
|
||
class ResumeIn(BaseModel):
|
||
response: Any = None
|
||
approval_id: str | None = None
|
||
|
||
|
||
@router.post("/{run_id}/resume", response_model=RunOut)
|
||
async def resume_run(
|
||
run_id: str, payload: ResumeIn, x_actor: str | None = Header(default=None)
|
||
) -> Run:
|
||
try:
|
||
# approval_id 以前定义了却从没传下去——一次运行里有多条待审批时，
|
||
# 引擎无从知道这次回复的是哪一条
|
||
return await run_manager.resume(
|
||
run_id, payload.response,
|
||
approval_id=payload.approval_id,
|
||
actor=(x_actor or "").strip() or None,
|
||
)
|
||
except KeyError as e:
|
||
raise HTTPException(404, str(e)) from e
|
||
except ValueError as e:
|
||
raise HTTPException(409, str(e)) from e
|
||
|
||
|
||
class ContinueIn(BaseModel):
|
||
"""接着跑。graph 只能带改过配置的同一张图，结构必须一致。"""
|
||
|
||
graph: dict[str, Any] | None = None
|
||
|
||
|
||
@router.post("/{run_id}/continue", response_model=RunOut)
|
||
async def continue_run(
|
||
run_id: str, payload: ContinueIn, x_actor: str | None = Header(default=None)
|
||
) -> Run:
|
||
"""从失败的节点接着跑，前面跑过的不重来。
|
||
|
||
和 /resume 不是一回事：那条是人工介入的断点（status=interrupted），这条是
|
||
真的挂了（status=failed）。挡住后者的原来只是 resume 里的一行状态判定，
|
||
而 checkpoint 一直都在——改个模型 id 就得把前面两分钟的取数重跑一遍，
|
||
纯属白等。
|
||
"""
|
||
try:
|
||
return await run_manager.continue_failed(
|
||
run_id, graph=payload.graph, actor=(x_actor or "").strip() or None,
|
||
)
|
||
except KeyError as e:
|
||
raise HTTPException(404, str(e)) from e
|
||
except ValueError as e:
|
||
raise HTTPException(409, str(e)) from e
|
||
|
||
|
||
@router.get("/{run_id}/state")
|
||
async def get_state(run_id: str) -> dict[str, Any]:
|
||
"""读取 checkpoint 里的当前状态。调试中断中的 run 时很有用。"""
|
||
from app.engine.compiler import compile_graph
|
||
from app.engine.context import RunContext
|
||
from app.engine.schema import GraphSpec
|
||
|
||
async with SessionLocal() as session:
|
||
run = await session.get(Run, run_id)
|
||
if not run:
|
||
raise HTTPException(404, "运行记录不存在")
|
||
|
||
spec = GraphSpec.model_validate(run.graph)
|
||
app = compile_graph(spec, RunContext(run_id=run_id, thread_id=run_id, spec=spec)).compile(
|
||
checkpointer=run_manager.checkpointer
|
||
)
|
||
snapshot = await app.aget_state({"configurable": {"thread_id": run_id}})
|
||
values = dict(snapshot.values or {})
|
||
values.pop("messages", None) # 消息对象不好直接序列化，前端也用不到
|
||
return {
|
||
"values": _jsonable(values),
|
||
"next": list(snapshot.next or ()),
|
||
"interrupts": [
|
||
{"id": getattr(i, "id", None), "value": _jsonable(getattr(i, "value", None))}
|
||
for i in (snapshot.interrupts or ())
|
||
],
|
||
}
|
||
|
||
|
||
@router.get("/{run_id}/verify")
|
||
async def verify_run(run_id: str) -> dict[str, Any]:
|
||
"""核对事件流和封存时的清单哈希是否一致：事后被改过、删过、插过都会对不上。"""
|
||
from app.engine.runner import verify_manifest
|
||
|
||
try:
|
||
return await verify_manifest(run_id)
|
||
except KeyError as e:
|
||
raise HTTPException(404, str(e)) from e
|
||
|
||
|
||
@router.get("/{run_id}/history")
|
||
async def get_history(run_id: str, limit: int = 50) -> list[dict[str, Any]]:
|
||
"""checkpoint 历史 —— 可以看到每一步之后的状态快照，用来做时间旅行式调试。"""
|
||
from app.engine.compiler import compile_graph
|
||
from app.engine.context import RunContext
|
||
from app.engine.schema import GraphSpec
|
||
|
||
async with SessionLocal() as session:
|
||
run = await session.get(Run, run_id)
|
||
if not run:
|
||
raise HTTPException(404, "运行记录不存在")
|
||
|
||
spec = GraphSpec.model_validate(run.graph)
|
||
app = compile_graph(spec, RunContext(run_id=run_id, thread_id=run_id, spec=spec)).compile(
|
||
checkpointer=run_manager.checkpointer
|
||
)
|
||
out: list[dict[str, Any]] = []
|
||
async for snapshot in app.aget_state_history({"configurable": {"thread_id": run_id}}):
|
||
values = dict(snapshot.values or {})
|
||
values.pop("messages", None)
|
||
out.append(
|
||
{
|
||
"checkpoint_id": snapshot.config.get("configurable", {}).get("checkpoint_id"),
|
||
"next": list(snapshot.next or ()),
|
||
"created_at": snapshot.created_at,
|
||
"trail": _jsonable(values.get("trail") or [])[-1:],
|
||
"nodes_done": list((values.get("nodes") or {}).keys()),
|
||
}
|
||
)
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
@router.delete("/{run_id}", status_code=204)
|
||
async def delete_run(run_id: str, session: AsyncSession = Depends(get_session)) -> None:
|
||
run = await session.get(Run, run_id)
|
||
if not run:
|
||
raise HTTPException(404, "运行记录不存在")
|
||
await session.delete(run)
|
||
await session.commit()
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 实时事件流
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@router.websocket("/{run_id}/stream")
|
||
async def stream_run(websocket: WebSocket, run_id: str) -> None:
|
||
"""把一次运行的事件推给前端。
|
||
|
||
先补历史再接实时，所以中途打开页面、或者刷新之后，时间线都是完整的。
|
||
"""
|
||
await websocket.accept()
|
||
after = int(websocket.query_params.get("after", 0) or 0)
|
||
|
||
# 先占订阅位，再读历史。反过来的话，两步之间图正好跑出来的事件既不在
|
||
# 历史里、也不在实时流里——永远丢了。实测就是这样：一次跑图的开头
|
||
# 两三条 node.started/node.finished 消失，界面上表现为某个节点永远
|
||
# 在转圈、它的子步骤跑到了顶层。
|
||
subscription = bus.attach(run_id)
|
||
try:
|
||
last_seq = after
|
||
async with SessionLocal() as session:
|
||
rows = await session.execute(
|
||
select(RunEvent)
|
||
.where(RunEvent.run_id == run_id, RunEvent.seq > after)
|
||
.order_by(RunEvent.seq)
|
||
)
|
||
for event in rows.scalars():
|
||
last_seq = max(last_seq, event.seq or 0)
|
||
await websocket.send_json(
|
||
{
|
||
"seq": event.seq,
|
||
"type": event.type,
|
||
"node_id": event.node_id,
|
||
"ts": event.ts,
|
||
"data": event.data,
|
||
"replay": True,
|
||
}
|
||
)
|
||
run = await session.get(Run, run_id)
|
||
|
||
# 已经结束的 run 没有后续事件，补完历史就关掉
|
||
if run and run.status in ("succeeded", "failed", "cancelled"):
|
||
await websocket.send_json({"type": "stream.end", "status": run.status})
|
||
await websocket.close()
|
||
return
|
||
|
||
# 占位期间攒下的那几条历史里也有，按 seq 滤掉，否则前端同一步收两遍
|
||
pump = asyncio.create_task(_pump(websocket, subscription.stream(after=last_seq)))
|
||
try:
|
||
# 客户端断开时读操作会抛异常，用它来结束这条连接
|
||
while True:
|
||
await websocket.receive_text()
|
||
except WebSocketDisconnect:
|
||
pass
|
||
finally:
|
||
pump.cancel()
|
||
with contextlib.suppress(asyncio.CancelledError):
|
||
await pump
|
||
except WebSocketDisconnect:
|
||
return
|
||
except Exception: # noqa: BLE001
|
||
with contextlib.suppress(Exception):
|
||
await websocket.close()
|
||
finally:
|
||
subscription.close()
|
||
|
||
|
||
async def _pump(websocket: WebSocket, stream: Any) -> None:
|
||
try:
|
||
async for event in stream:
|
||
await websocket.send_json(event.to_wire())
|
||
except (WebSocketDisconnect, RuntimeError):
|
||
return
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 审批
|
||
# --------------------------------------------------------------------------
|
||
|
||
approvals_router = APIRouter(prefix="/api/approvals", tags=["approvals"])
|
||
|
||
|
||
class ApprovalOut(BaseModel):
|
||
id: str
|
||
run_id: str
|
||
node_id: str
|
||
mode: str
|
||
title: str
|
||
payload: dict[str, Any]
|
||
status: str
|
||
response: dict[str, Any]
|
||
created_at: Any = None
|
||
|
||
model_config = {"from_attributes": True}
|
||
|
||
|
||
@approvals_router.get("", response_model=list[ApprovalOut])
|
||
async def list_approvals(
|
||
status: str = "pending",
|
||
run_id: str | None = None,
|
||
session: AsyncSession = Depends(get_session),
|
||
) -> list[Approval]:
|
||
stmt = select(Approval).order_by(Approval.created_at.desc()).limit(100)
|
||
if status != "all":
|
||
stmt = stmt.where(Approval.status == status)
|
||
if run_id:
|
||
stmt = stmt.where(Approval.run_id == run_id)
|
||
return list((await session.execute(stmt)).scalars())
|
||
|
||
|
||
class DecideIn(BaseModel):
|
||
approved: bool = True
|
||
note: str = ""
|
||
value: Any = None
|
||
args: dict[str, Any] | None = None
|
||
|
||
|
||
@approvals_router.post("/{approval_id}/decide", response_model=RunOut)
|
||
async def decide(
|
||
approval_id: str,
|
||
payload: DecideIn,
|
||
session: AsyncSession = Depends(get_session),
|
||
x_actor: str | None = Header(default=None),
|
||
) -> Run:
|
||
approval = await session.get(Approval, approval_id)
|
||
if not approval:
|
||
raise HTTPException(404, "审批请求不存在")
|
||
if approval.status != "pending":
|
||
raise HTTPException(409, "该请求已经处理过了")
|
||
|
||
# 责任归属先落库再恢复执行——恢复失败也要留下"谁试图批的"
|
||
approval.resolved_by = (x_actor or "").strip() or None
|
||
await session.commit()
|
||
|
||
response: dict[str, Any] = {"approved": payload.approved, "note": payload.note}
|
||
if payload.value is not None:
|
||
response["value"] = payload.value
|
||
if payload.args is not None:
|
||
response["args"] = payload.args
|
||
|
||
try:
|
||
return await run_manager.resume(
|
||
approval.run_id, response,
|
||
approval_id=approval.id,
|
||
actor=approval.resolved_by,
|
||
)
|
||
except ValueError as e:
|
||
raise HTTPException(409, str(e)) from e
|
||
|
||
|
||
def _jsonable(value: Any) -> Any:
|
||
if isinstance(value, dict):
|
||
return {str(k): _jsonable(v) for k, v in value.items()}
|
||
if isinstance(value, (list, tuple)):
|
||
return [_jsonable(v) for v in value]
|
||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||
return value
|
||
return str(value)
|