mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 05:21:57 +00:00
ThearchyHelios
03c18c4d78
一、文案（前端约 1600 处，后端传到界面的约 880 处） - 术语统一：工作流层面用「运行」（继续运行、重新运行、试运行），节点和工具层面用「执行」，不再用「跑」； 总时长 / 执行时长 / 等待审批；服务端；助手；Agent；token；工作流；固定版本；模型接入；系统提示；批准 / 驳回；受管级别 - 口语改书面：取不到、没取回来、读不懂、连不上、多半、照样、兜底、揉成这类说法改成「无法 / 未能 / 失败 / 请」；去掉拟人写法 - 不再露出实现细节：界面上不写 checkpoint、同一拍、哈希（凭证处除外），不写改版历史（「以前…」「老数据」）； 报错、校验、门禁、自动修复里裸写的配置键名和枚举值换成界面上的中文标签，由新的 backend/app/engine/labels.py 统一提供； 发布弹窗里结论句的选项不再是 off / require_citation - 同一件事只留一种说法：生成阶段文案、出具档位说明、画布锁定提示各合成一份 - 去掉界面上的工业场景示例（MES、ERP、「张工」「王工」）和真实产品名，换成通用示例 二、说错的提示（都有测试证明） - 数据库测连接失败时的专用提示以前从不出现（按报错文字全等比较，永远比不上）：改为按错误类型判断，不再拼出重复的前缀 - 「设置 → 供应商」改指「设置 → 模型接入」；「在设置里建数据源」改指「数据」页 - 缺少 API Key、模型接入已停用的报错不再夹英文，写明去哪里填 - 两处原因说错（「这种引用在后续版本支持」「这一版还不支持」）改成真实原因 - 节点说明里不准确的说法改正：代码节点在 microVM 或系统沙箱中执行；报告撰写、口径卡的说明不再说过头 - 导出的证据 CSV 里两列同名，第三列改为「片段编号」 三、前后端对齐 - 前端按后端原文识别日志、报错的地方，新旧两种写法都认得（历史运行里存的是旧文字），逐条补了新旧原文的核对 - 后端交回模型的原指令放进单独的 for_model 字段，前端在解析响应时去掉，不会显示到界面上 四、检查 - 检查脚本里按原文匹配的断言改成新文案；改完会变成永远成立的反向断言逐条修正，并做了变异验证 - 新增 check-copy（静态扫描前端界面字符串和后端上界面的文案）：再出现「跑」「后端」「多半」「令牌」、 界面上的 Copilot、裸写的配置键名、半角标点等就报错；确需保留的写进 scripts/copy-allow.json 并注明理由
289 lines
12 KiB
Python
289 lines
12 KiB
Python
"""运行收尾和几条并发路径上的竞态。
|
|
|
|
终态事件和运行记录要同时可见。两边都有人在读：
|
|
- 收到 run.finished 的客户端马上 GET 这次运行取完整成果（事件里的 output 是截断的）；
|
|
以前先落事件、隔一个事务才提交状态，每次都读到 running 和空的 output，问数据页
|
|
于是把截断过的成果标成残缺存进会话；
|
|
- 读到 failed 的一方马上去翻事件，要翻得到 run.failed。更早以前是先提交状态，这一边
|
|
偶尔翻不到。
|
|
两边都要成立，只能放进同一个事务。
|
|
|
|
恢复 / 接着跑和「放弃这次运行」抢同一次运行：读到 interrupted 之后隔了好几次 await
|
|
才写 running，放弃正好在这中间提交的话，一次已经取消、封存、发过 run.cancelled 的
|
|
运行会被改回 running，又拉起来跑。
|
|
|
|
刚停到审批上、还在落 interrupted 状态的那一刻点「放弃」：以前直接取消执行任务，把
|
|
收尾从中间打断——状态停在 running、审批还挂着、任务没了，再点一次只剩 409。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from app.core.bus import bus
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Approval, Run, RunEvent
|
|
from app.engine import compiler, runner
|
|
from app.engine.context import NodeError
|
|
from app.engine.runner import RunManager, run_manager
|
|
from app.engine.schema import NodeType
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
|
"data": {"label": nid, "config": config}}
|
|
|
|
|
|
def chain(*nodes):
|
|
return {"nodes": list(nodes),
|
|
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
@pytest.fixture
|
|
def slow(monkeypatch):
|
|
"""transform 节点慢一点，好让订阅先接上；id 为 boom 的直接失败，id 为 stuck 的一直卡着。"""
|
|
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
|
|
|
async def run(state, ctx):
|
|
await asyncio.sleep(30 if ctx.node.id == "stuck" else 0.15)
|
|
if ctx.node.id == "boom":
|
|
raise NodeError(ctx.node.id, "炸了")
|
|
return await original(state, ctx)
|
|
|
|
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, run)
|
|
|
|
|
|
def _graph(work_id: str = "work", *, gate: bool = False) -> dict:
|
|
nodes = [node("start", "input", fields=[{"name": "q"}]),
|
|
node(work_id, "transform", mode="template", template="成果全文")]
|
|
if gate:
|
|
nodes.append(node("gate", "human", mode="approve", title="放行吗"))
|
|
nodes.append(node("out", "output", fields=[{"name": "r", "value": f"{{{{ nodes.{work_id} }}}}"}]))
|
|
return chain(*nodes)
|
|
|
|
|
|
async def _status(run_id: str) -> str:
|
|
async with SessionLocal() as session:
|
|
return (await session.get(Run, run_id)).status
|
|
|
|
|
|
async def _until(check, timeout: float = 15.0) -> None:
|
|
deadline = asyncio.get_event_loop().time() + timeout
|
|
while asyncio.get_event_loop().time() < deadline:
|
|
if await check():
|
|
return
|
|
await asyncio.sleep(0.02)
|
|
raise AssertionError("等超时了")
|
|
|
|
|
|
async def _events(run_id: str) -> list[RunEvent]:
|
|
async with SessionLocal() as session:
|
|
return list((await session.execute(
|
|
select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
|
|
)).scalars())
|
|
|
|
|
|
async def _row_when(run_id: str, terminal: str, act=None) -> Run:
|
|
"""订阅这次运行，一收到 terminal 事件就读运行记录——客户端就是这么做的。"""
|
|
sub = bus.attach(run_id)
|
|
acting = asyncio.ensure_future(act()) if act is not None else None
|
|
try:
|
|
async for event in sub.stream():
|
|
if str(event.type) == terminal:
|
|
async with SessionLocal() as session:
|
|
return await session.get(Run, run_id)
|
|
finally:
|
|
sub.close()
|
|
if acting is not None:
|
|
await acting
|
|
raise AssertionError(f"没等到 {terminal}")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 终态事件和运行记录同时可见
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_the_run_is_final_when_run_finished_arrives(slow):
|
|
for _ in range(5):
|
|
run = await run_manager.start(graph=_graph(), input_payload={"q": "x"})
|
|
row = await _row_when(run.id, "run.finished")
|
|
assert row.status == "succeeded", f"收到 run.finished 时运行还是 {row.status}"
|
|
assert row.output == {"r": "成果全文"}, row.output
|
|
assert row.finished_at is not None and row.usage.get("wall_ms") is not None
|
|
await run_manager.wait_idle(run.id)
|
|
|
|
|
|
async def test_the_run_is_final_when_run_failed_arrives(slow):
|
|
for _ in range(5):
|
|
run = await run_manager.start(graph=_graph("boom"), input_payload={"q": "x"})
|
|
row = await _row_when(run.id, "run.failed")
|
|
assert row.status == "failed" and "炸了" in (row.error or ""), (row.status, row.error)
|
|
assert row.error_node_id == "boom"
|
|
await run_manager.wait_idle(run.id)
|
|
|
|
|
|
async def test_the_terminal_event_is_there_once_the_status_is(slow):
|
|
"""反方向：读到终态状态时，终态事件一定已经在库里。"""
|
|
for work_id, terminal in (("work", "run.finished"), ("boom", "run.failed")):
|
|
for _ in range(3):
|
|
run = await run_manager.start(graph=_graph(work_id), input_payload={"q": "x"})
|
|
|
|
async def settled(run_id=run.id) -> bool:
|
|
return await _status(run_id) in ("succeeded", "failed")
|
|
|
|
await _until(settled)
|
|
assert (await _events(run.id))[-1].type == terminal
|
|
await run_manager.wait_idle(run.id)
|
|
|
|
|
|
async def test_the_run_is_final_when_a_stop_arrives(slow):
|
|
run = await run_manager.start(graph=_graph("stuck"), input_payload={"q": "x"})
|
|
await _until(lambda: _started(run.id, "stuck"))
|
|
row = await _row_when(run.id, "run.cancelled", act=lambda: run_manager.cancel(run.id))
|
|
assert row.status == "cancelled" and row.finished_at is not None
|
|
|
|
|
|
async def test_the_run_is_final_when_an_abandon_arrives(slow):
|
|
run = await run_manager.start(graph=_graph(gate=True), input_payload={"q": "x"})
|
|
await _until(lambda: _is(run.id, "interrupted"))
|
|
await run_manager.wait_idle(run.id)
|
|
row = await _row_when(run.id, "run.cancelled", act=lambda: run_manager.cancel(run.id))
|
|
assert row.status == "cancelled" and row.finished_at is not None
|
|
assert row.last_seq == (await _events(run.id))[-1].seq
|
|
|
|
|
|
async def _started(run_id: str, node_id: str) -> bool:
|
|
return any(e.type == "node.started" and e.node_id == node_id for e in await _events(run_id))
|
|
|
|
|
|
async def _is(run_id: str, status: str) -> bool:
|
|
return await _status(run_id) == status
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 恢复 / 接着跑 和 放弃 抢同一次运行
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def _suspended_run() -> str:
|
|
"""服务重启挂起的运行：interrupted，没有待审批，断点完好。"""
|
|
run = await run_manager.start(graph=_graph("stuck"), input_payload={"q": "x"})
|
|
await _until(lambda: _started(run.id, "stuck"))
|
|
await run_manager.shutdown()
|
|
await run_manager.setup()
|
|
assert await _status(run.id) == "interrupted"
|
|
return run.id
|
|
|
|
|
|
def _abandon_in_between(monkeypatch) -> None:
|
|
"""在「读到 interrupted」和「写 running」之间，放弃这次运行的请求先提交了。"""
|
|
async def racing(run_id, spec, snapshot):
|
|
assert await run_manager._abandon(run_id, "另一个人")
|
|
return None
|
|
|
|
monkeypatch.setattr(runner, "_stale_replay", racing)
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["resume", "continue"])
|
|
async def test_a_run_abandoned_mid_way_is_not_revived(slow, monkeypatch, path):
|
|
run_id = await _suspended_run()
|
|
_abandon_in_between(monkeypatch)
|
|
with pytest.raises(ValueError, match="已取消"):
|
|
if path == "resume":
|
|
await run_manager.resume(run_id, None)
|
|
else:
|
|
await run_manager.continue_failed(run_id)
|
|
assert not run_manager.is_active(run_id), "取消了的运行又被拉起来跑了"
|
|
assert await _status(run_id) == "cancelled"
|
|
events = await _events(run_id)
|
|
assert events[-1].type == "run.cancelled" and not [e for e in events if e.type == "run.resumed"]
|
|
|
|
|
|
async def test_an_approval_closed_by_an_abandon_is_not_answered_again(slow, monkeypatch):
|
|
run = await run_manager.start(graph=_graph(gate=True), input_payload={"q": "x"})
|
|
await _until(lambda: _is(run.id, "interrupted"))
|
|
await run_manager.wait_idle(run.id)
|
|
_abandon_in_between(monkeypatch)
|
|
with pytest.raises(ValueError):
|
|
await run_manager.resume(run.id, {"approved": True})
|
|
async with SessionLocal() as session:
|
|
approval = (await session.execute(
|
|
select(Approval).where(Approval.run_id == run.id))).scalar_one()
|
|
assert approval.status == "cancelled", approval.status
|
|
assert await _status(run.id) == "cancelled" and not run_manager.is_active(run.id)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 刚停到审批上、还在落状态时点「放弃」
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_abandoning_while_the_pause_is_being_recorded(slow, monkeypatch):
|
|
original = RunManager._finalize
|
|
recording = asyncio.Event()
|
|
|
|
async def slow_finalize(self, run_id, status, *args, **kwargs):
|
|
if status == "interrupted":
|
|
recording.set()
|
|
await asyncio.sleep(0.4) # 落 interrupted 状态的这一段被拉长
|
|
return await original(self, run_id, status, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(RunManager, "_finalize", slow_finalize)
|
|
run = await run_manager.start(graph=_graph(gate=True), input_payload={"q": "x"})
|
|
await asyncio.wait_for(recording.wait(), 10)
|
|
assert run.id in run_manager._finalizing
|
|
|
|
assert await run_manager.cancel(run.id, actor="审核员") == "cancelled"
|
|
assert await _status(run.id) == "cancelled"
|
|
async with SessionLocal() as session:
|
|
approval = (await session.execute(
|
|
select(Approval).where(Approval.run_id == run.id))).scalar_one()
|
|
assert approval.status == "cancelled" and approval.resolved_by == "审核员"
|
|
assert (await _events(run.id))[-1].type == "run.cancelled"
|
|
|
|
|
|
async def test_an_approval_closed_just_before_it_is_marked_answered(slow, monkeypatch):
|
|
"""放弃恰好提交在「读到这条审批还是 pending」和「把它记成已回复」之间。"""
|
|
import sqlite3
|
|
|
|
from sqlalchemy import event
|
|
|
|
from app.db.base import engine
|
|
|
|
run = await run_manager.start(graph=_graph(gate=True), input_payload={"q": "x"})
|
|
await _until(lambda: _is(run.id, "interrupted"))
|
|
await run_manager.wait_idle(run.id)
|
|
armed = {"on": True}
|
|
|
|
def abandon_first(conn, cursor, statement, params, context, executemany):
|
|
if armed["on"] and statement.lstrip().upper().startswith("UPDATE APPROVALS"):
|
|
armed["on"] = False
|
|
other = sqlite3.connect(engine.url.database, timeout=5)
|
|
other.execute("UPDATE approvals SET status='cancelled' WHERE run_id=?", (run.id,))
|
|
other.execute("UPDATE runs SET status='cancelled' WHERE id=?", (run.id,))
|
|
other.commit()
|
|
other.close()
|
|
|
|
event.listen(engine.sync_engine, "before_cursor_execute", abandon_first)
|
|
try:
|
|
with pytest.raises(ValueError, match="已处理过|放弃"):
|
|
await run_manager.resume(run.id, {"approved": True})
|
|
finally:
|
|
event.remove(engine.sync_engine, "before_cursor_execute", abandon_first)
|
|
assert not armed["on"]
|
|
async with SessionLocal() as session:
|
|
approval = (await session.execute(
|
|
select(Approval).where(Approval.run_id == run.id))).scalar_one()
|
|
assert approval.status == "cancelled", "关掉的审批被改回了已回复"
|
|
assert not run_manager.is_active(run.id)
|