mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-20 17:02:01 +00:00
进程重启后接号不再撞上已有的序号:以前恢复、继续运行、放弃、启动时补日志都只按 run.last_seq 接号,而 last_seq 会被写小。按需裁判记判定(api/evidence.py 的 _append_judged)的条件更新不带 last_seq 条件:它取了号还没写,封存后追加(run_append)抢先提交了,它再提交时把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不碍事;重启后计数从 0 起,继续运行写出的 run.resumed 就和封存后追加的事件同号,重新封存时两条同号的事件一起进了清单,而清单按序号排序算哈希。
- runner 加 _seq_floor:接号起点取 last_seq 和库里实际最大序号中大的那个。恢复、继续运行、放弃都改用它,run_append 原来手写的同一段兜底也换成它 - 启动时(setup)先把落后的 last_seq 按库里实际的最后一条改正,再处理上次留下的运行中记录。_append_judged 规格不许改,它重启后也只按 last_seq 接号,不改正的话它第一次追加就撞号;有 (run_id, seq) 索引,五千次运行、一百五十万条事件实测约 7 毫秒。updated_at 不动,这是补记录,不是运行有了变化 - _stage 不带运行对象时的更新只往大里改:另开会话记的事件和执行任务同时在写,号大的先提交、号小的后提交,以前会把 last_seq 写小 测试:last_seq 写小、清掉本进程计数之后继续运行、恢复、放弃都接在实际的最后一条后面,全流无重复序号、verify 通过;_stage 先提交大号再提交小号,last_seq 不变;按真实的交错(裁判停在条件更新之前,封存后追加先提交)把 last_seq 写小,重启(shutdown、清计数、setup)后 last_seq 改正、别的运行原样不动,再裁判、继续运行都不撞号。去掉启动时的改正,只有裁判那条变红;去掉 _seq_floor,继续运行、恢复、放弃三条变红;_stage 改回无条件写,对应那条变红
This commit is contained in:
@@ -25,7 +25,7 @@ from app.core.bus import bus
|
||||
from app.core.events import EventType, RunEventModel
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent
|
||||
from app.engine.runner import _stage, run_manager
|
||||
from app.engine.runner import _seq_floor, _stage, run_manager
|
||||
|
||||
#: 运行的终态。封存过的运行只会停在这几种上
|
||||
_TERMINAL = ("succeeded", "failed", "cancelled")
|
||||
@@ -104,10 +104,9 @@ async def append_events(
|
||||
return None
|
||||
guard = (Run.status == "interrupted",)
|
||||
last = run.last_seq
|
||||
# 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,另取一次
|
||||
# 实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号
|
||||
stored = await session.scalar(select(func.max(RunEvent.seq)).where(RunEvent.run_id == run_id))
|
||||
bus.set_seq(run_id, max(last or 0, stored or 0))
|
||||
# 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面。last_seq 按说就是最后一条,_seq_floor
|
||||
# 另取一次实际的最大序号兜底——别处的追加把 last_seq 写小了,也不至于在这里撞号
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, last))
|
||||
events = [run_manager._event(run_id, kind, node_id=node_id, data=data)
|
||||
for kind, node_id, data in entries]
|
||||
# 核对和写入是同一行上的条件更新,和恢复、放弃、继续运行(runner._claim / _abandon)抢的是同一行
|
||||
|
||||
@@ -13,7 +13,7 @@ from typing import Any
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
from langgraph.errors import GraphRecursionError
|
||||
from langgraph.types import Command
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import case, func, select, update
|
||||
|
||||
from app.core.bus import bus
|
||||
from app.core.config import settings
|
||||
@@ -66,6 +66,15 @@ class RunManager:
|
||||
# 一个 checkpoint 都没写过,"恢复"它只会从 START 用默认值重跑一遍,
|
||||
# 还丢掉 run.input——那不是恢复,是伪造一份结果。
|
||||
async with SessionLocal() as session:
|
||||
# 先把落后于库里实际最后一条的 last_seq 改正(为什么会落后见 _seq_floor)。内存里的序号计数刚清零,
|
||||
# 接下来各处都按 last_seq 接号:本模块的几处另外兜了底,按需裁判记判定(api/evidence.py 的
|
||||
# _append_judged)没有,不改正的话它重启后第一次追加就撞上已有的序号。有 (run_id, seq) 索引,
|
||||
# 每次运行一次索引查找。updated_at 不动:这是补记录,不是运行有了变化
|
||||
latest = select(func.max(RunEvent.seq)).where(RunEvent.run_id == Run.id).scalar_subquery()
|
||||
await session.execute(
|
||||
update(Run).where(func.coalesce(Run.last_seq, 0) < latest)
|
||||
.values(last_seq=latest, updated_at=Run.updated_at)
|
||||
.execution_options(synchronize_session=False))
|
||||
stranded = list((await session.execute(
|
||||
select(Run.id, Run.last_seq).where(Run.status == "running")
|
||||
)).all())
|
||||
@@ -392,7 +401,7 @@ class RunManager:
|
||||
.values(status="resolved")
|
||||
)
|
||||
run = await session.get(Run, run_id)
|
||||
bus.set_seq(run_id, run.last_seq)
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
|
||||
await session.commit()
|
||||
await session.refresh(run)
|
||||
|
||||
@@ -492,7 +501,7 @@ class RunManager:
|
||||
**({"graph": spec.model_dump(mode="json")} if graph is not None else {}),
|
||||
**({"started_by": actor} if actor else {}))
|
||||
run = await session.get(Run, run_id)
|
||||
bus.set_seq(run_id, run.last_seq)
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
|
||||
await session.commit()
|
||||
await session.refresh(run)
|
||||
|
||||
@@ -558,7 +567,7 @@ class RunManager:
|
||||
usage = _with_inquiry(usage, inquiry)
|
||||
run.usage = {**usage, "duration_ms": timing["active_ms"], **timing}
|
||||
run.error = "用户取消:等待审批时放弃了这次运行" if closed else "用户取消:挂起期间放弃了这次运行"
|
||||
bus.set_seq(run_id, run.last_seq or 0)
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq))
|
||||
# 和状态同一个事务提交,理由同 _finalize
|
||||
event = self._event(run_id, EventType.RUN_CANCELLED, data={
|
||||
"timing": timing, "actor": actor,
|
||||
@@ -1260,11 +1269,27 @@ async def _stage(session: Any, event: RunEventModel, run: Run | None = None) ->
|
||||
if run is not None:
|
||||
run.last_seq = event.seq
|
||||
else:
|
||||
# 只往大里改:另开会话记的事件(note)和执行任务同时在写时,取号和提交的先后可以不一样,号小的后提交,
|
||||
# 无条件写会把 last_seq 写小(后果见 _seq_floor)
|
||||
await session.execute(
|
||||
update(Run).where(Run.id == event.run_id).values(last_seq=event.seq)
|
||||
update(Run).where(Run.id == event.run_id)
|
||||
.values(last_seq=case((Run.last_seq > event.seq, Run.last_seq), else_=event.seq))
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
|
||||
|
||||
async def _seq_floor(session: Any, run_id: str, last_seq: int | None) -> int:
|
||||
"""接着一次运行往下编号的起点:last_seq 和库里实际最大的序号取大的。
|
||||
|
||||
last_seq 按说就是最后一条,但有的写入会把它写小:按需裁判记判定(api/evidence.py 的 _append_judged)
|
||||
的条件更新不带 last_seq 条件,它取了号还没写、封存后追加(engine/run_append.py)抢先提交了,它再提交时
|
||||
把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不碍事;进程重启后计数从 0 起,只按
|
||||
last_seq 接号就会撞上已有的序号。清单按序号排序算哈希,撞了号封存核对就时对时不对。
|
||||
"""
|
||||
stored = await session.scalar(select(func.max(RunEvent.seq)).where(RunEvent.run_id == run_id))
|
||||
return max(last_seq or 0, stored or 0)
|
||||
|
||||
|
||||
async def _close_approvals(session: Any, run_id: str, actor: str | None, now: datetime) -> int:
|
||||
"""运行已经结束,它还没处理完的审批一并关掉。返回关掉了几条。"""
|
||||
rows = list((await session.execute(
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
- 已封存的运行追加:带 post_seal、落在 manifest_seq 之后,verify 照旧一致;封存不是这次结束之后封的、
|
||||
封存和调用方开始时看到的不同,都不追加
|
||||
- 连续追加、并发追加,序号严格递增,整条事件流没有重复的序号
|
||||
- last_seq 落后于库里实际的最后一条(按需裁判的追加和封存后追加交错提交时会这样):进程重启后恢复、
|
||||
继续运行、放弃、按需裁判都接在实际的最后一条后面,不撞号;runner 记事件不把 last_seq 写小
|
||||
- 恢复后的重放不受影响:前面的节点不重复执行,Agent 里批过的工具不重复执行(含升级前写下的断点)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -66,6 +68,9 @@ GATED = chain(node("start", "input"),
|
||||
node("gate", "human", mode="approve", title="这份周报可以出具吗"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.work }}"}]))
|
||||
SIMPLE = chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "完成"}]))
|
||||
#: 「boom」节点按 flaky 的开关失败:先让运行失败、封存,再关掉开关继续运行
|
||||
FLAKY = chain(node("start", "input"), node("boom", "transform", mode="template", template="门店 17 家"),
|
||||
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.boom }}"}]))
|
||||
|
||||
|
||||
def inquiry(question: str) -> tuple:
|
||||
@@ -86,6 +91,21 @@ def executed(monkeypatch) -> Counter:
|
||||
return seen
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def flaky(monkeypatch) -> dict:
|
||||
"""FLAKY 里的「boom」在 fail 为真时失败。"""
|
||||
state = {"fail": True}
|
||||
original = compiler.RUNNERS[NodeType.TRANSFORM]
|
||||
|
||||
async def boom(st, ctx):
|
||||
if ctx.node.id == "boom" and state["fail"]:
|
||||
raise NodeError(ctx.node.id, "数据源暂时不可用")
|
||||
return await original(st, ctx)
|
||||
|
||||
monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, boom)
|
||||
return state
|
||||
|
||||
|
||||
async def _row(run_id: str) -> Run:
|
||||
async with SessionLocal() as session:
|
||||
return await session.get(Run, run_id)
|
||||
@@ -128,6 +148,16 @@ async def _no_repeated_seq(run_id: str) -> None:
|
||||
assert len(seqs) == len(set(seqs)), f"事件流里有重复的序号:{seqs}"
|
||||
|
||||
|
||||
def _sql(statement: str, *params) -> None:
|
||||
"""另开一条连接改库:模拟别处(恢复、继续运行、另一个写入者)抢在中间提交。"""
|
||||
db = sqlite3.connect(settings.db_path, timeout=5)
|
||||
try:
|
||||
db.execute(statement, params)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 停在审批时追加
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -421,6 +451,134 @@ async def test_a_restarted_process_continues_after_the_stored_events():
|
||||
await _no_repeated_seq(paused.id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["continue", "resume", "abandon"])
|
||||
async def test_the_runner_numbers_after_the_stored_events_when_last_seq_lags(flaky, action):
|
||||
"""last_seq 比库里实际的最后一条小(怎么来的见下一条),本进程又没有这次运行的序号计数(重启过):继续运行、
|
||||
恢复、放弃都接在实际的最后一条后面。只按 last_seq 接,写出的 run.resumed / run.cancelled 撞上已有的序号,
|
||||
重新封存时两条同号的事件一起进了清单。"""
|
||||
if action == "continue":
|
||||
run = await _finished(FLAKY)
|
||||
assert run.status == "failed"
|
||||
else:
|
||||
run = await _paused()
|
||||
stored = (await _events(run.id))[-1].seq
|
||||
_sql("UPDATE runs SET last_seq = last_seq - 3 WHERE id = ?", run.id)
|
||||
bus._seq.pop(run.id, None)
|
||||
|
||||
if action == "continue":
|
||||
flaky["fail"] = False
|
||||
await run_manager.continue_failed(run.id)
|
||||
first_new = "run.resumed"
|
||||
elif action == "resume":
|
||||
await run_manager.resume(run.id, {"approved": True})
|
||||
first_new = "run.resumed"
|
||||
else:
|
||||
assert await run_manager.cancel(run.id) == "cancelled"
|
||||
first_new = "run.cancelled"
|
||||
final = await _until(run.id, ("succeeded", "cancelled"), sealed=True)
|
||||
assert final.status == ("cancelled" if action == "abandon" else "succeeded"), final.error
|
||||
assert [e.seq for e in await _events(run.id, first_new)][-1] == stored + 1
|
||||
await _no_repeated_seq(run.id)
|
||||
assert (await verify_manifest(run.id))["ok"] is True
|
||||
|
||||
|
||||
async def test_runner_event_writes_never_lower_last_seq():
|
||||
"""runner 记事件时 last_seq 只往大里改。取号和提交的先后可以不一样:别处另开会话记的事件(run_manager.note)
|
||||
和执行任务同时在写,号大的先提交、号小的后提交,以前后提交的那条把 last_seq 写回小的。"""
|
||||
from app.engine.runner import _stage
|
||||
|
||||
paused = await _paused()
|
||||
early = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "先取号"})
|
||||
late = run_manager._event(paused.id, EventType.LOG, data={"level": "info", "message": "后取号"})
|
||||
for event in (late, early):
|
||||
async with SessionLocal() as session:
|
||||
await _stage(session, event)
|
||||
await session.commit()
|
||||
assert (await _row(paused.id)).last_seq == late.seq
|
||||
|
||||
|
||||
def _judged_args(run: Run) -> tuple:
|
||||
"""直接调按需裁判记判定的那一步(api/evidence.py 的 _append_judged)要的参数:问过一次模型、什么都没判成。"""
|
||||
from app.api import evidence
|
||||
|
||||
sealed = evidence._Sealed(run_id=run.id, graph={}, seal={"sealed": True, "ok": True}, events=[],
|
||||
manifest_hash=run.manifest_hash, run_manifest_seq=run.manifest_seq)
|
||||
report = evidence._Report(node_id="out", doc_artifact="doc-1", ok=None, repairs=None, stats=None, doc=None,
|
||||
hash_ok=None, fields=[])
|
||||
outcome = {key: {} for key in ("verdicts", "keys", "unjudged", "skipped", "budget")}
|
||||
outcome.update(model="judge-model", priced=True, same_model=False, writer_model=None, cost_usd=0.0021,
|
||||
calls=1, duration_ms=1200, limits_hit=[], gaps=[], notes=[])
|
||||
return sealed, report, ["u1"], outcome
|
||||
|
||||
|
||||
async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(flaky, monkeypatch):
|
||||
"""按需裁判记判定(_append_judged)的条件更新不带 last_seq 条件:它取了号、还没写,封存后追加抢先提交了,
|
||||
它再提交时把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不会撞号;进程重启后计数从 0 起,
|
||||
各处又按 last_seq 接号——裁判自己也是。所以启动时按库里实际的最后一条改正 last_seq:之后再裁判、继续运行,
|
||||
都接在实际的最后一条后面。"""
|
||||
from app.api import evidence
|
||||
from app.engine.run_append import append_events
|
||||
|
||||
first = await _finished(FLAKY)
|
||||
assert first.status == "failed"
|
||||
held, opened = asyncio.Event(), evidence.SessionLocal
|
||||
|
||||
class HeldBeforeUpdate:
|
||||
"""裁判的会话:执行条件更新之前停住,等封存后追加先提交。"""
|
||||
|
||||
def __init__(self):
|
||||
self._cm = opened()
|
||||
|
||||
async def __aenter__(self):
|
||||
session = await self._cm.__aenter__()
|
||||
execute = session.execute
|
||||
|
||||
async def held_execute(statement, *args, **kwargs):
|
||||
if getattr(statement, "is_update", False):
|
||||
await held.wait()
|
||||
return await execute(statement, *args, **kwargs)
|
||||
|
||||
session.execute = held_execute
|
||||
return session
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return await self._cm.__aexit__(*exc)
|
||||
|
||||
monkeypatch.setattr(evidence, "SessionLocal", HeldBeforeUpdate)
|
||||
judging = asyncio.create_task(evidence._append_judged(*_judged_args(first)))
|
||||
for _ in range(200): # 裁判取到号了
|
||||
if bus._seq.get(first.id, 0) > first.last_seq:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
written = await append_events(first.id, [inquiry("这次为什么失败")], require="sealed")
|
||||
held.set()
|
||||
assert await judging == {"seq": first.last_seq + 1}
|
||||
assert written is not None and written[0].seq == first.last_seq + 2
|
||||
assert (await _row(first.id)).last_seq == first.last_seq + 1, "裁判后提交,last_seq 被写回了小的号"
|
||||
monkeypatch.setattr(evidence, "SessionLocal", opened)
|
||||
|
||||
# 进程重启:内存里的序号计数没了,启动时 setup 照常执行一遍。只改落后的那条,别的运行原样不动
|
||||
lagging = await _row(first.id)
|
||||
others = [await _finished(), await _paused()]
|
||||
await run_manager.shutdown()
|
||||
bus._seq.pop(first.id, None)
|
||||
await run_manager.setup()
|
||||
repaired = await _row(first.id)
|
||||
assert (repaired.last_seq, repaired.updated_at) == (written[-1].seq, lagging.updated_at)
|
||||
for other in others:
|
||||
now = await _row(other.id)
|
||||
assert (now.last_seq, now.updated_at) == (other.last_seq, other.updated_at)
|
||||
assert await evidence._append_judged(*_judged_args(first)) == {"seq": written[-1].seq + 1}
|
||||
await _no_repeated_seq(first.id)
|
||||
|
||||
flaky["fail"] = False
|
||||
await run_manager.continue_failed(first.id)
|
||||
final = await _until(first.id, ("succeeded",), sealed=True)
|
||||
assert final.manifest_seq > written[-1].seq + 1
|
||||
await _no_repeated_seq(first.id)
|
||||
assert (await verify_manifest(first.id))["ok"] is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 恢复后的重放:Agent 里批过的工具不重复执行
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user