mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 21:31:58 +00:00
复核 R-b15(隔离后等不到重新登记的运行原因记错、任务没摘):_drive 里 fence.ready() 等不到时照实收尾——排队中的(一步都没执行过)收成失败,写明「本进程隔离后没能重新登记,这次运行没有开始执行」并记 run.failed;要接着执行的(继续、批了审批)收成中断、断点完好可继续运行;条件更新,状态、属主变了(别的进程先接手)就不动;执行任务从 run_manager._tasks 里摘掉。补测试:隔离后新身份一直没登记上,新发起的运行收成失败、原因写对、run.failed 记下、任务摘掉
This commit is contained in:
@@ -58,6 +58,9 @@ REJECTED_CONTINUE = "被驳回的运行不能继续运行:改好内容后重
|
||||
REJECTED_RESUME = "这次运行已被驳回,没有待处理的审批,无法恢复:改好内容后重新发起"
|
||||
#: 服务重启时还在排队的运行改成失败时写的话(run.error 和补发的 run.failed 是同一句)
|
||||
QUEUED_AT_RESTART = "服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行"
|
||||
#: 本进程隔离(续不上租约)之后没能重新登记,这一段执行没有开始(cluster/fence.ready 等不到)
|
||||
NOT_REREGISTERED_QUEUED = "本进程隔离后没能重新登记,这次运行没有开始执行,请重新发起运行"
|
||||
NOT_REREGISTERED_RESUMABLE = "本进程隔离后没能重新登记,这次运行没有接着执行,已挂起,可从断点继续运行"
|
||||
|
||||
|
||||
class ContinueRefused(ValueError):
|
||||
@@ -798,7 +801,7 @@ class RunManager:
|
||||
try:
|
||||
await fence.ready()
|
||||
except Fenced:
|
||||
logger.error("运行 %s:本进程隔离之后一直没重新登记上,不开始执行(由别的进程按「属主不在了」处理)", run_id)
|
||||
await self._not_reregistered(run_id)
|
||||
return
|
||||
fence.bind()
|
||||
mine = identity.owner()
|
||||
@@ -1006,6 +1009,45 @@ class RunManager:
|
||||
finally:
|
||||
await slot.__aexit__(None, None, None)
|
||||
|
||||
async def _not_reregistered(self, run_id: str) -> None:
|
||||
"""隔离之后新身份一直没登记上、这一段执行不开始了(fence.ready 等不到,复核 R-b15):照实收尾,不留给别的进程按「服务
|
||||
重启时还在排队」去标。排队中的(一步都没执行过)收成失败;已经执行过、停在断点上又要接着执行的收成中断(断点完好,可以
|
||||
继续运行)。条件更新:状态、属主还是刚才看到的才改(别的进程先接手了就不动)。"""
|
||||
from app.cluster.claims import same
|
||||
|
||||
logger.error("运行 %s:本进程隔离之后一直没重新登记上,这一段执行不开始", run_id)
|
||||
try:
|
||||
event: RunEventModel | None = None
|
||||
async with SessionLocal() as session:
|
||||
seen = (await session.execute(select(Run.status, Run.owner).where(Run.id == run_id))).first()
|
||||
if seen is None or seen.status not in ("queued", "running"):
|
||||
return
|
||||
if seen.status == "queued":
|
||||
values: dict[str, Any] = {"status": "failed", "error": NOT_REREGISTERED_QUEUED}
|
||||
else:
|
||||
values = {"status": "interrupted", "error": NOT_REREGISTERED_RESUMABLE}
|
||||
claimed = await session.execute(update(Run).where(
|
||||
Run.id == run_id, Run.status == seen.status, same(Run.owner, seen.owner))
|
||||
.values(owner=None, **values).execution_options(synchronize_session=False))
|
||||
if claimed.rowcount != 1:
|
||||
await session.rollback()
|
||||
return
|
||||
run = await session.get(Run, run_id)
|
||||
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq if run else 0))
|
||||
event = (self._event(run_id, EventType.RUN_FAILED, data={"error": NOT_REREGISTERED_QUEUED})
|
||||
if seen.status == "queued" else
|
||||
self._event(run_id, EventType.LOG, data={"level": "warn", "code": "not_reregistered",
|
||||
"message": NOT_REREGISTERED_RESUMABLE}))
|
||||
await _stage(session, event, run)
|
||||
await session.commit()
|
||||
await bus.publish(event)
|
||||
await bus.close(run_id)
|
||||
except Exception: # noqa: BLE001 - 收不了尾的留给别的进程按「属主不在了」处理
|
||||
logger.exception("运行 %s:收尾时出错,留给别的进程按属主不在了处理", run_id)
|
||||
finally:
|
||||
self._tasks.pop(run_id, None)
|
||||
self._stopped_by.pop(run_id, None)
|
||||
|
||||
async def _stopped_in_queue(self, run_id: str) -> None:
|
||||
"""还在等并发名额时被取消。关停:什么都不写(还是排队中 / 刚认领的样子),照「属主不在了」的规矩由下次启动或别的
|
||||
进程接手(排队的标失败、认领了的标中断);人点了停止:收成 cancelled,同执行中停止(run.cancelled 带上是谁停的、封存)。"""
|
||||
|
||||
@@ -170,6 +170,29 @@ async def test_a_wall_clock_jump_does_not_fence_a_running_run(slow, monkeypatch,
|
||||
assert fence.generation() == before, "没有隔离"
|
||||
|
||||
|
||||
async def test_a_run_that_cannot_start_after_fencing_is_failed_with_the_real_reason(monkeypatch):
|
||||
"""复核 R-b15:隔离之后新身份一直没登记上,新发起的运行等不到(fence.ready)——这一行照实收成失败(写明是隔离后没能重新
|
||||
登记,不是「服务重启时还在排队」),run.failed 记下来,执行任务也从 run_manager 里摘掉。"""
|
||||
from app.cluster import workers
|
||||
from app.engine.runner import NOT_REREGISTERED_QUEUED
|
||||
|
||||
monkeypatch.setattr(workers, "lease_s", lambda: 0.3)
|
||||
fence.trip("测试") # 隔离了,新身份还没登记上
|
||||
run = await run_manager.start(graph=GRAPH, input_payload={"question": "q"})
|
||||
for _ in range(100):
|
||||
if run.id not in run_manager._tasks:
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
assert run.id not in run_manager._tasks, "执行任务摘掉了"
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
assert row.status == "failed" and row.error == NOT_REREGISTERED_QUEUED and row.owner is None
|
||||
events = await _events(run.id)
|
||||
assert [e.type for e in events if e.type == "run.failed"] == ["run.failed"]
|
||||
assert events[-1].data["error"] == NOT_REREGISTERED_QUEUED
|
||||
fence.renewed()
|
||||
|
||||
|
||||
async def test_the_final_write_checks_the_owner():
|
||||
async with SessionLocal() as session:
|
||||
run = Run(status="running", owner="peer-host:1:abcd", graph={}, input={})
|
||||
|
||||
Reference in New Issue
Block a user