复核 R-b15(隔离后等不到重新登记的运行原因记错、任务没摘):_drive 里 fence.ready() 等不到时照实收尾——排队中的(一步都没执行过)收成失败,写明「本进程隔离后没能重新登记,这次运行没有开始执行」并记 run.failed;要接着执行的(继续、批了审批)收成中断、断点完好可继续运行;条件更新,状态、属主变了(别的进程先接手)就不动;执行任务从 run_manager._tasks 里摘掉。补测试:隔离后新身份一直没登记上,新发起的运行收成失败、原因写对、run.failed 记下、任务摘掉

This commit is contained in:
ThearchyHelios
2026-10-19 15:08:39 +08:00
parent eb9af51abb
commit c0f7aae8ad
2 changed files with 66 additions and 1 deletions
+43 -1
View File
@@ -58,6 +58,9 @@ REJECTED_CONTINUE = "被驳回的运行不能继续运行:改好内容后重
REJECTED_RESUME = "这次运行已被驳回,没有待处理的审批,无法恢复:改好内容后重新发起"
#: 服务重启时还在排队的运行改成失败时写的话(run.error 和补发的 run.failed 是同一句)
QUEUED_AT_RESTART = "服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行"
#: 本进程隔离(续不上租约)之后没能重新登记,这一段执行没有开始(cluster/fence.ready 等不到)
NOT_REREGISTERED_QUEUED = "本进程隔离后没能重新登记,这次运行没有开始执行,请重新发起运行"
NOT_REREGISTERED_RESUMABLE = "本进程隔离后没能重新登记,这次运行没有接着执行,已挂起,可从断点继续运行"
class ContinueRefused(ValueError):
@@ -798,7 +801,7 @@ class RunManager:
try:
await fence.ready()
except Fenced:
logger.error("运行 %s:本进程隔离之后一直没重新登记上,不开始执行(由别的进程按「属主不在了」处理)", run_id)
await self._not_reregistered(run_id)
return
fence.bind()
mine = identity.owner()
@@ -1006,6 +1009,45 @@ class RunManager:
finally:
await slot.__aexit__(None, None, None)
async def _not_reregistered(self, run_id: str) -> None:
"""隔离之后新身份一直没登记上、这一段执行不开始了(fence.ready 等不到,复核 R-b15):照实收尾,不留给别的进程按「服务
重启时还在排队」去标。排队中的(一步都没执行过)收成失败;已经执行过、停在断点上又要接着执行的收成中断(断点完好,可以
继续运行)。条件更新:状态、属主还是刚才看到的才改(别的进程先接手了就不动)。"""
from app.cluster.claims import same
logger.error("运行 %s:本进程隔离之后一直没重新登记上,这一段执行不开始", run_id)
try:
event: RunEventModel | None = None
async with SessionLocal() as session:
seen = (await session.execute(select(Run.status, Run.owner).where(Run.id == run_id))).first()
if seen is None or seen.status not in ("queued", "running"):
return
if seen.status == "queued":
values: dict[str, Any] = {"status": "failed", "error": NOT_REREGISTERED_QUEUED}
else:
values = {"status": "interrupted", "error": NOT_REREGISTERED_RESUMABLE}
claimed = await session.execute(update(Run).where(
Run.id == run_id, Run.status == seen.status, same(Run.owner, seen.owner))
.values(owner=None, **values).execution_options(synchronize_session=False))
if claimed.rowcount != 1:
await session.rollback()
return
run = await session.get(Run, run_id)
bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq if run else 0))
event = (self._event(run_id, EventType.RUN_FAILED, data={"error": NOT_REREGISTERED_QUEUED})
if seen.status == "queued" else
self._event(run_id, EventType.LOG, data={"level": "warn", "code": "not_reregistered",
"message": NOT_REREGISTERED_RESUMABLE}))
await _stage(session, event, run)
await session.commit()
await bus.publish(event)
await bus.close(run_id)
except Exception: # noqa: BLE001 - 收不了尾的留给别的进程按「属主不在了」处理
logger.exception("运行 %s:收尾时出错,留给别的进程按属主不在了处理", run_id)
finally:
self._tasks.pop(run_id, None)
self._stopped_by.pop(run_id, None)
async def _stopped_in_queue(self, run_id: str) -> None:
"""还在等并发名额时被取消。关停:什么都不写(还是排队中 / 刚认领的样子),照「属主不在了」的规矩由下次启动或别的
进程接手(排队的标失败、认领了的标中断);人点了停止:收成 cancelled,同执行中停止(run.cancelled 带上是谁停的、封存)。"""
+23
View File
@@ -170,6 +170,29 @@ async def test_a_wall_clock_jump_does_not_fence_a_running_run(slow, monkeypatch,
assert fence.generation() == before, "没有隔离"
async def test_a_run_that_cannot_start_after_fencing_is_failed_with_the_real_reason(monkeypatch):
"""复核 R-b15:隔离之后新身份一直没登记上,新发起的运行等不到(fence.ready)——这一行照实收成失败(写明是隔离后没能重新
登记,不是「服务重启时还在排队」),run.failed 记下来,执行任务也从 run_manager 里摘掉。"""
from app.cluster import workers
from app.engine.runner import NOT_REREGISTERED_QUEUED
monkeypatch.setattr(workers, "lease_s", lambda: 0.3)
fence.trip("测试") # 隔离了,新身份还没登记上
run = await run_manager.start(graph=GRAPH, input_payload={"question": "q"})
for _ in range(100):
if run.id not in run_manager._tasks:
break
await asyncio.sleep(0.05)
assert run.id not in run_manager._tasks, "执行任务摘掉了"
async with SessionLocal() as session:
row = await session.get(Run, run.id)
assert row.status == "failed" and row.error == NOT_REREGISTERED_QUEUED and row.owner is None
events = await _events(run.id)
assert [e.type for e in events if e.type == "run.failed"] == ["run.failed"]
assert events[-1].data["error"] == NOT_REREGISTERED_QUEUED
fence.renewed()
async def test_the_final_write_checks_the_owner():
async with SessionLocal() as session:
run = Run(status="running", owner="peer-host:1:abcd", graph={}, input={})