diff --git a/backend/app/cluster/service.py b/backend/app/cluster/service.py index 2d97a18b..4bb6cf29 100644 --- a/backend/app/cluster/service.py +++ b/backend/app/cluster/service.py @@ -221,6 +221,48 @@ class _Service: fence.renewed() logger.warning("隔离之后换了身份重新登记:%s → %s(之前认领的行由当班进程按「属主不在了」接手)", "、".join(retired), identity.owner()) + try: + await self.tidy_retired(retired) + except Exception: # noqa: BLE001 - 收拾不了的等旧身份的租约过期、按「属主不在了」处理 + logger.exception("收拾旧身份留下的登记、名额、请求时出错(等它们的租约过期再按属主不在了处理)") + + async def tidy_retired(self, retired: list[str]) -> dict[str, int]: + """新身份登记上之后,收拾旧身份留下的(复核 R-b10):它的执行都已停下、写不了了,不必等它的租约过期——删掉它的登记(别的 + 进程马上按「属主不在了」接手它认领的运行、任务)、它占着的运行名额和同源队的位置,交出它拿着的当班租约;发给它的请求转到 + 新身份名下照办(停止请求对已经不在本进程执行的运行照停止的规矩办:属主不在了就接手、放弃)。""" + from sqlalchemy import delete, update + + from app.db.base import SessionLocal + from app.db.models_cluster import ClusterLease, ClusterRequest, ClusterWorker, LaneTicket, RunSlot + + if not retired: + return {} + me = identity.owner() + async with SessionLocal() as session: + out = { + "workers": (await session.execute(delete(ClusterWorker).where(ClusterWorker.id.in_(retired)))).rowcount, + "slots": (await session.execute(delete(RunSlot).where(RunSlot.owner.in_(retired)))).rowcount, + "lanes": (await session.execute(delete(LaneTicket).where(LaneTicket.owner.in_(retired)))).rowcount, + "requests": (await session.execute(update(ClusterRequest).where(ClusterRequest.owner.in_(retired)) + .values(owner=me).execution_options(synchronize_session=False))).rowcount, + "leases": (await session.execute(update(ClusterLease).where(ClusterLease.owner.in_(retired)) + .values(owner=None, lease_until=None) + .execution_options(synchronize_session=False))).rowcount, + } + await session.commit() + out = {k: int(v or 0) for k, v in out.items()} + for old in retired: + signals.publish("gone", old) + if out["slots"]: + signals.publish("slot") + if out["lanes"]: + signals.publish("lane") + if out["requests"]: + signals.publish("q", me) + self.reap_now = True + logger.warning("收拾旧身份 %s:删了登记 %d 条、运行名额 %d 个、同源队的位置 %d 个,%d 条发给它的请求转给 %s,交出当班租约 %d 份", + "、".join(retired), out["workers"], out["slots"], out["lanes"], out["requests"], me, out["leases"]) + return out # ---------------- 当班、心跳的其余活 ---------------- diff --git a/backend/app/cluster/slots.py b/backend/app/cluster/slots.py index d363dedb..48878c6d 100644 --- a/backend/app/cluster/slots.py +++ b/backend/app/cluster/slots.py @@ -196,6 +196,8 @@ class _Holder: woke: asyncio.Event = field(default_factory=asyncio.Event) #: 还在排队等名额(发下来之前) queued: bool = True + #: 要名额时本进程的身份(名额行记的属主):放的时候按它删——隔离之后换了身份也删得到(复核 R-b10) + owner: str = "" _holders: dict[str, _Holder] = {} @@ -341,27 +343,30 @@ async def _granted(run_id: str) -> bool | None: return await session.scalar(select(RunSlot.granted).where(RunSlot.run_id == run_id)) -async def _request(run_id: str, klass: str, priority: int) -> None: +async def _request(run_id: str, klass: str, priority: int) -> str: + """排进名额的队。交回这一行记的属主(本进程此刻的身份)。""" + me = identity.owner() if _in_memory(): - _mem[run_id] = _MemRow(run_id, klass, priority, utcnow(), identity.owner()) - return + _mem[run_id] = _MemRow(run_id, klass, priority, utcnow(), me) + return me from app.db.models_cluster import RunSlot async with SessionLocal() as session: await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id)) - session.add(RunSlot(run_id=run_id, klass=klass, priority=priority, owner=identity.owner(), granted=False, + session.add(RunSlot(run_id=run_id, klass=klass, priority=priority, owner=me, granted=False, requested_at=utcnow())) await session.commit() + return me -async def _release(run_id: str) -> None: +async def _release(run_id: str, owner: str) -> None: if _in_memory(): _mem.pop(run_id, None) return from app.db.models_cluster import RunSlot async with SessionLocal() as session: - await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id, RunSlot.owner == identity.owner())) + await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id, RunSlot.owner == owner)) await session.commit() @@ -375,7 +380,7 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]: _holders[run_id] = holder poll = POLL_S.get(settings.db_dialect, 5.0) try: - await _request(run_id, klass, priority) + holder.owner = await _request(run_id, klass, priority) while True: holder.woke.clear() await grant() @@ -384,7 +389,7 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]: holder.queued = False break if got is None: - await _request(run_id, klass, priority) + holder.owner = await _request(run_id, klass, priority) continue with contextlib.suppress(asyncio.TimeoutError): await asyncio.wait_for(holder.woke.wait(), timeout=poll) @@ -393,7 +398,7 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]: if _holders.get(run_id) is holder: _holders.pop(run_id, None) try: - await asyncio.shield(_release(run_id)) + await asyncio.shield(_release(run_id, holder.owner or identity.owner())) except Exception: # noqa: BLE001 - 放不掉的名额下次发的时候按「本进程里已经没人用」收回 logger.exception("运行 %s 的名额没放掉,下次发名额时收回", run_id) wake_local() diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 5d4c4af7..aa547ef7 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -1825,8 +1825,9 @@ def _register_requests() -> None: @requests.handle("stop", "run") async def _stop(run_id: str, actor: str | None, _payload: dict[str, Any]) -> None: - if run_manager.is_active(run_id): - await run_manager.cancel(run_id, actor=actor) + # 不在本进程执行的也照停止的规矩办(cancel 自己看属主):在别的活着的进程里执行就转给它,属主不在了就接手、放弃,停在 + # 断点上的收成 cancelled,已经结束的什么都不做。隔离之后转到新身份名下的停止请求就是这样落下的(复核 R-b10) + await run_manager.cancel(run_id, actor=actor) @requests.handle("note", "run") async def _note(run_id: str, _actor: str | None, payload: dict[str, Any]) -> None: diff --git a/backend/tests/test_cluster.py b/backend/tests/test_cluster.py index b92a68fd..580e5377 100644 --- a/backend/tests/test_cluster.py +++ b/backend/tests/test_cluster.py @@ -337,6 +337,63 @@ async def test_stopping_a_run_whose_owner_is_gone_takes_it_over_and_abandons_it( assert row.status == "cancelled" and row.end_reason == "stopped" +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3)") +async def test_a_slot_is_released_under_the_identity_it_was_taken_with(monkeypatch): + """复核 R-b10:拿着名额的时候隔离换了身份,放名额照样删得到那一行(按要名额时的身份删),不用等旧身份过期才收回。""" + _policy(monkeypatch, 2, interactive=(2, 40)) + monkeypatch.setattr(identity, "OWNER", identity.OWNER) + async with slots.hold("rb10-run", "chat"): + identity.regenerate() + async with SessionLocal() as session: + left = list((await session.execute(select(RunSlot).where(RunSlot.run_id == "rb10-run"))).scalars()) + assert left == [] + + +async def test_registering_again_tidies_up_what_the_retired_identity_left(monkeypatch): + """复核 R-b10:隔离之后新身份登记上,旧身份的登记、名额、同源队的位置删掉,当班租约交出,发给它的请求转到新身份名下照办—— + 停止请求对停在断点上的那次运行照停止的规矩收成 cancelled。""" + from app.cluster import requests, version + from app.cluster import service as service_mod + from app.cluster.fence import now as fence_now + + monkeypatch.setattr(identity, "OWNER", identity.OWNER) + old = identity.owner() + now = utcnow() + async with SessionLocal() as session: + session.add(ClusterWorker(id=old, host="this-host", pid=1, started_at=now, heartbeat_at=now, + lease_until=now + timedelta(seconds=60), code_version=version.code_version(), + schema_version=version.schema_version())) + session.add(RunSlot(run_id="retired-slot", klass="interactive", priority=40, owner=old, granted=True, + requested_at=now)) + session.add(LaneTicket(source_id="src-retired", kind="profile", job_id="retired-job", owner=old, granted=True)) + session.add(ClusterLease(name="test-retired", owner=old, lease_until=now + timedelta(seconds=60), epoch=1)) + run = Run(status="interrupted", owner=old, graph={}, input={}) + session.add(run) + await session.flush() + session.add(ClusterRequest(owner=old, kind="stop", target="run", target_id=run.id, actor="张工", payload={})) + await session.commit() + svc = service_mod._Service() + svc.retired = [old] + identity.regenerate() + svc.pending_since = fence_now() + await svc.register_again() + new = identity.owner() + async with SessionLocal() as session: + ids = set((await session.execute(select(ClusterWorker.id))).scalars()) + slot_rows = list((await session.execute(select(RunSlot).where(RunSlot.owner == old))).scalars()) + lane_rows = list((await session.execute(select(LaneTicket).where(LaneTicket.owner == old))).scalars()) + lease = await session.get(ClusterLease, "test-retired") + sent = list((await session.execute(select(ClusterRequest.owner).where(ClusterRequest.target_id == run.id))).scalars()) + assert old not in ids and new in ids, "旧身份的登记删掉:别的进程马上按属主不在了接手" + assert slot_rows == [] and lane_rows == [] and lease.owner is None + assert sent == [new], "发给旧身份的请求转到新身份名下" + await requests.drain() + await requests.settle() + async with SessionLocal() as session: + row = await session.get(Run, run.id) + assert row.status == "cancelled" and row.end_reason == "stopped", "停止请求照停止的规矩落下了" + + async def test_leadership_is_handed_over_when_the_lease_runs_out(): me = leader_mod.Leadership("test-lead") assert await me.claim() and me.held()