mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 20:01:59 +00:00
复核 R-b10(隔离换身份后旧身份的名额、请求没人收拾):要名额时把当时的身份记在 holder 上,放的时候按它删,换了身份也删得到;新身份登记上之后收拾旧身份(tidy_retired)——它的执行都已停下、写不了了,不必等它的租约过期:删掉它的登记(发 gone,别的进程马上按属主不在了接手它认领的运行、任务)、它占着的运行名额和同源队的位置,交出它拿着的当班租约,发给它的请求转到新身份名下照办,照实记一条日志;停止请求对不在本进程执行的运行也照停止的规矩办(cancel 自己看属主:转给正在执行它的进程、属主不在了就接手放弃、停在断点上的收成 cancelled)。补测试:拿着名额时换了身份照样放得掉(改前的写法这条不过);新身份登记上之后旧身份的登记、名额、队里的位置、当班租约都收拾掉,转过来的停止请求把停在断点上的那次运行收成 cancelled
This commit is contained in:
@@ -221,6 +221,48 @@ class _Service:
|
||||
fence.renewed()
|
||||
logger.warning("隔离之后换了身份重新登记:%s → %s(之前认领的行由当班进程按「属主不在了」接手)",
|
||||
"、".join(retired), identity.owner())
|
||||
try:
|
||||
await self.tidy_retired(retired)
|
||||
except Exception: # noqa: BLE001 - 收拾不了的等旧身份的租约过期、按「属主不在了」处理
|
||||
logger.exception("收拾旧身份留下的登记、名额、请求时出错(等它们的租约过期再按属主不在了处理)")
|
||||
|
||||
async def tidy_retired(self, retired: list[str]) -> dict[str, int]:
|
||||
"""新身份登记上之后,收拾旧身份留下的(复核 R-b10):它的执行都已停下、写不了了,不必等它的租约过期——删掉它的登记(别的
|
||||
进程马上按「属主不在了」接手它认领的运行、任务)、它占着的运行名额和同源队的位置,交出它拿着的当班租约;发给它的请求转到
|
||||
新身份名下照办(停止请求对已经不在本进程执行的运行照停止的规矩办:属主不在了就接手、放弃)。"""
|
||||
from sqlalchemy import delete, update
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models_cluster import ClusterLease, ClusterRequest, ClusterWorker, LaneTicket, RunSlot
|
||||
|
||||
if not retired:
|
||||
return {}
|
||||
me = identity.owner()
|
||||
async with SessionLocal() as session:
|
||||
out = {
|
||||
"workers": (await session.execute(delete(ClusterWorker).where(ClusterWorker.id.in_(retired)))).rowcount,
|
||||
"slots": (await session.execute(delete(RunSlot).where(RunSlot.owner.in_(retired)))).rowcount,
|
||||
"lanes": (await session.execute(delete(LaneTicket).where(LaneTicket.owner.in_(retired)))).rowcount,
|
||||
"requests": (await session.execute(update(ClusterRequest).where(ClusterRequest.owner.in_(retired))
|
||||
.values(owner=me).execution_options(synchronize_session=False))).rowcount,
|
||||
"leases": (await session.execute(update(ClusterLease).where(ClusterLease.owner.in_(retired))
|
||||
.values(owner=None, lease_until=None)
|
||||
.execution_options(synchronize_session=False))).rowcount,
|
||||
}
|
||||
await session.commit()
|
||||
out = {k: int(v or 0) for k, v in out.items()}
|
||||
for old in retired:
|
||||
signals.publish("gone", old)
|
||||
if out["slots"]:
|
||||
signals.publish("slot")
|
||||
if out["lanes"]:
|
||||
signals.publish("lane")
|
||||
if out["requests"]:
|
||||
signals.publish("q", me)
|
||||
self.reap_now = True
|
||||
logger.warning("收拾旧身份 %s:删了登记 %d 条、运行名额 %d 个、同源队的位置 %d 个,%d 条发给它的请求转给 %s,交出当班租约 %d 份",
|
||||
"、".join(retired), out["workers"], out["slots"], out["lanes"], out["requests"], me, out["leases"])
|
||||
return out
|
||||
|
||||
# ---------------- 当班、心跳的其余活 ----------------
|
||||
|
||||
|
||||
@@ -196,6 +196,8 @@ class _Holder:
|
||||
woke: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
#: 还在排队等名额(发下来之前)
|
||||
queued: bool = True
|
||||
#: 要名额时本进程的身份(名额行记的属主):放的时候按它删——隔离之后换了身份也删得到(复核 R-b10)
|
||||
owner: str = ""
|
||||
|
||||
|
||||
_holders: dict[str, _Holder] = {}
|
||||
@@ -341,27 +343,30 @@ async def _granted(run_id: str) -> bool | None:
|
||||
return await session.scalar(select(RunSlot.granted).where(RunSlot.run_id == run_id))
|
||||
|
||||
|
||||
async def _request(run_id: str, klass: str, priority: int) -> None:
|
||||
async def _request(run_id: str, klass: str, priority: int) -> str:
|
||||
"""排进名额的队。交回这一行记的属主(本进程此刻的身份)。"""
|
||||
me = identity.owner()
|
||||
if _in_memory():
|
||||
_mem[run_id] = _MemRow(run_id, klass, priority, utcnow(), identity.owner())
|
||||
return
|
||||
_mem[run_id] = _MemRow(run_id, klass, priority, utcnow(), me)
|
||||
return me
|
||||
from app.db.models_cluster import RunSlot
|
||||
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id))
|
||||
session.add(RunSlot(run_id=run_id, klass=klass, priority=priority, owner=identity.owner(), granted=False,
|
||||
session.add(RunSlot(run_id=run_id, klass=klass, priority=priority, owner=me, granted=False,
|
||||
requested_at=utcnow()))
|
||||
await session.commit()
|
||||
return me
|
||||
|
||||
|
||||
async def _release(run_id: str) -> None:
|
||||
async def _release(run_id: str, owner: str) -> None:
|
||||
if _in_memory():
|
||||
_mem.pop(run_id, None)
|
||||
return
|
||||
from app.db.models_cluster import RunSlot
|
||||
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id, RunSlot.owner == identity.owner()))
|
||||
await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id, RunSlot.owner == owner))
|
||||
await session.commit()
|
||||
|
||||
|
||||
@@ -375,7 +380,7 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]:
|
||||
_holders[run_id] = holder
|
||||
poll = POLL_S.get(settings.db_dialect, 5.0)
|
||||
try:
|
||||
await _request(run_id, klass, priority)
|
||||
holder.owner = await _request(run_id, klass, priority)
|
||||
while True:
|
||||
holder.woke.clear()
|
||||
await grant()
|
||||
@@ -384,7 +389,7 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]:
|
||||
holder.queued = False
|
||||
break
|
||||
if got is None:
|
||||
await _request(run_id, klass, priority)
|
||||
holder.owner = await _request(run_id, klass, priority)
|
||||
continue
|
||||
with contextlib.suppress(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(holder.woke.wait(), timeout=poll)
|
||||
@@ -393,7 +398,7 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]:
|
||||
if _holders.get(run_id) is holder:
|
||||
_holders.pop(run_id, None)
|
||||
try:
|
||||
await asyncio.shield(_release(run_id))
|
||||
await asyncio.shield(_release(run_id, holder.owner or identity.owner()))
|
||||
except Exception: # noqa: BLE001 - 放不掉的名额下次发的时候按「本进程里已经没人用」收回
|
||||
logger.exception("运行 %s 的名额没放掉,下次发名额时收回", run_id)
|
||||
wake_local()
|
||||
|
||||
@@ -1825,8 +1825,9 @@ def _register_requests() -> None:
|
||||
|
||||
@requests.handle("stop", "run")
|
||||
async def _stop(run_id: str, actor: str | None, _payload: dict[str, Any]) -> None:
|
||||
if run_manager.is_active(run_id):
|
||||
await run_manager.cancel(run_id, actor=actor)
|
||||
# 不在本进程执行的也照停止的规矩办(cancel 自己看属主):在别的活着的进程里执行就转给它,属主不在了就接手、放弃,停在
|
||||
# 断点上的收成 cancelled,已经结束的什么都不做。隔离之后转到新身份名下的停止请求就是这样落下的(复核 R-b10)
|
||||
await run_manager.cancel(run_id, actor=actor)
|
||||
|
||||
@requests.handle("note", "run")
|
||||
async def _note(run_id: str, _actor: str | None, payload: dict[str, Any]) -> None:
|
||||
|
||||
@@ -337,6 +337,63 @@ async def test_stopping_a_run_whose_owner_is_gone_takes_it_over_and_abandons_it(
|
||||
assert row.status == "cancelled" and row.end_reason == "stopped"
|
||||
|
||||
|
||||
@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3)")
|
||||
async def test_a_slot_is_released_under_the_identity_it_was_taken_with(monkeypatch):
|
||||
"""复核 R-b10:拿着名额的时候隔离换了身份,放名额照样删得到那一行(按要名额时的身份删),不用等旧身份过期才收回。"""
|
||||
_policy(monkeypatch, 2, interactive=(2, 40))
|
||||
monkeypatch.setattr(identity, "OWNER", identity.OWNER)
|
||||
async with slots.hold("rb10-run", "chat"):
|
||||
identity.regenerate()
|
||||
async with SessionLocal() as session:
|
||||
left = list((await session.execute(select(RunSlot).where(RunSlot.run_id == "rb10-run"))).scalars())
|
||||
assert left == []
|
||||
|
||||
|
||||
async def test_registering_again_tidies_up_what_the_retired_identity_left(monkeypatch):
|
||||
"""复核 R-b10:隔离之后新身份登记上,旧身份的登记、名额、同源队的位置删掉,当班租约交出,发给它的请求转到新身份名下照办——
|
||||
停止请求对停在断点上的那次运行照停止的规矩收成 cancelled。"""
|
||||
from app.cluster import requests, version
|
||||
from app.cluster import service as service_mod
|
||||
from app.cluster.fence import now as fence_now
|
||||
|
||||
monkeypatch.setattr(identity, "OWNER", identity.OWNER)
|
||||
old = identity.owner()
|
||||
now = utcnow()
|
||||
async with SessionLocal() as session:
|
||||
session.add(ClusterWorker(id=old, host="this-host", pid=1, started_at=now, heartbeat_at=now,
|
||||
lease_until=now + timedelta(seconds=60), code_version=version.code_version(),
|
||||
schema_version=version.schema_version()))
|
||||
session.add(RunSlot(run_id="retired-slot", klass="interactive", priority=40, owner=old, granted=True,
|
||||
requested_at=now))
|
||||
session.add(LaneTicket(source_id="src-retired", kind="profile", job_id="retired-job", owner=old, granted=True))
|
||||
session.add(ClusterLease(name="test-retired", owner=old, lease_until=now + timedelta(seconds=60), epoch=1))
|
||||
run = Run(status="interrupted", owner=old, graph={}, input={})
|
||||
session.add(run)
|
||||
await session.flush()
|
||||
session.add(ClusterRequest(owner=old, kind="stop", target="run", target_id=run.id, actor="张工", payload={}))
|
||||
await session.commit()
|
||||
svc = service_mod._Service()
|
||||
svc.retired = [old]
|
||||
identity.regenerate()
|
||||
svc.pending_since = fence_now()
|
||||
await svc.register_again()
|
||||
new = identity.owner()
|
||||
async with SessionLocal() as session:
|
||||
ids = set((await session.execute(select(ClusterWorker.id))).scalars())
|
||||
slot_rows = list((await session.execute(select(RunSlot).where(RunSlot.owner == old))).scalars())
|
||||
lane_rows = list((await session.execute(select(LaneTicket).where(LaneTicket.owner == old))).scalars())
|
||||
lease = await session.get(ClusterLease, "test-retired")
|
||||
sent = list((await session.execute(select(ClusterRequest.owner).where(ClusterRequest.target_id == run.id))).scalars())
|
||||
assert old not in ids and new in ids, "旧身份的登记删掉:别的进程马上按属主不在了接手"
|
||||
assert slot_rows == [] and lane_rows == [] and lease.owner is None
|
||||
assert sent == [new], "发给旧身份的请求转到新身份名下"
|
||||
await requests.drain()
|
||||
await requests.settle()
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
assert row.status == "cancelled" and row.end_reason == "stopped", "停止请求照停止的规矩落下了"
|
||||
|
||||
|
||||
async def test_leadership_is_handed_over_when_the_lease_runs_out():
|
||||
me = leader_mod.Leadership("test-lead")
|
||||
assert await me.claim() and me.held()
|
||||
|
||||
Reference in New Issue
Block a user