复核 R-b3(SQLite 单进程多出来的写库):SQLite 时守卫已经确认只有本进程——登记只写一次、不续租约、本进程直接当班(不写库)、接手只在启动时做,心跳只做本进程自己攒着的活;运行的名额在 SQLite 下放在进程里(规矩同一套:优先级、类别上限、交互留底,只是不落库),一段运行不再为要名额多写三次库。顺带(复核「另外两处」之二):PostgreSQL 发名额持着全站的锁时不再另取连接查「谁还活着」,拿锁之前备好,不认识的属主这一次当活着。补测试:SQLite 空闲十来拍心跳、要一次名额放一次,PRAGMA data_version 不变(名额落库的写法这条不过);库里的名额行那两条只在 PostgreSQL 上跑

This commit is contained in:
ThearchyHelios
2026-10-19 13:22:05 +08:00
parent 96bb372ef3
commit 5f7100906f
2 changed files with 123 additions and 28 deletions
+92 -27
View File
@@ -225,44 +225,83 @@ def _grant_lock() -> asyncio.Lock:
return lock
@dataclass
class _MemRow:
run_id: str
klass: str
priority: int
requested_at: Any
owner: str = ""
granted: bool = False
#: SQLite 时的名额(复核 R-b3):只有一个进程,规矩照同一套、只是不落库——一段运行不为要名额多写三次库
_mem: dict[str, _MemRow] = {}
def _in_memory() -> bool:
return settings.db_dialect == "sqlite"
def _pick(rows: list[Any], pol: dict[str, Any]) -> list[str]:
"""按规矩挑这一次发给谁(库里的行、进程里的行同一套):优先级高的先、同级先来先得,类别上限、全站上限、交互留底。"""
used = Counter(r.klass for r in rows if r.granted)
total = sum(used.values())
# 交互以外的三类合起来用了几个:到了「全站上限 − 交互留底」就不再发给它们(留底永远给交互)
others = total - used["interactive"]
others_cap = borrowable(pol)
limits = {k: int(v["limit"]) for k, v in pol["classes"].items()}
granted: list[str] = []
waiting = sorted((r for r in rows if not r.granted), key=lambda r: (-int(r.priority or 0), r.requested_at, r.run_id))
for row in waiting:
if total >= int(pol["max_total"]):
break
if used[row.klass] >= limits.get(row.klass, int(pol["max_total"])):
continue
if row.klass != "interactive" and others >= others_cap:
continue
used[row.klass] += 1
total += 1
if row.klass != "interactive":
others += 1
granted.append(row.run_id)
return granted
async def grant() -> list[str]:
"""把等着的按规矩过一遍,够格的改成已发;属主不在了的行删掉。交回这一次发出去的。全站串行(见模块说明)。"""
if _in_memory():
pol = await policy()
for run_id in [r for r in _mem if not _local(r)]:
_mem.pop(run_id, None)
granted = _pick(list(_mem.values()), pol)
for run_id in granted:
_mem[run_id].granted = True
if granted:
wake_local()
return granted
from app.cluster import state
from app.db.models_cluster import RunSlot
pol = await policy()
# 拿锁之前先备好「谁还活着」(必要时这时候查库),持锁期间只用持锁的那条连接
live = await state.current()
known_dead = await _known_dead(live)
async with _grant_lock():
async with SessionLocal() as session:
if session.get_bind().dialect.name == "postgresql":
await session.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": _GRANT_KEY})
rows = list((await session.execute(select(RunSlot))).scalars())
live = await state.live_for([r.owner for r in rows])
dead = [r.run_id for r in rows if not live.alive(r.owner, local=_local(r.run_id))]
# 持着全站的锁时不另取连接查库(复核:连接池用尽时会卡到取连接超时):用锁之前拿好的那一份「谁还活着」,不认识的属主
# 这一次当活着(不删它的行),下一次手上那份更新了再说
me = identity.owner()
dead = [r.run_id for r in rows
if (r.owner == me and not _local(r.run_id)) or (r.owner != me and r.owner in known_dead)]
if dead:
await session.execute(delete(RunSlot).where(RunSlot.run_id.in_(dead)))
rows = [r for r in rows if r.run_id not in set(dead)]
used = Counter(r.klass for r in rows if r.granted)
total = sum(used.values())
# 交互以外的三类合起来用了几个:到了「全站上限 − 交互留底」就不再发给它们(留底永远给交互)
others = total - used["interactive"]
others_cap = borrowable(pol)
limits = {k: int(v["limit"]) for k, v in pol["classes"].items()}
granted: list[str] = []
waiting = sorted((r for r in rows if not r.granted),
key=lambda r: (-int(r.priority or 0), r.requested_at, r.run_id))
granted = _pick(rows, pol)
now = utcnow()
for row in waiting:
if total >= int(pol["max_total"]):
break
if used[row.klass] >= limits.get(row.klass, int(pol["max_total"])):
continue
if row.klass != "interactive" and others >= others_cap:
continue
used[row.klass] += 1
total += 1
if row.klass != "interactive":
others += 1
granted.append(row.run_id)
if granted:
await session.execute(update(RunSlot).where(RunSlot.run_id.in_(granted))
.values(granted=True, granted_at=now).execution_options(synchronize_session=False))
@@ -275,8 +314,25 @@ async def grant() -> list[str]:
return granted
async def _known_dead(live: Any) -> set[str]:
"""名额表里属主不在手上那份「活着」里的那几个,拿锁之前当场再查一次登记:仍不在的算没了。"""
from app.cluster import state
from app.db.models_cluster import RunSlot
async with SessionLocal() as session:
owners = set((await session.execute(select(RunSlot.owner).distinct())).scalars())
unknown = {o for o in owners if o and o != identity.owner() and o not in live.others}
if not unknown:
return set()
fresh = await state.current(fresh=True)
return {o for o in unknown if o not in fresh.others}
async def _granted(run_id: str) -> bool | None:
"""这一行发了没有:True 发了,False 还在等,None 这一行不见了(被当成属主不在了删掉)。"""
if _in_memory():
row = _mem.get(run_id)
return None if row is None else row.granted
from app.db.models_cluster import RunSlot
async with SessionLocal() as session:
@@ -284,6 +340,9 @@ async def _granted(run_id: str) -> bool | None:
async def _request(run_id: str, klass: str, priority: int) -> None:
if _in_memory():
_mem[run_id] = _MemRow(run_id, klass, priority, utcnow(), identity.owner())
return
from app.db.models_cluster import RunSlot
async with SessionLocal() as session:
@@ -294,6 +353,9 @@ async def _request(run_id: str, klass: str, priority: int) -> None:
async def _release(run_id: str) -> None:
if _in_memory():
_mem.pop(run_id, None)
return
from app.db.models_cluster import RunSlot
async with SessionLocal() as session:
@@ -337,11 +399,14 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]:
async def snapshot() -> dict[str, Any]:
"""现在各类在跑、在等几个(系统信息、验收脚本看)。"""
from app.db.models_cluster import RunSlot
if _in_memory():
rows = [(r.klass, r.granted) for r in _mem.values() if _local(r.run_id)]
else:
from app.db.models_cluster import RunSlot
async with SessionLocal() as session:
rows = list((await session.execute(select(RunSlot.klass, RunSlot.granted))).all())
out = {k: {"running": 0, "waiting": 0} for k in CLASSES}
async with SessionLocal() as session:
rows = list((await session.execute(select(RunSlot.klass, RunSlot.granted))).all())
out = {k: {"running": 0, "waiting": 0} for k in CLASSES} # noqa: C420
for klass, granted in rows:
out.setdefault(klass, {"running": 0, "waiting": 0})["running" if granted else "waiting"] += 1
return {"policy": await policy(), "classes": out}
+31 -1
View File
@@ -115,7 +115,7 @@ async def test_a_class_at_its_cap_does_not_hold_up_the_others(monkeypatch):
await _until(lambda: "chat" in got)
assert len({"e1", "e2"} & set(got)) == 1, "系统类到了上限:另一个等着"
snapshot = await slots.snapshot()
assert snapshot["classes"]["system"] == {"running": 1, "waiting": 1}
assert snapshot["classes"]["system"] == {"running": 1, "waiting": 1}, snapshot
gate.set()
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
@@ -174,6 +174,7 @@ async def test_interactive_takes_the_reserve_at_once_and_it_is_never_lent(monkey
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3),没有别的进程的名额行")
async def test_slots_held_by_a_process_that_is_gone_are_taken_back(monkeypatch):
_policy(monkeypatch, 1, interactive=(1, 40))
async with SessionLocal() as session:
@@ -185,6 +186,7 @@ async def test_slots_held_by_a_process_that_is_gone_are_taken_back(monkeypatch):
assert rows == ["mine"], "没了的进程占着的名额收回了"
@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3),没有别的进程的名额行")
async def test_slots_of_a_live_peer_are_left_alone(monkeypatch):
_policy(monkeypatch, 1, interactive=(1, 40))
await _peer(PEER)
@@ -451,3 +453,31 @@ async def test_preflight_refuses_a_beating_peer_and_waits_out_a_stale_one():
with pytest.raises(version.VersionMismatch):
await _peer_version(PEER, "0.0.9+old+aaaaaaaaaaaa")
await workers.register()
@pytest.mark.sqlite_only
async def test_an_idle_single_process_writes_nothing_for_coordination(monkeypatch):
"""复核 R-b3:SQLite(只有一个进程)空闲时协调不写库——不续租约、不写当班、不周期接手;要名额、放名额也不落库。用另一条连接
读 PRAGMA data_version:别的连接提交过写事务它就会变。"""
import sqlite3
from app import cluster
from app.core.config import settings
monkeypatch.setattr(settings, "worker_heartbeat_s", 0.05)
await cluster.start()
try:
await asyncio.sleep(0.2)
probe = sqlite3.connect(settings.db_path)
try:
before = probe.execute("PRAGMA data_version").fetchone()[0]
await asyncio.sleep(0.6) # 十来拍心跳
async with slots.hold("idle-check", "chat"): # 一段运行要一个名额、用完放掉
pass
after = probe.execute("PRAGMA data_version").fetchone()[0]
finally:
probe.close()
assert before == after, "空闲的单进程为协调写了库"
assert cluster.leading(), "只有本进程:直接当班"
finally:
await cluster.stop()