From 5f7100906f69f5f4b1fa28f86244457d70c16d79 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 19 Oct 2026 13:22:05 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=8D=E6=A0=B8=20R-b3=EF=BC=88SQLite=20?= =?UTF-8?q?=E5=8D=95=E8=BF=9B=E7=A8=8B=E5=A4=9A=E5=87=BA=E6=9D=A5=E7=9A=84?= =?UTF-8?q?=E5=86=99=E5=BA=93=EF=BC=89=EF=BC=9ASQLite=20=E6=97=B6=E5=AE=88?= =?UTF-8?q?=E5=8D=AB=E5=B7=B2=E7=BB=8F=E7=A1=AE=E8=AE=A4=E5=8F=AA=E6=9C=89?= =?UTF-8?q?=E6=9C=AC=E8=BF=9B=E7=A8=8B=E2=80=94=E2=80=94=E7=99=BB=E8=AE=B0?= =?UTF-8?q?=E5=8F=AA=E5=86=99=E4=B8=80=E6=AC=A1=E3=80=81=E4=B8=8D=E7=BB=AD?= =?UTF-8?q?=E7=A7=9F=E7=BA=A6=E3=80=81=E6=9C=AC=E8=BF=9B=E7=A8=8B=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E5=BD=93=E7=8F=AD=EF=BC=88=E4=B8=8D=E5=86=99=E5=BA=93?= =?UTF-8?q?=EF=BC=89=E3=80=81=E6=8E=A5=E6=89=8B=E5=8F=AA=E5=9C=A8=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E6=97=B6=E5=81=9A=EF=BC=8C=E5=BF=83=E8=B7=B3=E5=8F=AA?= =?UTF-8?q?=E5=81=9A=E6=9C=AC=E8=BF=9B=E7=A8=8B=E8=87=AA=E5=B7=B1=E6=94=92?= =?UTF-8?q?=E7=9D=80=E7=9A=84=E6=B4=BB=EF=BC=9B=E8=BF=90=E8=A1=8C=E7=9A=84?= =?UTF-8?q?=E5=90=8D=E9=A2=9D=E5=9C=A8=20SQLite=20=E4=B8=8B=E6=94=BE?= =?UTF-8?q?=E5=9C=A8=E8=BF=9B=E7=A8=8B=E9=87=8C=EF=BC=88=E8=A7=84=E7=9F=A9?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E5=A5=97=EF=BC=9A=E4=BC=98=E5=85=88=E7=BA=A7?= =?UTF-8?q?=E3=80=81=E7=B1=BB=E5=88=AB=E4=B8=8A=E9=99=90=E3=80=81=E4=BA=A4?= =?UTF-8?q?=E4=BA=92=E7=95=99=E5=BA=95=EF=BC=8C=E5=8F=AA=E6=98=AF=E4=B8=8D?= =?UTF-8?q?=E8=90=BD=E5=BA=93=EF=BC=89=EF=BC=8C=E4=B8=80=E6=AE=B5=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E4=B8=8D=E5=86=8D=E4=B8=BA=E8=A6=81=E5=90=8D=E9=A2=9D?= =?UTF-8?q?=E5=A4=9A=E5=86=99=E4=B8=89=E6=AC=A1=E5=BA=93=E3=80=82=E9=A1=BA?= =?UTF-8?q?=E5=B8=A6=EF=BC=88=E5=A4=8D=E6=A0=B8=E3=80=8C=E5=8F=A6=E5=A4=96?= =?UTF-8?q?=E4=B8=A4=E5=A4=84=E3=80=8D=E4=B9=8B=E4=BA=8C=EF=BC=89=EF=BC=9A?= =?UTF-8?q?PostgreSQL=20=E5=8F=91=E5=90=8D=E9=A2=9D=E6=8C=81=E7=9D=80?= =?UTF-8?q?=E5=85=A8=E7=AB=99=E7=9A=84=E9=94=81=E6=97=B6=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E5=8F=A6=E5=8F=96=E8=BF=9E=E6=8E=A5=E6=9F=A5=E3=80=8C=E8=B0=81?= =?UTF-8?q?=E8=BF=98=E6=B4=BB=E7=9D=80=E3=80=8D=EF=BC=8C=E6=8B=BF=E9=94=81?= =?UTF-8?q?=E4=B9=8B=E5=89=8D=E5=A4=87=E5=A5=BD=EF=BC=8C=E4=B8=8D=E8=AE=A4?= =?UTF-8?q?=E8=AF=86=E7=9A=84=E5=B1=9E=E4=B8=BB=E8=BF=99=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E5=BD=93=E6=B4=BB=E7=9D=80=E3=80=82=E8=A1=A5=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=EF=BC=9ASQLite=20=E7=A9=BA=E9=97=B2=E5=8D=81=E6=9D=A5=E6=8B=8D?= =?UTF-8?q?=E5=BF=83=E8=B7=B3=E3=80=81=E8=A6=81=E4=B8=80=E6=AC=A1=E5=90=8D?= =?UTF-8?q?=E9=A2=9D=E6=94=BE=E4=B8=80=E6=AC=A1=EF=BC=8CPRAGMA=20data=5Fve?= =?UTF-8?q?rsion=20=E4=B8=8D=E5=8F=98=EF=BC=88=E5=90=8D=E9=A2=9D=E8=90=BD?= =?UTF-8?q?=E5=BA=93=E7=9A=84=E5=86=99=E6=B3=95=E8=BF=99=E6=9D=A1=E4=B8=8D?= =?UTF-8?q?=E8=BF=87=EF=BC=89=EF=BC=9B=E5=BA=93=E9=87=8C=E7=9A=84=E5=90=8D?= =?UTF-8?q?=E9=A2=9D=E8=A1=8C=E9=82=A3=E4=B8=A4=E6=9D=A1=E5=8F=AA=E5=9C=A8?= =?UTF-8?q?=20PostgreSQL=20=E4=B8=8A=E8=B7=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/cluster/slots.py | 119 ++++++++++++++++++++++++++-------- backend/tests/test_cluster.py | 32 ++++++++- 2 files changed, 123 insertions(+), 28 deletions(-) diff --git a/backend/app/cluster/slots.py b/backend/app/cluster/slots.py index cf870f7d..a7c83136 100644 --- a/backend/app/cluster/slots.py +++ b/backend/app/cluster/slots.py @@ -225,44 +225,83 @@ def _grant_lock() -> asyncio.Lock: return lock +@dataclass +class _MemRow: + run_id: str + klass: str + priority: int + requested_at: Any + owner: str = "" + granted: bool = False + + +#: SQLite 时的名额(复核 R-b3):只有一个进程,规矩照同一套、只是不落库——一段运行不为要名额多写三次库 +_mem: dict[str, _MemRow] = {} + + +def _in_memory() -> bool: + return settings.db_dialect == "sqlite" + + +def _pick(rows: list[Any], pol: dict[str, Any]) -> list[str]: + """按规矩挑这一次发给谁(库里的行、进程里的行同一套):优先级高的先、同级先来先得,类别上限、全站上限、交互留底。""" + used = Counter(r.klass for r in rows if r.granted) + total = sum(used.values()) + # 交互以外的三类合起来用了几个:到了「全站上限 − 交互留底」就不再发给它们(留底永远给交互) + others = total - used["interactive"] + others_cap = borrowable(pol) + limits = {k: int(v["limit"]) for k, v in pol["classes"].items()} + granted: list[str] = [] + waiting = sorted((r for r in rows if not r.granted), key=lambda r: (-int(r.priority or 0), r.requested_at, r.run_id)) + for row in waiting: + if total >= int(pol["max_total"]): + break + if used[row.klass] >= limits.get(row.klass, int(pol["max_total"])): + continue + if row.klass != "interactive" and others >= others_cap: + continue + used[row.klass] += 1 + total += 1 + if row.klass != "interactive": + others += 1 + granted.append(row.run_id) + return granted + + async def grant() -> list[str]: """把等着的按规矩过一遍,够格的改成已发;属主不在了的行删掉。交回这一次发出去的。全站串行(见模块说明)。""" + if _in_memory(): + pol = await policy() + for run_id in [r for r in _mem if not _local(r)]: + _mem.pop(run_id, None) + granted = _pick(list(_mem.values()), pol) + for run_id in granted: + _mem[run_id].granted = True + if granted: + wake_local() + return granted from app.cluster import state from app.db.models_cluster import RunSlot pol = await policy() + # 拿锁之前先备好「谁还活着」(必要时这时候查库),持锁期间只用持锁的那条连接 + live = await state.current() + known_dead = await _known_dead(live) async with _grant_lock(): async with SessionLocal() as session: if session.get_bind().dialect.name == "postgresql": await session.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": _GRANT_KEY}) rows = list((await session.execute(select(RunSlot))).scalars()) - live = await state.live_for([r.owner for r in rows]) - dead = [r.run_id for r in rows if not live.alive(r.owner, local=_local(r.run_id))] + # 持着全站的锁时不另取连接查库(复核:连接池用尽时会卡到取连接超时):用锁之前拿好的那一份「谁还活着」,不认识的属主 + # 这一次当活着(不删它的行),下一次手上那份更新了再说 + me = identity.owner() + dead = [r.run_id for r in rows + if (r.owner == me and not _local(r.run_id)) or (r.owner != me and r.owner in known_dead)] if dead: await session.execute(delete(RunSlot).where(RunSlot.run_id.in_(dead))) rows = [r for r in rows if r.run_id not in set(dead)] - used = Counter(r.klass for r in rows if r.granted) - total = sum(used.values()) - # 交互以外的三类合起来用了几个:到了「全站上限 − 交互留底」就不再发给它们(留底永远给交互) - others = total - used["interactive"] - others_cap = borrowable(pol) - limits = {k: int(v["limit"]) for k, v in pol["classes"].items()} - granted: list[str] = [] - waiting = sorted((r for r in rows if not r.granted), - key=lambda r: (-int(r.priority or 0), r.requested_at, r.run_id)) + granted = _pick(rows, pol) now = utcnow() - for row in waiting: - if total >= int(pol["max_total"]): - break - if used[row.klass] >= limits.get(row.klass, int(pol["max_total"])): - continue - if row.klass != "interactive" and others >= others_cap: - continue - used[row.klass] += 1 - total += 1 - if row.klass != "interactive": - others += 1 - granted.append(row.run_id) if granted: await session.execute(update(RunSlot).where(RunSlot.run_id.in_(granted)) .values(granted=True, granted_at=now).execution_options(synchronize_session=False)) @@ -275,8 +314,25 @@ async def grant() -> list[str]: return granted +async def _known_dead(live: Any) -> set[str]: + """名额表里属主不在手上那份「活着」里的那几个,拿锁之前当场再查一次登记:仍不在的算没了。""" + from app.cluster import state + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + owners = set((await session.execute(select(RunSlot.owner).distinct())).scalars()) + unknown = {o for o in owners if o and o != identity.owner() and o not in live.others} + if not unknown: + return set() + fresh = await state.current(fresh=True) + return {o for o in unknown if o not in fresh.others} + + async def _granted(run_id: str) -> bool | None: """这一行发了没有:True 发了,False 还在等,None 这一行不见了(被当成属主不在了删掉)。""" + if _in_memory(): + row = _mem.get(run_id) + return None if row is None else row.granted from app.db.models_cluster import RunSlot async with SessionLocal() as session: @@ -284,6 +340,9 @@ async def _granted(run_id: str) -> bool | None: async def _request(run_id: str, klass: str, priority: int) -> None: + if _in_memory(): + _mem[run_id] = _MemRow(run_id, klass, priority, utcnow(), identity.owner()) + return from app.db.models_cluster import RunSlot async with SessionLocal() as session: @@ -294,6 +353,9 @@ async def _request(run_id: str, klass: str, priority: int) -> None: async def _release(run_id: str) -> None: + if _in_memory(): + _mem.pop(run_id, None) + return from app.db.models_cluster import RunSlot async with SessionLocal() as session: @@ -337,11 +399,14 @@ async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]: async def snapshot() -> dict[str, Any]: """现在各类在跑、在等几个(系统信息、验收脚本看)。""" - from app.db.models_cluster import RunSlot + if _in_memory(): + rows = [(r.klass, r.granted) for r in _mem.values() if _local(r.run_id)] + else: + from app.db.models_cluster import RunSlot - async with SessionLocal() as session: - rows = list((await session.execute(select(RunSlot.klass, RunSlot.granted))).all()) - out = {k: {"running": 0, "waiting": 0} for k in CLASSES} + async with SessionLocal() as session: + rows = list((await session.execute(select(RunSlot.klass, RunSlot.granted))).all()) + out = {k: {"running": 0, "waiting": 0} for k in CLASSES} # noqa: C420 for klass, granted in rows: out.setdefault(klass, {"running": 0, "waiting": 0})["running" if granted else "waiting"] += 1 return {"policy": await policy(), "classes": out} diff --git a/backend/tests/test_cluster.py b/backend/tests/test_cluster.py index a819a16f..870742a9 100644 --- a/backend/tests/test_cluster.py +++ b/backend/tests/test_cluster.py @@ -115,7 +115,7 @@ async def test_a_class_at_its_cap_does_not_hold_up_the_others(monkeypatch): await _until(lambda: "chat" in got) assert len({"e1", "e2"} & set(got)) == 1, "系统类到了上限:另一个等着" snapshot = await slots.snapshot() - assert snapshot["classes"]["system"] == {"running": 1, "waiting": 1} + assert snapshot["classes"]["system"] == {"running": 1, "waiting": 1}, snapshot gate.set() await asyncio.wait_for(asyncio.gather(*tasks), timeout=10) @@ -174,6 +174,7 @@ async def test_interactive_takes_the_reserve_at_once_and_it_is_never_lent(monkey await asyncio.wait_for(asyncio.gather(*tasks), timeout=10) +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3),没有别的进程的名额行") async def test_slots_held_by_a_process_that_is_gone_are_taken_back(monkeypatch): _policy(monkeypatch, 1, interactive=(1, 40)) async with SessionLocal() as session: @@ -185,6 +186,7 @@ async def test_slots_held_by_a_process_that_is_gone_are_taken_back(monkeypatch): assert rows == ["mine"], "没了的进程占着的名额收回了" +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3),没有别的进程的名额行") async def test_slots_of_a_live_peer_are_left_alone(monkeypatch): _policy(monkeypatch, 1, interactive=(1, 40)) await _peer(PEER) @@ -451,3 +453,31 @@ async def test_preflight_refuses_a_beating_peer_and_waits_out_a_stale_one(): with pytest.raises(version.VersionMismatch): await _peer_version(PEER, "0.0.9+old+aaaaaaaaaaaa") await workers.register() + + +@pytest.mark.sqlite_only +async def test_an_idle_single_process_writes_nothing_for_coordination(monkeypatch): + """复核 R-b3:SQLite(只有一个进程)空闲时协调不写库——不续租约、不写当班、不周期接手;要名额、放名额也不落库。用另一条连接 + 读 PRAGMA data_version:别的连接提交过写事务它就会变。""" + import sqlite3 + + from app import cluster + from app.core.config import settings + + monkeypatch.setattr(settings, "worker_heartbeat_s", 0.05) + await cluster.start() + try: + await asyncio.sleep(0.2) + probe = sqlite3.connect(settings.db_path) + try: + before = probe.execute("PRAGMA data_version").fetchone()[0] + await asyncio.sleep(0.6) # 十来拍心跳 + async with slots.hold("idle-check", "chat"): # 一段运行要一个名额、用完放掉 + pass + after = probe.execute("PRAGMA data_version").fetchone()[0] + finally: + probe.close() + assert before == after, "空闲的单进程为协调写了库" + assert cluster.leading(), "只有本进程:直接当班" + finally: + await cluster.stop()