运行名额改成给交互留底、其余可借(用户拍板):交互留底缺省全站上限的 1/4、至少 1、至多全站上限减 1,应用、定时、系统三类合起来永远用不到留底;三类各自的上限缺省 = 全站上限 − 留底(设置里可调小),没人问数据时评测能开满;交互上限缺省 = 全站上限;等名额照旧按优先级、同级先来先得。存着的设置原样存、缺省按全站上限和留底现算;留底不小于全站上限时 422。设置页「运行并发」加「给交互留底」一项,写明留底和借用是什么,README 同步。补测试:缺省值、空闲时系统类借满到 全站 − 留底、留底不借给应用类、问数据一来马上拿到留底;多进程端到端加 5b(评测借满时问数据拿留底、不等评测跑完)

This commit is contained in:
ThearchyHelios
2026-10-19 12:53:28 +08:00
parent 319b787688
commit 6ed782a84b
6 changed files with 204 additions and 48 deletions
+59 -3
View File
@@ -47,9 +47,11 @@ async def _peer(owner: str = PEER, *, alive: bool = True) -> None:
state.forget()
def _policy(monkeypatch, total: int, **classes: tuple[int, int]) -> None:
value = slots.values({"max_total": total, "classes": {k: {"limit": lim, "priority": pri}
for k, (lim, pri) in classes.items()}})
def _policy(monkeypatch, total: int, reserve: int | None = None, **classes: tuple[int, int]) -> None:
stored: dict = {"max_total": total, "classes": {k: {"limit": lim, "priority": pri} for k, (lim, pri) in classes.items()}}
if reserve is not None:
stored["interactive_reserve"] = reserve
value = slots.values(stored)
async def fixed(*, fresh: bool = False): # noqa: ARG001
return value
@@ -117,6 +119,60 @@ async def test_a_class_at_its_cap_does_not_hold_up_the_others(monkeypatch):
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
def test_the_defaults_keep_a_reserve_for_interactive_and_lend_the_rest():
pol = slots.defaults(20)
assert pol["interactive_reserve"] == 5 and pol["classes"]["interactive"]["limit"] == 20
assert {pol["classes"][k]["limit"] for k in ("app", "schedule", "system")} == {15}, "别的类缺省借满到 全站 − 留底"
assert slots.defaults(2)["interactive_reserve"] == 1 and slots.defaults(1)["interactive_reserve"] == 0, \
"至少留 1 个;全站只有 1 个名额时不留(别的类才跑得起来)"
assert slots.values({"max_total": 8})["classes"]["system"]["limit"] == 6, "缺省按存着的全站上限现算"
assert slots.problem({"max_total": 4, "interactive_reserve": 4}) is not None
async def _holders(names_origins: list[tuple[str, str]], got: list[str], gate: asyncio.Event) -> list[asyncio.Task]:
async def run(name: str, origin: str) -> None:
async with slots.hold(f"run-{name}", origin):
got.append(name)
await gate.wait()
tasks = []
for name, origin in names_origins:
tasks.append(asyncio.create_task(run(name, origin)))
await asyncio.sleep(0.05)
return tasks
async def test_idle_system_runs_borrow_all_but_the_interactive_reserve(monkeypatch):
"""全站 4 个、留底 1 个:没人问数据时评测能同时跑 3 个(全站 − 留底),第 4 个等着。"""
_policy(monkeypatch, 4, reserve=1)
got: list[str] = []
gate = asyncio.Event()
tasks = await _holders([(f"e{i}", "eval") for i in range(4)], got, gate)
await _until(lambda: len(got) == 3)
await asyncio.sleep(0.3)
assert len(got) == 3, "留底那一个不借给评测"
assert (await slots.snapshot())["classes"]["system"] == {"running": 3, "waiting": 1}
gate.set()
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
async def test_interactive_takes_the_reserve_at_once_and_it_is_never_lent(monkeypatch):
"""评测借满了(3 个在跑),应用类的再来也借不到留底(全站还空着 1 个);问数据一来马上拿到留底,不用等评测跑完。"""
_policy(monkeypatch, 4, reserve=1)
got: list[str] = []
gate = asyncio.Event()
tasks = await _holders([("e1", "eval"), ("e2", "eval"), ("e3", "eval")], got, gate)
await _until(lambda: len(got) == 3)
tasks += await _holders([("app", "app_view")], got, gate)
await asyncio.sleep(0.3)
assert "app" not in got, "留底不借给应用类"
tasks += await _holders([("chat", "chat")], got, gate)
await _until(lambda: "chat" in got)
assert "app" not in got and set(got) == {"e1", "e2", "e3", "chat"}
gate.set()
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
async def test_slots_held_by_a_process_that_is_gone_are_taken_back(monkeypatch):
_policy(monkeypatch, 1, interactive=(1, 40))
async with SessionLocal() as session:
+51 -2
View File
@@ -6,8 +6,9 @@
进程只执行一次;两边的事件流都按 seq 递增、不重不漏,封存核对得上;
3. 在 A 上跑的运行,从 B 停止:照本进程的停止办(run.cancelled 带上是谁停的);
4. 定时任务到点只触发一次(两个进程都起着调度循环,只有当班的做);
5. 评测和问数据并发时优先级生效:全站只给 1 个名额,评测的作答运行(系统类,由本测试进程作为第三个登记的工作进程以评测的
来源发起——工作进程里发起评测要真的去调模型建图)排着队时,后来的问数据运行(交互类,经 A 的接口发起)先拿到名额;
5. 评测和问数据并发时优先级生效:全站只给 1 个名额(没有留底),评测的作答运行(系统类,由本测试进程作为第三个登记的工作进程
以评测的来源发起——工作进程里发起评测要真的去调模型建图)排着队时,后来的问数据运行(交互类,经 A 的接口发起)先拿到名额;
全站 2 个、给交互留底 1 个:评测借满了别的名额、还有题排着,问数据一来马上拿到留底,不等评测跑完;
6. 每日花费上限跨进程不越过:两个进程同时往同一本账上记,记下的合计不超过上限;
7. 强杀 B:租约过期后,B 上在跑的运行被照实标中断(可从断点继续),停在审批上的运行照常能在 A 上批、接着跑完;B 当着调度班的
话 A 接班。
@@ -404,6 +405,54 @@ async def test_5_chat_goes_before_queued_evaluation_runs(workers, monkeypatch):
await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": slots.defaults()}})
async def test_5b_the_interactive_reserve_is_there_when_evaluations_fill_the_rest(workers, monkeypatch):
"""全站 2 个名额、留底 1 个:评测(系统类)只借得到 1 个,第二题排着;问数据一来马上拿到留底,评测的第一题还在跑。"""
from app import cluster
from app.cluster import slots
from app.cluster.relay import relay
from app.core.config import settings
from app.db.base import SessionLocal
from app.db.models import Run
from app.engine.runner import run_manager
a = workers["A"]
async with await _client() as client:
put = await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": {
"max_total": 2, "interactive_reserve": 1, "classes": {}}}})
assert put.status_code == 200, put.text
assert put.json()["concurrency"]["classes"]["system"]["limit"] == 1, "别的类缺省借满到 全站 − 留底"
slots.forget_policy()
monkeypatch.setattr(settings, "sandbox_backend", "local")
await cluster.start()
await relay.start()
await run_manager.setup()
try:
evals = [await run_manager.start(graph=slow_graph(3), input_payload={"question": f"评测 {i}"},
origin_kind="eval", origin={"kind": "eval"}) for i in range(2)]
async def first_running() -> Any:
async with SessionLocal() as session:
return (await session.get(Run, evals[0].id)).status == "running"
await _until(first_running, what="第一题的作答运行拿到名额")
chat_id = (await client.post(f"{a.base}/api/runs", json={
"graph": slow_graph(0.2), "input": {"question": "问数据"}, "origin": "chat"})).json()["id"]
async def all_done() -> Any:
async with SessionLocal() as session:
rows = [await session.get(Run, rid, populate_existing=True) for rid in [*[e.id for e in evals], chat_id]]
return rows if all(r.status in ("succeeded", "failed") for r in rows) else None
e1, e2, chat = await _until(all_done, timeout=60, what="都跑完")
assert chat.started_at < e1.finished_at, "问数据拿到的是留底:评测的第一题还没跑完它就开始了"
assert e2.started_at >= e1.finished_at, "评测的第二题借不到留底,等第一题跑完"
finally:
await run_manager.shutdown()
await relay.stop()
await cluster.stop()
await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": slots.defaults()}})
def test_6_the_daily_cap_holds_across_processes(workers):
"""两个进程同时往评测那本账上各记 20 次、每次 0.1 美元,上限 1 美元:合计记下的正好 10 次、不超过上限。"""
script = (