mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 03:12:00 +00:00
运行名额改成给交互留底、其余可借(用户拍板):交互留底缺省全站上限的 1/4、至少 1、至多全站上限减 1,应用、定时、系统三类合起来永远用不到留底;三类各自的上限缺省 = 全站上限 − 留底(设置里可调小),没人问数据时评测能开满;交互上限缺省 = 全站上限;等名额照旧按优先级、同级先来先得。存着的设置原样存、缺省按全站上限和留底现算;留底不小于全站上限时 422。设置页「运行并发」加「给交互留底」一项,写明留底和借用是什么,README 同步。补测试:缺省值、空闲时系统类借满到 全站 − 留底、留底不借给应用类、问数据一来马上拿到留底;多进程端到端加 5b(评测借满时问数据拿留底、不等评测跑完)
This commit is contained in:
@@ -47,9 +47,11 @@ async def _peer(owner: str = PEER, *, alive: bool = True) -> None:
|
||||
state.forget()
|
||||
|
||||
|
||||
def _policy(monkeypatch, total: int, **classes: tuple[int, int]) -> None:
|
||||
value = slots.values({"max_total": total, "classes": {k: {"limit": lim, "priority": pri}
|
||||
for k, (lim, pri) in classes.items()}})
|
||||
def _policy(monkeypatch, total: int, reserve: int | None = None, **classes: tuple[int, int]) -> None:
|
||||
stored: dict = {"max_total": total, "classes": {k: {"limit": lim, "priority": pri} for k, (lim, pri) in classes.items()}}
|
||||
if reserve is not None:
|
||||
stored["interactive_reserve"] = reserve
|
||||
value = slots.values(stored)
|
||||
|
||||
async def fixed(*, fresh: bool = False): # noqa: ARG001
|
||||
return value
|
||||
@@ -117,6 +119,60 @@ async def test_a_class_at_its_cap_does_not_hold_up_the_others(monkeypatch):
|
||||
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
|
||||
|
||||
|
||||
def test_the_defaults_keep_a_reserve_for_interactive_and_lend_the_rest():
|
||||
pol = slots.defaults(20)
|
||||
assert pol["interactive_reserve"] == 5 and pol["classes"]["interactive"]["limit"] == 20
|
||||
assert {pol["classes"][k]["limit"] for k in ("app", "schedule", "system")} == {15}, "别的类缺省借满到 全站 − 留底"
|
||||
assert slots.defaults(2)["interactive_reserve"] == 1 and slots.defaults(1)["interactive_reserve"] == 0, \
|
||||
"至少留 1 个;全站只有 1 个名额时不留(别的类才跑得起来)"
|
||||
assert slots.values({"max_total": 8})["classes"]["system"]["limit"] == 6, "缺省按存着的全站上限现算"
|
||||
assert slots.problem({"max_total": 4, "interactive_reserve": 4}) is not None
|
||||
|
||||
|
||||
async def _holders(names_origins: list[tuple[str, str]], got: list[str], gate: asyncio.Event) -> list[asyncio.Task]:
|
||||
async def run(name: str, origin: str) -> None:
|
||||
async with slots.hold(f"run-{name}", origin):
|
||||
got.append(name)
|
||||
await gate.wait()
|
||||
|
||||
tasks = []
|
||||
for name, origin in names_origins:
|
||||
tasks.append(asyncio.create_task(run(name, origin)))
|
||||
await asyncio.sleep(0.05)
|
||||
return tasks
|
||||
|
||||
|
||||
async def test_idle_system_runs_borrow_all_but_the_interactive_reserve(monkeypatch):
|
||||
"""全站 4 个、留底 1 个:没人问数据时评测能同时跑 3 个(全站 − 留底),第 4 个等着。"""
|
||||
_policy(monkeypatch, 4, reserve=1)
|
||||
got: list[str] = []
|
||||
gate = asyncio.Event()
|
||||
tasks = await _holders([(f"e{i}", "eval") for i in range(4)], got, gate)
|
||||
await _until(lambda: len(got) == 3)
|
||||
await asyncio.sleep(0.3)
|
||||
assert len(got) == 3, "留底那一个不借给评测"
|
||||
assert (await slots.snapshot())["classes"]["system"] == {"running": 3, "waiting": 1}
|
||||
gate.set()
|
||||
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
|
||||
|
||||
|
||||
async def test_interactive_takes_the_reserve_at_once_and_it_is_never_lent(monkeypatch):
|
||||
"""评测借满了(3 个在跑),应用类的再来也借不到留底(全站还空着 1 个);问数据一来马上拿到留底,不用等评测跑完。"""
|
||||
_policy(monkeypatch, 4, reserve=1)
|
||||
got: list[str] = []
|
||||
gate = asyncio.Event()
|
||||
tasks = await _holders([("e1", "eval"), ("e2", "eval"), ("e3", "eval")], got, gate)
|
||||
await _until(lambda: len(got) == 3)
|
||||
tasks += await _holders([("app", "app_view")], got, gate)
|
||||
await asyncio.sleep(0.3)
|
||||
assert "app" not in got, "留底不借给应用类"
|
||||
tasks += await _holders([("chat", "chat")], got, gate)
|
||||
await _until(lambda: "chat" in got)
|
||||
assert "app" not in got and set(got) == {"e1", "e2", "e3", "chat"}
|
||||
gate.set()
|
||||
await asyncio.wait_for(asyncio.gather(*tasks), timeout=10)
|
||||
|
||||
|
||||
async def test_slots_held_by_a_process_that_is_gone_are_taken_back(monkeypatch):
|
||||
_policy(monkeypatch, 1, interactive=(1, 40))
|
||||
async with SessionLocal() as session:
|
||||
|
||||
@@ -6,8 +6,9 @@
|
||||
进程只执行一次;两边的事件流都按 seq 递增、不重不漏,封存核对得上;
|
||||
3. 在 A 上跑的运行,从 B 停止:照本进程的停止办(run.cancelled 带上是谁停的);
|
||||
4. 定时任务到点只触发一次(两个进程都起着调度循环,只有当班的做);
|
||||
5. 评测和问数据并发时优先级生效:全站只给 1 个名额,评测的作答运行(系统类,由本测试进程作为第三个登记的工作进程以评测的
|
||||
来源发起——工作进程里发起评测要真的去调模型建图)排着队时,后来的问数据运行(交互类,经 A 的接口发起)先拿到名额;
|
||||
5. 评测和问数据并发时优先级生效:全站只给 1 个名额(没有留底),评测的作答运行(系统类,由本测试进程作为第三个登记的工作进程
|
||||
以评测的来源发起——工作进程里发起评测要真的去调模型建图)排着队时,后来的问数据运行(交互类,经 A 的接口发起)先拿到名额;
|
||||
全站 2 个、给交互留底 1 个:评测借满了别的名额、还有题排着,问数据一来马上拿到留底,不等评测跑完;
|
||||
6. 每日花费上限跨进程不越过:两个进程同时往同一本账上记,记下的合计不超过上限;
|
||||
7. 强杀 B:租约过期后,B 上在跑的运行被照实标中断(可从断点继续),停在审批上的运行照常能在 A 上批、接着跑完;B 当着调度班的
|
||||
话 A 接班。
|
||||
@@ -404,6 +405,54 @@ async def test_5_chat_goes_before_queued_evaluation_runs(workers, monkeypatch):
|
||||
await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": slots.defaults()}})
|
||||
|
||||
|
||||
async def test_5b_the_interactive_reserve_is_there_when_evaluations_fill_the_rest(workers, monkeypatch):
|
||||
"""全站 2 个名额、留底 1 个:评测(系统类)只借得到 1 个,第二题排着;问数据一来马上拿到留底,评测的第一题还在跑。"""
|
||||
from app import cluster
|
||||
from app.cluster import slots
|
||||
from app.cluster.relay import relay
|
||||
from app.core.config import settings
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run
|
||||
from app.engine.runner import run_manager
|
||||
|
||||
a = workers["A"]
|
||||
async with await _client() as client:
|
||||
put = await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": {
|
||||
"max_total": 2, "interactive_reserve": 1, "classes": {}}}})
|
||||
assert put.status_code == 200, put.text
|
||||
assert put.json()["concurrency"]["classes"]["system"]["limit"] == 1, "别的类缺省借满到 全站 − 留底"
|
||||
slots.forget_policy()
|
||||
monkeypatch.setattr(settings, "sandbox_backend", "local")
|
||||
await cluster.start()
|
||||
await relay.start()
|
||||
await run_manager.setup()
|
||||
try:
|
||||
evals = [await run_manager.start(graph=slow_graph(3), input_payload={"question": f"评测 {i}"},
|
||||
origin_kind="eval", origin={"kind": "eval"}) for i in range(2)]
|
||||
|
||||
async def first_running() -> Any:
|
||||
async with SessionLocal() as session:
|
||||
return (await session.get(Run, evals[0].id)).status == "running"
|
||||
|
||||
await _until(first_running, what="第一题的作答运行拿到名额")
|
||||
chat_id = (await client.post(f"{a.base}/api/runs", json={
|
||||
"graph": slow_graph(0.2), "input": {"question": "问数据"}, "origin": "chat"})).json()["id"]
|
||||
|
||||
async def all_done() -> Any:
|
||||
async with SessionLocal() as session:
|
||||
rows = [await session.get(Run, rid, populate_existing=True) for rid in [*[e.id for e in evals], chat_id]]
|
||||
return rows if all(r.status in ("succeeded", "failed") for r in rows) else None
|
||||
|
||||
e1, e2, chat = await _until(all_done, timeout=60, what="都跑完")
|
||||
assert chat.started_at < e1.finished_at, "问数据拿到的是留底:评测的第一题还没跑完它就开始了"
|
||||
assert e2.started_at >= e1.finished_at, "评测的第二题借不到留底,等第一题跑完"
|
||||
finally:
|
||||
await run_manager.shutdown()
|
||||
await relay.stop()
|
||||
await cluster.stop()
|
||||
await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": slots.defaults()}})
|
||||
|
||||
|
||||
def test_6_the_daily_cap_holds_across_processes(workers):
|
||||
"""两个进程同时往评测那本账上各记 20 次、每次 0.1 美元,上限 1 美元:合计记下的正好 10 次、不超过上限。"""
|
||||
script = (
|
||||
|
||||
Reference in New Issue
Block a user