diff --git a/backend/app/engine/deadline.py b/backend/app/engine/deadline.py new file mode 100644 index 00000000..90078d92 --- /dev/null +++ b/backend/app/engine/deadline.py @@ -0,0 +1,98 @@ +"""这一段执行的截止时刻:核查节点据此在运行的时长上限之内收尾,不让整次运行被中止。 + +runner 给每一段执行(两次人工介入之间的那一段)套一道 asyncio.timeout(max_run_seconds),到点整次运行判失败。核查 +节点一个就能把这一段的时间用光:反方审稿逐句串行反查,默认 5 句、每句 90 秒再加 30 秒收尾宽限,最坏就是 600 秒;双盲 +复算每项 120 秒加宽限,指标多了也一样。被中止时节点里装着反查、盲算的那个 @task 还没做完、没进 checkpoint,「继续 +运行」把它从头再做一遍,多花一份钱,再次超时。 + +所以 runner 进入时长上限时把截止时刻放进这里的 ContextVar(segment),退出时清掉:节点、节点里的 @task、子工作流都 +在这个上下文里执行,都看得到。核查节点每开始一项(一句反查、一项盲算)之前问 allot():给下游留出余量之后还够不够 +再做一项,这一项的时长上限收紧到多少;不够就不再开始,剩下的照各自「没能核查」的路径记下来,节点正常收尾。决定都 +发生在节点自己的 @task 里,接着运行时取 checkpoint 里的结果,重放确定。 + +截止时刻用事件循环的钟(loop.time()),和 asyncio.timeout 同一个钟。没有上限时(不经 runner 执行的节点:单元测试、 +审批质询接口)为 None,allot() 原样放行,节点的行为和没有这道检查时一样。 +""" +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator +from contextvars import ContextVar +from dataclasses import dataclass + +#: 给下游节点留的余量:核查节点后面通常还有成果节点(出具契约复核报告),也可能还有写报告、发通知这类节点。核查把 +#: 时间用到最后一刻,后面的节点一开始就撞上时长上限,整次运行照样被中止。取 RESERVE_S 和这一段总时长的 RESERVE_SHARE +#: 中大的:60 秒够成果节点复核、再做一两次普通的模型调用;管理员调大了时长上限的,多半是因为节点本来就慢,余量跟着放大 +RESERVE_S = 60.0 +RESERVE_SHARE = 0.1 + + +@dataclass(frozen=True) +class Segment: + """这一段执行:截止时刻(事件循环时间)和总时长(秒,说明里写、按它算余量)。""" + + deadline: float + seconds: float + + +_SEGMENT: ContextVar[Segment | None] = ContextVar("agentlab_segment", default=None) + + +@contextlib.asynccontextmanager +async def segment(deadline: float | None, seconds: float) -> AsyncIterator[None]: + """runner 用:这一段执行的截止时刻(asyncio.Timeout.when())和总时长,退出时清掉。写成异步的上下文管理器,好和 + 时长上限写在同一个 async with 里。deadline 为 None(没有上限)时什么都不放。""" + token = _SEGMENT.set(Segment(float(deadline), float(seconds)) if deadline is not None else None) + try: + yield + finally: + _SEGMENT.reset(token) + + +def current() -> Segment | None: + """这一段执行的截止时刻;不在 runner 的执行里时为 None。""" + return _SEGMENT.get() + + +def reserve(seconds: float) -> float: + """给下游节点留的余量(秒),理由见 RESERVE_S。""" + return max(RESERVE_S, RESERVE_SHARE * seconds) + + +def _now() -> float: + return asyncio.get_running_loop().time() + + +@dataclass(frozen=True) +class Allot: + """allot() 的答复。start:够不够再开始一项。timeout_s:这一项的时长上限,没收紧时就是传进来的那个值(不限时为 + None)。tightened:按剩余时间收紧过(比配置的短,或者配置的是不限)。seconds:这一段的总时长,不再开始、收紧过的 + 时候说明里写;没有上限时为 None。""" + + start: bool + timeout_s: float | None + tightened: bool = False + seconds: float | None = None + + +def allot(timeout_s: float | None, *, grace: float) -> Allot: + """开始一项核查之前问一句:够不够,这一项的时长上限收紧到多少。timeout_s 是这一项配置的时长上限(None 不限), + grace 是到点之后收尾那次调用的宽限(verify_agent.SETTLE_GRACE_S)。 + + 一项最坏要用「时长上限 + 收尾宽限」。剩下的时间先扣掉给下游的余量、再扣掉收尾宽限,就是这一项查库最多能用的时长: + 比配置的短就收紧到它。收紧之后不到 min(配置的时长上限, 收尾宽限) 就不再开始:收尾宽限是留给一次模型调用的时间, + 查库的时长比它还短,连一轮都做不完,只会白花一次收尾的钱、记成没能核查。配置的时长上限本来就比它短的,按配置的 + 整段算。没有上限时原样放行。""" + found = _SEGMENT.get() + if found is None: + return Allot(True, timeout_s) + room = found.deadline - _now() - reserve(found.seconds) - grace + limit = room if timeout_s is None else min(timeout_s, room) + floor = grace if timeout_s is None else min(timeout_s, grace) + if limit < floor: + return Allot(False, None, seconds=found.seconds) + return Allot(True, limit, tightened=timeout_s is None or room < timeout_s, seconds=found.seconds) + + +__all__ = ["RESERVE_S", "RESERVE_SHARE", "Allot", "Segment", "allot", "current", "reserve", "segment"] diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 4677078b..2e65f504 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -24,6 +24,7 @@ from app.db.models import Approval, Run, RunEvent, Workflow from app.engine.approval import read_decision from app.engine.compiler import compile_graph, initial_state from app.engine.context import NodeError, RunContext +from app.engine.deadline import segment from app.engine.replay import PROTOCOL, PROTOCOL_KEY, protocol_of from app.engine.schema import GraphSpec, NodeType, loop_steps, topology_of, validate_graph from app.tools.datasource import RUN_VERSIONS_KEY @@ -714,7 +715,8 @@ class RunManager: # 代码执行它——一个挂住的模型或工具能把运行和它占着的并发名额一直拖着。 # 按"这一段执行"计:等人工审批时 _drive 已经结束,那段时间不算。 deadline = asyncio.timeout(settings.max_run_seconds) - async with deadline: + # 截止时刻同时交给节点(engine/deadline.py):核查节点据此在上限之内收尾,不把整次运行拖到被中止 + async with deadline, segment(deadline.when(), settings.max_run_seconds): async for mode, chunk in app.astream( payload, config=config, stream_mode=["custom", "updates"] ): diff --git a/backend/tests/test_run_deadline.py b/backend/tests/test_run_deadline.py new file mode 100644 index 00000000..078ca1ed --- /dev/null +++ b/backend/tests/test_run_deadline.py @@ -0,0 +1,179 @@ +"""这一段执行的截止时刻(engine/deadline.py):runner 进入时长上限时放进 ContextVar、退出时清掉,核查节点开始每一项之前 +问 allot() 够不够、时长收紧到多少。 + +- 没有截止时刻(不经 runner 执行):原样放行,时长上限还是传进来的那个值 +- 有截止时刻:先给下游留出余量(60 秒或这一段总时长的 10%,取大的)、再扣收尾宽限;比配置的时长短就收紧,收紧之后 + 不到 min(配置的时长, 收尾宽限) 就不再开始;配置的是不限时也按剩余时间收紧 +- runner 每一段执行都放一份:节点、节点里的 @task 都看得到,截止时刻就是 asyncio.timeout 的那一个;审批恢复后的那一段 + 是新的截止时刻;运行结束后清掉 +""" +from __future__ import annotations + +import asyncio + +import pytest +from langgraph.func import task +from sqlalchemy import select + +from app.core.config import settings +from app.db.base import SessionLocal +from app.db.models import Approval, Run +from app.engine import compiler, deadline +from app.engine.runner import run_manager +from app.engine.schema import NodeType + + +class Clock: + """假的事件循环钟:allot() 看到的「现在」由测试定。""" + + def __init__(self, monkeypatch, now: float = 1000.0) -> None: + self.now = now + monkeypatch.setattr(deadline, "_now", lambda: self.now) + + +@pytest.fixture +def clock(monkeypatch) -> Clock: + return Clock(monkeypatch) + + +async def test_without_a_deadline_everything_passes_unchanged(clock): + assert deadline.current() is None + for timeout in (90, 0.4, None): + answer = deadline.allot(timeout, grace=30.0) + assert answer == deadline.Allot(True, timeout), "不经 runner 执行:照旧放行,时长不动" + assert answer.timeout_s is timeout, "传进来的就是那个值(int 不变成 float),和没有这道检查时逐字一样" + + +async def test_plenty_of_time_keeps_the_configured_limit(clock): + async with deadline.segment(clock.now + 600, 600): + answer = deadline.allot(90, grace=30.0) + assert answer.start and answer.timeout_s == 90 and isinstance(answer.timeout_s, int) and not answer.tightened + assert answer.seconds == 600 + + +@pytest.mark.parametrize("left, start, limit", [ + (180.0, True, 90), # 60 秒余量 + 90 秒 + 30 秒宽限:刚好够整段 + (150.0, True, 60.0), # 够收紧到 60 秒 + (120.0, True, 30.0), # 收紧到 30 秒:和收尾宽限一样长,还开始 + (119.0, False, None), # 不到 30 秒:连一次模型调用都不够,不再开始 + (10.0, False, None), + (-5.0, False, None), # 截止时刻已经过了 +]) +async def test_the_limit_is_tightened_by_what_is_left(clock, left, start, limit): + async with deadline.segment(clock.now + left, 600): + answer = deadline.allot(90, grace=30.0) + assert answer.start is start + assert answer.timeout_s == limit + assert answer.tightened is (start and limit != 90) + assert answer.seconds == 600, "不再开始、收紧过的说明里要写这一段的总时长" + + +async def test_a_configured_limit_shorter_than_the_grace_runs_whole_or_not_at_all(clock): + """配置的时长上限本来就比收尾宽限短(10 秒):不收紧,够整段才开始。""" + async with deadline.segment(clock.now + 100, 600): + assert deadline.allot(10, grace=30.0) == deadline.Allot(True, 10, seconds=600) + async with deadline.segment(clock.now + 99, 600): + assert deadline.allot(10, grace=30.0) == deadline.Allot(False, None, seconds=600) + + +async def test_an_unlimited_item_is_bounded_by_the_deadline(clock): + """配置的是不限(None):有截止时刻时照样按剩余时间收紧,不然它能一直查到运行被中止。""" + async with deadline.segment(clock.now + 400, 600): + answer = deadline.allot(None, grace=30.0) + assert answer.start and answer.tightened and answer.timeout_s == pytest.approx(310.0) + async with deadline.segment(clock.now + 119, 600): + assert not deadline.allot(None, grace=30.0).start + + +@pytest.mark.parametrize("seconds, kept", [(600, 60.0), (300, 60.0), (1800, 180.0), (3600, 360.0)]) +def test_the_reserve_is_sixty_seconds_or_a_tenth(seconds, kept): + assert deadline.reserve(seconds) == kept + + +async def test_the_reserve_follows_the_segment_length(clock): + """时长上限调到 1800 秒:给下游留 180 秒,剩 300 秒时这一项最多 90 秒(300 − 180 − 30)。""" + async with deadline.segment(clock.now + 300, 1800): + assert deadline.allot(120, grace=30.0).timeout_s == pytest.approx(90.0) + + +async def test_the_segment_is_cleared_on_the_way_out(clock): + async with deadline.segment(clock.now + 600, 600): + assert deadline.current() == deadline.Segment(clock.now + 600, 600.0) + async with deadline.segment(None, 600): + assert deadline.current() is None, "没有上限的一段:里面看不到外面的截止时刻" + assert deadline.current() is not None + assert deadline.current() is None + with pytest.raises(RuntimeError): + async with deadline.segment(clock.now + 600, 600): + raise RuntimeError("节点出错") + assert deadline.current() is None, "出错退出也清掉" + + +# -------------------------------------------------------------------------- +# runner 放进来的截止时刻 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def wait(run_id: str, statuses=("succeeded", "failed", "interrupted")) -> Run: + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + await run_manager.wait_idle(run_id) + return row + raise AssertionError(f"运行 {run_id} 停在 {row.status}:{row.error}") + + +@task +async def _peek_in_task() -> object: + return deadline.current() + + +async def test_the_runner_hands_each_segment_its_deadline(monkeypatch, engine_up): + """节点、节点里的 @task 都看得到这一段的截止时刻:就是 asyncio.timeout 的那一个(开始时刻 + max_run_seconds)。 + 审批恢复后的那一段是新的截止时刻;运行结束后发起它的地方看不到。""" + seen: list[tuple[str, object, object, float]] = [] + + async def peek(state, ctx): + seen.append((ctx.node.id, deadline.current(), await _peek_in_task(), asyncio.get_running_loop().time())) + return {} + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, peek) + monkeypatch.setattr(settings, "max_run_seconds", 321) + peeked = {"mode": "template", "template": "好", "assign_to": "x"} + spec = chain(node("start", "input"), node("before", "transform", **peeked), + node("gate", "human", mode="approve", title="继续吗"), node("after", "transform", **peeked), + node("out", "output", fields=[{"name": "结果", "value": "好"}])) + run = await run_manager.start(graph=spec, input_payload={}) + row = await wait(run.id) + assert row.status == "interrupted", row.error + async with SessionLocal() as session: + approval = (await session.execute(select(Approval).where(Approval.run_id == run.id))).scalar_one() + await run_manager.resume(run.id, {"approved": True}, approval_id=approval.id) + row = await wait(run.id, ("succeeded", "failed")) + assert row.status == "succeeded", row.error + + (first_id, first, first_task, first_at), (second_id, second, second_task, second_at) = seen + assert (first_id, second_id) == ("before", "after") + for segment, in_task, at in ((first, first_task, first_at), (second, second_task, second_at)): + assert isinstance(segment, deadline.Segment) and segment.seconds == 321 + assert in_task == segment, "节点里的 @task 看到的是同一个截止时刻" + assert 300 < segment.deadline - at <= 321, "截止时刻是这一段开始时刻 + max_run_seconds,用事件循环的钟" + assert second.deadline > first.deadline, "审批恢复后的那一段从恢复时重新计时" + assert deadline.current() is None, "截止时刻只在 runner 的执行里,发起运行的地方看不到"