diff --git a/backend/app/engine/deadline.py b/backend/app/engine/deadline.py index 7e71785a..da112e03 100644 --- a/backend/app/engine/deadline.py +++ b/backend/app/engine/deadline.py @@ -12,21 +12,44 @@ runner 给每一段执行(两次人工介入之间的那一段)套一道 asy 截止时刻用事件循环的钟(loop.time()),和 asyncio.timeout 同一个钟。没有上限时(不经 runner 执行的节点:单元测试、 审批质询接口)为 None,allot() 原样放行,节点的行为和没有这道检查时一样。 + +因此没核成的,节点发日志时带上 cause: RUN_DEADLINE 和这一段的总时长(run_seconds),另起一条,不和别的原因混在一起: +界面据此说「运行时长上限将至」、给对的下一步——调高核查自己的上限只会更快撞线,检查模型、数据源也无济于事。 """ from __future__ import annotations import asyncio import contextlib import math -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from contextvars import ContextVar from dataclasses import dataclass +from typing import Any -#: 给下游节点留的余量:核查节点后面通常还有成果节点(出具契约复核报告),也可能还有写报告、发通知这类节点。核查把 -#: 时间用到最后一刻,后面的节点一开始就撞上时长上限,整次运行照样被中止。取 RESERVE_S 和这一段总时长的 RESERVE_SHARE -#: 中大的:60 秒够成果节点复核、再做一两次普通的模型调用;管理员调大了时长上限的,多半是因为节点本来就慢,余量跟着放大 +from app.core.events import EventType +from app.engine.schema import GraphSpec, NodeType + +#: 给下游节点留的余量,按份算。一份给收尾的节点:核查节点后面通常还有成果节点(出具契约复核报告),核查把时间用到 +#: 最后一刻,后面的节点一开始就撞上时长上限,整次运行照样被中止。一份取 RESERVE_S 和这一段总时长的 RESERVE_SHARE 中 +#: 大的:60 秒够成果节点复核、再做一两次普通的模型调用;管理员调大了时长上限的,多半是因为节点本来就慢,余量跟着放大 RESERVE_S = 60.0 RESERVE_SHARE = 0.1 +#: 这一段不到 RESERVE_S / SHORT_SHARE(240 秒)时,一份按总时长的 SHORT_SHARE 算:时长上限调得很短(比如 180 秒)时, +#: 固定的 60 秒余量加上收尾宽限,前面的节点用掉一分钟核查就一项都开始不了,比没有这道检查时还差 +SHORT_SHARE = 0.25 +#: 下游这一段里每个还要调模型的节点再各加一份:Copilot 指南里的惯常接法是「口径卡 → 双盲复算 → 写报告 → 反方审稿 → +#: 成果」,双盲复算只留一份的话,写报告那次模型调用一慢,运行就在写报告时被中止。反方审稿也记一份:默认 600 秒的一段 +#: 一份是 60 秒,正好够它开始一句(它自己那份余量之外,还要收尾宽限和最短的查库时长各 30 秒)。估得粗,宁可多留: +#: 核查少做几项记成没核成,好过整次运行被中止 +MODEL_NODES = frozenset({NodeType.LLM, NodeType.AGENT, NodeType.SUPERVISOR, NodeType.SUBGRAPH, NodeType.REPORT, + NodeType.DUAL_CHECK, NodeType.CHALLENGE}) +#: 余量合计最多占这一段总时长的这么多:下游节点再多,核查节点也总还能开始几项 +RESERVE_CAP = 0.5 +#: 日志里的 cause:这几项没核成是因为运行时长上限将至(不再开始、收紧后到了点) +RUN_DEADLINE = "run_deadline" +#: 收紧过的核查到了点:核查 agent 自己发的 verify_limit 改成这一句(relabel) +SQUEEZED_LIMIT = ("核查已达上限(本次运行即将达到 {seconds:g} 秒的时长上限,时长已按剩余时间收紧到 {timeout:g} 秒)," + "已停止查询,按已有结果收尾") @dataclass(frozen=True) @@ -56,9 +79,35 @@ def current() -> Segment | None: return _SEGMENT.get() -def reserve(seconds: float) -> float: - """给下游节点留的余量(秒),理由见 RESERVE_S。""" - return max(RESERVE_S, RESERVE_SHARE * seconds) +def reserve(seconds: float, after: int = 0) -> float: + """给下游节点留的余量(秒):一份给收尾的节点,after 是下游这一段里还要调模型的节点数(models_after),各再加一份; + 合计不超过总时长的 RESERVE_CAP。一份多大、为什么,见 RESERVE_S、SHORT_SHARE、MODEL_NODES。""" + share = max(min(RESERVE_S, SHORT_SHARE * seconds), RESERVE_SHARE * seconds) + return min(share * (1 + max(after, 0)), RESERVE_CAP * seconds) + + +def models_after(spec: GraphSpec, node_id: str) -> int: + """这个节点下游、这一段执行里还要调模型的节点(MODEL_NODES)有几个。沿连线往下走,人工审批节点不往后走:审批之后是 + 新的一段,重新计时。分支汇合、环路里的节点只算一次,自己不算;分支上不一定执行的也算上,宁可多留。""" + nodes = spec.node_map() + children: dict[str, list[str]] = {} + for edge in spec.edges: + children.setdefault(edge.source, []).append(edge.target) + seen: set[str] = {node_id} + stack = list(children.get(node_id, [])) + count = 0 + while stack: + cur = stack.pop() + node = nodes.get(cur) + if cur in seen or node is None: + continue + seen.add(cur) + if node.type == NodeType.HUMAN: + continue + if node.type in MODEL_NODES: + count += 1 + stack.extend(children.get(cur, [])) + return count def _now() -> float: @@ -77,9 +126,10 @@ class Allot: seconds: float | None = None -def allot(timeout_s: float | None, *, grace: float) -> Allot: +def allot(timeout_s: float | None, *, grace: float, after: int = 0) -> Allot: """开始一项核查之前问一句:够不够,这一项的时长上限收紧到多少。timeout_s 是这一项配置的时长上限(None 不限), - grace 是到点之后收尾那次调用的宽限(verify_agent.SETTLE_GRACE_S)。 + grace 是到点之后收尾那次调用的宽限(verify_agent.SETTLE_GRACE_S),after 是下游这一段里还要调模型的节点数 + (models_after,按它多留余量)。 一项最坏要用「时长上限 + 收尾宽限」。剩下的时间先扣掉给下游的余量、再扣掉收尾宽限,就是这一项查库最多能用的时长: 比配置的短就收紧到它(取到 0.1 秒,往下取:时间线上写「时长上限 69.7 秒」,不写一串小数)。收紧之后不到 @@ -89,7 +139,7 @@ def allot(timeout_s: float | None, *, grace: float) -> Allot: found = _SEGMENT.get() if found is None: return Allot(True, timeout_s) - room = math.floor((found.deadline - _now() - reserve(found.seconds) - grace) * 10) / 10 + room = math.floor((found.deadline - _now() - reserve(found.seconds, after) - grace) * 10) / 10 limit = room if timeout_s is None else min(timeout_s, room) floor = grace if timeout_s is None else min(timeout_s, grace) if limit < floor: @@ -97,4 +147,21 @@ def allot(timeout_s: float | None, *, grace: float) -> Allot: return Allot(True, limit, tightened=timeout_s is None or room < timeout_s, seconds=found.seconds) -__all__ = ["RESERVE_S", "RESERVE_SHARE", "Allot", "Segment", "allot", "current", "reserve", "segment"] +def relabel(emit: Callable[..., Any], allot: Allot) -> Callable[..., Any]: + """交给核查 agent 的 emit。这一项的时长收紧过时,它到点自己发的 verify_limit 写的是收紧后的秒数(「时长上限 52.3 + 秒」),和任何设置都对不上,读的人会去找这个数是在哪儿配的、甚至去调大它:改说运行时长上限将至,带上 cause 和这一段 + 的总时长。别的事件、别的上限原样转发;没收紧时原样交回 emit。""" + if not allot.tightened: + return emit + + def wrapped(kind: Any, **data: Any) -> Any: + if str(kind) == EventType.LOG and data.get("code") == "verify_limit" and data.get("limit") == "timeout_s": + data = {**data, "cause": RUN_DEADLINE, "run_seconds": allot.seconds, + "message": SQUEEZED_LIMIT.format(seconds=allot.seconds, timeout=allot.timeout_s)} + return emit(kind, **data) + + return wrapped + + +__all__ = ["MODEL_NODES", "RESERVE_CAP", "RESERVE_S", "RESERVE_SHARE", "RUN_DEADLINE", "SHORT_SHARE", "Allot", + "Segment", "allot", "current", "models_after", "relabel", "reserve", "segment"] diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index bbc0e9d7..b5c555e3 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -36,8 +36,9 @@ 逐句串行反查:并行的话几句的工具调用在时间线上交错,看不出哪条查询在审哪一句;每日余额按先后扣,结果也才确定。 串行就可能撞上运行的时长上限(默认 5 句、每句 90 秒再加 30 秒收尾宽限,最坏 600 秒,正好是一段执行的上限),整次运行 被中止。所以每句开始之前看这一段执行还剩多少时间(engine/deadline.py):给下游留出余量后不够再审一句,这句和后面的 -都不再开始,记为未能反查(challenge_unreviewed 写明运行时长上限将至),节点照常收尾;够的话这句的时长上限按剩余时间 -收紧,到点照样按查到的结果收尾。 +都不再开始,记为未能反查,节点照常收尾;够的话这句的时长上限按剩余时间收紧,到点照样按查到的结果收尾。这两种在 +时间线上都带 cause: run_deadline(没开始的另起一条 challenge_unreviewed,收紧后到点的是 challenge_limit 里单独的 +一条),界面据此说运行时长上限将至,不叫人调大每句的上限,也不叫人去查模型和数据源。 整个反查连同存审稿记录、发日志都在一个 @task 里:接着跑时取 checkpoint 里的结果,不重复查库、不重复记账,时间线 上也不再发一遍。结果带着输入指纹,接着跑时改了审稿本身的配置就重审;同一次执行里的重试沿用已经审完的(见 _reviewed)。 """ @@ -98,7 +99,7 @@ _REVIEWS_KEY = "challenge_reviews" _LATE_CLAIM = "本次运行即将达到 {seconds:g} 秒的时长上限,为保证运行按时完成,这句未反查" _LATE_REST = "本次运行即将达到 {seconds:g} 秒的时长上限,为保证运行按时完成,剩下 {count} 句未审" #: 时长按剩余时间收紧过的句子触了时长上限:单记一类(_limit_words 说运行时长上限将至,不说「每句时长上限 90 秒」—— -#: 那会叫人调大每句的上限,调大了只会更快撞线),challenge_limit 照记 timeout_s,界面认得 +#: 那会叫人调大每句的上限,调大了只会更快撞线),challenge_limit 照记 timeout_s(老界面认得),另带 cause _SQUEEZED = "squeezed" CHALLENGE_SYSTEM = ( @@ -518,7 +519,7 @@ class _Review: max_claims, max_queries, timeout_s, daily_max_usd self.spec, self.masks, self.writer_model = spec, masks, writer_model self.fetch = _fetcher() - #: 这一段执行的总时长(engine/deadline.py):有句子的时长按剩余时间收紧过时记下,那几句触顶时说明里写 + #: 这一段执行的总时长(engine/deadline.py):有句子的时长按剩余时间收紧过、或者不再开始时记下,说明和日志里写 self.run_seconds: float | None = None #: 这份报告目录里的查询编号:摘录里指向它们的写法标上「原报告」(mark_original) self.queries = {a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"} @@ -575,6 +576,9 @@ class _Review: unreviewed: list[str] = [] # 没能反查的句子各自的原因(触顶的由 challenge_limit 说,不在这里) partial: list[list[str]] = [] # 反查了、但该查的源有停用了的句子:各自没查成的源 late: tuple[str, str] | None = None # 运行时长上限将至:(每句的理由, 时间线上的原因),后面的句子都照这个记 + overdue = 0 # 因此没开始的句数:时间线另起一条(带 cause),不混进 unreviewed + # 下游这一段里还要调模型的节点:各多留一份余量(engine/deadline.py)。图在运行里不变,算一次 + after = deadline.models_after(ctx.run.spec, ctx.node.id) def unreached(cand: judging.Candidate, rationale: str, why: str) -> None: claims.append(_claim(cand, "inconclusive", rationale)) @@ -589,7 +593,8 @@ class _Review: unreached(cand, f"反查未完成:{broken}", broken) continue if late is not None: - unreached(cand, *late) + claims.append(_claim(cand, "inconclusive", late[0])) + overdue += 1 continue try: own = claim_sources(cand.aliases, self.catalog, self.fetch) @@ -613,19 +618,22 @@ class _Review: continue # 派出去之前看这一段执行还剩多少时间:给下游留出余量后不够再审一句,这句和后面的都不再开始、记为未能反查, # 节点照常收尾,运行不被中止;够的话这句的时长上限按剩余时间收紧。在 @task 里定,接着运行时不再重新定 - allot = deadline.allot(self.timeout_s, grace=verify_agent.SETTLE_GRACE_S) + allot = deadline.allot(self.timeout_s, grace=verify_agent.SETTLE_GRACE_S, after=after) if not allot.start: + self.run_seconds = allot.seconds late = (_LATE_CLAIM.format(seconds=allot.seconds), _LATE_REST.format(seconds=allot.seconds, count=len(self.picked) - at)) - unreached(cand, *late) + claims.append(_claim(cand, "inconclusive", late[0])) + overdue += 1 continue if allot.tightened: self.run_seconds = allot.seconds + # 收紧过的:核查 agent 到点自己发的 verify_limit 改说运行时长上限将至(deadline.relabel),不写收紧后的秒数 outcome = await run_verify_agent( spec=self.spec, system=CHALLENGE_SYSTEM, task=task_text, sources=live, output_schema=CHALLENGE_SCHEMA, max_queries=self.max_queries, timeout_s=allot.timeout_s, - daily_max_usd=self.daily_max_usd, emit=ctx.emit, run_id=ctx.run.run_id, node_id=ctx.node.id, - data_versions=run_versions(ctx.run)) + daily_max_usd=self.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id, + node_id=ctx.node.id, data_versions=run_versions(ctx.run)) if outcome.error and not outcome.model: broken = outcome.error # 连模型都没建成(没配好、接入出错):换一句也一样 model = model or outcome.model @@ -655,9 +663,9 @@ class _Review: unreviewed.append(why) counts = {v: sum(1 for c in claims if c["verdict"] == v) for v in VERDICTS} - # 没能反查的句数:没派出去、派出去却没查成的(unreviewed 里各有原因),加上触顶时没能给出结论的。都记为无法判断, - # 可「无法判断 2 句」既可能是查了、数据不够,也可能是根本没查:审批卡要分开说 - missed = len(unreviewed) + sum(skipped for _, skipped in hit.values()) + # 没能反查的句数:没派出去、派出去却没查成的(unreviewed 里各有原因),运行时长上限将至没开始的,加上触顶时没能给出 + # 结论的。都记为无法判断,可「无法判断 2 句」既可能是查了、数据不够,也可能是根本没查:审批卡要分开说 + missed = len(unreviewed) + overdue + sum(skipped for _, skipped in hit.values()) artifact = await self._store({"report": self.report_id, "doc_artifact": self.doc_artifact, "claims": claims, "model": model}) for limit, (touched, skipped) in hit.items(): @@ -665,13 +673,24 @@ class _Review: message = f"已达上限({words}),{skipped} 句未反查" if limit == "daily_max_usd" else \ f"有 {touched} 句反查已达上限({words}),已按查到的结果收尾" + \ (f",其中 {skipped} 句未能给出结论" if skipped else "") + # 收紧过的那一组照记 timeout_s(老界面认得),另带 cause、这一段的总时长和触顶的句数:界面据此说运行时长 + # 上限将至,不叫人调高每句的上限;同一节点里还有句子触了自己的上限时,两条也分得开 + squeezed = {"cause": deadline.RUN_DEADLINE, "run_seconds": self.run_seconds, "claims": touched} \ + if limit == _SQUEEZED else {} ctx.emit(EventType.LOG, level="warn", code="challenge_limit", - limit="timeout_s" if limit == _SQUEEZED else limit, claims_skipped=skipped, message=message) + limit="timeout_s" if limit == _SQUEEZED else limit, claims_skipped=skipped, message=message, + **squeezed) if unreviewed: more = "等" if len(set(unreviewed)) > 1 else "" ctx.emit(EventType.LOG, level="warn", code="challenge_unreviewed", claims_skipped=len(unreviewed), reason=unreviewed[0], message=f"反方审稿有 {len(unreviewed)} 句结论未能反查,已记为无法判断:{unreviewed[0]}{more}") + if overdue and late is not None: + # 运行时长上限将至没开始的另起一条,带 cause:不被前面句子的原因盖掉、藏进「等」里,界面也不会为它叫人去查 + # 模型和数据源 + ctx.emit(EventType.LOG, level="warn", code="challenge_unreviewed", cause=deadline.RUN_DEADLINE, + run_seconds=self.run_seconds, claims_skipped=overdue, reason=late[1], + message=f"反方审稿有 {overdue} 句结论未能反查,已记为无法判断:{late[1]}") if partial: names = list(dict.fromkeys(s for gone in partial for s in gone)) ctx.emit(EventType.LOG, level="warn", code="challenge_sources_lost", claims=len(partial), sources=names, diff --git a/backend/app/engine/nodes/dual_check.py b/backend/app/engine/nodes/dual_check.py index 02b95481..1b2506fb 100644 --- a/backend/app/engine/nodes/dual_check.py +++ b/backend/app/engine/nodes/dual_check.py @@ -24,7 +24,8 @@ SQL,就跟着差异说明进了审批卡、对照工件和目录日志。完 按发起审批时的目录各会往目录里写什么、不写的为什么(dual_check_facts.preview,和真写同一套判断);几个指标落到目录 同一项上时每种选择组合合起来会怎样写(preview_joint),放在对照工件的 catalog_joint 里。 盲算触顶、出错、交不出数,或者输入本身没法单独复算的指标按「无法复算」记缺口,发 dual_check_unchecked。运行这一段的 -时长上限将至时不再开始新的盲算、开始的按剩余时间收紧时长(engine/deadline.py),没来得及的照此记,运行不被中止。 +时长上限将至时不再开始新的盲算、开始的按剩余时间收紧时长(engine/deadline.py),没来得及的照此记,运行不被中止; +这几个另发一条 dual_check_unchecked,带 cause: run_deadline,界面据此说原因、给对的下一步。 **产出的状态。** 全部一致是 agreed;没有不一致、但有无法复算的:一个都没核成是 unchecked,核成了一部分是 partial; 有不一致、人裁决过的是 resolved。产出和台账都带 counts(一致、不一致、无法复算各几个)。 @@ -121,6 +122,9 @@ GAP_UNCHECKED = "指标「{name}」无法复算:{reason}" #: 运行时长上限将至(engine/deadline.py):没来得及开始的盲算、时长按剩余时间收紧后到了点的盲算,都按无法复算记 LATE = "本次运行即将达到 {seconds:g} 秒的时长上限,为保证运行按时完成,未进行盲算" SQUEEZED = "本次运行即将达到 {seconds:g} 秒的时长上限,盲算时长已按剩余时间收紧,到时未能完成" +#: 这几个指标另起的那条 dual_check_unchecked(带 cause)的原话 +LATE_UNCHECKED = ("本次运行即将达到 {seconds:g} 秒的时长上限,为保证运行按时完成,双盲复算有 {count} 个指标未能复算:" + "{names},按无法复算计入缺口") BLIND_SYSTEM = ( "你在做双盲复算:另一条流程已经算出了口径卡上的一个指标,你要在看不到它的 SQL 和结果的情况下,按业务定义独立算出" @@ -418,7 +422,7 @@ async def _check(state: GraphState, ctx: NodeContext, targets: list[dict[str, An if agreed and not any(r["status"] == DIFFERENT for r in rows): ctx.emit(EventType.LOG, level="info", code="dual_check_agreed", metrics=agreed, message=f"双盲复算一致:{agreed} 个指标两边的结果相同,已放行") - if unchecked := [r for r in rows if r["status"] == UNCHECKED]: + if unchecked := [r for r in rows if r["status"] == UNCHECKED and not r.get("cause")]: # 结构上没法单独复算的(合并查询、遮罩列)以前一条 dual_check_* 日志都没有,时间线上看不出为什么没核成 parts = ";".join(f"「{r['name']}」({r.get('reason') or '原因不明'})" for r in unchecked[:5]) ctx.emit(EventType.LOG, level="warn", code="dual_check_unchecked", @@ -426,6 +430,15 @@ async def _check(state: GraphState, ctx: NodeContext, targets: list[dict[str, An for r in unchecked], message=f"双盲复算未能核对 {len(unchecked)} 个指标:{parts}" + ("等" if len(unchecked) > 5 else "") + ",按无法复算计入缺口") + if late := [r for r in rows if r["status"] == UNCHECKED and r.get("cause")]: + # 运行时长上限将至没核成的(_blind 记下 cause)另起一条:界面据此说原因、给下一步,调高每项的上限只会更快撞线 + seconds = late[0].get("run_seconds") + ctx.emit(EventType.LOG, level="warn", code="dual_check_unchecked", cause=late[0]["cause"], run_seconds=seconds, + metrics=[{"id": r["metric_id"], "name": r["name"], "reason": r.get("reason") or "原因不明"} + for r in late], + message=LATE_UNCHECKED.format(seconds=seconds or 0, count=len(late), + names="".join(f"「{r['name']}」" for r in late[:5]) + + ("等" if len(late) > 5 else ""))) return {"rows": rows, "results": results, "model": model, "same_model": same, "usage": usage} @@ -730,22 +743,27 @@ async def _blind(ctx: NodeContext, jobs: dict[str, dict[str, Any]], options: _Op catalogs[job["source"]] = await _catalog_text(sources, job["source"]) outcomes: dict[str, Any] = {} squeezed: dict[str, float] = {} # 时长按剩余时间收紧过的盲算 → 这一段执行的总时长(到了点时说明里写) + late: dict[str, float] = {} # 运行时长上限将至、没开始的盲算 → 同上 + # 下游这一段里还要调模型的节点(写报告、反方审稿……):各多留一份余量(engine/deadline.py) + after = deadline.models_after(ctx.run.spec, ctx.node.id) async def one(key: str, job: dict[str, Any]) -> None: # 给下游留出余量后不够再做一项:不再开始,交回一个没有模型的结果,按无法复算记——第一项就不够时,和核查模型 # 用不了一样,后面的不再一个个去试。在 @task 里定,重放时不再重新定 - allot = deadline.allot(options.timeout_s, grace=verify_agent.SETTLE_GRACE_S) + allot = deadline.allot(options.timeout_s, grace=verify_agent.SETTLE_GRACE_S, after=after) if not allot.start: + late[key] = allot.seconds outcomes[key] = VerifyOutcome(result=None, cells=[], queries=[], rationale=None, usage={}, error=LATE.format(seconds=allot.seconds)) return if allot.tightened: squeezed[key] = allot.seconds + # 收紧过的:核查 agent 到点自己发的 verify_limit 改说运行时长上限将至(deadline.relabel),不写收紧后的秒数 outcomes[key] = await run_verify_agent( spec=spec, system=BLIND_SYSTEM, task=_task_text(job, catalogs[job["source"]]), sources=[job["source"]], output_schema=BLIND_SCHEMA, max_queries=options.max_queries, timeout_s=allot.timeout_s, - daily_max_usd=options.daily_max_usd, emit=ctx.emit, run_id=ctx.run.run_id, node_id=ctx.node.id, - data_versions=run_versions(ctx.run)) + daily_max_usd=options.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id, + node_id=ctx.node.id, data_versions=run_versions(ctx.run)) keys = list(jobs) await one(keys[0], jobs[keys[0]]) @@ -767,10 +785,14 @@ async def _blind(ctx: NodeContext, jobs: dict[str, dict[str, Any]], options: _Op results[key] = dict(results[keys[0]]) continue results[key] = _slim(outcome) - if key in squeezed and results[key]["limit"] == "timeout_s": + if key in late: + # 没开始的:带上 cause 和这一段的总时长,指标按它记无法复算时时间线另起一条(_check) + results[key].update(cause=deadline.RUN_DEADLINE, run_seconds=late[key]) + elif key in squeezed and results[key]["limit"] == "timeout_s": # 收紧过的时长到了点:照样按无法复算记(同触了每项时长上限),原因说运行时长上限将至。不留 limit,免得发 # dual_check_limit——它叫人调高每项的上限,调高了只会更快撞线 - results[key].update(value=None, limit=None, reason=SQUEEZED.format(seconds=squeezed[key])) + results[key].update(value=None, limit=None, reason=SQUEEZED.format(seconds=squeezed[key]), + cause=deadline.RUN_DEADLINE, run_seconds=squeezed[key]) for k in usage: usage[k] += outcome.usage.get(k) or 0 if outcome.model and model is None: @@ -900,7 +922,8 @@ def _compare(row: dict[str, Any], metric: dict[str, Any], results: dict[str, Any row.update(status=UNCHECKED, reason="盲算" + _limit_text(found["limit"], options)) return if found.get("value") is None: - row.update(status=UNCHECKED, reason=found.get("reason") or "盲算没有交回可用的数值") + row.update(status=UNCHECKED, reason=found.get("reason") or "盲算没有交回可用的数值", + **{k: found[k] for k in ("cause", "run_seconds") if k in found}) # 运行时长上限将至(_blind) return values[inp["path"]] = found["value"] outcome = recompute.recompute(metric, recompute.Changes(inputs=values), recompute.Sources( diff --git a/backend/tests/test_run_deadline.py b/backend/tests/test_run_deadline.py index 06c83407..2275163d 100644 --- a/backend/tests/test_run_deadline.py +++ b/backend/tests/test_run_deadline.py @@ -2,8 +2,10 @@ 问 allot() 够不够、时长收紧到多少。 - 没有截止时刻(不经 runner 执行):原样放行,时长上限还是传进来的那个值 -- 有截止时刻:先给下游留出余量(60 秒或这一段总时长的 10%,取大的)、再扣收尾宽限;比配置的时长短就收紧,收紧之后 - 不到 min(配置的时长, 收尾宽限) 就不再开始;配置的是不限时也按剩余时间收紧 +- 有截止时刻:先给下游留出余量(一份取 60 秒或这一段总时长的 10% 中大的,这一段不到 240 秒时按 25%;下游这一段里每个 + 还要调模型的节点再各加一份,合计不超过一半)、再扣收尾宽限;比配置的时长短就收紧,收紧之后不到 min(配置的时长, + 收尾宽限) 就不再开始;配置的是不限时也按剩余时间收紧 +- 收紧过的核查到了点,核查 agent 自己发的 verify_limit 改说运行时长上限将至,带上 cause(relabel) - runner 每一段执行都放一份:节点、节点里的 @task 都看得到,截止时刻就是 asyncio.timeout 的那一个;审批恢复后的那一段 是新的截止时刻;运行结束后清掉 """ @@ -16,11 +18,12 @@ from langgraph.func import task from sqlalchemy import select from app.core.config import settings +from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import Approval, Run from app.engine import compiler, deadline from app.engine.runner import run_manager -from app.engine.schema import NodeType +from app.engine.schema import GraphSpec, NodeType class Clock: @@ -100,12 +103,99 @@ def test_the_reserve_is_sixty_seconds_or_a_tenth(seconds, kept): assert deadline.reserve(seconds) == kept +@pytest.mark.parametrize("seconds, kept", [(240, 60.0), (200, 50.0), (180, 45.0), (120, 30.0), (60, 15.0)]) +def test_a_short_segment_keeps_a_quarter(seconds, kept): + """这一段不到 240 秒:一份余量按总时长的 25% 算,不再固定 60 秒——不然 180 秒的一段,前面的节点用掉 60 秒之后核查 + 一项都开始不了,比没有这道检查时还差。""" + assert deadline.reserve(seconds) == kept + + +@pytest.mark.parametrize("seconds, after, kept", [ + (600, 1, 120.0), (600, 2, 180.0), (600, 4, 300.0), (600, 9, 300.0), # 最多留一半 + (1800, 2, 540.0), (180, 1, 90.0), (180, 3, 90.0), +]) +def test_each_model_node_downstream_adds_a_share(seconds, after, kept): + """下游这一段里每个还要调模型的节点再各加一份:双盲复算后面接着写报告、反方审稿时,只留 60 秒的话写报告的那次模型 + 调用一慢,运行照样被中止。合计不超过这一段总时长的一半,核查节点总还能开始几项。""" + assert deadline.reserve(seconds, after=after) == kept + + async def test_the_reserve_follows_the_segment_length(clock): """时长上限调到 1800 秒:给下游留 180 秒,剩 300 秒时这一项最多 90 秒(300 − 180 − 30)。""" async with deadline.segment(clock.now + 300, 1800): assert deadline.allot(120, grace=30.0).timeout_s == pytest.approx(90.0) +async def test_downstream_model_nodes_take_their_share(clock): + """下游还有写报告、反方审稿:剩 300 秒时给下游留 180 秒,这一项最多 90 秒(300 − 180 − 30);只剩 230 秒时不再开始。""" + async with deadline.segment(clock.now + 300, 600): + assert deadline.allot(120, grace=30.0, after=2) == deadline.Allot(True, 90.0, tightened=True, seconds=600) + assert deadline.allot(120, grace=30.0).timeout_s == 120, "下游没有调模型的节点:照旧只留一份" + async with deadline.segment(clock.now + 239, 600): + assert not deadline.allot(120, grace=30.0, after=2).start + + +async def test_a_short_segment_still_starts(clock): + """单次运行时限只有 150 秒、前面用掉 50 秒:留 37.5 秒余量、30 秒收尾宽限,这一项还能查 32.5 秒,照样开始。""" + async with deadline.segment(clock.now + 100, 150): + answer = deadline.allot(90, grace=30.0) + assert answer.start and answer.timeout_s == pytest.approx(32.5) and answer.tightened + + +def spec_of(*nodes, edges) -> GraphSpec: + return GraphSpec.model_validate({"nodes": [node(nid, ntype) for nid, ntype in nodes], + "edges": [{"source": a, "target": b} for a, b in edges]}) + + +def test_model_nodes_after_count_this_segment_only(): + """数下游这一段里还要调模型的节点:写报告、反方审稿、模型、Agent……各算一个;取数、整形、成果这类不算;人工审批之后 + 是新的一段(重新计时),不算;分支汇合到同一个节点只算一次。""" + spec = spec_of(("start", "input"), ("fetch", "tool"), ("card", "metrics"), ("dual", "dual_check"), + ("write", "report"), ("chal", "challenge"), ("shape", "transform"), ("out", "output"), + edges=[("start", "fetch"), ("fetch", "card"), ("card", "dual"), ("dual", "write"), + ("write", "chal"), ("chal", "shape"), ("shape", "out")]) + assert deadline.models_after(spec, "dual") == 2 + assert deadline.models_after(spec, "write") == 1 + assert deadline.models_after(spec, "chal") == 0 + gated = spec_of(("dual", "dual_check"), ("ask", "llm"), ("gate", "human"), ("write", "report"), ("out", "output"), + edges=[("dual", "ask"), ("dual", "gate"), ("gate", "write"), ("write", "out"), ("ask", "out")]) + assert deadline.models_after(gated, "dual") == 1, "人工审批之后的写报告在下一段,不占这一段的时间" + forked = spec_of(("dual", "dual_check"), ("a", "agent"), ("b", "agent"), ("join", "report"), + edges=[("dual", "a"), ("dual", "b"), ("a", "join"), ("b", "join")]) + assert deadline.models_after(forked, "dual") == 3 + looped = spec_of(("chal", "challenge"), ("ask", "llm"), edges=[("chal", "ask"), ("ask", "chal")]) + assert deadline.models_after(looped, "chal") == 1, "环回到自己:自己不算" + assert deadline.models_after(looped, "nowhere") == 0 + + +async def test_a_squeezed_check_says_the_run_deadline_when_it_stops(): + """收紧过的核查到了点:核查 agent 发的 verify_limit 写的是收紧后的秒数(「时长上限 52.3 秒」),和任何设置都对不上。 + relabel 包住 emit,把这一条改说运行时长上限将至、带上 cause 和这一段的总时长;别的事件、别的上限原样转发;没收紧 + 时原样交回 emit。""" + sent: list[tuple[str, dict]] = [] + + def emit(kind, **data): + sent.append((str(kind), data)) + + assert deadline.relabel(emit, deadline.Allot(True, 90)) is emit + assert deadline.relabel(emit, deadline.Allot(True, 90, seconds=600)) is emit, "有截止时刻但没收紧:照旧" + wrapped = deadline.relabel(emit, deadline.Allot(True, 52.3, tightened=True, seconds=600)) + wrapped(EventType.LOG, level="warn", code="verify_limit", limit="timeout_s", + message="核查已达上限(时长上限 52.3 秒),已停止查询,按已有结果收尾") + wrapped(EventType.LOG, level="warn", code="verify_limit", limit="max_queries", + message="核查已达上限(最多查询 6 次),已停止查询,按已有结果收尾") + wrapped(EventType.TOOL_END, tool="db_query__shop", limit="timeout_s") + squeezed, queries, tool = sent + assert squeezed == (EventType.LOG, { + "level": "warn", "code": "verify_limit", "limit": "timeout_s", "cause": deadline.RUN_DEADLINE, + "run_seconds": 600, + "message": "核查已达上限(本次运行即将达到 600 秒的时长上限,时长已按剩余时间收紧到 52.3 秒),已停止查询," + "按已有结果收尾"}) + assert deadline.RUN_DEADLINE == "run_deadline" + assert queries[1]["message"] == "核查已达上限(最多查询 6 次),已停止查询,按已有结果收尾" and "cause" not in queries[1] + assert tool == (EventType.TOOL_END, {"tool": "db_query__shop", "limit": "timeout_s"}) + + async def test_the_segment_is_cleared_on_the_way_out(clock): async with deadline.segment(clock.now + 600, 600): assert deadline.current() == deadline.Segment(clock.now + 600, 600.0) diff --git a/backend/tests/test_verify_deadline.py b/backend/tests/test_verify_deadline.py index cfa205f3..cd7aca75 100644 --- a/backend/tests/test_verify_deadline.py +++ b/backend/tests/test_verify_deadline.py @@ -5,6 +5,10 @@ 运行时长上限将至,不说「每句时长上限 90 秒」;接着运行时取 checkpoint 里的审稿,不重审 - 双盲复算:同理,没来得及开始的盲算按无法复算记(照现有的路径记缺口);收紧过的时长到点也按无法复算记,不发提醒 调高上限的 dual_check_limit;审批恢复后重放,盲算不再执行、哪几项没来得及照旧 +- 因运行时长上限将至没核成的,时间线另起一条、带 cause: run_deadline 和这一段的总时长(run_seconds):界面据此说清 + 原因、给对的下一步(调高核查自己的上限只会更快撞线,检查模型、数据源也无济于事),不和别的原因混在一条里、被「等」 + 盖掉;收紧过的核查 agent 自己发的 verify_limit 也改说运行时长上限将至 +- 下游还有写报告这类要调模型的节点时,核查给它们多留一份余量:双盲复算后面接着写报告,运行不在写报告时被中止 - 时间充裕时(默认 600 秒的一段)时长上限照传原值:和没有这道检查时一样 两种钟:精确的计数用假的钟(每派出一项就把 allot() 看到的钟往前拨若干秒,真实只过去几毫秒,runner 的 asyncio.timeout @@ -24,10 +28,11 @@ from app.core import artifact_store from app.core.config import settings from app.db.base import SessionLocal from app.db.models import Approval, Run -from app.engine import deadline, verify_agent +from app.engine import compiler, deadline, verify_agent from app.engine.nodes import challenge as challenge_mod from app.engine.nodes import dual_check as dual_check_mod from app.engine.runner import run_manager, verify_manifest +from app.engine.schema import NodeType from app.main import app from app.providers.mock_model import MockChatModel from tests.fixtures.dual_check_shop import WEEK, Blind, add_source, build @@ -116,6 +121,7 @@ async def test_five_claims_with_time_for_two_review_two(monkeypatch, shop): [warn] = await logs(row.id, "challenge_unreviewed") assert warn["claims_skipped"] == 3 and warn["reason"] == LATE_REST + assert warn["cause"] == deadline.RUN_DEADLINE and warn["run_seconds"] == 600, "界面按 cause 说原因、给下一步" assert warn["message"] == f"反方审稿有 3 句结论未能反查,已记为无法判断:{LATE_REST}" assert await logs(row.id, "challenge_limit") == [], "没有哪一句触了自己的上限" [done] = await logs(row.id, "challenge_done") @@ -137,13 +143,52 @@ async def test_a_claim_squeezed_by_the_deadline_says_so(monkeypatch, shop): [limit] = await logs(row.id, "challenge_limit") assert limit["limit"] == "timeout_s" and limit["claims_skipped"] == 0 + assert limit["cause"] == deadline.RUN_DEADLINE and limit["run_seconds"] == 600 and limit["claims"] == 1, \ + "界面认 cause:不叫人调高每句的上限" assert "本次运行即将达到 600 秒的时长上限" in limit["message"] and "收紧" in limit["message"] assert "每句时长上限" not in limit["message"] out = await output_of(row.id, "chal") assert out["counts"]["refuted"] == 1, "收紧后按查到的结果收尾:反方给的结论照常采信" assert out["unreviewed"] == 3 [warn] = await logs(row.id, "challenge_unreviewed") - assert warn["claims_skipped"] == 3 + assert warn["claims_skipped"] == 3 and warn["cause"] == deadline.RUN_DEADLINE + + +async def test_the_run_deadline_gets_its_own_line(monkeypatch, shop): + """第 1 句反查一条查询都没执行成功(另一个原因),第 3 句起来不及审:时间线两条 challenge_unreviewed,各说各的原因。 + 运行时长上限将至的那条带 cause,不被前一句的原因盖掉、藏进「等」里——界面也就不会为了它叫人去查模型和数据源。""" + plays = dict(FIVE_PLAYS) + plays["居各区域之首"] = Play([], "weakened", "没有查询。") + models = Models(monkeypatch, shop, writer=(FIVE,), plays=plays) + Ticker(monkeypatch, challenge_mod, step=250) + row = await finish(graph(shop)) + assert row.status == "succeeded", row.error + assert models.reviewed() == ORDER[:2] + out = await output_of(row.id, "chal") + assert out["unreviewed"] == 4 and out["counts"]["inconclusive"] == 4 + + other, late = await logs(row.id, "challenge_unreviewed") + assert other["claims_skipped"] == 1 and other["reason"] == "反查没有执行成功任何查询" and "cause" not in other + assert other["message"] == "反方审稿有 1 句结论未能反查,已记为无法判断:反查没有执行成功任何查询" + assert late["claims_skipped"] == 3 and late["reason"] == LATE_REST + assert late["cause"] == deadline.RUN_DEADLINE and late["run_seconds"] == 600 + assert late["message"] == f"反方审稿有 3 句结论未能反查,已记为无法判断:{LATE_REST}" + + +async def test_its_own_limit_and_the_run_deadline_are_told_apart(monkeypatch, shop): + """第 1 句触了自己的每句时长上限(90 秒,没收紧),第 2 句的时长按剩余时间收紧后也到了点:两条 challenge_limit 都是 + timeout_s,只有收紧的那条带 cause——时间线上不是两行一样的「反方审稿已达时长上限」,叫人调高每句上限的只有第 1 条。""" + Models(monkeypatch, shop, writer=(FIVE,), plays=FIVE_PLAYS) + ticker = Ticker(monkeypatch, challenge_mod, step=440, hit=lambda timeout: True) + row = await finish(graph(shop)) + assert row.status == "succeeded", row.error + first, second = ticker.timeouts + assert first == 90 and second < 90, ticker.timeouts + own, squeezed = await logs(row.id, "challenge_limit") + assert own["limit"] == squeezed["limit"] == "timeout_s" + assert "cause" not in own and "每句时长上限 90 秒" in own["message"] + assert squeezed["cause"] == deadline.RUN_DEADLINE and squeezed["run_seconds"] == 600 and squeezed["claims"] == 1 + assert "每句时长上限" not in squeezed["message"] async def test_no_time_at_all_reviews_nothing_but_finishes(monkeypatch, shop): @@ -156,7 +201,7 @@ async def test_no_time_at_all_reviews_nothing_but_finishes(monkeypatch, shop): out = await output_of(row.id, "chal") assert out["counts"] == {"stands": 0, "weakened": 0, "refuted": 0, "inconclusive": 5} and out["unreviewed"] == 5 [warn] = await logs(row.id, "challenge_unreviewed") - assert warn["claims_skipped"] == 5 and "剩下 5 句未审" in warn["reason"] + assert warn["claims_skipped"] == 5 and "剩下 5 句未审" in warn["reason"] and warn["cause"] == deadline.RUN_DEADLINE assert row.output["_issuance"]["tier"] == "formal" @@ -215,7 +260,13 @@ async def test_a_slow_review_no_longer_aborts_the_run(monkeypatch, shop): assert out["unreviewed"] == len(late) [warn] = await logs(row.id, "challenge_unreviewed") assert warn["claims_skipped"] == len(late) and f"剩下 {len(late)} 句未审" in warn["reason"] - assert "本次运行即将达到 7 秒的时长上限" in warn["reason"] + assert "本次运行即将达到 7 秒的时长上限" in warn["reason"] and warn["cause"] == deadline.RUN_DEADLINE + # 核查 agent 到点自己也发 verify_limit:时长收紧过的那几句说运行时长上限将至,写的秒数不会是一个和设置对不上的数 + for said in await logs(row.id, "verify_limit"): + if said.get("cause") == deadline.RUN_DEADLINE: + assert "本次运行即将达到 7 秒的时长上限,时长已按剩余时间收紧到" in said["message"] and said["run_seconds"] == 7 + else: + assert "时长上限 1.5 秒" in said["message"] or said["limit"] != "timeout_s", said # -------------------------------------------------------------------------- @@ -289,10 +340,31 @@ async def test_five_metrics_with_time_for_two_check_two(monkeypatch, orders): [unchecked] = await logs(row.id, "dual_check_unchecked") assert unchecked["metrics"] == [{"id": m, "name": NAMES[m], "reason": LATE_BLIND} for m in ("buyers", "top", "low")] + assert unchecked["cause"] == deadline.RUN_DEADLINE and unchecked["run_seconds"] == 600, "界面按 cause 说原因、给下一步" + assert unchecked["message"] == ("本次运行即将达到 600 秒的时长上限,为保证运行按时完成,双盲复算有 3 个指标未能复算:" + "「付款客户数」「最大单笔」「最小单笔」,按无法复算计入缺口") assert await logs(row.id, "dual_check_limit") == [] assert (await verify_manifest(row.id))["ok"] is True +async def test_blind_checks_cut_by_the_deadline_get_their_own_line(monkeypatch, orders): + """「订单数」的盲算没查到这个数(另一个原因),后 3 项来不及开始:时间线两条 dual_check_unchecked。运行时长上限将至的 + 那条带 cause、只列那 3 项,界面据此说原因、给下一步;另一条照旧,不带 cause。""" + Blind(monkeypatch, orders, {name: plan for name, plan in PLANS.items() if name != "订单数"}) + Ticker(monkeypatch, verify_agent, step=260) + row = await dual_run(dual_graph(orders)) + assert row.status == "succeeded", row.error + out = await output_of(row.id, "dual") + assert unchecked_reasons(out) == {"orders": "盲算没有查到这个数", "buyers": LATE_BLIND, "top": LATE_BLIND, + "low": LATE_BLIND} + other, late = await logs(row.id, "dual_check_unchecked") + assert other["metrics"] == [{"id": "orders", "name": "订单数", "reason": "盲算没有查到这个数"}] and "cause" not in other + assert other["message"] == "双盲复算未能核对 1 个指标:「订单数」(盲算没有查到这个数),按无法复算计入缺口" + assert late["cause"] == deadline.RUN_DEADLINE and late["run_seconds"] == 600 + assert [m["id"] for m in late["metrics"]] == ["buyers", "top", "low"] + assert all(m["reason"] == LATE_BLIND for m in late["metrics"]) + + async def test_no_time_for_even_the_first_blind_check(monkeypatch, orders): """一开始就只剩 40 秒:第一项都不开始(后面的也就不再一个个去试),五项都按无法复算记,节点状态是一个都没核成。""" blind = Blind(monkeypatch, orders, PLANS) @@ -323,6 +395,28 @@ async def test_a_blind_check_squeezed_by_the_deadline_says_so(monkeypatch, order assert "每项时长上限" not in reasons["orders"] assert {reasons[m] for m in ("buyers", "top", "low")} == {LATE_BLIND} assert await logs(row.id, "dual_check_limit") == [] + [unchecked] = await logs(row.id, "dual_check_unchecked") + assert unchecked["cause"] == deadline.RUN_DEADLINE and [m["id"] for m in unchecked["metrics"]] == \ + ["orders", "buyers", "top", "low"], "收紧后到点的和没来得及开始的是同一个原因:一条说完" + + +def dual_report_graph(source: str) -> dict: + """取数 → 口径卡 → 双盲复算 → 写报告 → 成果:Copilot 指南里的惯常接法,双盲复算下游还有一次写报告的模型调用。""" + nodes = dual_graph(source)["nodes"] + nodes.insert(4, node("write", "report", "写周报", instructions="写周报")) + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +@pytest.mark.parametrize("build, low, high", [(dual_graph, 100, 110), (dual_report_graph, 40, 50)]) +async def test_a_report_downstream_gets_its_share(monkeypatch, orders, build, low, high): + """第 3 项开始时只剩约 200 秒:下游只有成果节点时留 60 秒余量,这一项还能查 110 秒左右;下游还要写报告时再给它留 + 60 秒,这一项只查 50 秒左右——双盲复算用到最后一刻的话,写报告那次模型调用一慢,整次运行照样被中止。""" + Blind(monkeypatch, orders, PLANS) + ticker = Ticker(monkeypatch, verify_agent, step=200) + row = await dual_run(build(orders)) + assert row.status == "succeeded", row.error + first, second, third = ticker.timeouts + assert first == second == 120 and low < third < high, ticker.timeouts async def test_resuming_after_the_review_replays_the_blind_checks(monkeypatch, orders): @@ -383,3 +477,36 @@ async def test_slow_blind_checks_no_longer_abort_the_run(monkeypatch, orders): "一项接一项:来得及的在前" [unchecked] = await logs(row.id, "dual_check_unchecked") assert {m["id"] for m in unchecked["metrics"]} == set(reasons) + + +async def test_a_slow_report_after_the_blind_checks_still_finishes(monkeypatch, orders): + """真的钟:双盲复算 → 写报告 → 成果,这一段只有 10 秒。盲算一项接一项,每项都查到时长用完(查不完的查询);写报告要 + 3 秒。双盲复算给下游的写报告多留了一份余量(4 秒),收尾之后报告写得完,运行照常完成——只留一份(2 秒)的话,盲算 + 用到最后,报告写到一半整次运行就被中止。几项来得及随机器快慢变,只断言不变的。""" + monkeypatch.setattr(settings, "max_run_seconds", 10) + monkeypatch.setattr(deadline, "RESERVE_S", 2.0) + monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.4) + monkeypatch.setattr(dual_check_mod, "BLIND_CONCURRENCY", 1) + + async def slow_model(session, spec): + return MockChatModel(model_name="slow-verify", latency=0.1), "slow-verify" + + monkeypatch.setattr(verify_agent, "get_chat_model", slow_model) + endless = [f"SELECT COUNT(*) AS blind_v FROM orders WHERE id > {i}" for i in range(300)] + Blind(monkeypatch, orders, {name: (endless, "blind_v") for name in NAMES.values()}) + wrote: list[float] = [] + + async def slow_report(state, ctx): + await asyncio.sleep(3.0) + wrote.append(asyncio.get_running_loop().time()) + return {} + + monkeypatch.setitem(compiler.RUNNERS, NodeType.REPORT, slow_report) + row = await dual_run(dual_report_graph(orders), + options={"verify": {"dual_check": {"timeout_s": 3, "max_queries": None}}}) + assert row.status == "succeeded", row.error + assert "时长上限,已中止" not in (row.error or "") + assert len(wrote) == 1, "报告写完了" + out = await output_of(row.id, "dual") + reasons = unchecked_reasons(out) + assert any("本次运行即将达到 10 秒的时长上限" in why for why in reasons.values()), reasons